diff options
Diffstat (limited to 'taskcluster')
143 files changed, 14228 insertions, 0 deletions
diff --git a/taskcluster/ci/artifact-build/kind.yml b/taskcluster/ci/artifact-build/kind.yml new file mode 100644 index 000000000..46e522ff5 --- /dev/null +++ b/taskcluster/ci/artifact-build/kind.yml @@ -0,0 +1,39 @@ +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, You can obtain one at http://mozilla.org/MPL/2.0/. + +implementation: taskgraph.task.transform:TransformTask + +transforms: + - taskgraph.transforms.build_attrs:transforms + - taskgraph.transforms.job:transforms + - taskgraph.transforms.task:transforms + +jobs: + linux64-artifact/opt: + description: "Linux64 Opt Artifact Build" + index: + product: firefox + job-name: linux64-artifact-opt + treeherder: + platform: linux64/opt + kind: build + symbol: AB + tier: 2 + worker-type: aws-provisioner-v1/gecko-{level}-b-linux + worker: + implementation: docker-worker + docker-image: {in-tree: desktop-build} + max-run-time: 36000 + run: + using: mozharness + actions: [get-secrets build generate-build-stats] + config: + - builds/releng_sub_linux_configs/64_artifact.py + - balrog/production.py + script: "mozharness/scripts/fx_desktop_build.py" + secrets: true + tooltool-downloads: public + need-xvfb: true + keep-artifacts: false + diff --git a/taskcluster/ci/build-signing/kind.yml b/taskcluster/ci/build-signing/kind.yml new file mode 100644 index 000000000..89d6e0220 --- /dev/null +++ b/taskcluster/ci/build-signing/kind.yml @@ -0,0 +1,8 @@ +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, You can obtain one at http://mozilla.org/MPL/2.0/. + +implementation: 'taskgraph.task.signing:SigningTask' + +kind-dependencies: + - build diff --git a/taskcluster/ci/build/android.yml b/taskcluster/ci/build/android.yml new file mode 100644 index 000000000..74088e3d3 --- /dev/null +++ b/taskcluster/ci/build/android.yml @@ -0,0 +1,137 @@ +android-api-15/debug: + description: "Android 4.0 API15+ Debug" + index: + product: mobile + job-name: + buildbot: android-api-15-debug + gecko-v2: android-api-15-debug + treeherder: + platform: android-4-0-armv7-api15/debug + symbol: tc(B) + worker-type: aws-provisioner-v1/gecko-{level}-b-android + worker: + implementation: docker-worker + max-run-time: 7200 + run: + using: mozharness + actions: [get-secrets build multi-l10n update] + config: + - builds/releng_base_android_64_builds.py + - disable_signing.py + - platform_supports_post_upload_to_latest.py + script: "mozharness/scripts/fx_desktop_build.py" + custom-build-variant-cfg: api-15-debug + tooltool-downloads: internal + +android-x86/opt: + description: "Android 4.2 x86 Opt" + index: + product: mobile + job-name: android-x86-opt + treeherder: + platform: android-4-2-x86/opt + symbol: tc(B) + tier: 1 + worker-type: aws-provisioner-v1/gecko-{level}-b-android + worker: + implementation: docker-worker + max-run-time: 7200 + run: + using: mozharness + actions: [get-secrets build multi-l10n update] + config: + - builds/releng_base_android_64_builds.py + - disable_signing.py + - platform_supports_post_upload_to_latest.py + script: "mozharness/scripts/fx_desktop_build.py" + custom-build-variant-cfg: x86 + tooltool-downloads: internal + +android-api-15/opt: + description: "Android 4.0 API15+ Opt" + index: + product: mobile + job-name: android-api-15-opt + treeherder: + platform: android-4-0-armv7-api15/opt + symbol: tc(B) + tier: 2 + worker-type: aws-provisioner-v1/gecko-{level}-b-android + worker: + implementation: docker-worker + max-run-time: 7200 + run: + using: mozharness + actions: [get-secrets build multi-l10n update] + config: + - builds/releng_base_android_64_builds.py + - disable_signing.py + - platform_supports_post_upload_to_latest.py + script: "mozharness/scripts/fx_desktop_build.py" + custom-build-variant-cfg: api-15 + tooltool-downloads: internal + +android-api-15-nightly/opt: + description: "Android 4.0 API15+ Nightly" + attributes: + nightly: true + index: + product: mobile + job-name: android-api-15-nightly-opt + treeherder: + platform: android-4-0-armv7-api15/opt + symbol: tc(N) + tier: 2 + worker-type: aws-provisioner-v1/gecko-{level}-b-android + worker: + implementation: docker-worker + max-run-time: 7200 + run: + using: mozharness + actions: [get-secrets build multi-l10n update] + config: + - builds/releng_base_android_64_builds.py + - disable_signing.py + - platform_supports_post_upload_to_latest.py + - taskcluster_nightly.py + script: "mozharness/scripts/fx_desktop_build.py" + custom-build-variant-cfg: api-15 + tooltool-downloads: internal + run-on-projects: [] + +android-api-15-gradle/opt: + description: "Android 4.0 API15+ (Gradle) Opt" + index: + product: mobile + job-name: android-api-15-gradle-opt + treeherder: + platform: android-4-0-armv7-api15/opt + symbol: tc(Bg) + tier: 2 + worker-type: aws-provisioner-v1/gecko-{level}-b-android + worker: + implementation: docker-worker + max-run-time: 7200 + env: + # Bug 1292762 - Set GRADLE_USER_HOME to avoid sdk-manager-plugin intermittent + GRADLE_USER_HOME: /home/worker/workspace/build/src/dotgradle + artifacts: + - name: public/android/maven + path: /home/worker/workspace/build/src/obj-firefox/gradle/build/mobile/android/geckoview/maven/ + type: directory + - name: public/android/geckoview_example.apk + path: /home/worker/workspace/build/src/obj-firefox/gradle/build/mobile/android/geckoview_example/outputs/apk/geckoview_example-withGeckoBinaries.apk + type: file + - name: public/build + path: /home/worker/artifacts/ + type: directory + run: + using: mozharness + actions: [get-secrets build multi-l10n update] + config: + - builds/releng_base_android_64_builds.py + - disable_signing.py + - platform_supports_post_upload_to_latest.py + script: "mozharness/scripts/fx_desktop_build.py" + custom-build-variant-cfg: api-15-gradle + tooltool-downloads: internal diff --git a/taskcluster/ci/build/kind.yml b/taskcluster/ci/build/kind.yml new file mode 100644 index 000000000..acb8548c9 --- /dev/null +++ b/taskcluster/ci/build/kind.yml @@ -0,0 +1,16 @@ +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, You can obtain one at http://mozilla.org/MPL/2.0/. + +implementation: taskgraph.task.transform:TransformTask + +transforms: + - taskgraph.transforms.build:transforms + - taskgraph.transforms.build_attrs:transforms + - taskgraph.transforms.job:transforms + - taskgraph.transforms.task:transforms + +jobs-from: + - linux.yml + - macosx.yml + - windows.yml diff --git a/taskcluster/ci/build/linux.yml b/taskcluster/ci/build/linux.yml new file mode 100644 index 000000000..7143522c8 --- /dev/null +++ b/taskcluster/ci/build/linux.yml @@ -0,0 +1,254 @@ +linux64/opt: + description: "Linux64 Opt" + index: + product: firefox + job-name: linux64-opt + treeherder: + platform: linux64/opt + symbol: tc(B) + tier: 2 + worker-type: aws-provisioner-v1/gecko-{level}-b-linux + worker: + implementation: docker-worker + max-run-time: 36000 + run: + using: mozharness + actions: [get-secrets build check-test generate-build-stats update] + config: + - builds/releng_base_linux_64_builds.py + - balrog/production.py + script: "mozharness/scripts/fx_desktop_build.py" + secrets: true + tooltool-downloads: public + need-xvfb: true + +linux64/pgo: + description: "Linux64 PGO" + index: + product: firefox + job-name: + buildbot: linux64-pgo + gecko-v2: linux64-pgo + treeherder: + platform: linux64/pgo + symbol: tc(B) + tier: 2 + worker-type: aws-provisioner-v1/gecko-{level}-b-linux + worker: + implementation: docker-worker + max-run-time: 36000 + coalesce-name: linux64-pgo + run: + using: mozharness + actions: [get-secrets build check-test generate-build-stats update] + options: [enable-pgo] + config: + - builds/releng_base_linux_64_builds.py + - balrog/production.py + script: "mozharness/scripts/fx_desktop_build.py" + secrets: true + tooltool-downloads: public + need-xvfb: true + +linux64/debug: + description: "Linux64 Debug" + index: + product: firefox + job-name: + buildbot: linux64-debug + gecko-v2: linux64-debug + treeherder: + platform: linux64/debug + symbol: tc(B) + worker-type: aws-provisioner-v1/gecko-{level}-b-linux + worker: + implementation: docker-worker + max-run-time: 36000 + run: + using: mozharness + actions: [get-secrets build check-test generate-build-stats update] + config: + - builds/releng_base_linux_64_builds.py + - balrog/production.py + script: "mozharness/scripts/fx_desktop_build.py" + secrets: true + custom-build-variant-cfg: debug + tooltool-downloads: public + need-xvfb: true + +linux/opt: + description: "Linux32 Opt" + index: + product: firefox + job-name: linux-opt + treeherder: + platform: linux32/opt + symbol: tc(B) + tier: 2 + worker-type: aws-provisioner-v1/gecko-{level}-b-linux + worker: + implementation: docker-worker + max-run-time: 36000 + coalesce-name: opt_linux32 + run: + using: mozharness + actions: [get-secrets build check-test generate-build-stats update] + config: + - builds/releng_base_linux_32_builds.py + - balrog/production.py + script: "mozharness/scripts/fx_desktop_build.py" + secrets: true + tooltool-downloads: public + need-xvfb: true + +linux/debug: + description: "Linux32 Debug" + index: + product: firefox + job-name: linux-debug + treeherder: + platform: linux32/debug + symbol: tc(B) + tier: 2 + worker-type: aws-provisioner-v1/gecko-{level}-b-linux + worker: + implementation: docker-worker + max-run-time: 36000 + coalesce-name: dbg_linux32 + run: + using: mozharness + actions: [get-secrets build check-test generate-build-stats update] + config: + - builds/releng_base_linux_32_builds.py + - balrog/production.py + script: "mozharness/scripts/fx_desktop_build.py" + secrets: true + custom-build-variant-cfg: debug + tooltool-downloads: public + need-xvfb: true + +linux/pgo: + description: "Linux32 PGO" + index: + product: firefox + job-name: + gecko-v2: linux-pgo + treeherder: + platform: linux32/pgo + symbol: tc(B) + tier: 2 + worker-type: aws-provisioner-v1/gecko-{level}-b-linux + worker: + implementation: docker-worker + max-run-time: 36000 + coalesce-name: linux32-pgo + run: + using: mozharness + actions: [get-secrets build check-test generate-build-stats update] + options: [enable-pgo] + config: + - builds/releng_base_linux_32_builds.py + - balrog/production.py + script: "mozharness/scripts/fx_desktop_build.py" + secrets: true + tooltool-downloads: public + need-xvfb: true + +linux64-asan/opt: + description: "Linux64 Opt ASAN" + index: + product: firefox + job-name: linux64-asan-opt + treeherder: + platform: linux64/asan + symbol: tc(Bo) + worker-type: aws-provisioner-v1/gecko-{level}-b-linux + worker: + implementation: docker-worker + max-run-time: 36000 + run: + using: mozharness + actions: [get-secrets build check-test generate-build-stats update] + config: + - builds/releng_base_linux_64_builds.py + - balrog/production.py + script: "mozharness/scripts/fx_desktop_build.py" + secrets: true + custom-build-variant-cfg: asan-tc + tooltool-downloads: public + need-xvfb: true + +linux64-asan/debug: + description: "Linux64 Debug ASAN" + index: + product: firefox + job-name: linux64-asan-debug + treeherder: + platform: linux64/asan + symbol: tc(Bd) + worker-type: aws-provisioner-v1/gecko-{level}-b-linux + worker: + implementation: docker-worker + max-run-time: 36000 + run: + using: mozharness + actions: [get-secrets build check-test generate-build-stats update] + config: + - builds/releng_base_linux_64_builds.py + - balrog/production.py + script: "mozharness/scripts/fx_desktop_build.py" + secrets: true + custom-build-variant-cfg: asan-tc-and-debug + tooltool-downloads: public + need-xvfb: true + +linux64-jsdcov/opt: + description: "Linux64-JSDCov Opt" + index: + product: firefox + job-name: linux64-jsdcov-opt + treeherder: + platform: linux64/jsdcov + symbol: tc(B) + tier: 2 + run-on-projects: [ ] + worker-type: aws-provisioner-v1/gecko-{level}-b-linux + worker: + implementation: docker-worker + max-run-time: 36000 + run: + using: mozharness + actions: [get-secrets build check-test generate-build-stats update] + config: + - builds/releng_base_linux_64_builds.py + - balrog/production.py + script: "mozharness/scripts/fx_desktop_build.py" + secrets: true + tooltool-downloads: public + need-xvfb: true + +linux64-ccov/opt: + description: "Linux64-CCov Opt" + index: + product: firefox + job-name: linux64-ccov-opt + treeherder: + platform: linux64/ccov + symbol: tc(B) + tier: 2 + run-on-projects: [ ] + worker-type: aws-provisioner-v1/gecko-{level}-b-linux + worker: + implementation: docker-worker + max-run-time: 36000 + run: + using: mozharness + actions: [get-secrets build check-test generate-build-stats update] + config: + - builds/releng_base_linux_64_builds.py + - balrog/production.py + script: "mozharness/scripts/fx_desktop_build.py" + secrets: true + custom-build-variant-cfg: code-coverage + tooltool-downloads: public + need-xvfb: true diff --git a/taskcluster/ci/build/macosx.yml b/taskcluster/ci/build/macosx.yml new file mode 100644 index 000000000..b8d03669c --- /dev/null +++ b/taskcluster/ci/build/macosx.yml @@ -0,0 +1,71 @@ +macosx64/debug: + description: "MacOS X x64 Cross-compile" + index: + product: firefox + job-name: macosx64-debug + treeherder: + platform: osx-10-7/debug + symbol: tc(B) + tier: 2 + worker-type: aws-provisioner-v1/gecko-{level}-b-macosx64 + worker: + implementation: docker-worker + max-run-time: 36000 + run: + using: mozharness + actions: [get-secrets build generate-build-stats update] + config: + - builds/releng_base_mac_64_cross_builds.py + - balrog/production.py + script: "mozharness/scripts/fx_desktop_build.py" + secrets: true + custom-build-variant-cfg: cross-debug + tooltool-downloads: internal + +macosx64/opt: + description: "MacOS X x64 Cross-compile" + index: + product: firefox + job-name: macosx64-opt + treeherder: + platform: osx-10-7/opt + symbol: tc(B) + tier: 2 + run-on-projects: [ ] + worker-type: aws-provisioner-v1/gecko-{level}-b-macosx64 + worker: + implementation: docker-worker + max-run-time: 36000 + run: + using: mozharness + actions: [get-secrets build generate-build-stats update] + config: + - builds/releng_base_mac_64_cross_builds.py + - balrog/production.py + script: "mozharness/scripts/fx_desktop_build.py" + secrets: true + tooltool-downloads: internal + +macosx64-universal/opt: + description: "MacOS X Universal Cross-compile" + index: + product: firefox + job-name: macosx64-opt + treeherder: + platform: osx-10-7/opt + symbol: tc(Bu) + tier: 2 + worker-type: aws-provisioner-v1/gecko-{level}-b-macosx64 + worker: + implementation: docker-worker + max-run-time: 36000 + run: + using: mozharness + actions: [get-secrets build generate-build-stats update] + config: + - builds/releng_base_mac_64_cross_builds.py + - balrog/production.py + script: "mozharness/scripts/fx_desktop_build.py" + secrets: true + custom-build-variant-cfg: cross-universal + tooltool-downloads: internal diff --git a/taskcluster/ci/build/windows.yml b/taskcluster/ci/build/windows.yml new file mode 100644 index 000000000..a3211219c --- /dev/null +++ b/taskcluster/ci/build/windows.yml @@ -0,0 +1,122 @@ +win32/debug: + description: "Win32 Debug" + index: + product: firefox + job-name: + gecko-v2: win32-debug + treeherder: + platform: windows2012-32/debug + symbol: tc(B) + tier: 2 + worker-type: aws-provisioner-v1/gecko-{level}-b-win2012 + worker: + implementation: generic-worker + max-run-time: 7200 + run: + using: mozharness + script: mozharness/scripts/fx_desktop_build.py + config: + - builds/taskcluster_firefox_win32_debug.py + +win32/opt: + description: "Win32 Opt" + index: + product: firefox + job-name: + gecko-v2: win32-opt + treeherder: + platform: windows2012-32/opt + symbol: tc(B) + tier: 2 + worker-type: aws-provisioner-v1/gecko-{level}-b-win2012 + worker: + implementation: generic-worker + max-run-time: 7200 + run: + using: mozharness + script: mozharness/scripts/fx_desktop_build.py + config: + - builds/taskcluster_firefox_win32_opt.py + +win32/pgo: + description: "Win32 Opt PGO" + index: + product: firefox + job-name: + gecko-v2: win32-pgo + treeherder: + platform: windows2012-32/pgo + symbol: tc(B) + tier: 2 + worker-type: aws-provisioner-v1/gecko-{level}-b-win2012 + worker: + implementation: generic-worker + max-run-time: 9000 + run: + using: mozharness + options: [enable-pgo] + script: mozharness/scripts/fx_desktop_build.py + config: + - builds/taskcluster_firefox_win32_opt.py + +win64/debug: + description: "Win64 Debug" + index: + product: firefox + job-name: + gecko-v2: win64-debug + treeherder: + platform: windows2012-64/debug + symbol: tc(B) + tier: 2 + worker-type: aws-provisioner-v1/gecko-{level}-b-win2012 + worker: + implementation: generic-worker + max-run-time: 7200 + run: + using: mozharness + script: mozharness/scripts/fx_desktop_build.py + config: + - builds/taskcluster_firefox_win64_debug.py + +win64/opt: + description: "Win64 Opt" + index: + product: firefox + job-name: + gecko-v2: win64-opt + treeherder: + platform: windows2012-64/opt + symbol: tc(B) + tier: 2 + worker-type: aws-provisioner-v1/gecko-{level}-b-win2012 + worker: + implementation: generic-worker + max-run-time: 7200 + run: + using: mozharness + script: mozharness/scripts/fx_desktop_build.py + config: + - builds/taskcluster_firefox_win64_opt.py + +win64/pgo: + description: "Win64 Opt PGO" + index: + product: firefox + job-name: + gecko-v2: win64-pgo + treeherder: + platform: windows2012-64/pgo + symbol: tc(B) + tier: 2 + worker-type: aws-provisioner-v1/gecko-{level}-b-win2012 + worker: + implementation: generic-worker + max-run-time: 10800 + run: + using: mozharness + options: [enable-pgo] + script: mozharness/scripts/fx_desktop_build.py + config: + - builds/taskcluster_firefox_win64_opt.py + diff --git a/taskcluster/ci/desktop-test/kind.yml b/taskcluster/ci/desktop-test/kind.yml new file mode 100644 index 000000000..31e305cbe --- /dev/null +++ b/taskcluster/ci/desktop-test/kind.yml @@ -0,0 +1,12 @@ +implementation: taskgraph.task.test:TestTask + +kind-dependencies: + - build + +transforms: + - taskgraph.transforms.tests.test_description:validate + - taskgraph.transforms.tests.desktop_test:transforms + - taskgraph.transforms.tests.all_kinds:transforms + - taskgraph.transforms.tests.test_description:validate + - taskgraph.transforms.tests.make_task_description:transforms + - taskgraph.transforms.task:transforms diff --git a/taskcluster/ci/desktop-test/test-platforms.yml b/taskcluster/ci/desktop-test/test-platforms.yml new file mode 100644 index 000000000..b03d3c41d --- /dev/null +++ b/taskcluster/ci/desktop-test/test-platforms.yml @@ -0,0 +1,73 @@ +# This file maps build platforms to test platforms. In some cases, a +# single build may be tested on multiple test platforms, but a single test +# platform can only link to one build platform. Both build and test platforms +# are represented as <platform>/<type>, where <type> is what Treeherder calls a +# collection. +# +# Each test platform further specifies the set of tests that will be scheduled +# for the platform, referring to tests defined in test-sets.yml. +# +# Note that set does not depend on the tree; tree-dependent job selection +# should be performed in the target task selection phase of task-graph +# generation. + +linux64/debug: + build-platform: linux64/debug + test-set: all-tests +linux64/opt: + build-platform: linux64/opt + test-set: all-tests + +# TODO: use 'pgo' and 'asan' labels here, instead of -pgo/opt +linux64-pgo/opt: + build-platform: linux64-pgo/opt + test-set: all-tests +linux64-asan/opt: + build-platform: linux64-asan/opt + test-set: asan-tests + +linux64-ccov/opt: + build-platform: linux64-ccov/opt + test-set: ccov-code-coverage-tests +linux64-jsdcov/opt: + build-platform: linux64-jsdcov/opt + test-set: jsdcov-code-coverage-tests + +# win32 vm +windows7-32-vm/debug: + build-platform: win32/debug + test-set: windows-vm-tests +windows7-32-vm/opt: + build-platform: win32/opt + test-set: windows-vm-tests + +# win32 gpu +#windows7-32/debug: +# build-platform: win32/debug +# test-set: windows-gpu-tests +#windows7-32/opt: +# build-platform: win32/opt +# test-set: windows-gpu-tests + +# win64 vm +windows10-64-vm/debug: + build-platform: win64/debug + test-set: windows-vm-tests +windows10-64-vm/opt: + build-platform: win64/opt + test-set: windows-vm-tests + +# win64 gpu +#windows10-64/debug: +# build-platform: win64/debug +# test-set: windows-gpu-tests +#windows10-64/opt: +# build-platform: win64/opt +# test-set: windows-gpu-tests + +# macosx64/debug: +# build-platform: macosx64/debug +# test-set: macosx64-tests +# macosx64/opt: +# build-platform: macosx64/opt +# test-set: macosx64-tests diff --git a/taskcluster/ci/desktop-test/test-sets.yml b/taskcluster/ci/desktop-test/test-sets.yml new file mode 100644 index 000000000..da1ea6263 --- /dev/null +++ b/taskcluster/ci/desktop-test/test-sets.yml @@ -0,0 +1,118 @@ +# Each key in this file specifies a set of tests to run. Different test sets +# may, for example, be bound to different test platforms. +# +# Note that set does not depend on the tree; tree-dependent job selection +# should be performed in the target task selection phase of task-graph +# generation. +# +# A test set has a name, and a list of tests that it contains. +# +# Test names given here reference tests.yml. + +all-tests: + - cppunit + - crashtest + - firefox-ui-functional-local + - firefox-ui-functional-remote + - gtest + - jittest + - jsreftest + - marionette + - mochitest + - mochitest-a11y + - mochitest-browser-chrome + - mochitest-chrome + - mochitest-clipboard + - mochitest-devtools-chrome + - mochitest-gpu + - mochitest-jetpack + - mochitest-media + - mochitest-webgl + - reftest + - reftest-no-accel + - web-platform-tests + - web-platform-tests-reftests + - web-platform-tests-wdspec + - xpcshell + +asan-tests: + - cppunit + - crashtest + - firefox-ui-functional-local + - firefox-ui-functional-remote + - gtest + - jittest + - jsreftest + - marionette + - mochitest + - mochitest-a11y + - mochitest-browser-chrome + - mochitest-chrome + - mochitest-clipboard + - mochitest-devtools-chrome + - mochitest-gpu + - mochitest-jetpack + - mochitest-media + - mochitest-webgl + - reftest + - reftest-no-accel + - xpcshell + +windows-vm-tests: + - cppunit + #- crashtest + - external-media-tests + #- gtest + #- jittest + #- jsreftest + #- marionette + #- mochitest + #- mochitest-browser-chrome + #- mochitest-devtools-chrome + #- mochitest-jetpack + #- mochitest-media + #- web-platform-tests + #- web-platform-tests-reftests + #- xpcshell + +# windows-gpu-tests: +# - reftest +# - reftest-no-accel +# - mochitest-webgl + +# these tests currently run on hardware, but may migrate above when validated +# note: on win, mochitest-a11y and mochitest-chrome come under mochitest-other +# windows-hw-tests: +# - mochitest-clipboard +# - mochitest-gpu +# - mochitest-other + +ccov-code-coverage-tests: + - mochitest + - mochitest-browser-chrome + - mochitest-devtools-chrome + - xpcshell + +jsdcov-code-coverage-tests: + - mochitest-browser-chrome + - mochitest-devtools-chrome + +macosx64-tests: + - cppunit + - crashtest + # - gtest + - jsreftest + # - marionette + # - mochitest + # - mochitest-browser-chrome + # - mochitest-clipboard + # - mochitest-devtools-chrome + # - mochitest-gpu + # - mochitest-jetpack + # - mochitest-media + # - mochitest-other + - mochitest-webgl + # - reftest + # - web-platform-tests + # - web-platform-tests-reftests + # - xpcshell diff --git a/taskcluster/ci/desktop-test/tests.yml b/taskcluster/ci/desktop-test/tests.yml new file mode 100644 index 000000000..edfb30909 --- /dev/null +++ b/taskcluster/ci/desktop-test/tests.yml @@ -0,0 +1,721 @@ +# Each stanza here describes a particular test suite or sub-suite. These are +# processed through the transformations described in kind.yml to produce a +# bunch of tasks. See the schema in `test-descriptions.py` for a description +# of the fields used here. + +# Note that these are in lexical order + +cppunit: + description: "CPP Unit Tests" + suite: cppunittest + treeherder-symbol: tc(Cpp) + e10s: false + run-on-projects: + by-test-platform: + windows.*: ['mozilla-central', 'try'] + default: ['all'] + mozharness: + script: desktop_unittest.py + no-read-buildbot-config: true + config: + by-test-platform: + win.*: + - unittests/win_taskcluster_unittest.py + macosx.*: + - remove_executables.py + - unittests/mac_unittest.py + default: + - unittests/linux_unittest.py + - remove_executables.py + extra-options: + - --cppunittest-suite=cppunittest + +crashtest: + description: "Crashtest run" + suite: reftest/crashtest + treeherder-symbol: tc-R(C) + docker-image: {"in-tree": "desktop1604-test"} + e10s: + by-test-platform: + # Bug 1304435 + win.*: false + default: both + mozharness: + script: desktop_unittest.py + chunked: true + no-read-buildbot-config: true + config: + by-test-platform: + win.*: + - unittests/win_taskcluster_unittest.py + macosx.*: + - remove_executables.py + - unittests/mac_unittest.py + default: + - unittests/linux_unittest.py + - remove_executables.py + extra-options: + - --reftest-suite=crashtest + +external-media-tests: + description: "External Media Test run" + suite: external-media-tests + treeherder-symbol: tc-VP(b-m) + e10s: false + tier: 2 + max-run-time: 5400 + run-on-projects: + by-test-platform: + windows.*: ['mozilla-central', 'try'] + default: ['all'] + mozharness: + script: firefox_media_tests_taskcluster.py + config: + by-test-platform: + win.*: + - mediatests/taskcluster_windows_config.py + default: + - mediatests/taskcluster_posix_config.py + - remove_executables.py + +firefox-ui-functional-local: + description: "Firefox-ui-tests functional run" + suite: "firefox-ui/functional local" + treeherder-symbol: tc-Fxfn-l(en-US) + max-run-time: 5400 + tier: 1 + docker-image: {"in-tree": "desktop1604-test"} + mozharness: + script: firefox_ui_tests/functional.py + config: + - firefox_ui_tests/taskcluster.py + - remove_executables.py + extra-options: + - "--tag" + - "local" + +firefox-ui-functional-remote: + description: "Firefox-ui-tests functional run" + suite: "firefox-ui/functional remote" + treeherder-symbol: tc-Fxfn-r(en-US) + max-run-time: 5400 + tier: 2 + docker-image: {"in-tree": "desktop1604-test"} + mozharness: + script: firefox_ui_tests/functional.py + config: + - firefox_ui_tests/taskcluster.py + - remove_executables.py + extra-options: + - "--tag" + - "remote" + +gtest: + description: "GTests run" + suite: gtest + treeherder-symbol: tc(GTest) + e10s: false + instance-size: xlarge + mozharness: + script: desktop_unittest.py + no-read-buildbot-config: true + config: + by-test-platform: + win.*: + - unittests/win_taskcluster_unittest.py + macosx.*: + - remove_executables.py + - unittests/mac_unittest.py + default: + - unittests/linux_unittest.py + - remove_executables.py + extra-options: + - --gtest-suite=gtest + +jittest: + description: "JIT Test run" + suite: jittest/jittest-chunked + treeherder-symbol: tc(Jit) + e10s: false + chunks: + by-test-platform: + win.*: 1 + default: 6 + mozharness: + script: desktop_unittest.py + no-read-buildbot-config: true + config: + by-test-platform: + win.*: + - unittests/win_taskcluster_unittest.py + macosx.*: + - remove_executables.py + - unittests/mac_unittest.py + default: + - unittests/linux_unittest.py + - remove_executables.py + extra-options: + - --jittest-suite=jittest-chunked + +jsreftest: + description: "JS Reftest run" + suite: reftest/jsreftest + treeherder-symbol: tc-R(J) + chunks: + by-test-platform: + win.*: 1 + default: 2 + mozharness: + script: desktop_unittest.py + no-read-buildbot-config: true + config: + by-test-platform: + win.*: + - unittests/win_taskcluster_unittest.py + macosx.*: + - remove_executables.py + - unittests/mac_unittest.py + default: + - unittests/linux_unittest.py + - remove_executables.py + extra-options: + - --reftest-suite=jsreftest + +marionette: + description: "Marionette unittest run" + suite: marionette + treeherder-symbol: tc(Mn) + max-run-time: 5400 + docker-image: {"in-tree": "desktop1604-test"} + mozharness: + script: marionette.py + no-read-buildbot-config: true + config: + by-test-platform: + win.*: + - marionette/windows_taskcluster_config.py + default: + - marionette/prod_config.py + - remove_executables.py + +mochitest: + description: "Mochitest plain run" + suite: mochitest/plain-chunked + treeherder-symbol: tc-M() + loopback-video: true + run-on-projects: + by-test-platform: + linux64-ccov/opt: [] + default: ['all'] + chunks: + by-test-platform: + macosx.*: 5 + win.*: 5 + default: 10 + e10s: + by-test-platform: + linux64-ccov/opt: false + default: both + max-run-time: 5400 + mozharness: + script: desktop_unittest.py + no-read-buildbot-config: true + chunked: true + config: + by-test-platform: + win.*: + - unittests/win_taskcluster_unittest.py + macosx.*: + - remove_executables.py + - unittests/mac_unittest.py + default: + - unittests/linux_unittest.py + - remove_executables.py + extra-options: + by-test-platform: + linux64-ccov/opt: + - --mochitest-suite=plain-chunked + - --code-coverage + default: + - --mochitest-suite=plain-chunked + # Bug 1281241: migrating to m3.large instances + instance-size: legacy + allow-software-gl-layers: false + +mochitest-a11y: + description: "Mochitest a11y run" + suite: mochitest/a11y + treeherder-symbol: tc-M(a11y) + loopback-video: true + e10s: false + mozharness: + script: desktop_unittest.py + no-read-buildbot-config: true + chunked: true + config: + by-test-platform: + win.*: + - unittests/win_taskcluster_unittest.py + macosx.*: + - remove_executables.py + - unittests/mac_unittest.py + default: + - unittests/linux_unittest.py + - remove_executables.py + extra-options: + - --mochitest-suite=a11y + +mochitest-browser-chrome: + description: "Mochitest browser-chrome run" + suite: + by-test-platform: + linux64-jsdcov/opt: mochitest/browser-chrome-coverage + default: mochitest/browser-chrome-chunked + treeherder-symbol: tc-M(bc) + loopback-video: true + run-on-projects: + by-test-platform: + linux64-jsdcov/opt: [] + linux64-ccov/opt: [] + default: ['all'] + chunks: + by-test-platform: + linux64-jsdcov/opt: 35 + default: 10 + e10s: + by-test-platform: + linux64-jsdcov/opt: false + linux64-ccov/opt: false + default: both + max-run-time: + by-test-platform: + linux64-jsdcov/opt: 7200 + linux64-ccov/opt: 7200 + linux64/debug: 5400 + default: 3600 + mozharness: + script: desktop_unittest.py + no-read-buildbot-config: true + config: + by-test-platform: + win.*: + - unittests/win_taskcluster_unittest.py + macosx.*: + - remove_executables.py + - unittests/mac_unittest.py + default: + - unittests/linux_unittest.py + - remove_executables.py + extra-options: + by-test-platform: + linux64-jsdcov/opt: + - --mochitest-suite=browser-chrome-coverage + linux64-ccov/opt: + - --mochitest-suite=browser-chrome-chunked + - --code-coverage + default: + - --mochitest-suite=browser-chrome-chunked + # Bug 1281241: migrating to m3.large instances + instance-size: + by-test-platform: + linux64-jsdcov/opt: xlarge + linux64-ccov/opt: xlarge + default: legacy + allow-software-gl-layers: false + +mochitest-chrome: + description: "Mochitest chrome run" + suite: mochitest/chrome + treeherder-symbol: tc-M(c) + loopback-video: true + chunks: + by-test-platform: + macosx.*: 1 + default: 3 + e10s: false + mozharness: + script: desktop_unittest.py + no-read-buildbot-config: true + config: + by-test-platform: + win.*: + - unittests/win_taskcluster_unittest.py + macosx.*: + - remove_executables.py + - unittests/mac_unittest.py + default: + - unittests/linux_unittest.py + - remove_executables.py + extra-options: + - --mochitest-suite=chrome + +mochitest-clipboard: + description: "Mochitest clipboard run" + suite: mochitest/plain-clipboard,chrome-clipboard,browser-chrome-clipboard,jetpack-package-clipboard + treeherder-symbol: tc-M(cl) + loopback-video: true + instance-size: xlarge + mozharness: + script: desktop_unittest.py + no-read-buildbot-config: true + chunked: true + config: + by-test-platform: + win.*: + - unittests/win_taskcluster_unittest.py + macosx.*: + - remove_executables.py + - unittests/mac_unittest.py + default: + - unittests/linux_unittest.py + - remove_executables.py + extra-options: + - --mochitest-suite=plain-clipboard,chrome-clipboard,browser-chrome-clipboard,jetpack-package-clipboard + +mochitest-devtools-chrome: + description: "Mochitest devtools-chrome run" + suite: + by-test-platform: + linux64-jsdcov/opt: mochitest/mochitest-devtools-chrome-coverage + default: mochitest/mochitest-devtools-chrome-chunked + treeherder-symbol: tc-M(dt) + loopback-video: true + max-run-time: 5400 + chunks: + by-test-platform: + win.*: 8 + default: 10 + run-on-projects: + by-test-platform: + linux64-ccov/opt: [] + linux64-jsdcov/opt: [] + windows.*: ['mozilla-central', 'try'] + default: ['all'] + e10s: + by-test-platform: + # Bug 1242986: linux64/debug mochitest-devtools-chrome e10s is not greened up yet + linux64/debug: false + linux64-ccov/opt: false + linux64-jsdcov/opt: false + default: both + mozharness: + script: desktop_unittest.py + no-read-buildbot-config: true + config: + by-test-platform: + win.*: + - unittests/win_taskcluster_unittest.py + macosx.*: + - remove_executables.py + - unittests/mac_unittest.py + default: + - unittests/linux_unittest.py + - remove_executables.py + extra-options: + by-test-platform: + linux64-ccov/opt: + - --mochitest-suite=mochitest-devtools-chrome-chunked + - --code-coverage + linux64-jsdcov: + - --mochitest-suite=mochitest-devtools-chrome-coverage + default: + - --mochitest-suite=mochitest-devtools-chrome-chunked + instance-size: + by-test-platform: + # Bug 1281241: migrating to m3.large instances + linux64-asan/opt: legacy + default: default + # Bug 1296086: high number of intermittents observed with software GL and large instances + allow-software-gl-layers: false + +mochitest-gpu: + description: "Mochitest GPU run" + suite: mochitest/plain-gpu,chrome-gpu,browser-chrome-gpu + treeherder-symbol: tc-M(gpu) + loopback-video: true + e10s: + by-test-platform: + win.*: both + default: true + mozharness: + script: desktop_unittest.py + no-read-buildbot-config: true + chunked: true + config: + by-test-platform: + win.*: + - unittests/win_taskcluster_unittest.py + macosx.*: + - remove_executables.py + - unittests/mac_unittest.py + default: + - unittests/linux_unittest.py + - remove_executables.py + extra-options: + - --mochitest-suite=plain-gpu,chrome-gpu,browser-chrome-gpu + +mochitest-jetpack: + description: "Mochitest jetpack run" + suite: mochitest/jetpack-package + treeherder-symbol: tc-M(JP) + loopback-video: true + e10s: false + max-run-time: 5400 + run-on-projects: + by-test-platform: + windows.*: ['mozilla-central', 'try'] + default: ['all'] + mozharness: + script: desktop_unittest.py + no-read-buildbot-config: true + chunked: true + config: + by-test-platform: + win.*: + - unittests/win_taskcluster_unittest.py + macosx.*: + - remove_executables.py + - unittests/mac_unittest.py + default: + - unittests/linux_unittest.py + - remove_executables.py + extra-options: + - --mochitest-suite=jetpack-package + - --mochitest-suite=jetpack-addon + +mochitest-media: + description: "Mochitest media run" + suite: mochitest/mochitest-media + treeherder-symbol: tc-M(mda) + max-run-time: 5400 + loopback-video: true + instance-size: large + docker-image: {"in-tree": "desktop1604-test"} + mozharness: + script: desktop_unittest.py + no-read-buildbot-config: true + chunked: true + config: + by-test-platform: + win.*: + - unittests/win_taskcluster_unittest.py + macosx.*: + - remove_executables.py + - unittests/mac_unittest.py + default: + - unittests/linux_unittest.py + - remove_executables.py + extra-options: + - --mochitest-suite=mochitest-media + +mochitest-other: + description: "Mochitest other" + suite: mochitest/other + treeherder-symbol: tc-M(oth) + e10s: false + max-run-time: 5400 + mozharness: + script: mozharness/scripts/desktop_unittest.py + no-read-buildbot-config: true + include-blob-upload-branch: true + chunked: true + config: + by-test-platform: + macosx.*: + - remove_executables.py + - unittests/mac_unittest.py + default: + - unittests/linux_unittest.py + - remove_executables.py + extra-options: + - --mochitest-suite=chrome,a11y + +mochitest-webgl: + description: "Mochitest webgl run" + suite: mochitest/mochitest-gl + treeherder-symbol: tc-M(gl) + chunks: 3 + loopback-video: true + mozharness: + script: desktop_unittest.py + no-read-buildbot-config: true + chunked: true + config: + by-test-platform: + win.*: + - unittests/win_taskcluster_unittest.py + macosx.*: + - remove_executables.py + - unittests/mac_unittest.py + default: + - unittests/linux_unittest.py + - remove_executables.py + extra-options: + - --mochitest-suite=mochitest-gl + # Bug 1296733: llvmpipe with mesa 9.2.1 lacks thread safety + allow-software-gl-layers: false + +reftest: + description: "Reftest run" + suite: reftest/reftest + treeherder-symbol: tc-R(R) + docker-image: {"in-tree": "desktop1604-test"} + chunks: + by-test-platform: + macosx.*: 1 + default: 8 + mozharness: + script: desktop_unittest.py + no-read-buildbot-config: true + config: + by-test-platform: + win.*: + - unittests/win_taskcluster_unittest.py + macosx.*: + - remove_executables.py + - unittests/mac_unittest.py + default: + - unittests/linux_unittest.py + - remove_executables.py + extra-options: + - --reftest-suite=reftest + +reftest-no-accel: + description: "Reftest not accelerated run" + suite: reftest/reftest-no-accel + treeherder-symbol: tc-R(Ru) + docker-image: {"in-tree": "desktop1604-test"} + chunks: + by-test-platform: + macosx.*: 1 + default: 8 + mozharness: + script: desktop_unittest.py + no-read-buildbot-config: true + config: + by-test-platform: + win.*: + - unittests/win_taskcluster_unittest.py + macosx.*: + - remove_executables.py + - unittests/mac_unittest.py + default: + - unittests/linux_unittest.py + - remove_executables.py + extra-options: + - --reftest-suite=reftest-no-accel + +web-platform-tests: + description: "Web platform test run" + suite: web-platform-tests + treeherder-symbol: tc-W() + chunks: + by-test-platform: + macosx.*: 5 + default: 12 + max-run-time: 7200 + instance-size: xlarge + docker-image: {"in-tree": "desktop1604-test"} + checkout: true + run-on-projects: + by-test-platform: + windows.*: ['mozilla-central', 'try'] + default: ['all'] + mozharness: + script: web_platform_tests.py + no-read-buildbot-config: true + config: + by-test-platform: + win.*: + - web_platform_tests/prod_config_windows_taskcluster.py + default: + - web_platform_tests/prod_config.py + - remove_executables.py + extra-options: + - --test-type=testharness + +web-platform-tests-reftests: + description: "Web platform reftest run" + suite: web-platform-tests-reftests + treeherder-symbol: tc-W(Wr) + max-run-time: 5400 + instance-size: xlarge + docker-image: {"in-tree": "desktop1604-test"} + checkout: true + run-on-projects: + by-test-platform: + windows.*: ['mozilla-central', 'try'] + default: ['all'] + mozharness: + script: web_platform_tests.py + no-read-buildbot-config: true + config: + by-test-platform: + win.*: + - web_platform_tests/prod_config_windows_taskcluster.py + default: + - web_platform_tests/prod_config.py + - remove_executables.py + extra-options: + - --test-type=reftest + +web-platform-tests-wdspec: + description: "Web platform webdriver-spec run" + suite: web-platform-tests-wdspec + treeherder-symbol: tc-W(Wd) + max-run-time: 5400 + instance-size: xlarge + docker-image: {"in-tree": "desktop1604-test"} + checkout: true + mozharness: + script: web_platform_tests.py + no-read-buildbot-config: true + config: + by-test-platform: + win.*: + - web_platform_tests/prod_config_windows_taskcluster.py + default: + - web_platform_tests/prod_config.py + - remove_executables.py + extra-options: + - --test-type=wdspec + +xpcshell: + description: "xpcshell test run" + suite: xpcshell + treeherder-symbol: tc-X() + run-on-projects: + by-test-platform: + linux64-ccov/opt: [] + default: ['all'] + chunks: + by-test-platform: + # win.*: 1 + macosx.*: 1 + linux64/debug: 10 + default: 8 + max-run-time: 5400 + e10s: false + mozharness: + script: desktop_unittest.py + no-read-buildbot-config: true + config: + by-test-platform: + win.*: + - unittests/win_taskcluster_unittest.py + macosx.*: + - remove_executables.py + - unittests/mac_unittest.py + default: + - unittests/linux_unittest.py + - remove_executables.py + extra-options: + by-test-platform: + linux64-ccov/opt: + - --xpcshell-suite=xpcshell + - --code-coverage + default: + - --xpcshell-suite=xpcshell + # Bug 1281241: migrating to m3.large instances + instance-size: legacy + allow-software-gl-layers: false diff --git a/taskcluster/ci/docker-image/image.yml b/taskcluster/ci/docker-image/image.yml new file mode 100644 index 000000000..f9a1113f0 --- /dev/null +++ b/taskcluster/ci/docker-image/image.yml @@ -0,0 +1,68 @@ +--- +task: + created: + relative-datestamp: "0 seconds" + deadline: + relative-datestamp: "24 hours" + metadata: + name: 'Docker Image Build: {{image_name}}' + description: 'Build the docker image {{image_name}} for use by dependent tasks' + source: '{{source}}' + owner: mozilla-taskcluster-maintenance@mozilla.com + tags: + createdForUser: '{{owner}}' + + workerType: taskcluster-images + provisionerId: aws-provisioner-v1 + schedulerId: task-graph-scheduler + + routes: + # Indexing routes to avoid building the same image twice + - index.{{index_image_prefix}}.level-{{level}}.{{image_name}}.latest + - index.{{index_image_prefix}}.level-{{level}}.{{image_name}}.pushdate.{{year}}.{{month}}-{{day}}-{{pushtime}} + - index.{{index_image_prefix}}.level-{{level}}.{{image_name}}.hash.{{context_hash}} + # Treeherder routes + - tc-treeherder.v2.{{project}}.{{head_rev}}.{{pushlog_id}} + - tc-treeherder-stage.v2.{{project}}.{{head_rev}}.{{pushlog_id}} + + scopes: + - secrets:get:project/taskcluster/gecko/hgfingerprint + - docker-worker:cache:level-{{level}}-imagebuilder-v1 + + payload: + env: + HASH: '{{context_hash}}' + PROJECT: '{{project}}' + CONTEXT_URL: '{{context_url}}' + IMAGE_NAME: '{{image_name}}' + GECKO_BASE_REPOSITORY: '{{base_repository}}' + GECKO_HEAD_REPOSITORY: '{{head_repository}}' + GECKO_HEAD_REV: '{{head_rev}}' + HG_STORE_PATH: '/home/worker/checkouts/hg-store' + cache: + 'level-{{level}}-imagebuilder-v1': '/home/worker/checkouts' + features: + dind: true + chainOfTrust: true + taskclusterProxy: true + image: '{{#docker_image}}image_builder{{/docker_image}}' + maxRunTime: 3600 + artifacts: + '{{artifact_path}}': + type: 'file' + path: '/home/worker/workspace/artifacts/image.tar.zst' + expires: + relative-datestamp: "1 year" + extra: + imageMeta: # Useful when converting back from JSON in action tasks + level: '{{level}}' + contextHash: '{{context_hash}}' + imageName: '{{image_name}}' + treeherderEnv: + - staging + - production + treeherder: + jobKind: other + build: + platform: 'taskcluster-images' + groupSymbol: 'I' diff --git a/taskcluster/ci/docker-image/kind.yml b/taskcluster/ci/docker-image/kind.yml new file mode 100644 index 000000000..d1c118c8e --- /dev/null +++ b/taskcluster/ci/docker-image/kind.yml @@ -0,0 +1,19 @@ +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, You can obtain one at http://mozilla.org/MPL/2.0/. + +implementation: 'taskgraph.task.docker_image:DockerImageTask' +images_path: '../../../testing/docker' + +# make a task for each docker-image we might want. For the moment, since we +# write artifacts for each, these are whitelisted, but ideally that will change +# (to use subdirectory clones of the proper directory), at which point we can +# generate tasks for every docker image in the directory, secure in the +# knowledge that unnecessary images will be omitted from the target task graph +images: + desktop-test: dt + desktop1604-test: dt16t + desktop-build: db + tester: tst + lint: lnt + android-gradle-build: agb diff --git a/taskcluster/ci/hazard/kind.yml b/taskcluster/ci/hazard/kind.yml new file mode 100644 index 000000000..4849cdbd6 --- /dev/null +++ b/taskcluster/ci/hazard/kind.yml @@ -0,0 +1,58 @@ +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, You can obtain one at http://mozilla.org/MPL/2.0/. + +implementation: taskgraph.task.transform:TransformTask + +transforms: + - taskgraph.transforms.build_attrs:transforms + - taskgraph.transforms.job:transforms + - taskgraph.transforms.task:transforms + +job-defaults: + treeherder: + kind: build + tier: 1 + worker-type: aws-provisioner-v1/gecko-{level}-b-linux + worker: + implementation: docker-worker + max-run-time: 36000 + docker-image: {in-tree: desktop-build} + +jobs: + linux64-shell-haz/debug: + description: "JS Shell Hazard Analysis Linux" + index: + product: firefox + job-name: + gecko-v2: shell-haz-debug + treeherder: + platform: linux64/debug + symbol: SM-tc(H) + run: + using: hazard + tooltool-manifest: "browser/config/tooltool-manifests/linux64/hazard.manifest" + command: > + cd /home/worker/checkouts/gecko/taskcluster/scripts/builder + && ./build-haz-linux.sh --project shell $HOME/workspace + when: + files-changed: + - js/public/** + - js/src/** + + linux64-haz/debug: + description: "Browser Hazard Analysis Linux" + index: + product: firefox + job-name: + gecko-v2: browser-haz-debug + treeherder: + platform: linux64/debug + symbol: tc(H) + run: + using: hazard + tooltool-manifest: "browser/config/tooltool-manifests/linux64/hazard.manifest" + mozconfig: "browser/config/mozconfigs/linux64/hazards" + command: > + cd /home/worker/checkouts/gecko/taskcluster/scripts/builder + && ./build-haz-linux.sh --project browser $HOME/workspace diff --git a/taskcluster/ci/l10n/kind.yml b/taskcluster/ci/l10n/kind.yml new file mode 100644 index 000000000..dfd011e9a --- /dev/null +++ b/taskcluster/ci/l10n/kind.yml @@ -0,0 +1,89 @@ +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, You can obtain one at http://mozilla.org/MPL/2.0/. + +# NOTE: please write a description of this kind in taskcluster/docs/kinds.rst + +implementation: taskgraph.task.transform:TransformTask + +transforms: + - taskgraph.transforms.l10n:transforms + - taskgraph.transforms.build_attrs:transforms + - taskgraph.transforms.job:transforms + - taskgraph.transforms.task:transforms + +job-defaults: + index: + product: firefox + treeherder: + kind: build + tier: 2 + worker-type: aws-provisioner-v1/gecko-{level}-b-linux + worker: + implementation: docker-worker + docker-image: {in-tree: desktop-build} + max-run-time: 36000 + + # don't run anywhere by default, but still available via try + run-on-projects: [] + run: + # NOTE: this should really be a different "using" since it's using + # build-l10n.sh instead of build-linux.sh. Preferably, build-linux.sh + # and the mozharness run implementation should be modified to support + # the functionality that l10n needs + using: mozharness + job-script: taskcluster/scripts/builder/build-l10n.sh + when: + files-changed: + - browser/locales/all-locales + - python/compare-locales/** + - testing/mozharness/configs/single_locale/** + - testing/mozharness/mozharness/mozilla/l10n/locales.py + - testing/mozharness/scripts/desktop_l10n.py + - toolkit/locales/** + - toolkit/mozapps/installer/** + +jobs: + linux-l10n/opt: + description: "Localization" + index: + job-name: + gecko-v2: linux32-l10n-opt + treeherder: + platform: linux32/opt + symbol: tc(L10n) + run: + script: mozharness/scripts/desktop_l10n.py + actions: [clone-locales list-locales setup repack summary] + config: + - single_locale/tc_linux32.py + options: + - environment-config=single_locale/production.py + - branch-config=single_locale/{project}.py + - platform-config=single_locale/linux32.py + - total-chunks=1 + - this-chunk=1 + tooltool-downloads: public + need-xvfb: true + + linux64-l10n/opt: + description: "Localization" + index: + job-name: + gecko-v2: linux64-l10n-opt + treeherder: + platform: linux64/opt + symbol: tc(L10n) + run: + script: mozharness/scripts/desktop_l10n.py + actions: [clone-locales list-locales setup repack summary] + config: + - single_locale/tc_linux64.py + options: + - environment-config=single_locale/production.py + - branch-config=single_locale/{project}.py + - platform-config=single_locale/linux64.py + - total-chunks=1 + - this-chunk=1 + tooltool-downloads: public + need-xvfb: true diff --git a/taskcluster/ci/marionette-harness/kind.yml b/taskcluster/ci/marionette-harness/kind.yml new file mode 100644 index 000000000..8ec8189b9 --- /dev/null +++ b/taskcluster/ci/marionette-harness/kind.yml @@ -0,0 +1,51 @@ +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, You can obtain one at http://mozilla.org/MPL/2.0/. + +# NOTE: please write a description of this kind in taskcluster/docs/kinds.rst + +implementation: taskgraph.task.transform:TransformTask + +transforms: + - taskgraph.transforms.marionette_harness:transforms + - taskgraph.transforms.task:transforms + +# NOTE: this task should be refactored so that it is invoked as a job either +# with a run.using of "mozharness", and combined with the source-check kind. + +jobs: + marionette-harness/opt: + description: "Marionette harness unit test" + attributes: + build_platform: marionette-harness + build_type: opt + treeherder: + platform: linux64/opt + kind: test + tier: 2 + symbol: tc(Mn-h) + worker-type: aws-provisioner-v1/gecko-t-linux-xlarge + worker: + implementation: docker-worker + docker-image: {in-tree: desktop-build} # NOTE: better to use the lint image + env: + JOB_SCRIPT: "taskcluster/scripts/tester/harness-test-linux.sh" + MOZHARNESS_SCRIPT: "testing/mozharness/scripts/marionette_harness_tests.py" + TOOLS_DISABLE: "true" + TOOLTOOL_REPO: "https://github.com/mozilla/build-tooltool" + TOOLTOOL_REV: "master" + artifacts: + - name: public/logs/ + path: /home/worker/workspace/mozharness_workspace/upload/logs/ + type: directory + command: + - "bash" + - "/home/worker/bin/build.sh" + - "--tests=testing/marionette/harness/marionette_harness/tests/harness_unit" + - "--work-dir=mozharness_workspace" + max-run-time: 1800 + when: + files-changed: + - "testing/marionette/harness/**" + - "testing/mozharness/scripts/marionette_harness_tests.py" + - "testing/config/marionette_harness_test_requirements.txt" diff --git a/taskcluster/ci/source-check/doc.yml b/taskcluster/ci/source-check/doc.yml new file mode 100644 index 000000000..0ab91c518 --- /dev/null +++ b/taskcluster/ci/source-check/doc.yml @@ -0,0 +1,32 @@ +sphinx/opt: + description: Generate the Sphinx documentation + treeherder: + symbol: tc(Doc) + kind: test + tier: 1 + platform: lint/opt + worker-type: aws-provisioner-v1/b2gtest + worker: + implementation: docker-worker + docker-image: {in-tree: "lint"} + max-run-time: 1800 + artifacts: + - type: file + name: public/docs.tar.gz + path: /home/worker/checkouts/gecko/docs.tar.gz + run: + using: run-task + command: > + cd /home/worker/checkouts/gecko && + ./mach doc --outdir docs-out --no-open && + rm -rf docs-out/html/Mozilla_Source_Tree_Docs/_venv && + mv docs-out/html/Mozilla_Source_Tree_Docs docs && + tar -czf docs.tar.gz docs + run-on-projects: + - integration + - release + when: + files-changed: + - '**/*.py' + - '**/*.rst' + - 'tools/docs/**' diff --git a/taskcluster/ci/source-check/kind.yml b/taskcluster/ci/source-check/kind.yml new file mode 100644 index 000000000..6bc2b4b83 --- /dev/null +++ b/taskcluster/ci/source-check/kind.yml @@ -0,0 +1,15 @@ +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, You can obtain one at http://mozilla.org/MPL/2.0/. + +implementation: taskgraph.task.transform:TransformTask + +transforms: + - taskgraph.transforms.build_attrs:transforms + - taskgraph.transforms.job:transforms + - taskgraph.transforms.task:transforms + +jobs-from: + - python-tests.yml + - mozlint.yml + - doc.yml diff --git a/taskcluster/ci/source-check/mozlint.yml b/taskcluster/ci/source-check/mozlint.yml new file mode 100644 index 000000000..fd22a9f8f --- /dev/null +++ b/taskcluster/ci/source-check/mozlint.yml @@ -0,0 +1,97 @@ +mozlint-eslint/opt: + description: JS lint check + treeherder: + symbol: ES + kind: test + tier: 1 + platform: lint/opt + worker-type: aws-provisioner-v1/b2gtest + worker: + implementation: docker-worker + docker-image: {in-tree: "lint"} + max-run-time: 1800 + run: + using: run-task + command: > + cd /home/worker/checkouts/gecko/tools/lint/eslint && + /build/tooltool.py fetch -m manifest.tt && + tar xvfz eslint.tar.gz && + rm eslint.tar.gz && + ln -s ../eslint-plugin-mozilla node_modules && + cd ../../.. && + ./mach lint -l eslint -f treeherder --quiet + run-on-projects: + - integration + - release + when: + files-changed: + # Files that are likely audited. + - '**/*.js' + - '**/*.jsm' + - '**/*.jsx' + - '**/*.html' + - '**/*.xhtml' + - '**/*.xml' + # Run when eslint policies change. + - '**/.eslintignore' + - '**/*eslintrc*' + # The plugin implementing custom checks. + - 'tools/lint/eslint/eslint-plugin-mozilla/**' + # Other misc lint related files. + - 'python/mozlint/**' + - 'tools/lint/**' + - 'testing/docker/lint/**' + +mozlint-flake8/opt: + description: flake8 run over the gecko codebase + treeherder: + symbol: f8 + kind: test + tier: 1 + platform: lint/opt + worker-type: aws-provisioner-v1/b2gtest + worker: + implementation: docker-worker + docker-image: {in-tree: "lint"} + max-run-time: 1800 + run: + using: mach + mach: lint -l flake8 -f treeherder + run-on-projects: + - integration + - release + when: + files-changed: + - '**/*.py' + - '**/.flake8' + - 'python/mozlint/**' + - 'tools/lint/**' + - 'testing/docker/lint/**' + +wptlint-gecko/opt: + description: web-platform-tests linter + treeherder: + symbol: W + kind: test + tier: 1 + platform: lint/opt + worker-type: aws-provisioner-v1/b2gtest + worker: + implementation: docker-worker + docker-image: {in-tree: "lint"} + max-run-time: 1800 + run: + using: mach + mach: lint -l wpt -l wpt_manifest -f treeherder + run-on-projects: + - integration + - release + when: + files-changed: + - 'testing/web-platform/tests/**' + - 'testing/web-platform/mozilla/tests/**' + - 'testing/web-platform/meta/MANIFEST.json' + - 'testing/web-platform/mozilla/meta/MANIFEST.json' + - 'python/mozlint/**' + - 'tools/lint/**' + - 'testing/docker/lint/**' diff --git a/taskcluster/ci/source-check/python-tests.yml b/taskcluster/ci/source-check/python-tests.yml new file mode 100644 index 000000000..2f580f251 --- /dev/null +++ b/taskcluster/ci/source-check/python-tests.yml @@ -0,0 +1,49 @@ +taskgraph-tests/opt: + description: taskcluster/taskgraph unit tests + treeherder: + symbol: tg + kind: test + tier: 2 + platform: linux64/opt + worker-type: aws-provisioner-v1/b2gtest + worker: + implementation: docker-worker + docker-image: {in-tree: "lint"} + max-run-time: 1800 + run: + using: mach + mach: taskgraph python-tests + run-on-projects: + - integration + - release + when: + files-changed: + - 'taskcluster/**/*.py' + - 'config/mozunit.py' + - 'python/mach/**/*.py' + +mozharness/opt: + description: mozharness integration tests + treeherder: + symbol: MH + kind: test + tier: 2 + platform: lint/opt + worker-type: aws-provisioner-v1/b2gtest + worker: + implementation: docker-worker + docker-image: {in-tree: "lint"} + max-run-time: 1800 + run: + using: run-task + cache-dotcache: true + command: > + cd /home/worker/checkouts/gecko/testing/mozharness && + /usr/bin/pip2 install tox && + /home/worker/.local/bin/tox -e py27-hg3.7 + run-on-projects: + - integration + - release + when: + files-changed: + - 'testing/mozharness/**' diff --git a/taskcluster/ci/spidermonkey/kind.yml b/taskcluster/ci/spidermonkey/kind.yml new file mode 100644 index 000000000..9e8f9a682 --- /dev/null +++ b/taskcluster/ci/spidermonkey/kind.yml @@ -0,0 +1,199 @@ +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, You can obtain one at http://mozilla.org/MPL/2.0/. + +implementation: taskgraph.task.transform:TransformTask + +transforms: + - taskgraph.transforms.build_attrs:transforms + - taskgraph.transforms.job:transforms + - taskgraph.transforms.task:transforms + +job-defaults: + treeherder: + platform: linux64/opt + kind: build + tier: 1 + index: + product: firefox + worker-type: aws-provisioner-v1/gecko-{level}-b-linux + worker: + implementation: docker-worker + max-run-time: 36000 + docker-image: {in-tree: desktop-build} + run: + using: spidermonkey + when: + files-changed: + # any when.files-changed specified below in a job will be + # appended to this list + - js/public/** + - js/src/** + +jobs: + sm-package/opt: + description: "Spidermonkey source package and test" + index: + job-name: + buildbot: sm-plain + gecko-v2: sm-package-opt + treeherder: + symbol: SM-tc(pkg) + run: + using: spidermonkey-package + spidermonkey-variant: plain + when: + files-changed: + - build/** + - config/** + - configure.py + - dom/bindings/** + - intl/icu/** + - js/moz.configure + - layout/tools/reftest/reftest/** + - Makefile.in + - media/webrtc/trunk/tools/gyp/** + - memory/** + - mfbt/** + - modules/fdlibm/** + - modules/zlib/src/** + - mozglue/** + - moz.build + - moz.configure + - nsprpub/** + - python/** + - taskcluster/moz.build + - testing/mozbase/** + - test.mozbuild + - toolkit/mozapps/installer/package-name.mk + - toolkit/mozapps/installer/upload-files.mk + + sm-mozjs-sys/debug: + description: "Build js/src as the mozjs_sys Rust crate" + index: + job-name: + gecko-v2: sm-mozjs-sys-debug + treeherder: + symbol: SM-tc(mozjs-crate) + run: + using: spidermonkey-mozjs-crate + spidermonkey-variant: plain + run-on-projects: + - integration + - release + - try + + sm-plain/debug: + description: "Spidermonkey Plain" + index: + job-name: + buildbot: sm-plain + gecko-v2: sm-plaindebug-debug + treeherder: + platform: linux64/debug + symbol: SM-tc(p) + run: + spidermonkey-variant: plaindebug + + sm-plain/opt: + description: "Spidermonkey Plain" + index: + job-name: sm-plain-opt + treeherder: + symbol: SM-tc(p) + run: + spidermonkey-variant: plain + + sm-arm-sim/debug: + description: "Spidermonkey ARM sim" + index: + job-name: + buildbot: sm-plain + gecko-v2: sm-arm-sim-debug + treeherder: + symbol: SM-tc(arm) + run: + spidermonkey-variant: arm-sim + + sm-arm64-sim/debug: + description: "Spidermonkey ARM64 sim" + index: + job-name: + buildbot: sm-plain + gecko-v2: sm-arm64-sim-debug + treeherder: + symbol: SM-tc(arm64) + run: + spidermonkey-variant: arm64-sim + + sm-asan/opt: + description: "Spidermonkey Address Sanitizer" + index: + job-name: + buildbot: sm-plain + gecko-v2: sm-asan-opt + treeherder: + symbol: SM-tc(asan) + run: + spidermonkey-variant: asan + tooltool-manifest: browser/config/tooltool-manifests/linux64/asan.manifest + + sm-compacting/debug: + description: "Spidermonkey Compacting" + index: + job-name: + buildbot: sm-plain + gecko-v2: sm-compacting-debug + treeherder: + symbol: SM-tc(cgc) + run: + spidermonkey-variant: compacting + + sm-msan/opt: + description: "Spidermonkey Memory Sanitizer" + index: + job-name: + buildbot: sm-plain + gecko-v2: sm-msan-opt + treeherder: + symbol: SM-tc(msan) + run: + spidermonkey-variant: msan + tooltool-manifest: browser/config/tooltool-manifests/linux64/msan.manifest + + sm-tsan/opt: + description: "Spidermonkey Thread Sanitizer" + index: + job-name: + buildbot: sm-plain + gecko-v2: sm-tsan-opt + treeherder: + symbol: SM-tc(tsan) + tier: 3 + run-on-projects: [] + run: + spidermonkey-variant: tsan + tooltool-manifest: browser/config/tooltool-manifests/linux64/tsan.manifest + + sm-rootanalysis/debug: + description: "Spidermonkey Root Analysis" + index: + job-name: + buildbot: sm-plain + gecko-v2: sm-rootanalysis-debug + treeherder: + symbol: SM-tc(r) + run: + spidermonkey-variant: rootanalysis + + sm-nonunified/debug: + description: "Spidermonkey Non-Unified Debug" + index: + job-name: + buildbot: sm-plain + gecko-v2: sm-nonunified-debug + treeherder: + platform: linux64/debug + symbol: SM-tc(nu) + run: + spidermonkey-variant: nonunified diff --git a/taskcluster/ci/static-analysis/kind.yml b/taskcluster/ci/static-analysis/kind.yml new file mode 100644 index 000000000..ddd312e70 --- /dev/null +++ b/taskcluster/ci/static-analysis/kind.yml @@ -0,0 +1,63 @@ +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, You can obtain one at http://mozilla.org/MPL/2.0/. + +implementation: taskgraph.task.transform:TransformTask + +transforms: + - taskgraph.transforms.build_attrs:transforms + - taskgraph.transforms.job:transforms + - taskgraph.transforms.task:transforms + +job-defaults: + index: + product: firefox + treeherder: + symbol: S + kind: build + tier: 1 + worker: + implementation: docker-worker + docker-image: {in-tree: desktop-build} + max-run-time: 36000 + +jobs: + macosx64-st-an/opt: + description: "MacOS X x64 Cross-compile Static Analysis" + index: + job-name: macosx64-st-an-opt + treeherder: + platform: osx-10-7/opt + worker-type: aws-provisioner-v1/gecko-{level}-b-macosx64 + run: + using: mozharness + actions: [get-secrets build generate-build-stats update] + config: + - builds/releng_base_mac_64_cross_builds.py + - balrog/production.py + # Note that, despite the name "cross-opt", this config variant + # enables static analysis. + custom-build-variant-cfg: cross-opt + script: "mozharness/scripts/fx_desktop_build.py" + secrets: true + tooltool-downloads: internal + keep-artifacts: false + + linux64-st-an/opt: + description: "Linux64 Opt Static Analysis" + index: + job-name: linux64-st-an-opt + treeherder: + platform: linux64/opt + worker-type: aws-provisioner-v1/gecko-{level}-b-linux + run: + using: mozharness + actions: [get-secrets build generate-build-stats] + config: + - builds/releng_sub_linux_configs/64_stat_and_opt.py + - balrog/production.py + script: "mozharness/scripts/fx_desktop_build.py" + secrets: true + tooltool-downloads: public + need-xvfb: true + keep-artifacts: false diff --git a/taskcluster/ci/toolchain/kind.yml b/taskcluster/ci/toolchain/kind.yml new file mode 100644 index 000000000..acc67a57f --- /dev/null +++ b/taskcluster/ci/toolchain/kind.yml @@ -0,0 +1,14 @@ +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, You can obtain one at http://mozilla.org/MPL/2.0/. + +implementation: taskgraph.task.transform:TransformTask + +transforms: + - taskgraph.transforms.build_attrs:transforms + - taskgraph.transforms.job:transforms + - taskgraph.transforms.task:transforms + +jobs-from: + - linux.yml + - windows.yml diff --git a/taskcluster/ci/toolchain/linux.yml b/taskcluster/ci/toolchain/linux.yml new file mode 100644 index 000000000..9eeadf3ef --- /dev/null +++ b/taskcluster/ci/toolchain/linux.yml @@ -0,0 +1,66 @@ +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, You can obtain one at http://mozilla.org/MPL/2.0/. + +linux64-clang/opt: + description: "Clang toolchain build" + treeherder: + kind: build + platform: linux64/opt + symbol: Cc(Clang) + tier: 1 + run: + using: toolchain-script + script: build-clang-linux.sh + worker-type: aws-provisioner-v1/gecko-{level}-b-linux + worker: + implementation: docker-worker + docker-image: {in-tree: desktop-build} + max-run-time: 36000 + when: + files-changed: + - 'build/build-clang/**' + - 'taskcluster/scripts/misc/build-clang-linux.sh' + - 'taskcluster/taskgraph/transforms/job/toolchain.py' + +linux64-gcc/opt: + description: "GCC toolchain build" + treeherder: + kind: build + platform: linux64/opt + symbol: Cc(GCC) + tier: 1 + run: + using: toolchain-script + script: build-gcc-linux.sh + worker-type: aws-provisioner-v1/gecko-{level}-b-linux + worker: + implementation: docker-worker + docker-image: {in-tree: desktop-build} + max-run-time: 36000 + when: + files-changed: + - 'build/unix/build-gcc/**' + - 'taskcluster/scripts/misc/build-gcc-linux.sh' + - 'taskcluster/taskgraph/transforms/job/toolchain.py' + +linux64-binutils/opt: + description: "Binutils toolchain build" + treeherder: + kind: build + platform: linux64/opt + symbol: Cc(binutils) + tier: 1 + run: + using: toolchain-script + script: build-binutils-linux.sh + worker-type: aws-provisioner-v1/gecko-{level}-b-linux + worker: + implementation: docker-worker + docker-image: {in-tree: desktop-build} + max-run-time: 36000 + when: + files-changed: + - 'build/unix/build-binutils/**' + - 'taskcluster/scripts/misc/build-binutils-linux.sh' + - 'taskcluster/taskgraph/transforms/job/toolchain.py' diff --git a/taskcluster/ci/toolchain/windows.yml b/taskcluster/ci/toolchain/windows.yml new file mode 100644 index 000000000..b5b950616 --- /dev/null +++ b/taskcluster/ci/toolchain/windows.yml @@ -0,0 +1,23 @@ +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, You can obtain one at http://mozilla.org/MPL/2.0/. + +win32-clang-cl/opt: + description: "Clang-cl toolchain build" + treeherder: + kind: build + platform: windows-2012-32/opt + symbol: Cc(ClangCL) + tier: 2 + worker-type: aws-provisioner-v1/gecko-{level}-b-win2012 + worker: + implementation: generic-worker + max-run-time: 36000 + run: + using: toolchain-script + script: build-clang-windows.sh + when: + files-changed: + - 'build/build-clang/**' + - 'taskcluster/scripts/misc/build-clang-windows.sh' + - 'taskcluster/taskgraph/transforms/job/toolchain.py' diff --git a/taskcluster/ci/upload-symbols/job-template.yml b/taskcluster/ci/upload-symbols/job-template.yml new file mode 100644 index 000000000..43d6736a0 --- /dev/null +++ b/taskcluster/ci/upload-symbols/job-template.yml @@ -0,0 +1,19 @@ +label: # see transforms +description: Upload Symbols +dependencies: # see transforms +expires-after: 7 days +deadline-after: 24 hours +run-on-projects: + - try +worker-type: aws-provisioner-v1/symbol-upload +worker: + implementation: docker-worker + max-run-time: 600 + command: ["/bin/bash", "bin/upload.sh"] + docker-image: taskclusterprivate/upload_symbols:0.0.4 + env: + GECKO_HEAD_REPOSITORY: # see transforms + GECKO_HEAD_REV: # see transforms + ARTIFACT_TASKID: {"task-reference": "<build>"} +scopes: + - docker-worker:image:taskclusterprivate/upload_symbols:0.0.4 diff --git a/taskcluster/ci/upload-symbols/kind.yml b/taskcluster/ci/upload-symbols/kind.yml new file mode 100644 index 000000000..b3e2ba5d8 --- /dev/null +++ b/taskcluster/ci/upload-symbols/kind.yml @@ -0,0 +1,19 @@ +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, You can obtain one at http://mozilla.org/MPL/2.0/. + +implementation: taskgraph.task.post_build:PostBuildTask + +transforms: + - taskgraph.transforms.upload_symbols:transforms + - taskgraph.transforms.task:transforms + +kind-dependencies: + - build + +job-template: job-template.yml + +only-for-build-platforms: + - linux64/opt + - linux64/debug + - android-api-15/opt diff --git a/taskcluster/ci/valgrind/kind.yml b/taskcluster/ci/valgrind/kind.yml new file mode 100644 index 000000000..77e37e89b --- /dev/null +++ b/taskcluster/ci/valgrind/kind.yml @@ -0,0 +1,40 @@ +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, You can obtain one at http://mozilla.org/MPL/2.0/. + +implementation: taskgraph.task.transform:TransformTask + +transforms: + - taskgraph.transforms.build_attrs:transforms + - taskgraph.transforms.job:transforms + - taskgraph.transforms.task:transforms + +jobs: + linux64-valgrind/opt: + description: "Linux64 Valgrind Opt" + index: + product: firefox + job-name: linux64-valgrind-opt + treeherder: + platform: linux64/opt + symbol: tc(V) + kind: build + tier: 1 + worker-type: aws-provisioner-v1/gecko-{level}-b-linux + worker: + implementation: docker-worker + docker-image: {in-tree: desktop-build} + max-run-time: 72000 + run: + using: mozharness + actions: [get-secrets build valgrind-test generate-build-stats] + custom-build-variant-cfg: valgrind + config: + - builds/releng_base_linux_64_builds.py + - balrog/production.py + script: "mozharness/scripts/fx_desktop_build.py" + secrets: true + tooltool-downloads: public + need-xvfb: true + + diff --git a/taskcluster/docs/attributes.rst b/taskcluster/docs/attributes.rst new file mode 100644 index 000000000..d93964d65 --- /dev/null +++ b/taskcluster/docs/attributes.rst @@ -0,0 +1,124 @@ +=============== +Task Attributes +=============== + +Tasks can be filtered, for example to support "try" pushes which only perform a +subset of the task graph or to link dependent tasks. This filtering is the +difference between a full task graph and a target task graph. + +Filtering takes place on the basis of attributes. Each task has a dictionary +of attributes and filters over those attributes can be expressed in Python. A +task may not have a value for every attribute. + +The attributes, and acceptable values, are defined here. In general, attribute +names and values are the short, lower-case form, with underscores. + +kind +==== + +A task's ``kind`` attribute gives the name of the kind that generated it, e.g., +``build`` or ``spidermonkey``. + +run_on_projects +=============== + +The projects where this task should be in the target task set. This is how +requirements like "only run this on inbound" get implemented. These are +either project names or the aliases + + * `integration` -- integration branches + * `release` -- release branches including mozilla-central + * `all` -- everywhere (the default) + +For try, this attribute applies only if ``-p all`` is specified. All jobs can +be specified by name regardless of ``run_on_projects``. + +If ``run_on_projects`` is set to an empty list, then the task will not run +anywhere, unless its build platform is specified explicitly in try syntax. + +task_duplicates +=============== + +This is used to indicate that we want multiple copies of the task created. +This feature is used to track down intermittent job failures. + +If this value is set to N, the task-creation machinery will create a total of N +copies of the task. Only the first copy will be included in the taskgraph +output artifacts, although all tasks will be contained in the same taskGroup. + +While most attributes are considered read-only, target task methods may alter +this attribute of tasks they include in the target set. + +build_platform +============== + +The build platform defines the platform for which the binary was built. It is +set for both build and test jobs, although test jobs may have a different +``test_platform``. + +build_type +========== + +The type of build being performed. This is a subdivision of ``build_platform``, +used for different kinds of builds that target the same platform. Values are + + * ``debug`` + * ``opt`` + +test_platform +============= + +The test platform defines the platform on which tests are run. It is only +defined for test jobs and may differ from ``build_platform`` when the same binary +is tested on several platforms (for example, on several versions of Windows). +This applies for both talos and unit tests. + +Unlike build_platform, the test platform is represented in a slash-separated +format, e.g., ``linux64/opt``. + +unittest_suite +============== + +This is the unit test suite being run in a unit test task. For example, +``mochitest`` or ``cppunittest``. + +unittest_flavor +=============== + +If a unittest suite has subdivisions, those are represented as flavors. Not +all suites have flavors, in which case this attribute should be set to match +the suite. Examples: ``mochitest-devtools-chrome-chunked`` or ``a11y``. + +unittest_try_name +================= + +This is the name used to refer to a unit test via try syntax. It +may not match either of ``unittest_suite`` or ``unittest_flavor``. + +talos_try_name +============== + +This is the name used to refer to a talos job via try syntax. + +test_chunk +========== + +This is the chunk number of a chunked test suite (talos or unittest). Note +that this is a string! + +e10s +==== + +For test suites which distinguish whether they run with or without e10s, this +boolean value identifies this particular run. + +image_name +========== + +For the ``docker_image`` kind, this attribute contains the docker image name. + +nightly +======= + +Signals whether the task is part of a nightly graph. Useful when filtering +out nightly tasks from full task set at target stage. diff --git a/taskcluster/docs/caches.rst b/taskcluster/docs/caches.rst new file mode 100644 index 000000000..9f19035d7 --- /dev/null +++ b/taskcluster/docs/caches.rst @@ -0,0 +1,43 @@ +.. taskcluster_caches: + +============= +Common Caches +============= + +There are various caches used by the in-tree tasks. This page attempts to +document them and their appropriate use. + +Version Control Caches +====================== + +``level-{{level}}-checkouts-{{version}}`` + This cache holds version control checkouts, each in a subdirectory named + after the repo (e.g., ``gecko``). + + Checkouts should be read-only. If a task needs to create new files from + content of a checkout, this content should be written in a separate + directory/cache (like a workspace). + + A ``version`` parameter appears in the cache name to allow + backwards-incompatible changes to the cache's behavior. + +``level-{{level}}-{{project}}-tc-vcs`` (deprecated) + This cache is used internally by ``tc-vcs``. This tool is deprecated and + should be replaced with ``hg robustcheckout``. + +Workspace Caches +================ + +``level-{{level}}-*-workspace`` + These caches (of various names typically ending with ``workspace``) + contain state to be shared between task invocations. Use cases are + dependent on the task. + +Other +===== + +``tooltool-cache`` + Tooltool invocations should use this cache. Tooltool will store files here + indexed by their hash, and will verify hashes before copying files from + this directory, so there is no concern with sharing the cache between jobs + of different levels. diff --git a/taskcluster/docs/docker-images.rst b/taskcluster/docs/docker-images.rst new file mode 100644 index 000000000..22dea4dea --- /dev/null +++ b/taskcluster/docs/docker-images.rst @@ -0,0 +1,42 @@ +.. taskcluster_dockerimages: + +============= +Docker Images +============= + +TaskCluster Docker images are defined in the source directory under +``testing/docker``. Each directory therein contains the name of an +image used as part of the task graph. + +Adding Extra Files to Images +============================ + +Dockerfile syntax has been extended to allow *any* file from the +source checkout to be added to the image build *context*. (Traditionally +you can only ``ADD`` files from the same directory as the Dockerfile.) + +Simply add the following syntax as a comment in a Dockerfile:: + + # %include <path> + +e.g. + + # %include mach + # %include testing/mozharness + +The argument to ``# %include`` is a relative path from the root level of +the source directory. It can be a file or a directory. If a file, only that +file will be added. If a directory, every file under that directory will be +added (even files that are untracked or ignored by version control). + +Files added using ``# %include`` syntax are available inside the build +context under the ``topsrcdir/`` path. + +Files are added as they exist on disk. e.g. executable flags should be +preserved. However, the file owner/group is changed to ``root`` and the +``mtime`` of the file is normalized. + +Here is an example Dockerfile snippet:: + + # %include mach + ADD topsrcdir/mach /home/worker/mach diff --git a/taskcluster/docs/how-tos.rst b/taskcluster/docs/how-tos.rst new file mode 100644 index 000000000..6b143dd42 --- /dev/null +++ b/taskcluster/docs/how-tos.rst @@ -0,0 +1,220 @@ +How Tos +======= + +All of this equipment is here to help you get your work done more efficiently. +However, learning how task-graphs are generated is probably not the work you +are interested in doing. This section should help you accomplish some of the +more common changes to the task graph with minimal fuss. + +.. important:: + + If you cannot accomplish what you need with the information provided here, + please consider whether you can achieve your goal in a different way. + Perhaps something simpler would cost a bit more in compute time, but save + the much more expensive resource of developers' mental bandwidth. + Task-graph generation is already complex enough! + + If you want to proceed, you may need to delve into the implementation of + task-graph generation. The documentation and code are designed to help, as + are the authors - ``hg blame`` may help track down helpful people. + + As you write your new transform or add a new kind, please consider the next + developer. Where possible, make your change data-driven and general, so + that others can make a much smaller change. Document the semantics of what + you are changing clearly, especially if it involves modifying a transform + schema. And if you are adding complexity temporarily while making a + gradual transition, please open a new bug to remind yourself to remove the + complexity when the transition is complete. + +Hacking Task Graphs +------------------- + +The recommended process for changing task graphs is this: + +1. Find a recent decision task on the project or branch you are working on, + and download its ``parameters.yml`` from the Task Inspector. This file + contains all of the inputs to the task-graph generation process. Its + contents are simple enough if you would like to modify it, and it is + documented in :doc:`parameters`. + +2. Run one of the ``mach taskgraph`` subcommands (see :doc:`taskgraph`) to + generate a baseline against which to measure your changes. For example: + + .. code-block:: none + + ./mach taskgraph tasks --json -p parameters.yml > old-tasks.json + +3. Make your modifications under ``taskcluster/``. + +4. Run the same ``mach taskgraph`` command, sending the output to a new file, + and use ``diff`` to compare the old and new files. Make sure your changes + have the desired effect and no undesirable side-effects. + +5. When you are satisfied with the changes, push them to try to ensure that the + modified tasks work as expected. + +Common Changes +-------------- + +Changing Test Characteristics +............................. + +First, find the test description. This will be in +``taskcluster/ci/*/tests.yml``, for the appropriate kind (consult +:doc:`kinds`). You will find a YAML stanza for each test suite, and each +stanza defines the test's characteristics. For example, the ``chunks`` +property gives the number of chunks to run. This can be specified as a simple +integer if all platforms have the same chunk count, or it can be keyed by test +platform. For example: + +.. code-block:: yaml + + chunks: + by-test-platform: + linux64/debug: 10 + default: 8 + +The full set of available properties is in +``taskcluster/taskgraph/transform/tests/test_description.py``. Some other +commonly-modified properties are ``max-run-time`` (useful if tests are being +killed for exceeding maxRunTime) and ``treeherder-symbol``. + +.. note:: + + Android tests are also chunked at the mozharness level, so you will need to + modify the relevant mozharness config, as well. + +Adding a Test Suite +................... + +To add a new test suite, you will need to know the proper mozharness invocation +for that suite, and which kind it fits into (consult :doc:`kinds`). + +Add a new stanza to ``taskcluster/ci/<kind>/tests.yml``, copying from the other +stanzas in that file. The meanings should be clear, but authoritative +documentation is in +``taskcluster/taskgraph/transform/tests/test_description.py`` should you need +it. The stanza name is the name by which the test will be referenced in try +syntax. + +Add your new test to a test set in ``test-sets.yml`` in the same directory. If +the test should only run on a limited set of platforms, you may need to define +a new test set and reference that from the appropriate platforms in +``test-platforms.yml``. If you do so, include some helpful comments in +``test-sets.yml`` for the next person. + +Greening Up a New Test +...................... + +When a test is not yet reliably green, configuration for that test should not +be landed on integration branches. Of course, you can control where the +configuration is landed! For many cases, it is easiest to green up a test in +try: push the configuration to run the test to try along with your work to fix +the remaining test failures. + +When working with a group, check out a "twig" repository to share among your +group, and land the test configuration in that repository. Once the test is +green, merge to an integration branch and the test will begin running there as +well. + +Adding a New Task +................. + +If you are adding a new task that is not a test suite, there are a number of +options. A few questions to consider: + + * Is this a new build platform or variant that will produce an artifact to + be run through the usual test suites? + + * Does this task depend on other tasks? Do other tasks depend on it? + + * Is this one of a few related tasks, or will you need to generate a large + set of tasks using some programmatic means (for example, chunking)? + + * How is the task actually excuted? Mozharness? Mach? + + * What kind of environment does the task require? + +Armed with that information, you can choose among a few options for +implementing this new task. Try to choose the simplest solution that will +satisfy your near-term needs. Since this is all implemented in-tree, it +is not difficult to refactor later when you need more generality. + +Existing Kind +````````````` + +The simplest option is to add your task to an existing kind. This is most +practical when the task "makes sense" as part of that kind -- for example, if +your task is building an installer for a new platform using mozharness scripts +similar to the existing build tasks, it makes most sense to add your task to +the ``build`` kind. If you need some additional functionality in the kind, +it's OK to modify the implementation as necessary, as long as the modification +is complete and useful to the next developer to come along. + +New Kind +```````` + +The next option to consider is adding a new kind. A distinct kind gives you +some isolation from other task types, which can be nice if you are adding an +experimental kind of task. + +Kinds can range in complexity. The simplest sort of kind uses the +``TransformTask`` implementation to read a list of jobs from the ``jobs`` key, +and applies the standard ``job`` and ``task`` transforms: + +.. code-block:: yaml + + implementation: taskgraph.task.transform:TransformTask + transforms: + - taskgraph.transforms.job:transforms + - taskgraph.transforms.task:transforms + jobs: + - ..your job description here.. + +Custom Kind Implementation +`````````````````````````` + +If your task depends on other tasks, then the decision of which tasks to create +may require some code. For example, the ``upload-symbols`` kind iterates over +the builds in the graph, generating a task for each one. This specific +post-build behavior is implemented in the general +``taskgraph.task.post_build:PostBuildTask`` kind implementation. If your task +needs something more purpose-specific, then it may be time to write a new kind +implementation. + +Custom Transforms +````````````````` + +If your task needs to create many tasks from a single description, for example +to implement chunking, it is time to implement some custom transforms. Ideally +those transforms will produce job descriptions, so you can use the existing ``job`` +and ``task`` transforms: + +.. code-block:: yaml + + transforms: + - taskgraph.transforms.my_stuff:transforms + - taskgraph.transforms.job:transforms + - taskgraph.transforms.task:transforms + +Similarly, if you need to include dynamic task defaults -- perhaps some feature +is only available in level-3 repositories, or on specific projects -- then +custom transforms are the appropriate tool. Try to keep transforms simple, +single-purpose and well-documented! + +Custom Run-Using +```````````````` + +If the way your task is executed is unique (so, not a mach command or +mozharness invocation), you can add a new implementation of the job +description's "run" section. Before you do this, consider that it might be a +better investment to modify your task to support invocation via mozharness or +mach, instead. If this is not possible, then adding a new file in +``taskcluster/taskgraph/transforms/jobs`` with a structure similar to its peers +will make the new run-using option available for job descriptions. + +Something Else? +............... + +If you make another change not described here that turns out to be simple or +common, please include an update to this file in your patch. diff --git a/taskcluster/docs/index.rst b/taskcluster/docs/index.rst new file mode 100644 index 000000000..d1a5c600b --- /dev/null +++ b/taskcluster/docs/index.rst @@ -0,0 +1,30 @@ +.. taskcluster_index: + +TaskCluster Task-Graph Generation +================================= + +The ``taskcluster`` directory contains support for defining the graph of tasks +that must be executed to build and test the Gecko tree. This is more complex +than you might suppose! This implementation supports: + + * A huge array of tasks + * Different behavior for different repositories + * "Try" pushes, with special means to select a subset of the graph for execution + * Optimization -- skipping tasks that have already been performed + * Extremely flexible generation of a variety of tasks using an approach of + incrementally transforming job descriptions into task definitions. + +This section of the documentation describes the process in some detail, +referring to the source where necessary. If you are reading this with a +particular goal in mind and would rather avoid becoming a task-graph expert, +check out the :doc:`how-to section <how-tos>`. + +.. toctree:: + + taskgraph + loading + transforms + yaml-templates + docker-images + how-tos + reference diff --git a/taskcluster/docs/kinds.rst b/taskcluster/docs/kinds.rst new file mode 100644 index 000000000..44bddb360 --- /dev/null +++ b/taskcluster/docs/kinds.rst @@ -0,0 +1,144 @@ +Task Kinds +========== + +This section lists and documents the available task kinds. + +build +------ + +Builds are tasks that produce an installer or other output that can be run by +users or automated tests. This is more restrictive than most definitions of +"build" in a Mozilla context: it does not include tasks that run build-like +actions for static analysis or to produce instrumented artifacts. + +artifact-build +-------------- + +This kind performs an artifact build: one based on precompiled binaries +discovered via the TaskCluster index. This task verifies that such builds +continue to work correctly. + +hazard +------ + +Hazard builds are similar to "regular' builds, but use a compiler extension to +extract a bunch of data from the build and then analyze that data looking for +hazardous behaviors. + +l10n +---- + +TBD (Callek) + +source-check +------------ + +Source-checks are tasks that look at the Gecko source directly to check +correctness. This can include linting, Python unit tests, source-code +analysis, or measurement work -- basically anything that does not require a +build. + +upload-symbols +-------------- + +Upload-symbols tasks run after builds and upload the symbols files generated by +build tasks to Socorro for later use in crash analysis. + +valgrind +-------- + +Valgrind tasks produce builds instrumented by valgrind. + +static-analysis +--------------- + +Static analysis builds use the compiler to perform some detailed analysis of +the source code while building. The useful output from these tasks are their +build logs, and while they produce a binary, they do not upload it as an +artifact. + +toolchain +--------- + +Toolchain builds create the compiler toolchains used to build Firefox. These +will eventually be dependencies of the builds themselves, but for the moment +are run manually via try pushes and the results uploaded to tooltool. + +spidermonkey +------------ + +Spidermonkey tasks check out the full gecko source tree, then compile only the +spidermonkey portion. Each task runs specific tests after the build. + +marionette-harness +------------------ + +TBD (Maja) + +Tests +----- + +Test tasks for Gecko products are divided into several kinds, but share a +common implementation. The process goes like this, based on a set of YAML +files named in ``kind.yml``: + + * For each build task, determine the related test platforms based on the build + platform. For example, a Windows 2010 build might be tested on Windows 7 + and Windows 10. Each test platform specifies a "test set" indicating which + tests to run. This is configured in the file named + ``test-platforms.yml``. + + * Each test set is expanded to a list of tests to run. This is configured in + the file named by ``test-sets.yml``. + + * Each named test is looked up in the file named by ``tests.yml`` to find a + test description. This test description indicates what the test does, how + it is reported to treeherder, and how to perform the test, all in a + platform-independent fashion. + + * Each test description is converted into one or more tasks. This is + performed by a sequence of transforms defined in the ``transforms`` key in + ``kind.yml``. See :doc:`transforms`: for more information on these + transforms. + + * The resulting tasks become a part of the task graph. + +.. important:: + + This process generates *all* test jobs, regardless of tree or try syntax. + It is up to a later stage of the task-graph generation (the target set) to + select the tests that will actually be performed. + +desktop-test +............ + +The ``desktop-test`` kind defines tests for Desktop builds. Its ``tests.yml`` +defines the full suite of desktop tests and their particulars, leaving it to +the transforms to determine how those particulars apply to Linux, OS X, and +Windows. + +android-test +............ + +The ``android-test`` kind defines tests for Android builds. + +It is very similar to ``desktop-test``, but the details of running the tests +differ substantially, so they are defined separately. + +docker-image +------------ + +Tasks of the ``docker-image`` kind build the Docker images in which other +Docker tasks run. + +The tasks to generate each docker image have predictable labels: +``build-docker-image-<name>``. + +Docker images are built from subdirectories of ``testing/docker``, using +``docker build``. There is currently no capability for one Docker image to +depend on another in-tree docker image, without uploading the latter to a +Docker repository + +The task definition used to create the image-building tasks is given in +``image.yml`` in the kind directory, and is interpreted as a :doc:`YAML +Template <yaml-templates>`. diff --git a/taskcluster/docs/loading.rst b/taskcluster/docs/loading.rst new file mode 100644 index 000000000..1fa3c50f1 --- /dev/null +++ b/taskcluster/docs/loading.rst @@ -0,0 +1,31 @@ +Loading Tasks +============= + +The full task graph generation involves creating tasks for each kind. Kinds +are ordered to satisfy ``kind-dependencies``, and then the ``implementation`` +specified in ``kind.yml`` is used to load the tasks for that kind. + +Specifically, the class's ``load_tasks`` class method is called, and returns a +list of new ``Task`` instances. + +TransformTask +------------- + +Most kinds generate their tasks by starting with a set of items describing the +jobs that should be performed and transforming them into task definitions. +This is the familiar ``transforms`` key in ``kind.yml`` and is further +documented in :doc:`transforms`. + +Such kinds generally specify their tasks in a common format: either based on a +``jobs`` property in ``kind.yml``, or on YAML files listed in ``jobs-from``. +This is handled by the ``TransformTask`` class in +``taskcluster/taskgraph/task/transform.py``. + +For kinds producing tasks that depend on other tasks -- for example, signing +tasks depend on build tasks -- ``TransformTask`` has a ``get_inputs`` method +that can be overridden in subclasses and written to return a set of items based +on tasks that already exist. You can see a nice example of this behavior in +``taskcluster/taskgraph/task/post_build.py``. + +For more information on how all of this works, consult the docstrings and +comments in the source code itself. diff --git a/taskcluster/docs/parameters.rst b/taskcluster/docs/parameters.rst new file mode 100644 index 000000000..8514259ce --- /dev/null +++ b/taskcluster/docs/parameters.rst @@ -0,0 +1,97 @@ +========== +Parameters +========== + +Task-graph generation takes a collection of parameters as input, in the form of +a JSON or YAML file. + +During decision-task processing, some of these parameters are supplied on the +command line or by environment variables. The decision task helpfully produces +a full parameters file as one of its output artifacts. The other ``mach +taskgraph`` commands can take this file as input. This can be very helpful +when working on a change to the task graph. + +When experimenting with local runs of the task-graph generation, it is always +best to find a recent decision task's ``parameters.yml`` file, and modify that +file if necessary, rather than starting from scratch. This ensures you have a +complete set of parameters. + +The properties of the parameters object are described here, divided rougly by +topic. + +Push Information +---------------- + +``triggered_by`` + The event that precipitated this decision task; one of ``"nightly"`` or + ``"push"``. + +``base_repository`` + The repository from which to do an initial clone, utilizing any available + caching. + +``head_repository`` + The repository containing the changeset to be built. This may differ from + ``base_repository`` in cases where ``base_repository`` is likely to be cached + and only a few additional commits are needed from ``head_repository``. + +``head_rev`` + The revision to check out; this can be a short revision string + +``head_ref`` + For Mercurial repositories, this is the same as ``head_rev``. For + git repositories, which do not allow pulling explicit revisions, this gives + the symbolic ref containing ``head_rev`` that should be pulled from + ``head_repository``. + +``owner`` + Email address indicating the person who made the push. Note that this + value may be forged and *must not* be relied on for authentication. + +``message`` + The commit message + +``pushlog_id`` + The ID from the ``hg.mozilla.org`` pushlog + +``pushdate`` + The timestamp of the push to the repository that triggered this decision + task. Expressed as an integer seconds since the UNIX epoch. + +``build_date`` + The timestamp of the build date. Defaults to ``pushdate`` and falls back to present time of + taskgraph invocation. Expressed as an integer seconds since the UNIX epoch. + +``moz_build_date`` + A formatted timestamp of ``build_date``. Expressed as a string with the following + format: %Y%m%d%H%M%S + +Tree Information +---------------- + +``project`` + Another name for what may otherwise be called tree or branch or + repository. This is the unqualified name, such as ``mozilla-central`` or + ``cedar``. + +``level`` + The `SCM level + <https://www.mozilla.org/en-US/about/governance/policies/commit/access-policy/>`_ + associated with this tree. This dictates the names of resources used in the + generated tasks, and those tasks will fail if it is incorrect. + +Target Set +---------- + +The "target set" is the set of task labels which must be included in a task +graph. The task graph generation process will include any tasks required by +those in the target set, recursively. In a decision task, this set can be +specified programmatically using one of a variety of methods (e.g., parsing try +syntax or reading a project-specific configuration file). + +``target_tasks_method`` + The method to use to determine the target task set. This is the suffix of + one of the functions in ``tascluster/taskgraph/target_tasks.py``. + +``optimize_target_tasks`` + If true, then target tasks are eligible for optimization. diff --git a/taskcluster/docs/reference.rst b/taskcluster/docs/reference.rst new file mode 100644 index 000000000..813a3f630 --- /dev/null +++ b/taskcluster/docs/reference.rst @@ -0,0 +1,12 @@ +Reference +========= + +These sections contain some reference documentation for various aspects of +taskgraph generation. + +.. toctree:: + + kinds + parameters + attributes + caches diff --git a/taskcluster/docs/taskgraph.rst b/taskcluster/docs/taskgraph.rst new file mode 100644 index 000000000..5d3e7c7d3 --- /dev/null +++ b/taskcluster/docs/taskgraph.rst @@ -0,0 +1,276 @@ +====================== +TaskGraph Mach Command +====================== + +The task graph is built by linking different kinds of tasks together, pruning +out tasks that are not required, then optimizing by replacing subgraphs with +links to already-completed tasks. + +Concepts +-------- + +* *Task Kind* - Tasks are grouped by kind, where tasks of the same kind do not + have interdependencies but have substantial similarities, and may depend on + tasks of other kinds. Kinds are the primary means of supporting diversity, + in that a developer can add a new kind to do just about anything without + impacting other kinds. + +* *Task Attributes* - Tasks have string attributes by which can be used for + filtering. Attributes are documented in :doc:`attributes`. + +* *Task Labels* - Each task has a unique identifier within the graph that is + stable across runs of the graph generation algorithm. Labels are replaced + with TaskCluster TaskIds at the latest time possible, facilitating analysis + of graphs without distracting noise from randomly-generated taskIds. + +* *Optimization* - replacement of a task in a graph with an equivalent, + already-completed task, or a null task, avoiding repetition of work. + +Kinds +----- + +Kinds are the focal point of this system. They provide an interface between +the large-scale graph-generation process and the small-scale task-definition +needs of different kinds of tasks. Each kind may implement task generation +differently. Some kinds may generate task definitions entirely internally (for +example, symbol-upload tasks are all alike, and very simple), while other kinds +may do little more than parse a directory of YAML files. + +A ``kind.yml`` file contains data about the kind, as well as referring to a +Python class implementing the kind in its ``implementation`` key. That +implementation may rely on lots of code shared with other kinds, or contain a +completely unique implementation of some functionality. + +The full list of pre-defined keys in this file is: + +``implementation`` + Class implementing this kind, in the form ``<module-path>:<object-path>``. + This class should be a subclass of ``taskgraph.kind.base:Kind``. + +``kind-dependencies`` + Kinds which should be loaded before this one. This is useful when the kind + will use the list of already-created tasks to determine which tasks to + create, for example adding an upload-symbols task after every build task. + +Any other keys are subject to interpretation by the kind implementation. + +The result is a nice segmentation of implementation so that the more esoteric +in-tree projects can do their crazy stuff in an isolated kind without making +the bread-and-butter build and test configuration more complicated. + +Dependencies +------------ + +Dependencies between tasks are represented as labeled edges in the task graph. +For example, a test task must depend on the build task creating the artifact it +tests, and this dependency edge is named 'build'. The task graph generation +process later resolves these dependencies to specific taskIds. + +Decision Task +------------- + +The decision task is the first task created when a new graph begins. It is +responsible for creating the rest of the task graph. + +The decision task for pushes is defined in-tree, in ``.taskcluster.yml``. That +task description invokes ``mach taskcluster decision`` with some metadata about +the push. That mach command determines the optimized task graph, then calls +the TaskCluster API to create the tasks. + +Note that this mach command is *not* designed to be invoked directly by humans. +Instead, use the mach commands described below, supplying ``parameters.yml`` +from a recent decision task. These commands allow testing everything the +decision task does except the command-line processing and the +``queue.createTask`` calls. + +Graph Generation +---------------- + +Graph generation, as run via ``mach taskgraph decision``, proceeds as follows: + +#. For all kinds, generate all tasks. The result is the "full task set" +#. Create dependency links between tasks using kind-specific mechanisms. The + result is the "full task graph". +#. Select the target tasks (based on try syntax or a tree-specific + specification). The result is the "target task set". +#. Based on the full task graph, calculate the transitive closure of the target + task set. That is, the target tasks and all requirements of those tasks. + The result is the "target task graph". +#. Optimize the target task graph based on kind-specific optimization methods. + The result is the "optimized task graph" with fewer nodes than the target + task graph. +#. Create tasks for all tasks in the optimized task graph. + +Transitive Closure +.................. + +Transitive closure is a fancy name for this sort of operation: + + * start with a set of tasks + * add all tasks on which any of those tasks depend + * repeat until nothing changes + +The effect is this: imagine you start with a linux32 test job and a linux64 test job. +In the first round, each test task depends on the test docker image task, so add that image task. +Each test also depends on a build, so add the linux32 and linux64 build tasks. + +Then repeat: the test docker image task is already present, as are the build +tasks, but those build tasks depend on the build docker image task. So add +that build docker image task. Repeat again: this time, none of the tasks in +the set depend on a task not in the set, so nothing changes and the process is +complete. + +And as you can see, the graph we've built now includes everything we wanted +(the test jobs) plus everything required to do that (docker images, builds). + +Optimization +------------ + +The objective of optimization to remove as many tasks from the graph as +possible, as efficiently as possible, thereby delivering useful results as +quickly as possible. For example, ideally if only a test script is modified in +a push, then the resulting graph contains only the corresponding test suite +task. + +A task is said to be "optimized" when it is either replaced with an equivalent, +already-existing task, or dropped from the graph entirely. + +A task can be optimized if all of its dependencies can be optimized and none of +its inputs have changed. For a task on which no other tasks depend (a "leaf +task"), the optimizer can determine what has changed by looking at the +version-control history of the push: if the relevant files are not modified in +the push, then it considers the inputs unchanged. For tasks on which other +tasks depend ("non-leaf tasks"), the optimizer must replace the task with +another, equivalent task, so it generates a hash of all of the inputs and uses +that to search for a matching, existing task. + +In some cases, such as try pushes, tasks in the target task set have been +explicitly requested and are thus excluded from optimization. In other cases, +the target task set is almost the entire task graph, so targetted tasks are +considered for optimization. This behavior is controlled with the +``optimize_target_tasks`` parameter. + +Action Tasks +------------ + +Action Tasks are tasks which help you to schedule new jobs via Treeherder's +"Add New Jobs" feature. The Decision Task creates a YAML file named +``action.yml`` which can be used to schedule Action Tasks after suitably replacing +``{{decision_task_id}}`` and ``{{task_labels}}``, which correspond to the decision +task ID of the push and a comma separated list of task labels which need to be +scheduled. + +This task invokes ``mach taskgraph action-task`` which builds up a task graph of +the requested tasks. This graph is optimized using the tasks running initially in +the same push, due to the decision task. + +So for instance, if you had already requested a build task in the ``try`` command, +and you wish to add a test which depends on this build, the original build task +is re-used. + +Action Tasks are currently scheduled by +[pulse_actions](https://github.com/mozilla/pulse_actions). This feature is only +present on ``try`` pushes for now. + +Mach commands +------------- + +A number of mach subcommands are available aside from ``mach taskgraph +decision`` to make this complex system more accesssible to those trying to +understand or modify it. They allow you to run portions of the +graph-generation process and output the results. + +``mach taskgraph tasks`` + Get the full task set + +``mach taskgraph full`` + Get the full task graph + +``mach taskgraph target`` + Get the target task set + +``mach taskgraph target-graph`` + Get the target task graph + +``mach taskgraph optimized`` + Get the optimized task graph + +Each of these commands taskes a ``--parameters`` option giving a file with +parameters to guide the graph generation. The decision task helpfully produces +such a file on every run, and that is generally the easiest way to get a +parameter file. The parameter keys and values are described in +:doc:`parameters`; using that information, you may modify an existing +``parameters.yml`` or create your own. + +Task Parameterization +--------------------- + +A few components of tasks are only known at the very end of the decision task +-- just before the ``queue.createTask`` call is made. These are specified +using simple parameterized values, as follows: + +``{"relative-datestamp": "certain number of seconds/hours/days/years"}`` + Objects of this form will be replaced with an offset from the current time + just before the ``queue.createTask`` call is made. For example, an + artifact expiration might be specified as ``{"relative-timestamp": "1 + year"}``. + +``{"task-reference": "string containing <dep-name>"}`` + The task definition may contain "task references" of this form. These will + be replaced during the optimization step, with the appropriate taskId for + the named dependency substituted for ``<dep-name>`` in the string. + Multiple labels may be substituted in a single string, and ``<<>`` can be + used to escape a literal ``<``. + +Taskgraph JSON Format +--------------------- + +Task graphs -- both the graph artifacts produced by the decision task and those +output by the ``--json`` option to the ``mach taskgraph`` commands -- are JSON +objects, keyed by label, or for optimized task graphs, by taskId. For +convenience, the decision task also writes out ``label-to-taskid.json`` +containing a mapping from label to taskId. Each task in the graph is +represented as a JSON object. + +Each task has the following properties: + +``task_id`` + The task's taskId (only for optimized task graphs) + +``label`` + The task's label + +``attributes`` + The task's attributes + +``dependencies`` + The task's in-graph dependencies, represented as an object mapping + dependency name to label (or to taskId for optimized task graphs) + +``task`` + The task's TaskCluster task definition. + +``kind_implementation`` + The module and the class name which was used to implement this particular task. + It is always of the form ``<module-path>:<object-path>`` + +The results from each command are in the same format, but with some differences +in the content: + +* The ``tasks`` and ``target`` subcommands both return graphs with no edges. + That is, just collections of tasks without any dependencies indicated. + +* The ``optimized`` subcommand returns tasks that have been assigned taskIds. + The dependencies array, too, contains taskIds instead of labels, with + dependencies on optimized tasks omitted. However, the ``task.dependencies`` + array is populated with the full list of dependency taskIds. All task + references are resolved in the optimized graph. + +The output of the ``mach taskgraph`` commands are suitable for processing with +the `jq <https://stedolan.github.io/jq/>`_ utility. For example, to extract all +tasks' labels and their dependencies: + +.. code-block:: shell + + jq 'to_entries | map({label: .value.label, dependencies: .value.dependencies})' + diff --git a/taskcluster/docs/transforms.rst b/taskcluster/docs/transforms.rst new file mode 100644 index 000000000..1679c5589 --- /dev/null +++ b/taskcluster/docs/transforms.rst @@ -0,0 +1,198 @@ +Transforms +========== + +Many task kinds generate tasks by a process of transforming job descriptions +into task definitions. The basic operation is simple, although the sequence of +transforms applied for a particular kind may not be! + +Overview +-------- + +To begin, a kind implementation generates a collection of items; see +:doc:`loading`. The items are simply Python dictionaries, and describe +"semantically" what the resulting task or tasks should do. + +The kind also defines a sequence of transformations. These are applied, in +order, to each item. Early transforms might apply default values or break +items up into smaller items (for example, chunking a test suite). Later +transforms rewrite the items entirely, with the final result being a task +definition. + +Transform Functions +................... + +Each transformation looks like this: + +.. code-block:: + + @transforms.add + def transform_an_item(config, items): + """This transform ...""" # always a docstring! + for item in items: + # .. + yield item + +The ``config`` argument is a Python object containing useful configuration for +the kind, and is a subclass of +:class:`taskgraph.transforms.base.TransformConfig`, which specifies a few of +its attributes. Kinds may subclass and add additional attributes if necessary. + +While most transforms yield one item for each item consumed, this is not always +true: items that are not yielded are effectively filtered out. Yielding +multiple items for each consumed item implements item duplication; this is how +test chunking is accomplished, for example. + +The ``transforms`` object is an instance of +:class:`taskgraph.transforms.base.TransformSequence`, which serves as a simple +mechanism to combine a sequence of transforms into one. + +Schemas +....... + +The items used in transforms are validated against some simple schemas at +various points in the transformation process. These schemas accomplish two +things: they provide a place to add comments about the meaning of each field, +and they enforce that the fields are actually used in the documented fashion. + +Keyed By +........ + +Several fields in the input items can be "keyed by" another value in the item. +For example, a test description's chunks may be keyed by ``test-platform``. +In the item, this looks like: + +.. code-block:: yaml + + chunks: + by-test-platform: + linux64/debug: 12 + linux64/opt: 8 + default: 10 + +This is a simple but powerful way to encode business rules in the items +provided as input to the transforms, rather than expressing those rules in the +transforms themselves. If you are implementing a new business rule, prefer +this mode where possible. The structure is easily resolved to a single value +using :func:`taskgraph.transform.base.get_keyed_by`. + +Organization +------------- + +Task creation operates broadly in a few phases, with the interfaces of those +stages defined by schemas. The process begins with the raw data structures +parsed from the YAML files in the kind configuration. This data can processed +by kind-specific transforms resulting, for test jobs, in a "test description". +For non-test jobs, the next step is a "job description". These transformations +may also "duplicate" tasks, for example to implement chunking or several +variations of the same task. + +In any case, shared transforms then convert this into a "task description", +which the task-generation transforms then convert into a task definition +suitable for ``queue.createTask``. + +Test Descriptions +----------------- + +The transforms configured for test kinds proceed as follows, based on +configuration in ``kind.yml``: + + * The test description is validated to conform to the schema in + ``taskcluster/taskgraph/transforms/tests/test_description.py``. This schema + is extensively documented and is a the primary reference for anyone + modifying tests. + + * Kind-specific transformations are applied. These may apply default + settings, split tests (e.g., one to run with feature X enabled, one with it + disabled), or apply across-the-board business rules such as "all desktop + debug test platforms should have a max-run-time of 5400s". + + * Transformations generic to all tests are applied. These apply policies + which apply to multiple kinds, e.g., for treeherder tiers. This is also the + place where most values which differ based on platform are resolved, and + where chunked tests are split out into a test per chunk. + + * The test is again validated against the same schema. At this point it is + still a test description, just with defaults and policies applied, and + per-platform options resolved. So transforms up to this point do not modify + the "shape" of the test description, and are still governed by the schema in + ``test_description.py``. + + * The ``taskgraph.transforms.tests.make_task_description:transforms`` then + take the test description and create a *task* description. This transform + embodies the specifics of how test runs work: invoking mozharness, various + worker options, and so on. + + * Finally, the ``taskgraph.transforms.task:transforms``, described above + under "Task-Generation Transforms", are applied. + +Test dependencies are produced in the form of a dictionary mapping dependency +name to task label. + +Job Descriptions +---------------- + +A job description says what to run in the task. It is a combination of a +``run`` section and all of the fields from a task description. The run section +has a ``using`` property that defines how this task should be run; for example, +``mozharness`` to run a mozharness script, or ``mach`` to run a mach command. +The remainder of the run section is specific to the run-using implementation. + +The effect of a job description is to say "run this thing on this worker". The +job description must contain enough information about the worker to identify +the workerType and the implementation (docker-worker, generic-worker, etc.). +Any other task-description information is passed along verbatim, although it is +augmented by the run-using implementation. + +The run-using implementations are all located in +``taskcluster/taskgraph/transforms/job``, along with the schemas for their +implementations. Those well-commented source files are the canonical +documentation for what constitutes a job description, and should be considered +part of the documentation. + +Task Descriptions +----------------- + +Every kind needs to create tasks, and all of those tasks have some things in +common. They all run on one of a small set of worker implementations, each +with their own idiosyncracies. And they all report to TreeHerder in a similar +way. + +The transforms in ``taskcluster/taskgraph/transforms/task.py`` implement +this common functionality. They expect a "task description", and produce a +task definition. The schema for a task description is defined at the top of +``task.py``, with copious comments. Go forth and read it now! + +In general, the task-description transforms handle functionality that is common +to all Gecko tasks. While the schema is the definitive reference, the +functionality includes: + +* TreeHerder metadata + +* Build index routes + +* Information about the projects on which this task should run + +* Optimizations + +* Defaults for ``expires-after`` and and ``deadline-after``, based on project + +* Worker configuration + +The parts of the task description that are specific to a worker implementation +are isolated in a ``task_description['worker']`` object which has an +``implementation`` property naming the worker implementation. Each worker +implementation has its own section of the schema describing the fields it +expects. Thus the transforms that produce a task description must be aware of +the worker implementation to be used, but need not be aware of the details of +its payload format. + +The ``task.py`` file also contains a dictionary mapping treeherder groups to +group names using an internal list of group names. Feel free to add additional +groups to this list as necessary. + +More Detail +----------- + +The source files provide lots of additional detail, both in the code itself and +in the comments and docstrings. For the next level of detail beyond this file, +consult the transform source under ``taskcluster/taskgraph/transforms``. diff --git a/taskcluster/docs/yaml-templates.rst b/taskcluster/docs/yaml-templates.rst new file mode 100644 index 000000000..515999e60 --- /dev/null +++ b/taskcluster/docs/yaml-templates.rst @@ -0,0 +1,49 @@ +Task Definition YAML Templates +============================== + +A few kinds of tasks are described using templated YAML files. These files +allow some limited forms of inheritance and template substitution as well as +the usual YAML features, as described below. + +Please do not use these features in new kinds. If you are tempted to use +variable substitution over a YAML file to define tasks, please instead +implement a new kind-specific transform to accopmlish your goal. For example, +if the current push-id must be included as an argument in +``task.payload.command``, write a transform function that makes that assignment +while building a job description, rather than parameterizing that value in the +input to the transforms. + +Inheritance +----------- + +One YAML file can "inherit" from another by including a top-level ``$inherits`` +key. That key specifies the parent file in ``from``, and optionally a +collection of variables in ``variables``. For example: + +.. code-block:: yaml + + $inherits: + from: 'tasks/builds/base_linux32.yml' + variables: + build_name: 'linux32' + build_type: 'dbg' + +Inheritance proceeds as follows: First, the child document has its template +substitutions performed and is parsed as YAML. Then, the parent document is +parsed, with substitutions specified by ``variables`` added to the template +substitutions. Finally, the child document is merged with the parent. + +To merge two JSON objects (dictionaries), each value is merged individually. +Lists are merged by concatenating the lists from the parent and child +documents. Atomic values (strings, numbers, etc.) are merged by preferring the +child document's value. + +Substitution +------------ + +Each document is expanded using the PyStache template engine before it is +parsed as YAML. The parameters for this expansion are specific to the task +kind. + +Simple value substitution looks like ``{{variable}}``. Function calls look +like ``{{#function}}argument{{/function}}``. diff --git a/taskcluster/mach_commands.py b/taskcluster/mach_commands.py new file mode 100644 index 000000000..b5515db14 --- /dev/null +++ b/taskcluster/mach_commands.py @@ -0,0 +1,290 @@ +# -*- coding: utf-8 -*- + +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, You can obtain one at http://mozilla.org/MPL/2.0/. + + +from __future__ import absolute_import, print_function, unicode_literals + +import json +import logging +import sys +import traceback + +from mach.decorators import ( + CommandArgument, + CommandProvider, + Command, + SubCommand, +) + +from mozbuild.base import MachCommandBase + +ARTIFACT_URL = 'https://queue.taskcluster.net/v1/task/{}/artifacts/{}' + + +class ShowTaskGraphSubCommand(SubCommand): + """A SubCommand with TaskGraph-specific arguments""" + + def __call__(self, func): + after = SubCommand.__call__(self, func) + args = [ + CommandArgument('--root', '-r', default='taskcluster/ci', + help="root of the taskgraph definition relative to topsrcdir"), + CommandArgument('--quiet', '-q', action="store_true", + help="suppress all logging output"), + CommandArgument('--verbose', '-v', action="store_true", + help="include debug-level logging output"), + CommandArgument('--json', '-J', action="store_const", + dest="format", const="json", + help="Output task graph as a JSON object"), + CommandArgument('--labels', '-L', action="store_const", + dest="format", const="labels", + help="Output the label for each task in the task graph (default)"), + CommandArgument('--parameters', '-p', required=True, + help="parameters file (.yml or .json; see " + "`taskcluster/docs/parameters.rst`)`"), + CommandArgument('--no-optimize', dest="optimize", action="store_false", + default="true", + help="do not remove tasks from the graph that are found in the " + "index (a.k.a. optimize the graph)"), + ] + for arg in args: + after = arg(after) + return after + + +@CommandProvider +class MachCommands(MachCommandBase): + + @Command('taskgraph', category="ci", + description="Manipulate TaskCluster task graphs defined in-tree") + def taskgraph(self): + """The taskgraph subcommands all relate to the generation of task graphs + for Gecko continuous integration. A task graph is a set of tasks linked + by dependencies: for example, a binary must be built before it is tested, + and that build may further depend on various toolchains, libraries, etc. + """ + + @SubCommand('taskgraph', 'python-tests', + description='Run the taskgraph unit tests') + def taskgraph_python_tests(self, **options): + import unittest + import mozunit + suite = unittest.defaultTestLoader.discover('taskgraph.test') + runner = mozunit.MozTestRunner(verbosity=2) + result = runner.run(suite) + if not result.wasSuccessful(): + sys.exit(1) + + @ShowTaskGraphSubCommand('taskgraph', 'tasks', + description="Show all tasks in the taskgraph") + def taskgraph_tasks(self, **options): + return self.show_taskgraph('full_task_set', options) + + @ShowTaskGraphSubCommand('taskgraph', 'full', + description="Show the full taskgraph") + def taskgraph_full(self, **options): + return self.show_taskgraph('full_task_graph', options) + + @ShowTaskGraphSubCommand('taskgraph', 'target', + description="Show the target task set") + def taskgraph_target(self, **options): + return self.show_taskgraph('target_task_set', options) + + @ShowTaskGraphSubCommand('taskgraph', 'target-graph', + description="Show the target taskgraph") + def taskgraph_target_taskgraph(self, **options): + return self.show_taskgraph('target_task_graph', options) + + @ShowTaskGraphSubCommand('taskgraph', 'optimized', + description="Show the optimized taskgraph") + def taskgraph_optimized(self, **options): + return self.show_taskgraph('optimized_task_graph', options) + + @SubCommand('taskgraph', 'decision', + description="Run the decision task") + @CommandArgument('--root', '-r', + default='taskcluster/ci', + help="root of the taskgraph definition relative to topsrcdir") + @CommandArgument('--base-repository', + required=True, + help='URL for "base" repository to clone') + @CommandArgument('--head-repository', + required=True, + help='URL for "head" repository to fetch revision from') + @CommandArgument('--head-ref', + required=True, + help='Reference (this is same as rev usually for hg)') + @CommandArgument('--head-rev', + required=True, + help='Commit revision to use from head repository') + @CommandArgument('--message', + required=True, + help='Commit message to be parsed. Example: "try: -b do -p all -u all"') + @CommandArgument('--revision-hash', + required=True, + help='Treeherder revision hash (long revision id) to attach results to') + @CommandArgument('--project', + required=True, + help='Project to use for creating task graph. Example: --project=try') + @CommandArgument('--pushlog-id', + dest='pushlog_id', + required=True, + default=0) + @CommandArgument('--pushdate', + dest='pushdate', + required=True, + type=int, + default=0) + @CommandArgument('--owner', + required=True, + help='email address of who owns this graph') + @CommandArgument('--level', + required=True, + help='SCM level of this repository') + @CommandArgument('--triggered-by', + choices=['nightly', 'push'], + default='push', + help='Source of execution of the decision graph') + @CommandArgument('--target-tasks-method', + help='method for selecting the target tasks to generate') + def taskgraph_decision(self, **options): + """Run the decision task: generate a task graph and submit to + TaskCluster. This is only meant to be called within decision tasks, + and requires a great many arguments. Commands like `mach taskgraph + optimized` are better suited to use on the command line, and can take + the parameters file generated by a decision task. """ + + import taskgraph.decision + try: + self.setup_logging() + return taskgraph.decision.taskgraph_decision(options) + except Exception: + traceback.print_exc() + sys.exit(1) + + @SubCommand('taskgraph', 'action-task', + description="Run the action task") + @CommandArgument('--root', '-r', + default='taskcluster/ci', + help="root of the taskgraph definition relative to topsrcdir") + @CommandArgument('--decision-id', + required=True, + help="Decision Task ID of the reference decision task") + @CommandArgument('--task-labels', + required=True, + help='Comma separated list of task labels to be scheduled') + def taskgraph_action(self, **options): + """Run the action task: Generates a task graph using the set of labels + provided in the task-labels parameter. It uses the full-task file of + the gecko decision task.""" + + import taskgraph.action + try: + self.setup_logging() + return taskgraph.action.taskgraph_action(options) + except Exception: + traceback.print_exc() + sys.exit(1) + + def setup_logging(self, quiet=False, verbose=True): + """ + Set up Python logging for all loggers, sending results to stderr (so + that command output can be redirected easily) and adding the typical + mach timestamp. + """ + # remove the old terminal handler + old = self.log_manager.replace_terminal_handler(None) + + # re-add it, with level and fh set appropriately + if not quiet: + level = logging.DEBUG if verbose else logging.INFO + self.log_manager.add_terminal_logging( + fh=sys.stderr, level=level, + write_interval=old.formatter.write_interval, + write_times=old.formatter.write_times) + + # all of the taskgraph logging is unstructured logging + self.log_manager.enable_unstructured() + + def show_taskgraph(self, graph_attr, options): + import taskgraph.parameters + import taskgraph.target_tasks + import taskgraph.generator + + try: + self.setup_logging(quiet=options['quiet'], verbose=options['verbose']) + parameters = taskgraph.parameters.load_parameters_file(options) + parameters.check() + + target_tasks_method = parameters.get('target_tasks_method', 'all_tasks') + target_tasks_method = taskgraph.target_tasks.get_method(target_tasks_method) + tgg = taskgraph.generator.TaskGraphGenerator( + root_dir=options['root'], + parameters=parameters, + target_tasks_method=target_tasks_method) + + tg = getattr(tgg, graph_attr) + + show_method = getattr(self, 'show_taskgraph_' + (options['format'] or 'labels')) + show_method(tg) + except Exception: + traceback.print_exc() + sys.exit(1) + + def show_taskgraph_labels(self, taskgraph): + for label in taskgraph.graph.visit_postorder(): + print(label) + + def show_taskgraph_json(self, taskgraph): + print(json.dumps(taskgraph.to_json(), + sort_keys=True, indent=2, separators=(',', ': '))) + + +@CommandProvider +class TaskClusterImagesProvider(object): + @Command('taskcluster-load-image', category="ci", + description="Load a pre-built Docker image") + @CommandArgument('--task-id', + help="Load the image at public/image.tar in this task," + "rather than searching the index") + @CommandArgument('image_name', nargs='?', + help="Load the image of this name based on the current" + "contents of the tree (as built for mozilla-central" + "or mozilla-inbound)") + def load_image(self, image_name, task_id): + from taskgraph.docker import load_image_by_name, load_image_by_task_id + if not image_name and not task_id: + print("Specify either IMAGE-NAME or TASK-ID") + sys.exit(1) + try: + if task_id: + ok = load_image_by_task_id(task_id) + else: + ok = load_image_by_name(image_name) + if not ok: + sys.exit(1) + except Exception: + traceback.print_exc() + sys.exit(1) + + @Command('taskcluster-build-image', category='ci', + description='Build a Docker image') + @CommandArgument('image_name', + help='Name of the image to build') + @CommandArgument('--context-only', + help="File name the context tarball should be written to." + "with this option it will only build the context.tar.", + metavar='context.tar') + def build_image(self, image_name, context_only): + from taskgraph.docker import build_image, build_context + try: + if context_only is None: + build_image(image_name) + else: + build_context(image_name, context_only) + except Exception: + traceback.print_exc() + sys.exit(1) diff --git a/taskcluster/moz.build b/taskcluster/moz.build new file mode 100644 index 000000000..d1e280706 --- /dev/null +++ b/taskcluster/moz.build @@ -0,0 +1,7 @@ +# -*- Mode: python; indent-tabs-mode: nil; tab-width: 40 -*- +# vim: set filetype=python: +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, You can obtain one at http://mozilla.org/MPL/2.0/. + +SPHINX_TREES['taskcluster'] = 'docs' diff --git a/taskcluster/scripts/builder/build-haz-linux.sh b/taskcluster/scripts/builder/build-haz-linux.sh new file mode 100755 index 000000000..1d5ef52ba --- /dev/null +++ b/taskcluster/scripts/builder/build-haz-linux.sh @@ -0,0 +1,89 @@ +#!/bin/bash -ex + +function usage() { + echo "Usage: $0 [--project <shell|browser>] <workspace-dir> flags..." + echo "flags are treated the same way as a commit message would be" + echo "(as in, they are scanned for directives just like a try: ... line)" +} + +PROJECT=shell +WORKSPACE= +DO_TOOLTOOL=1 +while [[ $# -gt 0 ]]; do + if [[ "$1" == "-h" ]] || [[ "$1" == "--help" ]]; then + usage + exit 0 + elif [[ "$1" == "--project" ]]; then + shift + PROJECT="$1" + shift + elif [[ "$1" == "--no-tooltool" ]]; then + shift + DO_TOOLTOOL= + elif [[ -z "$WORKSPACE" ]]; then + WORKSPACE=$( cd "$1" && pwd ) + shift + break + fi +done + +SCRIPT_FLAGS="$@" + +# Ensure all the scripts in this dir are on the path.... +DIRNAME=$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd ) +PATH=$DIRNAME:$PATH + +# Use GECKO_BASE_REPOSITORY as a signal for whether we are running in automation. +export AUTOMATION=${GECKO_BASE_REPOSITORY:+1} + +: ${GECKO_DIR:=$WORKSPACE/gecko} +: ${TOOLTOOL_MANIFEST:=browser/config/tooltool-manifests/linux64/hazard.manifest} +: ${TOOLTOOL_CACHE:=$WORKSPACE/tt-cache} + +if ! [ -d $GECKO_DIR ]; then + echo "GECKO_DIR must be set to a directory containing a gecko source checkout" >&2 + exit 1 +fi +GECKO_DIR=$( cd "$GECKO_DIR" && pwd ) + +# Directory to populate with tooltool-installed tools +export TOOLTOOL_DIR="$WORKSPACE" + +# Directory to hold the (useless) object files generated by the analysis. +export MOZ_OBJDIR="$WORKSPACE/obj-analyzed" +mkdir -p "$MOZ_OBJDIR" + +if [ -n "$DO_TOOLTOOL" ]; then + ( cd $TOOLTOOL_DIR; python $GECKO_DIR/testing/docker/recipes/tooltool.py --url https://api.pub.build.mozilla.org/tooltool/ -m $GECKO_DIR/$TOOLTOOL_MANIFEST fetch -c $TOOLTOOL_CACHE ) +fi + +export NO_MERCURIAL_SETUP_CHECK=1 + +if [[ "$PROJECT" = "browser" ]]; then ( + cd "$WORKSPACE" + set "$WORKSPACE" + . setup-ccache.sh + # Mozbuild config: + export MOZBUILD_STATE_PATH=$WORKSPACE/mozbuild/ + # Create .mozbuild so mach doesn't complain about this + mkdir -p $MOZBUILD_STATE_PATH +) fi +. hazard-analysis.sh + +build_js_shell + +# Artifacts folder is outside of the cache. +mkdir -p $HOME/artifacts/ || true + +function onexit () { + grab_artifacts "$WORKSPACE/analysis" "$HOME/artifacts" +} + +trap onexit EXIT + +configure_analysis "$WORKSPACE/analysis" +run_analysis "$WORKSPACE/analysis" "$PROJECT" + +check_hazards "$WORKSPACE/analysis" + +################################### script end ################################### diff --git a/taskcluster/scripts/builder/build-l10n.sh b/taskcluster/scripts/builder/build-l10n.sh new file mode 100755 index 000000000..be16955a5 --- /dev/null +++ b/taskcluster/scripts/builder/build-l10n.sh @@ -0,0 +1,98 @@ +#! /bin/bash -vex + +set -x -e + +echo "running as" $(id) + +. /home/worker/scripts/xvfb.sh + +#### +# Taskcluster friendly wrapper for performing fx desktop l10n repacks via mozharness. +# Based on ./build-linux.sh +#### + +# Inputs, with defaults + +: MOZHARNESS_SCRIPT ${MOZHARNESS_SCRIPT} +: MOZHARNESS_CONFIG ${MOZHARNESS_CONFIG} +: MOZHARNESS_ACTIONS ${MOZHARNESS_ACTIONS} +: MOZHARNESS_OPTIONS ${MOZHARNESS_OPTIONS} + +: TOOLTOOL_CACHE ${TOOLTOOL_CACHE:=/home/worker/tooltool-cache} + +: NEED_XVFB ${NEED_XVFB:=false} + +: WORKSPACE ${WORKSPACE:=/home/worker/workspace} + +set -v + +fail() { + echo # make sure error message is on a new line + echo "[build-l10n.sh:error]" "${@}" + exit 1 +} + +export MOZ_CRASHREPORTER_NO_REPORT=1 +export MOZ_OBJDIR=obj-firefox +export TINDERBOX_OUTPUT=1 + +# Ensure that in tree libraries can be found +export LIBRARY_PATH=$LIBRARY_PATH:$WORKSPACE/src/obj-firefox:$WORKSPACE/src/gcc/lib64 + +# test required parameters are supplied +if [[ -z ${MOZHARNESS_SCRIPT} ]]; then fail "MOZHARNESS_SCRIPT is not set"; fi +if [[ -z ${MOZHARNESS_CONFIG} ]]; then fail "MOZHARNESS_CONFIG is not set"; fi + +cleanup() { + local rv=$? + cleanup_xvfb + exit $rv +} +trap cleanup EXIT INT + +# run XVfb in the background, if necessary +if $NEED_XVFB; then + start_xvfb '1024x768x24' 2 +fi + +# set up mozharness configuration, via command line, env, etc. + +# $TOOLTOOL_CACHE bypasses mozharness completely and is read by tooltool_wrapper.sh to set the +# cache. However, only some mozharness scripts use tooltool_wrapper.sh, so this may not be +# entirely effective. +export TOOLTOOL_CACHE + +# support multiple, space delimited, config files +config_cmds="" +for cfg in $MOZHARNESS_CONFIG; do + config_cmds="${config_cmds} --config ${cfg}" +done + +# if MOZHARNESS_ACTIONS is given, only run those actions (completely overriding default_actions +# in the mozharness configuration) +if [ -n "$MOZHARNESS_ACTIONS" ]; then + actions="" + for action in $MOZHARNESS_ACTIONS; do + actions="$actions --$action" + done +fi + +# if MOZHARNESS_OPTIONS is given, append them to mozharness command line run +# e.g. enable-pgo +if [ -n "$MOZHARNESS_OPTIONS" ]; then + options="" + for option in $MOZHARNESS_OPTIONS; do + options="$options --$option" + done +fi + +cd /home/worker + +python2.7 $WORKSPACE/build/src/testing/${MOZHARNESS_SCRIPT} \ + --disable-mock \ + --revision ${GECKO_HEAD_REV} \ + $actions \ + $options \ + ${config_cmds} \ + --log-level=debug \ + --work-dir=$WORKSPACE/build \ diff --git a/taskcluster/scripts/builder/build-linux.sh b/taskcluster/scripts/builder/build-linux.sh new file mode 100755 index 000000000..8885abdec --- /dev/null +++ b/taskcluster/scripts/builder/build-linux.sh @@ -0,0 +1,122 @@ +#! /bin/bash -vex + +set -x -e + +echo "running as" $(id) + +. /home/worker/scripts/xvfb.sh + +#### +# Taskcluster friendly wrapper for performing fx desktop builds via mozharness. +#### + +# Inputs, with defaults + +: MOZHARNESS_SCRIPT ${MOZHARNESS_SCRIPT} +: MOZHARNESS_CONFIG ${MOZHARNESS_CONFIG} +: MOZHARNESS_ACTIONS ${MOZHARNESS_ACTIONS} +: MOZHARNESS_OPTIONS ${MOZHARNESS_OPTIONS} + +: TOOLTOOL_CACHE ${TOOLTOOL_CACHE:=/home/worker/tooltool-cache} + +: NEED_XVFB ${NEED_XVFB:=false} + +: MH_CUSTOM_BUILD_VARIANT_CFG ${MH_CUSTOM_BUILD_VARIANT_CFG} +: MH_BRANCH ${MH_BRANCH:=mozilla-central} +: MH_BUILD_POOL ${MH_BUILD_POOL:=staging} +: MOZ_SCM_LEVEL ${MOZ_SCM_LEVEL:=1} + +: WORKSPACE ${WORKSPACE:=/home/worker/workspace} + +set -v + +fail() { + echo # make sure error message is on a new line + echo "[build-linux.sh:error]" "${@}" + exit 1 +} + +export MOZ_CRASHREPORTER_NO_REPORT=1 +export MOZ_OBJDIR=obj-firefox +export TINDERBOX_OUTPUT=1 + +# use "simple" package names so that they can be hard-coded in the task's +# extras.locations +export MOZ_SIMPLE_PACKAGE_NAME=target + +# Do not try to upload symbols (see https://bugzilla.mozilla.org/show_bug.cgi?id=1164615) +export MOZ_AUTOMATION_UPLOAD_SYMBOLS=0 + +# Ensure that in tree libraries can be found +export LIBRARY_PATH=$LIBRARY_PATH:$WORKSPACE/src/obj-firefox:$WORKSPACE/src/gcc/lib64 + +# test required parameters are supplied +if [[ -z ${MOZHARNESS_SCRIPT} ]]; then fail "MOZHARNESS_SCRIPT is not set"; fi +if [[ -z ${MOZHARNESS_CONFIG} ]]; then fail "MOZHARNESS_CONFIG is not set"; fi + +cleanup() { + local rv=$? + cleanup_xvfb + exit $rv +} +trap cleanup EXIT INT + +# run XVfb in the background, if necessary +if $NEED_XVFB; then + start_xvfb '1024x768x24' 2 +fi + +# set up mozharness configuration, via command line, env, etc. + +debug_flag="" +if [ 0$DEBUG -ne 0 ]; then + debug_flag='--debug' +fi + +custom_build_variant_cfg_flag="" +if [ -n "${MH_CUSTOM_BUILD_VARIANT_CFG}" ]; then + custom_build_variant_cfg_flag="--custom-build-variant-cfg=${MH_CUSTOM_BUILD_VARIANT_CFG}" +fi + +# $TOOLTOOL_CACHE bypasses mozharness completely and is read by tooltool_wrapper.sh to set the +# cache. However, only some mozharness scripts use tooltool_wrapper.sh, so this may not be +# entirely effective. +export TOOLTOOL_CACHE + +# support multiple, space delimited, config files +config_cmds="" +for cfg in $MOZHARNESS_CONFIG; do + config_cmds="${config_cmds} --config ${cfg}" +done + +# if MOZHARNESS_ACTIONS is given, only run those actions (completely overriding default_actions +# in the mozharness configuration) +if [ -n "$MOZHARNESS_ACTIONS" ]; then + actions="" + for action in $MOZHARNESS_ACTIONS; do + actions="$actions --$action" + done +fi + +# if MOZHARNESS_OPTIONS is given, append them to mozharness command line run +# e.g. enable-pgo +if [ -n "$MOZHARNESS_OPTIONS" ]; then + options="" + for option in $MOZHARNESS_OPTIONS; do + options="$options --$option" + done +fi + +cd /home/worker + +python2.7 $WORKSPACE/build/src/testing/${MOZHARNESS_SCRIPT} ${config_cmds} \ + $debug_flag \ + $custom_build_variant_cfg_flag \ + --disable-mock \ + $actions \ + $options \ + --log-level=debug \ + --scm-level=$MOZ_SCM_LEVEL \ + --work-dir=$WORKSPACE/build \ + --branch=${MH_BRANCH} \ + --build-pool=${MH_BUILD_POOL} diff --git a/taskcluster/scripts/builder/build-sm-mozjs-crate.sh b/taskcluster/scripts/builder/build-sm-mozjs-crate.sh new file mode 100755 index 000000000..09c353084 --- /dev/null +++ b/taskcluster/scripts/builder/build-sm-mozjs-crate.sh @@ -0,0 +1,18 @@ +#!/usr/bin/env bash + +set -xe + +source $(dirname $0)/sm-tooltool-config.sh + +# Ensure that we have a .config/cargo that points us to our vendored crates +# rather than to crates.io. +cd "$SRCDIR/.cargo" +sed -e "s|@top_srcdir@|$SRCDIR|" < config.in | tee config + +cd "$SRCDIR/js/src" + +export PATH="$PATH:$TOOLTOOL_CHECKOUT/cargo/bin:$TOOLTOOL_CHECKOUT/rustc/bin" +export RUST_BACKTRACE=1 + +cargo build --verbose --frozen --features debugmozjs +cargo build --verbose --frozen diff --git a/taskcluster/scripts/builder/build-sm-package.sh b/taskcluster/scripts/builder/build-sm-package.sh new file mode 100755 index 000000000..6bb819f26 --- /dev/null +++ b/taskcluster/scripts/builder/build-sm-package.sh @@ -0,0 +1,28 @@ +#!/bin/bash + +set -xe + +source $(dirname $0)/sm-tooltool-config.sh + +mkdir -p $UPLOAD_DIR + +# Package up the sources into the release tarball. +AUTOMATION=1 DIST=$UPLOAD_DIR $SRCDIR/js/src/make-source-package.sh + +# Extract the tarball into a new directory in the workspace. + +PACKAGE_DIR=$WORK/sm-package +mkdir -p $PACKAGE_DIR +pushd $PACKAGE_DIR + +tar -xjvf $UPLOAD_DIR/mozjs-*.tar.bz2 + +: ${PYTHON:=python2.7} + +# Build the freshly extracted, packaged SpiderMonkey. +pushd ./mozjs-*/js/src +AUTOMATION=1 $PYTHON ./devtools/automation/autospider.py --skip-tests=checks $SPIDERMONKEY_VARIANT +popd + +# Copy artifacts for upload by TaskCluster +cp -rL ./mozjs-*/obj-spider/dist/bin/{js,jsapi-tests,js-gdb.py,libmozjs*} $UPLOAD_DIR diff --git a/taskcluster/scripts/builder/build-sm.sh b/taskcluster/scripts/builder/build-sm.sh new file mode 100755 index 000000000..d61a7a81c --- /dev/null +++ b/taskcluster/scripts/builder/build-sm.sh @@ -0,0 +1,20 @@ +#!/bin/bash + +set -x + +source $(dirname $0)/sm-tooltool-config.sh + +: ${PYTHON:=python2.7} + +# Run the script +export MOZ_UPLOAD_DIR="$UPLOAD_DIR" +AUTOMATION=1 $PYTHON $SRCDIR/js/src/devtools/automation/autospider.py $SPIDERMONKEY_VARIANT +BUILD_STATUS=$? + +# Ensure upload dir exists +mkdir -p $UPLOAD_DIR + +# Copy artifacts for upload by TaskCluster +cp -rL $SRCDIR/obj-spider/dist/bin/{js,jsapi-tests,js-gdb.py} $UPLOAD_DIR + +exit $BUILD_STATUS diff --git a/taskcluster/scripts/builder/desktop-setup.sh b/taskcluster/scripts/builder/desktop-setup.sh new file mode 100755 index 000000000..4b74a1201 --- /dev/null +++ b/taskcluster/scripts/builder/desktop-setup.sh @@ -0,0 +1,24 @@ +#!/bin/bash -ex + +test $MOZCONFIG # mozconfig is required... +test -d $1 # workspace must exist at this point... +WORKSPACE=$( cd "$1" && pwd ) + +. setup-ccache.sh + +# Gecko source: +export GECKO_DIR=$WORKSPACE/gecko +# Gaia source: +export GAIA_DIR=$WORKSPACE/gaia +# Mozbuild config: +export MOZBUILD_STATE_PATH=$WORKSPACE/mozbuild/ + +# Create .mozbuild so mach doesn't complain about this +mkdir -p $MOZBUILD_STATE_PATH + +### Install package dependencies +install-packages.sh ${TOOLTOOL_DIR:-$GECKO_DIR} + +# Ensure object-folder exists +export MOZ_OBJDIR=$WORKSPACE/object-folder/ +mkdir -p $MOZ_OBJDIR diff --git a/taskcluster/scripts/builder/get-objdir.py b/taskcluster/scripts/builder/get-objdir.py new file mode 100755 index 000000000..132e20d4f --- /dev/null +++ b/taskcluster/scripts/builder/get-objdir.py @@ -0,0 +1,20 @@ +#!/usr/bin/env python2.7 + +from __future__ import print_function +import sys +import os +import json +import subprocess +from StringIO import StringIO + +gecko_dir = sys.argv[1] +os.chdir(gecko_dir) + +result = subprocess.check_output(["./mach", "environment", "--format", "json"]) +environment = json.load(StringIO(result)) + +topobjdir = environment["mozconfig"]["topobjdir"] +if topobjdir is None: + topobjdir = sys.argv[2] + +print(topobjdir) diff --git a/taskcluster/scripts/builder/hazard-analysis.sh b/taskcluster/scripts/builder/hazard-analysis.sh new file mode 100755 index 000000000..d3e574742 --- /dev/null +++ b/taskcluster/scripts/builder/hazard-analysis.sh @@ -0,0 +1,149 @@ +#!/bin/bash -ex + +[ -n "$WORKSPACE" ] +[ -n "$MOZ_OBJDIR" ] +[ -n "$GECKO_DIR" ] + +HAZARD_SHELL_OBJDIR=$WORKSPACE/obj-haz-shell +JS_SRCDIR=$GECKO_DIR/js/src +ANALYSIS_SRCDIR=$JS_SRCDIR/devtools/rootAnalysis + +export CC="$TOOLTOOL_DIR/gcc/bin/gcc" +export CXX="$TOOLTOOL_DIR/gcc/bin/g++" + +PYTHON=python2.7 +if ! which $PYTHON; then + PYTHON=python +fi + + +function check_commit_msg () { + ( set +e; + if [[ -n "$AUTOMATION" ]]; then + hg --cwd "$GECKO_DIR" log -r. --template '{desc}\n' | grep -F -q -- "$1" + else + echo -- "$SCRIPT_FLAGS" | grep -F -q -- "$1" + fi + ) +} + +if check_commit_msg "--dep"; then + HAZ_DEP=1 +fi + +function build_js_shell () { + # Must unset MOZ_OBJDIR and MOZCONFIG here to prevent the build system from + # inferring that the analysis output directory is the current objdir. We + # need a separate objdir here to build the opt JS shell to use to run the + # analysis. + ( + unset MOZ_OBJDIR + unset MOZCONFIG + ( cd $JS_SRCDIR; autoconf-2.13 ) + if [[ -z "$HAZ_DEP" ]]; then + [ -d $HAZARD_SHELL_OBJDIR ] && rm -rf $HAZARD_SHELL_OBJDIR + fi + mkdir -p $HAZARD_SHELL_OBJDIR || true + cd $HAZARD_SHELL_OBJDIR + $JS_SRCDIR/configure --enable-optimize --disable-debug --enable-ctypes --enable-nspr-build --without-intl-api --with-ccache + make -j4 + ) # Restore MOZ_OBJDIR and MOZCONFIG +} + +function configure_analysis () { + local analysis_dir + analysis_dir="$1" + + if [[ -z "$HAZ_DEP" ]]; then + [ -d "$analysis_dir" ] && rm -rf "$analysis_dir" + fi + + mkdir -p "$analysis_dir" || true + ( + cd "$analysis_dir" + cat > defaults.py <<EOF +js = "$HAZARD_SHELL_OBJDIR/dist/bin/js" +analysis_scriptdir = "$ANALYSIS_SRCDIR" +objdir = "$MOZ_OBJDIR" +source = "$GECKO_DIR" +sixgill = "$TOOLTOOL_DIR/sixgill/usr/libexec/sixgill" +sixgill_bin = "$TOOLTOOL_DIR/sixgill/usr/bin" +EOF + + cat > run-analysis.sh <<EOF +#!/bin/sh +if [ \$# -eq 0 ]; then + set gcTypes +fi +export ANALYSIS_SCRIPTDIR="$ANALYSIS_SRCDIR" +exec "$ANALYSIS_SRCDIR/analyze.py" "\$@" +EOF + chmod +x run-analysis.sh + ) +} + +function run_analysis () { + local analysis_dir + analysis_dir="$1" + local build_type + build_type="$2" + + if [[ -z "$HAZ_DEP" ]]; then + [ -d $MOZ_OBJDIR ] && rm -rf $MOZ_OBJDIR + fi + + ( + cd "$analysis_dir" + $PYTHON "$ANALYSIS_SRCDIR/analyze.py" --buildcommand="$GECKO_DIR/testing/mozharness/scripts/spidermonkey/build.${build_type}" + ) +} + +function grab_artifacts () { + local analysis_dir + analysis_dir="$1" + local artifacts + artifacts="$2" + + ( + cd "$analysis_dir" + ls -lah + + # Do not error out if no files found + shopt -s nullglob + set +e + for f in *.txt *.lst; do + gzip -9 -c "$f" > "${artifacts}/$f.gz" + done + + # Check whether the user requested .xdb file upload in the top commit comment + if check_commit_msg "--upload-xdbs"; then + HAZ_UPLOAD_XDBS=1 + fi + + if [ -n "$HAZ_UPLOAD_XDBS" ]; then + for f in *.xdb; do + bzip2 -c "$f" > "${artifacts}/$f.bz2" + done + fi + ) +} + +function check_hazards () { + ( + set +e + NUM_HAZARDS=$(grep -c 'Function.*has unrooted.*live across GC call' "$1"/rootingHazards.txt) + NUM_UNSAFE=$(grep -c '^Function.*takes unsafe address of unrooted' "$1"/refs.txt) + NUM_UNNECESSARY=$(grep -c '^Function.* has unnecessary root' "$1"/unnecessary.txt) + + set +x + echo "TinderboxPrint: rooting hazards<br/>$NUM_HAZARDS" + echo "TinderboxPrint: unsafe references to unrooted GC pointers<br/>$NUM_UNSAFE" + echo "TinderboxPrint: unnecessary roots<br/>$NUM_UNNECESSARY" + + if [ $NUM_HAZARDS -gt 0 ]; then + echo "TEST-UNEXPECTED-FAIL $NUM_HAZARDS hazards detected" >&2 + echo "TinderboxPrint: documentation<br/><a href='https://wiki.mozilla.org/Javascript:Hazard_Builds'>static rooting hazard analysis failures</a>, visit \"Inspect Task\" link for hazard details" + exit 1 + fi + ) +} diff --git a/taskcluster/scripts/builder/install-packages.sh b/taskcluster/scripts/builder/install-packages.sh new file mode 100755 index 000000000..2f5cdf489 --- /dev/null +++ b/taskcluster/scripts/builder/install-packages.sh @@ -0,0 +1,13 @@ +#!/bin/bash -vex + +gecko_dir=$1 +test -d $gecko_dir +test -n "$TOOLTOOL_CACHE" +test -n "$TOOLTOOL_MANIFEST" +test -n "$TOOLTOOL_REPO" +test -n "$TOOLTOOL_REV" + +tc-vcs checkout $gecko_dir/tooltool $TOOLTOOL_REPO $TOOLTOOL_REPO $TOOLTOOL_REV + +(cd $gecko_dir; python $gecko_dir/tooltool/tooltool.py --url https://api.pub.build.mozilla.org/tooltool/ -m $gecko_dir/$TOOLTOOL_MANIFEST fetch -c $TOOLTOOL_CACHE) + diff --git a/taskcluster/scripts/builder/setup-ccache.sh b/taskcluster/scripts/builder/setup-ccache.sh new file mode 100644 index 000000000..3c03b2640 --- /dev/null +++ b/taskcluster/scripts/builder/setup-ccache.sh @@ -0,0 +1,9 @@ +#! /bin/bash -ex + +test -d $1 # workspace must exist at this point... +WORKSPACE=$( cd "$1" && pwd ) + +export CCACHE_DIR=$WORKSPACE/ccache + +ccache -M 12G +ccache -s diff --git a/taskcluster/scripts/builder/sm-tooltool-config.sh b/taskcluster/scripts/builder/sm-tooltool-config.sh new file mode 100755 index 000000000..b6a062858 --- /dev/null +++ b/taskcluster/scripts/builder/sm-tooltool-config.sh @@ -0,0 +1,50 @@ +#!/bin/bash + +set -xe + +: ${TOOLTOOL_SERVER:=https://api.pub.build.mozilla.org/tooltool/} +: ${SPIDERMONKEY_VARIANT:=plain} +: ${UPLOAD_DIR:=$HOME/artifacts/} +: ${WORK:=$HOME/workspace} +: ${SRCDIR:=$WORK/build/src} + +mkdir -p $WORK +cd $WORK + +# Need to install things from tooltool. Figure out what platform to use. + +case $(uname -m) in + i686 | arm ) + BITS=32 + ;; + *) + BITS=64 + ;; +esac + +case "$OSTYPE" in + darwin*) + PLATFORM_OS=macosx + ;; + linux-gnu) + PLATFORM_OS=linux + ;; + msys) + PLATFORM_OS=win + ;; + *) + echo "Unrecognized OSTYPE '$OSTYPE'" >&2 + PLATFORM_OS=linux + ;; +esac + +# Install everything needed for the browser on this platform. Not all of it is +# necessary for the JS shell, but it's less duplication to share tooltool +# manifests. +BROWSER_PLATFORM=$PLATFORM_OS$BITS +: ${TOOLTOOL_MANIFEST:=browser/config/tooltool-manifests/$BROWSER_PLATFORM/releng.manifest} + +: ${TOOLTOOL_CHECKOUT:=$WORK} +export TOOLTOOL_CHECKOUT + +(cd $TOOLTOOL_CHECKOUT && python ${SRCDIR}/testing/docker/recipes/tooltool.py --url $TOOLTOOL_SERVER -m $SRCDIR/$TOOLTOOL_MANIFEST fetch ${TOOLTOOL_CACHE:+ -c $TOOLTOOL_CACHE}) diff --git a/taskcluster/scripts/copy.sh b/taskcluster/scripts/copy.sh new file mode 100755 index 000000000..931145a3b --- /dev/null +++ b/taskcluster/scripts/copy.sh @@ -0,0 +1,9 @@ +#! /bin/bash -ex + +# This script copies the contents of the "scripts" folder into a docker +# container using tar/untar the container id must be passed. + +DIRNAME=$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd ) +docker exec $1 mkdir -p $2 +cd $DIRNAME +tar -cv * | docker exec -i $1 tar -x -C $2 diff --git a/taskcluster/scripts/misc/build-binutils-linux.sh b/taskcluster/scripts/misc/build-binutils-linux.sh new file mode 100755 index 000000000..da0eb2724 --- /dev/null +++ b/taskcluster/scripts/misc/build-binutils-linux.sh @@ -0,0 +1,16 @@ +#!/bin/bash +set -x -e -v + +# This script is for building binutils for Linux. + +WORKSPACE=$HOME/workspace +HOME_DIR=$WORKSPACE/build +UPLOAD_DIR=$WORKSPACE/artifacts + +cd $HOME_DIR/src + +build/unix/build-binutils/build-binutils.sh $HOME_DIR + +# Put a tarball in the artifacts dir +mkdir -p $UPLOAD_DIR +cp $HOME_DIR/binutils.tar.* $UPLOAD_DIR diff --git a/taskcluster/scripts/misc/build-cctools.sh b/taskcluster/scripts/misc/build-cctools.sh new file mode 100755 index 000000000..3eea0929d --- /dev/null +++ b/taskcluster/scripts/misc/build-cctools.sh @@ -0,0 +1,82 @@ +#!/bin/bash +set -x -e -v + +# This script is for building cctools (Apple's binutils) for Linux using +# crosstool-ng (https://github.com/diorcety/crosstool-ng). + +WORKSPACE=$HOME/workspace +UPLOAD_DIR=$WORKSPACE/artifacts + +# Repository info +: CROSSTOOL_NG_REPOSITORY ${CROSSTOOL_NG_REPOSITORY:=https://github.com/diorcety/crosstool-ng} +: CROSSTOOL_NG_REV ${CROSSTOOL_NG_REV:=master} + +# hacky +ln -s `which gcc` ~/bin/x86_64-linux-gnu-gcc +export PATH=$PATH:~/bin + +# Set some crosstools-ng directories +CT_TOP_DIR=$WORKSPACE/crosstool-ng-build +CT_PREFIX_DIR=$WORKSPACE/cctools +CT_SRC_DIR=$CT_TOP_DIR/src +CT_TARBALLS_DIR=$CT_TOP_DIR +CT_WORK_DIR=$CT_SRC_DIR +CT_LIB_DIR=$WORKSPACE/crosstool-ng +CT_BUILD_DIR=$CT_TOP_DIR/build +CT_LLVM_DIR=$WORKSPACE/clang +CT_BUILDTOOLS_PREFIX_DIR=$CT_PREFIX_DIR + +# Create our directories +rm -rf $CT_TOP_DIR +mkdir $CT_TOP_DIR +rm -rf $CT_PREFIX_DIR +mkdir $CT_PREFIX_DIR +mkdir -p $CT_SRC_DIR + +# Clone the crosstool-ng repo +tc-vcs checkout $CT_LIB_DIR $CROSSTOOL_NG_REPOSITORY $CROSSTOOL_NG_REPOSITORY $CROSSTOOL_NG_REV + +# Fetch clang from tooltool +cd $WORKSPACE +wget -O tooltool.py https://raw.githubusercontent.com/mozilla/build-tooltool/master/tooltool.py +chmod +x tooltool.py +: TOOLTOOL_CACHE ${TOOLTOOL_CACHE:=/home/worker/tooltool-cache} +export TOOLTOOL_CACHE + +wget ${GECKO_HEAD_REPOSITORY}/raw-file/${GECKO_HEAD_REV}/browser/config/tooltool-manifests/linux64/clang.manifest + +python tooltool.py -v --manifest=clang.manifest fetch + +# Copy clang into the crosstools-ng srcdir +cp -Rp $CT_LLVM_DIR $CT_SRC_DIR + +# Configure crosstools-ng +sed=sed +CT_CONNECT_TIMEOUT=5 +CT_BINUTILS_VERSION=809 +CT_PATCH_ORDER=bundled +CT_BUILD=x86_64-linux-gnu +CT_HOST=x86_64-linux-gnu +CT_TARGET=x86_64-apple-darwin10 +CT_LLVM_FULLNAME=clang + +cd $CT_TOP_DIR + +# gets a bit too verbose here +set +x + +. $CT_LIB_DIR/scripts/functions +. $CT_LIB_DIR/scripts/build/binutils/cctools.sh + +# Build cctools +do_binutils_get +do_binutils_extract +do_binutils_for_host + +set -x + +strip $CT_PREFIX_DIR/bin/* + +# Put a tarball in the artifacts dir +mkdir -p $UPLOAD_DIR +tar czf $UPLOAD_DIR/cctools.tar.gz -C $WORKSPACE `basename $CT_PREFIX_DIR` diff --git a/taskcluster/scripts/misc/build-clang-linux.sh b/taskcluster/scripts/misc/build-clang-linux.sh new file mode 100755 index 000000000..e1c6f2f0d --- /dev/null +++ b/taskcluster/scripts/misc/build-clang-linux.sh @@ -0,0 +1,30 @@ +#!/bin/bash +set -x -e -v + +# This script is for building clang for Linux. + +WORKSPACE=$HOME/workspace +HOME_DIR=$WORKSPACE/build +UPLOAD_DIR=$WORKSPACE/artifacts + +# Fetch our toolchain from tooltool +cd $HOME_DIR +wget -O tooltool.py https://raw.githubusercontent.com/mozilla/build-tooltool/master/tooltool.py +chmod +x tooltool.py +: TOOLTOOL_CACHE ${TOOLTOOL_CACHE:=/home/worker/tooltool-cache} +export TOOLTOOL_CACHE +cd src +$HOME_DIR/tooltool.py -m browser/config/tooltool-manifests/linux64/releng.manifest fetch + +# gets a bit too verbose here +set +x + +cd build/build-clang +# |mach python| sets up a virtualenv for us! +../../mach python ./build-clang.py -c clang-static-analysis-linux64.json + +set -x + +# Put a tarball in the artifacts dir +mkdir -p $UPLOAD_DIR +cp clang.tar.* $UPLOAD_DIR diff --git a/taskcluster/scripts/misc/build-clang-windows.sh b/taskcluster/scripts/misc/build-clang-windows.sh new file mode 100755 index 000000000..6d2acaa03 --- /dev/null +++ b/taskcluster/scripts/misc/build-clang-windows.sh @@ -0,0 +1,61 @@ +#!/bin/bash + +set -x -e -v + +# This script is for building clang-cl on Windows. + +# Fetch our toolchain from tooltool. +wget -O tooltool.py ${TOOLTOOL_REPO}/raw/${TOOLTOOL_REV}/tooltool.py +chmod +x tooltool.py +: TOOLTOOL_CACHE ${TOOLTOOL_CACHE:=/home/worker/tooltool-cache} +export TOOLTOOL_CACHE + +TOOLTOOL_AUTH_FILE=/c/builds/relengapi.tok +if [ ! -e ${TOOLTOOL_AUTH_FILE} ]; then + echo cannot find ${TOOLTOOL_AUTH_FILE} + exit 1 +fi + +TOOLTOOL_MANIFEST=build/src/browser/config/tooltool-manifests/win32/build-clang-cl.manifest +./tooltool.py --authentication-file="${TOOLTOOL_AUTH_FILE}" -m "${TOOLTOOL_MANIFEST}" fetch + +# Set up all the Visual Studio paths. +MSVC_DIR=vs2015u3 +VSWINPATH="$(cd ${MSVC_DIR} && pwd)" + +echo vswinpath ${VSWINPATH} + +export WINDOWSSDKDIR="${VSWINPATH}/SDK" +export WIN32_REDIST_DIR="${VSWINPATH}/VC/redist/x86/Microsoft.VC140.CRT" +export WIN_UCRT_REDIST_DIR="${VSWINPATH}/SDK/Redist/ucrt/DLLs/x86" + +export PATH="${VSWINPATH}/VC/bin/amd64_x86:${VSWINPATH}/VC/bin/amd64:${VSWINPATH}/VC/bin:${VSWINPATH}/SDK/bin/x86:${VSWINPATH}/SDK/bin/x64:${VSWINPATH}/DIA SDK/bin:${PATH}" +export PATH="${VSWINPATH}/VC/redist/x86/Microsoft.VC140.CRT:${VSWINPATH}/VC/redist/x64/Microsoft.VC140.CRT:${VSWINPATH}/SDK/Redist/ucrt/DLLs/x86:${VSWINPATH}/SDK/Redist/ucrt/DLLs/x64:${PATH}" + +export INCLUDE="${VSWINPATH}/VC/include:${VSWINPATH}/VC/atlmfc/include:${VSWINPATH}/SDK/Include/10.0.14393.0/ucrt:${VSWINPATH}/SDK/Include/10.0.14393.0/shared:${VSWINPATH}/SDK/Include/10.0.14393.0/um:${VSWINPATH}/SDK/Include/10.0.14393.0/winrt:${VSWINPATH}/DIA SDK/include" +export LIB="${VSWINPATH}/VC/lib:${VSWINPATH}/VC/atlmfc/lib:${VSWINPATH}/SDK/lib/10.0.14393.0/ucrt/x86:${VSWINPATH}/SDK/lib/10.0.14393.0/um/x86:${VSWINPATH}/DIA SDK/lib" + +export PATH="$(cd svn && pwd)/bin:${PATH}" +export PATH="$(cd cmake && pwd)/bin:${PATH}" +export PATH="$(cd ninja && pwd)/bin:${PATH}" + +# We use |mach python| to set up a virtualenv automatically for us. We create +# a dummy mozconfig, because the default machinery for config.guess-choosing +# of the objdir doesn't work very well. +MOZCONFIG="$(pwd)/mozconfig" +cat > ${MOZCONFIG} <<EOF +mk_add_options MOZ_OBJDIR=$(pwd)/objdir +EOF + +# gets a bit too verbose here +set +x + +BUILD_CLANG_DIR=build/src/build/build-clang +MOZCONFIG=${MOZCONFIG} build/src/mach python ${BUILD_CLANG_DIR}/build-clang.py -c ${BUILD_CLANG_DIR}/clang-static-analysis-win32.json + +set -x + +# Put a tarball in the artifacts dir +UPLOAD_PATH=public/build +mkdir -p ${UPLOAD_PATH} +cp clang.tar.* ${UPLOAD_PATH} diff --git a/taskcluster/scripts/misc/build-gcc-linux.sh b/taskcluster/scripts/misc/build-gcc-linux.sh new file mode 100755 index 000000000..7621ec4aa --- /dev/null +++ b/taskcluster/scripts/misc/build-gcc-linux.sh @@ -0,0 +1,16 @@ +#!/bin/bash +set -e + +# This script is for building GCC for Linux. + +WORKSPACE=$HOME/workspace +HOME_DIR=$WORKSPACE/build +UPLOAD_DIR=$WORKSPACE/artifacts + +cd $HOME_DIR/src + +build/unix/build-gcc/build-gcc.sh $HOME_DIR + +# Put a tarball in the artifacts dir +mkdir -p $UPLOAD_DIR +cp $HOME_DIR/gcc.tar.* $UPLOAD_DIR diff --git a/taskcluster/scripts/misc/minidump_stackwalk.sh b/taskcluster/scripts/misc/minidump_stackwalk.sh new file mode 100755 index 000000000..de4fd748c --- /dev/null +++ b/taskcluster/scripts/misc/minidump_stackwalk.sh @@ -0,0 +1,125 @@ +#!/bin/bash +# +# This script builds minidump_stackwalk binaries from the Google Breakpad +# source for all of the operating systems that we run Firefox tests on: +# Linux x86, Linux x86-64, Windows x86, OS X x86-64. +# +# It expects to be run in the luser/breakpad-builder:0.7 Docker image and +# needs access to the relengapiproxy to download internal tooltool files. + +set -v -e -x + +# This is a pain to support properly with gclient. +#: BREAKPAD_REPO ${BREAKPAD_REPO:=https://google-breakpad.googlecode.com/svn/trunk/} +: BREAKPAD_REV "${BREAKPAD_REV:=master}" +: STACKWALK_HTTP_REPO "${STACKWALK_HTTP_REPO:=https://hg.mozilla.org/users/tmielczarek_mozilla.com/stackwalk-http}" +: STACKWALK_HTTP_REV "${STACKWALK_HTTP_REV:=default}" + +ncpu=$(getconf _NPROCESSORS_ONLN) + +function build() +{ + cd /tmp + local platform=$1 + local strip_prefix=$2 + local configure_args=$3 + local make_args=$4 + local objdir=/tmp/obj-breakpad-$platform + local ext= + if test "$platform" = "win32"; then + ext=.exe + fi + rm -rf "$objdir" + mkdir "$objdir" + # First, build Breakpad + cd "$objdir" + # shellcheck disable=SC2086 + CFLAGS="-O2 $CFLAGS" CXXFLAGS="-O2 $CXXFLAGS" /tmp/breakpad/src/configure --disable-tools $configure_args + # shellcheck disable=SC2086 + make -j$ncpu $make_args src/libbreakpad.a src/third_party/libdisasm/libdisasm.a src/processor/stackwalk_common.o + # Second, build stackwalk-http + make -f /tmp/stackwalk-http/Makefile BREAKPAD_SRCDIR=/tmp/breakpad/src "BREAKPAD_OBJDIR=$(pwd)" "OS=$platform" "-j$ncpu" + "${strip_prefix}strip" "stackwalk${ext}" + cp "stackwalk${ext}" "/tmp/stackwalker/${platform}-minidump_stackwalk${ext}" +} + +function linux64() +{ + export LDFLAGS="-static-libgcc -static-libstdc++" + build linux64 + unset LDFLAGS +} + +function linux32() +{ + export LDFLAGS="-static-libgcc -static-libstdc++ -L/tmp/libcurl-i386/lib" + export CFLAGS="-m32 -I/tmp/libcurl-i386/include" + export CXXFLAGS="-m32 -I/tmp/libcurl-i386/include" + build linux32 "" "--enable-m32" + unset LDFLAGS CFLAGS CXXFLAGS +} + +function macosx64() +{ + cd /tmp + if ! test -d MacOSX10.7.sdk; then + python tooltool.py -v --manifest=macosx-sdk.manifest --url=http://relengapi/tooltool/ fetch + fi + export MACOSX_SDK=/tmp/MacOSX10.7.sdk + export CCTOOLS=/tmp/cctools + local FLAGS="-stdlib=libc++ -target x86_64-apple-darwin10 -mlinker-version=136 -B /tmp/cctools/bin -isysroot ${MACOSX_SDK} -mmacosx-version-min=10.7" + export CC="clang $FLAGS" + export CXX="clang++ $FLAGS -std=c++11" + local old_path="$PATH" + export PATH="/tmp/clang/bin:/tmp/cctools/bin/:$PATH" + export LD_LIBRARY_PATH=/usr/lib/llvm-3.6/lib/ + + build macosx64 "/tmp/cctools/bin/x86_64-apple-darwin10-" "--host=x86_64-apple-darwin10" "AR=/tmp/cctools/bin/x86_64-apple-darwin10-ar" + + unset CC CXX LD_LIBRARY_PATH MACOSX_SDK CCTOOLS + export PATH="$old_path" +} + +function win32() +{ + export LDFLAGS="-static-libgcc -static-libstdc++" + export CFLAGS="-D__USE_MINGW_ANSI_STDIO=1" + export CXXFLAGS="-D__USE_MINGW_ANSI_STDIO=1" + export ZLIB_DIR=/tmp/zlib-mingw + build win32 "i686-w64-mingw32-" "--host=i686-w64-mingw32" + unset LDFLAGS CFLAGS CXXFLAGS ZLIB_DIR +} + +cd /tmp +if ! test -d depot_tools; then + git clone https://chromium.googlesource.com/chromium/tools/depot_tools.git +else + (cd depot_tools; git pull origin master) +fi +export PATH=$(pwd)/depot_tools:"$PATH" +if ! test -d breakpad; then + mkdir breakpad + pushd breakpad + fetch breakpad + popd +else + pushd breakpad/src + git pull origin master + popd +fi +pushd breakpad/src +git checkout "${BREAKPAD_REV}" +gclient sync +popd + +(cd breakpad/src; git rev-parse master) +if ! test -d stackwalk-http; then + hg clone -u "$STACKWALK_HTTP_REV" "$STACKWALK_HTTP_REPO" +else + (cd stackwalk-http && hg pull "$STACKWALK_HTTP_REPO" && hg up "$STACKWALK_HTTP_REV") +fi +mkdir -p stackwalker +linux64 +linux32 +macosx64 +win32 diff --git a/taskcluster/scripts/misc/repackage-jdk-centos.sh b/taskcluster/scripts/misc/repackage-jdk-centos.sh new file mode 100755 index 000000000..2c952602b --- /dev/null +++ b/taskcluster/scripts/misc/repackage-jdk-centos.sh @@ -0,0 +1,45 @@ +#! /bin/bash + +set -e -x + +mkdir -p artifacts +pushd build + +rm -rf root && mkdir root && cd root + +# change these variables when updating java version +mirror_url_base="http://mirror.centos.org/centos/6.7/updates/x86_64/Packages" +openjdk=java-1.7.0-openjdk-1.7.0.85-2.6.1.3.el6_7.x86_64.rpm +openjdk_devel=java-1.7.0-openjdk-devel-1.7.0.85-2.6.1.3.el6_7.x86_64.rpm +jvm_openjdk_dir=java-1.7.0-openjdk-1.7.0.85.x86_64 + +# grab the rpm and unpack it +wget ${mirror_url_base}/${openjdk} +wget ${mirror_url_base}/${openjdk_devel} +rpm2cpio $openjdk | cpio -ivd +rpm2cpio $openjdk_devel | cpio -ivd + +cd usr/lib/jvm +mv $jvm_openjdk_dir java_home + +# cacerts is a relative symlink, which doesn't work when we repackage. Make it +# absolute. We could use tar's --dereference option, but there's a subtle +# difference between making the symlink absolute and using --dereference. +# Making the symlink absolute lets the consuming system set the cacerts; using +# --dereference takes the producing system's cacerts and sets them in stone. We +# prefer the flexibility of the former. +rm java_home/jre/lib/security/cacerts +ln -s /etc/pki/java/cacerts java_home/jre/lib/security/cacerts + +# document version this is based on +echo "Built from ${mirror_url_Base} + ${openjdk} + ${openjdk_devel} + +Run through rpm2cpio | cpio, and /usr/lib/jvm/${jvm_openjdk_dir} renamed to 'java_home'." > java_home/VERSION + +# tarball the unpacked rpm and put it in the taskcluster upload artifacts dir +tar -Jvcf java_home-${jvm_openjdk_dir}.tar.xz java_home +popd + +mv build/root/usr/lib/jvm/java_home-${jvm_openjdk_dir}.tar.xz artifacts diff --git a/taskcluster/scripts/tester/harness-test-linux.sh b/taskcluster/scripts/tester/harness-test-linux.sh new file mode 100644 index 000000000..b38ffa124 --- /dev/null +++ b/taskcluster/scripts/tester/harness-test-linux.sh @@ -0,0 +1,40 @@ +#! /bin/bash -vex + +set -x -e + +echo "running as" $(id) + +#### +# Taskcluster friendly wrapper for running a script in +# testing/mozharness/scripts in a source checkout (no build). +# Example use: Python-only harness unit tests +#### + +: WORKSPACE ${WORKSPACE:=/home/worker/workspace} +: SRC_ROOT ${SRC_ROOT:=$WORKSPACE/build/src} +# These paths should be relative to $SRC_ROOT +: MOZHARNESS_SCRIPT ${MOZHARNESS_SCRIPT} +: MOZHARNESS_CONFIG ${MOZHARNESS_CONFIG} +: mozharness args "${@}" + +set -v +cd $WORKSPACE + +fail() { + echo # make sure error message is on a new line + echo "[harness-test-linux.sh:error]" "${@}" + exit 1 +} + +if [[ -z ${MOZHARNESS_SCRIPT} ]]; then fail "MOZHARNESS_SCRIPT is not set"; fi + +# support multiple, space delimited, config files +config_cmds="" +for cfg in $MOZHARNESS_CONFIG; do + config_cmds="${config_cmds} --config-file ${SRC_ROOT}/${cfg}" +done + +python2.7 $SRC_ROOT/${MOZHARNESS_SCRIPT} ${config_cmds} "${@}" + + + diff --git a/taskcluster/scripts/tester/run-wizard b/taskcluster/scripts/tester/run-wizard new file mode 100755 index 000000000..5dafb0b62 --- /dev/null +++ b/taskcluster/scripts/tester/run-wizard @@ -0,0 +1,170 @@ +#!/usr/bin/env python +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this, +# file, You can obtain one at http://mozilla.org/MPL/2.0/. + +from __future__ import print_function, unicode_literals + +import datetime +import os +import subprocess +import sys +import time +from textwrap import wrap + +here = os.path.dirname(os.path.abspath(__file__)) + + +def call(cmd, **kwargs): + print(" ".join(cmd)) + return subprocess.call(cmd, **kwargs) + + +def wait_for_run_mozharness(timeout=30): + starttime = datetime.datetime.now() + while datetime.datetime.now() - starttime < datetime.timedelta(seconds=timeout): + if os.path.isfile(os.path.join(here, 'run-mozharness')): + break + time.sleep(0.2) + else: + print("Timed out after %d seconds waiting for the 'run-mozharness' binary" % timeout) + return 1 + + +def resume(): + wait_for_run_mozharness() + call(['run-mozharness']) + + +def setup(): + """Run the mozharness script without the 'run-tests' action. + + This will do all the necessary setup steps like creating a virtualenv and + downloading the tests and firefox binary. But it stops before running the + tests. + """ + wait_for_run_mozharness() + status = call(['run-mozharness', '--no-run-tests']) + + if status: + # something went wrong + return status + + build_dir = os.path.expanduser(os.path.join('~', 'workspace', 'build')) + mach_src = os.path.join(build_dir, 'tests', 'mach') + mach_dest = os.path.expanduser(os.path.join('~', 'bin', 'mach')) + + if os.path.exists(mach_dest): + os.remove(mach_dest) + os.symlink(mach_src, mach_dest) + + activate = os.path.join(build_dir, 'venv', 'bin', 'activate') + if os.path.isfile(activate): + # TODO Support other shells + bashrc = os.path.expanduser(os.path.join('~', '.bashrc')) + with open(bashrc, 'ab') as f: + f.write(". {}".format(activate)) + + print(""" +Mozharness has finished downloading the build and tests to: +{} + +A limited mach environment has also been set up and added to the $PATH, but +it may be missing the command you need. To see a list of commands, run: + $ mach help +""".lstrip().format(build_dir)) + + +def clone(): + """Clone the correct gecko repository and update to the proper revision.""" + base_repo = os.environ['GECKO_HEAD_REPOSITORY'] + dest = os.path.expanduser(os.path.join('~', 'gecko')) + + # Specify method to checkout a revision. This defaults to revisions as + # SHA-1 strings, but also supports symbolic revisions like `tip` via the + # branch flag. + if os.environ.get('GECKO_HEAD_REV'): + revision_flag = b'--revision' + revision = os.environ['GECKO_HEAD_REV'] + elif os.environ.get('GECKO_HEAD_REF'): + revision_flag = b'--branch' + revision = os.environ['GECKO_HEAD_REF'] + else: + print('revision is not specified for checkout') + return 1 + + # TODO Bug 1301382 - pin hg.mozilla.org fingerprint. + call([ + b'/usr/bin/hg', b'robustcheckout', + b'--sharebase', os.environ['HG_STORE_PATH'], + b'--purge', + b'--upstream', b'https://hg.mozilla.org/mozilla-unified', + revision_flag, revision, + base_repo, dest + ]) + print("Finished cloning to {} at revision {}.".format(dest, revision)) + + +def exit(): + pass + + +OPTIONS = [ + ('Resume task', resume, + "Resume the original task without modification. This can be useful for " + "passively monitoring it from another shell."), + ('Setup task', setup, + "Setup the task (download the application and tests) but don't run the " + "tests just yet. The tests can be run with a custom configuration later. " + "This will provide a mach environment (experimental)."), + ('Clone gecko', clone, + "Perform a clone of gecko using the task's repo and update it to the " + "task's revision."), + ('Exit', exit, "Exit this wizard and return to the shell.") +] + + +def _fmt_options(): + max_line_len = 60 + max_name_len = max(len(o[0]) for o in OPTIONS) + + # TODO Pad will be off if there are more than 9 options. + pad = ' ' * (max_name_len+6) + + msg = [] + for i, (name, _, desc) in enumerate(OPTIONS): + desc = wrap(desc, width=max_line_len) + desc = [desc[0]] + [pad + l for l in desc[1:]] + + optstr = '{}) {} - {}\n'.format( + i+1, name.ljust(max_name_len), '\n'.join(desc)) + msg.append(optstr) + msg.append("Select one of the above options: ") + return '\n'.join(msg) + + +def wizard(): + print("This wizard can help you get started with some common debugging " + "workflows.\nWhat would you like to do?\n") + print(_fmt_options(), end="") + choice = None + while True: + choice = raw_input().decode('utf8') + try: + choice = int(choice)-1 + if 0 <= choice < len(OPTIONS): + break + except ValueError: + pass + + print("Must provide an integer from 1-{}:".format(len(OPTIONS))) + + func = OPTIONS[choice][1] + ret = func() + + print("Use the 'run-wizard' command to start this wizard again.") + return ret + + +if __name__ == '__main__': + sys.exit(wizard()) diff --git a/taskcluster/scripts/tester/test-b2g.sh b/taskcluster/scripts/tester/test-b2g.sh new file mode 100644 index 000000000..43a54f93a --- /dev/null +++ b/taskcluster/scripts/tester/test-b2g.sh @@ -0,0 +1,118 @@ +#! /bin/bash -xe + +set -x -e + +echo "running as" $(id) + +#### +# Taskcluster friendly wrapper for performing fx desktop tests via mozharness. +#### + +# Inputs, with defaults + +: MOZHARNESS_URL ${MOZHARNESS_URL} +: MOZHARNESS_SCRIPT ${MOZHARNESS_SCRIPT} +: MOZHARNESS_CONFIG ${MOZHARNESS_CONFIG} +: NEED_XVFB ${NEED_XVFB:=true} +: NEED_PULSEAUDIO ${NEED_PULSEAUDIO:=false} +: NEED_PULL_GAIA ${NEED_PULL_GAIA:=false} +: SKIP_MOZHARNESS_RUN ${SKIP_MOZHARNESS_RUN:=false} +: WORKSPACE ${WORKSPACE:=/home/worker/workspace} +: mozharness args "${@}" + +set -v +cd $WORKSPACE + +# test required parameters are supplied +if [[ -z ${MOZHARNESS_URL} ]]; then exit 1; fi +if [[ -z ${MOZHARNESS_SCRIPT} ]]; then exit 1; fi +if [[ -z ${MOZHARNESS_CONFIG} ]]; then exit 1; fi + +mkdir -p ~/artifacts/public + +cleanup() { + if [ -n "$xvfb_pid" ]; then + kill $xvfb_pid || true + fi +} +trap cleanup EXIT INT + +# Unzip the mozharness ZIP file created by the build task +curl --fail -o mozharness.zip --retry 10 -L $MOZHARNESS_URL +rm -rf mozharness +unzip -q mozharness.zip +rm mozharness.zip + +if ! [ -d mozharness ]; then + echo "mozharness zip did not contain mozharness/" + exit 1 +fi + +# start up the pulseaudio daemon. Note that it's important this occur +# before the Xvfb startup. +if $NEED_PULSEAUDIO; then + pulseaudio --fail --daemonize --start + pactl load-module module-null-sink +fi + +# run XVfb in the background, if necessary +if $NEED_XVFB; then + Xvfb :0 -nolisten tcp -screen 0 1600x1200x24 \ + > ~/artifacts/public/xvfb.log 2>&1 & + export DISPLAY=:0 + xvfb_pid=$! + # Only error code 255 matters, because it signifies that no + # display could be opened. As long as we can open the display + # tests should work. We'll retry a few times with a sleep before + # failing. + retry_count=0 + max_retries=2 + xvfb_test=0 + until [ $retry_count -gt $max_retries ]; do + xvinfo || xvfb_test=$? + if [ $xvfb_test != 255 ]; then + retry_count=$(($max_retries + 1)) + else + retry_count=$(($retry_count + 1)) + echo "Failed to start Xvfb, retry: $retry_count" + sleep 2 + fi done + if [ $xvfb_test == 255 ]; then exit 255; fi +fi + +gaia_cmds="" +if $NEED_PULL_GAIA; then + # test required parameters are supplied + if [[ -z ${GAIA_BASE_REPOSITORY} ]]; then exit 1; fi + if [[ -z ${GAIA_HEAD_REPOSITORY} ]]; then exit 1; fi + if [[ -z ${GAIA_REV} ]]; then exit 1; fi + if [[ -z ${GAIA_REF} ]]; then exit 1; fi + + tc-vcs checkout \ + ${WORKSPACE}/gaia \ + ${GAIA_BASE_REPOSITORY} \ + ${GAIA_HEAD_REPOSITORY} \ + ${GAIA_REV} \ + ${GAIA_REF} + + gaia_cmds="--gaia-dir=${WORKSPACE}" +fi + +# support multiple, space delimited, config files +config_cmds="" +for cfg in $MOZHARNESS_CONFIG; do + config_cmds="${config_cmds} --config-file ${cfg}" +done + +if [ ${SKIP_MOZHARNESS_RUN} == true ]; then + # Skipping Mozharness is to allow the developer start the window manager + # properly and letting them change the execution of Mozharness without + # exiting the container + echo "We skipped running Mozharness." + echo "Make sure you export DISPLAY=:0 before calling Mozharness." + echo "Don't forget to call it with 'sudo -E -u worker'." +else + # run the given mozharness script and configs, but pass the rest of the + # arguments in from our own invocation + python2.7 $WORKSPACE/${MOZHARNESS_SCRIPT} ${config_cmds} ${gaia_cmds} "${@}" +fi diff --git a/taskcluster/scripts/tester/test-macosx.sh b/taskcluster/scripts/tester/test-macosx.sh new file mode 100644 index 000000000..8c2b758fb --- /dev/null +++ b/taskcluster/scripts/tester/test-macosx.sh @@ -0,0 +1,77 @@ +#! /bin/bash -xe + +set -x -e + +echo "running as" $(id) + +#### +# Taskcluster friendly wrapper for performing fx Mac OSX tests via mozharness. +#### + +# Inputs, with defaults + +: MOZHARNESS_URL ${MOZHARNESS_URL} +: MOZHARNESS_SCRIPT ${MOZHARNESS_SCRIPT} +: MOZHARNESS_CONFIG ${MOZHARNESS_CONFIG} + +WORKSPACE=$HOME +cd $WORKSPACE + +rm -rf artifacts +mkdir artifacts + +# test required parameters are supplied +if [[ -z ${MOZHARNESS_URL} ]]; then fail "MOZHARNESS_URL is not set"; fi +if [[ -z ${MOZHARNESS_SCRIPT} ]]; then fail "MOZHARNESS_SCRIPT is not set"; fi +if [[ -z ${MOZHARNESS_CONFIG} ]]; then fail "MOZHARNESS_CONFIG is not set"; fi + +# Download mozharness with exponential backoff +# curl already applies exponential backoff, but not for all +# failed cases, apparently, as we keep getting failed downloads +# with 404 code. +download_mozharness() { + local max_attempts=10 + local timeout=1 + local attempt=0 + + echo "Downloading mozharness" + + while [[ $attempt < $max_attempts ]]; do + if curl --fail -o mozharness.zip --retry 10 -L $MOZHARNESS_URL; then + rm -rf mozharness + if unzip -q mozharness.zip; then + return 0 + fi + echo "error unzipping mozharness.zip" >&2 + else + echo "failed to download mozharness zip" >&2 + fi + echo "Download failed, retrying in $timeout seconds..." >&2 + sleep $timeout + timeout=$((timeout*2)) + attempt=$((attempt+1)) + done + + fail "Failed to download and unzip mozharness" +} + +download_mozharness +rm mozharness.zip + +# For telemetry purposes, the build process wants information about the +# source it is running; tc-vcs obscures this a little, but we can provide +# it directly. +export MOZ_SOURCE_REPO="${GECKO_HEAD_REPOSITORY}" +export MOZ_SOURCE_CHANGESET="${GECKO_HEAD_REV}" + +# support multiple, space delimited, config files +config_cmds="" +for cfg in $MOZHARNESS_CONFIG; do + config_cmds="${config_cmds} --config-file ${cfg}" +done + +rm -rf build logs properties target.dmg + +# run the given mozharness script and configs, but pass the rest of the +# arguments in from our own invocation +python2.7 $WORKSPACE/mozharness/scripts/${MOZHARNESS_SCRIPT} ${config_cmds} "${@}" diff --git a/taskcluster/scripts/tester/test-ubuntu.sh b/taskcluster/scripts/tester/test-ubuntu.sh new file mode 100644 index 000000000..0c2ccc702 --- /dev/null +++ b/taskcluster/scripts/tester/test-ubuntu.sh @@ -0,0 +1,188 @@ +#! /bin/bash -xe + +set -x -e + +echo "running as" $(id) + +# Detect release version. +. /etc/lsb-release +if [ "${DISTRIB_RELEASE}" == "12.04" ]; then + UBUNTU_1204=1 +elif [ "${DISTRIB_RELEASE}" == "16.04" ]; then + UBUNTU_1604=1 +fi + +. /home/worker/scripts/xvfb.sh + +#### +# Taskcluster friendly wrapper for performing fx desktop tests via mozharness. +#### + +# Inputs, with defaults + +: MOZHARNESS_PATH ${MOZHARNESS_PATH} +: MOZHARNESS_URL ${MOZHARNESS_URL} +: MOZHARNESS_SCRIPT ${MOZHARNESS_SCRIPT} +: MOZHARNESS_CONFIG ${MOZHARNESS_CONFIG} +: NEED_XVFB ${NEED_XVFB:=true} +: NEED_WINDOW_MANAGER ${NEED_WINDOW_MANAGER:=false} +: NEED_PULSEAUDIO ${NEED_PULSEAUDIO:=false} +: START_VNC ${START_VNC:=false} +: TASKCLUSTER_INTERACTIVE ${TASKCLUSTER_INTERACTIVE:=false} +: WORKSPACE ${WORKSPACE:=/home/worker/workspace} +: mozharness args "${@}" + +set -v +cd $WORKSPACE + +fail() { + echo # make sure error message is on a new line + echo "[test-linux.sh:error]" "${@}" + exit 1 +} + +maybe_start_pulse() { + if $NEED_PULSEAUDIO; then + pulseaudio --fail --daemonize --start + pactl load-module module-null-sink + fi +} + +# test required parameters are supplied +if [ -z "${MOZHARNESS_PATH}" -a -z "${MOZHARNESS_URL}" ]; then + fail "MOZHARNESS_PATH or MOZHARNESS_URL must be defined"; +fi + +if [[ -z ${MOZHARNESS_SCRIPT} ]]; then fail "MOZHARNESS_SCRIPT is not set"; fi +if [[ -z ${MOZHARNESS_CONFIG} ]]; then fail "MOZHARNESS_CONFIG is not set"; fi + +mkdir -p ~/artifacts/public + +cleanup() { + local rv=$? + if [[ -s /home/worker/.xsession-errors ]]; then + # To share X issues + cp /home/worker/.xsession-errors ~/artifacts/public/xsession-errors.log + fi + cleanup_xvfb + exit $rv +} +trap cleanup EXIT INT + +# Download mozharness with exponential backoff +# curl already applies exponential backoff, but not for all +# failed cases, apparently, as we keep getting failed downloads +# with 404 code. +download_mozharness() { + local max_attempts=10 + local timeout=1 + local attempt=0 + + echo "Downloading mozharness" + + while [[ $attempt < $max_attempts ]]; do + if curl --fail -o mozharness.zip --retry 10 -L $MOZHARNESS_URL; then + rm -rf mozharness + if unzip -q mozharness.zip; then + return 0 + fi + echo "error unzipping mozharness.zip" >&2 + else + echo "failed to download mozharness zip" >&2 + fi + echo "Download failed, retrying in $timeout seconds..." >&2 + sleep $timeout + timeout=$((timeout*2)) + attempt=$((attempt+1)) + done + + fail "Failed to download and unzip mozharness" +} + +# Download mozharness if we're told to. +if [ ${MOZHARNESS_URL} ]; then + download_mozharness + rm mozharness.zip + + if ! [ -d mozharness ]; then + fail "mozharness zip did not contain mozharness/" + fi + + MOZHARNESS_PATH=`pwd`/mozharness +fi + +# pulseaudio daemon must be started before xvfb on Ubuntu 12.04. +if [ "${UBUNTU_1204}" ]; then + maybe_start_pulse +fi + +# run XVfb in the background, if necessary +if $NEED_XVFB; then + start_xvfb '1600x1200x24' 0 +fi + +if $START_VNC; then + x11vnc > ~/artifacts/public/x11vnc.log 2>&1 & +fi + +if $NEED_WINDOW_MANAGER; then + # This is read by xsession to select the window manager + echo DESKTOP_SESSION=ubuntu > /home/worker/.xsessionrc + + # note that doing anything with this display before running Xsession will cause sadness (like, + # crashes in compiz). Make sure that X has enough time to start + sleep 15 + # DISPLAY has already been set above + # XXX: it would be ideal to add a semaphore logic to make sure that the + # window manager is ready + /etc/X11/Xsession 2>&1 & + + # Turn off the screen saver and screen locking + gsettings set org.gnome.desktop.screensaver idle-activation-enabled false + gsettings set org.gnome.desktop.screensaver lock-enabled false + gsettings set org.gnome.desktop.screensaver lock-delay 3600 + # Disable the screen saver + xset s off s reset + + if [ "${UBUNTU_1604}" ]; then + # start compiz for our window manager + compiz 2>&1 & + #TODO: how to determine if compiz starts correctly? + fi +fi + +if [ "${UBUNTU_1604}" ]; then + maybe_start_pulse +fi + +# For telemetry purposes, the build process wants information about the +# source it is running; tc-vcs obscures this a little, but we can provide +# it directly. +export MOZ_SOURCE_REPO="${GECKO_HEAD_REPOSITORY}" +export MOZ_SOURCE_CHANGESET="${GECKO_HEAD_REV}" + +# support multiple, space delimited, config files +config_cmds="" +for cfg in $MOZHARNESS_CONFIG; do + config_cmds="${config_cmds} --config-file ${MOZHARNESS_PATH}/configs/${cfg}" +done + +mozharness_bin="/home/worker/bin/run-mozharness" + +# Save the computed mozharness command to a binary which is useful +# for interactive mode. +echo -e "#!/usr/bin/env bash +# Some mozharness scripts assume base_work_dir is in +# the current working directory, see bug 1279237 +cd $WORKSPACE +cmd=\"python2.7 ${MOZHARNESS_PATH}/scripts/${MOZHARNESS_SCRIPT} ${config_cmds} ${@} \${@}\" +echo \"Running: \${cmd}\" +exec \${cmd}" > ${mozharness_bin} +chmod +x ${mozharness_bin} + +# In interactive mode, the user will be prompted with options for what to do. +if ! $TASKCLUSTER_INTERACTIVE; then + # run the given mozharness script and configs, but pass the rest of the + # arguments in from our own invocation + ${mozharness_bin}; +fi diff --git a/taskcluster/taskgraph/__init__.py b/taskcluster/taskgraph/__init__.py new file mode 100644 index 000000000..e69de29bb --- /dev/null +++ b/taskcluster/taskgraph/__init__.py diff --git a/taskcluster/taskgraph/action.py b/taskcluster/taskgraph/action.py new file mode 100644 index 000000000..608fe3370 --- /dev/null +++ b/taskcluster/taskgraph/action.py @@ -0,0 +1,68 @@ +# -*- coding: utf-8 -*- + +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, You can obtain one at http://mozilla.org/MPL/2.0/. + +from __future__ import absolute_import, print_function, unicode_literals + +import json +import logging +import requests +import yaml + +from .create import create_tasks +from .decision import write_artifact +from .optimize import optimize_task_graph +from .taskgraph import TaskGraph + +logger = logging.getLogger(__name__) +TASKCLUSTER_QUEUE_URL = "https://queue.taskcluster.net/v1/task/" + + +def taskgraph_action(options): + """ + Run the action task. This function implements `mach taskgraph action-task`, + and is responsible for + + * creating taskgraph of tasks asked for in parameters with respect to + a given gecko decision task and schedule these jobs. + """ + + decision_task_id = options['decision_id'] + # read in the full graph for reference + full_task_json = get_artifact(decision_task_id, "public/full-task-graph.json") + decision_params = get_artifact(decision_task_id, "public/parameters.yml") + all_tasks, full_task_graph = TaskGraph.from_json(full_task_json) + + target_tasks = set(options['task_labels'].split(',')) + target_graph = full_task_graph.graph.transitive_closure(target_tasks) + target_task_graph = TaskGraph( + {l: all_tasks[l] for l in target_graph.nodes}, + target_graph) + + existing_tasks = get_artifact(decision_task_id, "public/label-to-taskid.json") + + # We don't want to optimize target tasks since they have been requested by user + # Hence we put `target_tasks under` `do_not_optimize` + optimized_graph, label_to_taskid = optimize_task_graph(target_task_graph=target_task_graph, + params=decision_params, + do_not_optimize=target_tasks, + existing_tasks=existing_tasks) + + # write out the optimized task graph to describe what will actually happen, + # and the map of labels to taskids + write_artifact('task-graph.json', optimized_graph.to_json()) + write_artifact('label-to-taskid.json', label_to_taskid) + # actually create the graph + create_tasks(optimized_graph, label_to_taskid, decision_params) + + +def get_artifact(task_id, path): + url = TASKCLUSTER_QUEUE_URL + task_id + "/artifacts/" + path + resp = requests.get(url=url) + if path.endswith('.json'): + artifact = json.loads(resp.text) + elif path.endswith('.yml'): + artifact = yaml.load(resp.text) + return artifact diff --git a/taskcluster/taskgraph/action.yml b/taskcluster/taskgraph/action.yml new file mode 100644 index 000000000..c816f4d5c --- /dev/null +++ b/taskcluster/taskgraph/action.yml @@ -0,0 +1,74 @@ +--- +created: '{{now}}' +deadline: '{{#from_now}}1 day{{/from_now}}' +expires: '{{#from_now}}14 day{{/from_now}}' +metadata: + owner: mozilla-taskcluster-maintenance@mozilla.com + source: 'https://hg.mozilla.org/{{project}}/file/{{head_rev}}/taskcluster/taskgraph/action.yml' + name: "[tc] Action Task" + description: Helps schedule new jobs without new push + +workerType: "gecko-decision" +provisionerId: "aws-provisioner-v1" +schedulerId: "gecko-level-{{level}}" + +tags: + createdForUser: {{owner}} + +scopes: + # Bug 1269443: cache scopes, etc. must be listed explicitly + - "docker-worker:cache:level-1-*" + - "docker-worker:cache:tooltool-cache" + - "secrets:get:project/taskcluster/gecko/hgfingerprint" + - "assume:repo:hg.mozilla.org/try:*" + +routes: + - "tc-treeherder.v2.{{project}}.{{head_rev}}.{{pushlog_id}}" + - "tc-treeherder-stage.v2.{{project}}.{{head_rev}}.{{pushlog_id}}" + +payload: + env: + GECKO_BASE_REPOSITORY: 'https://hg.mozilla.org/mozilla-unified' + GECKO_HEAD_REPOSITORY: '{{{head_repository}}}' + GECKO_HEAD_REF: '{{head_ref}}' + GECKO_HEAD_REV: '{{head_rev}}' + HG_STORE_PATH: /home/worker/checkouts/hg-store + + cache: + level-{{level}}-checkouts: /home/worker/checkouts + + features: + taskclusterProxy: true + + # Note: This task is built server side without the context or tooling that + # exist in tree so we must hard code the version + image: 'taskcluster/decision:0.1.7' + + # Virtually no network or other potentially risky operations happen as part + # of the task timeout aside from the initial clone. We intentionally have + # set this to a lower value _all_ decision tasks should use a root + # repository which is cached. + maxRunTime: 1800 + + command: + - /home/worker/bin/run-task + - '--vcs-checkout=/home/worker/checkouts/gecko' + - '--' + - bash + - -cx + - > + cd /home/worker/checkouts/gecko && + ln -s /home/worker/artifacts artifacts && + ./mach --log-no-times taskgraph action-task + --decision-id='{{decision_task_id}}' + --task-label='{{task_labels}}' + + artifacts: + 'public': + type: 'directory' + path: '/home/worker/artifacts' + expires: '{{#from_now}}7 days{{/from_now}}' + +extra: + treeherder: + symbol: A diff --git a/taskcluster/taskgraph/create.py b/taskcluster/taskgraph/create.py new file mode 100644 index 000000000..f577f8873 --- /dev/null +++ b/taskcluster/taskgraph/create.py @@ -0,0 +1,122 @@ +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, You can obtain one at http://mozilla.org/MPL/2.0/. + +from __future__ import absolute_import, print_function, unicode_literals + +import concurrent.futures as futures +import requests +import requests.adapters +import json +import os +import logging + +from slugid import nice as slugid +from taskgraph.util.time import ( + current_json_time, + json_time_from_now +) + +logger = logging.getLogger(__name__) + +# the maximum number of parallel createTask calls to make +CONCURRENCY = 50 + + +def create_tasks(taskgraph, label_to_taskid, params): + taskid_to_label = {t: l for l, t in label_to_taskid.iteritems()} + + session = requests.Session() + + # Default HTTPAdapter uses 10 connections. Mount custom adapter to increase + # that limit. Connections are established as needed, so using a large value + # should not negatively impact performance. + http_adapter = requests.adapters.HTTPAdapter(pool_connections=CONCURRENCY, + pool_maxsize=CONCURRENCY) + session.mount('https://', http_adapter) + session.mount('http://', http_adapter) + + decision_task_id = os.environ.get('TASK_ID') + + # when running as an actual decision task, we use the decision task's + # taskId as the taskGroupId. The process that created the decision task + # helpfully placed it in this same taskGroup. If there is no $TASK_ID, + # fall back to a slugid + task_group_id = decision_task_id or slugid() + scheduler_id = 'gecko-level-{}'.format(params['level']) + + with futures.ThreadPoolExecutor(CONCURRENCY) as e: + fs = {} + + # We can't submit a task until its dependencies have been submitted. + # So our strategy is to walk the graph and submit tasks once all + # their dependencies have been submitted. + # + # Using visit_postorder() here isn't the most efficient: we'll + # block waiting for dependencies of task N to submit even though + # dependencies for task N+1 may be finished. If we need to optimize + # this further, we can build a graph of task dependencies and walk + # that. + for task_id in taskgraph.graph.visit_postorder(): + task_def = taskgraph.tasks[task_id].task + attributes = taskgraph.tasks[task_id].attributes + # if this task has no dependencies, make it depend on this decision + # task so that it does not start immediately; and so that if this loop + # fails halfway through, none of the already-created tasks run. + if decision_task_id and not task_def.get('dependencies'): + task_def['dependencies'] = [decision_task_id] + + task_def['taskGroupId'] = task_group_id + task_def['schedulerId'] = scheduler_id + + # Wait for dependencies before submitting this. + deps_fs = [fs[dep] for dep in task_def.get('dependencies', []) + if dep in fs] + for f in futures.as_completed(deps_fs): + f.result() + + fs[task_id] = e.submit(_create_task, session, task_id, + taskid_to_label[task_id], task_def) + + # Schedule tasks as many times as task_duplicates indicates + for i in range(1, attributes.get('task_duplicates', 1)): + # We use slugid() since we want a distinct task id + fs[task_id] = e.submit(_create_task, session, slugid(), + taskid_to_label[task_id], task_def) + + # Wait for all futures to complete. + for f in futures.as_completed(fs.values()): + f.result() + + +def _create_task(session, task_id, label, task_def): + # create the task using 'http://taskcluster/queue', which is proxied to the queue service + # with credentials appropriate to this job. + + # Resolve timestamps + now = current_json_time(datetime_format=True) + task_def = resolve_timestamps(now, task_def) + + logger.debug("Creating task with taskId {} for {}".format(task_id, label)) + res = session.put('http://taskcluster/queue/v1/task/{}'.format(task_id), + data=json.dumps(task_def)) + if res.status_code != 200: + try: + logger.error(res.json()['message']) + except: + logger.error(res.text) + res.raise_for_status() + + +def resolve_timestamps(now, task_def): + def recurse(val): + if isinstance(val, list): + return [recurse(v) for v in val] + elif isinstance(val, dict): + if val.keys() == ['relative-datestamp']: + return json_time_from_now(val['relative-datestamp'], now) + else: + return {k: recurse(v) for k, v in val.iteritems()} + else: + return val + return recurse(task_def) diff --git a/taskcluster/taskgraph/decision.py b/taskcluster/taskgraph/decision.py new file mode 100644 index 000000000..d6460390f --- /dev/null +++ b/taskcluster/taskgraph/decision.py @@ -0,0 +1,181 @@ +# -*- coding: utf-8 -*- + +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, You can obtain one at http://mozilla.org/MPL/2.0/. + +from __future__ import absolute_import, print_function, unicode_literals + +import os +import json +import logging + +import time +import yaml + +from .generator import TaskGraphGenerator +from .create import create_tasks +from .parameters import Parameters +from .target_tasks import get_method +from .taskgraph import TaskGraph + +from taskgraph.util.templates import Templates +from taskgraph.util.time import ( + json_time_from_now, + current_json_time, +) + +logger = logging.getLogger(__name__) + +ARTIFACTS_DIR = 'artifacts' +GECKO = os.path.realpath(os.path.join(__file__, '..', '..', '..')) + +# For each project, this gives a set of parameters specific to the project. +# See `taskcluster/docs/parameters.rst` for information on parameters. +PER_PROJECT_PARAMETERS = { + 'try': { + 'target_tasks_method': 'try_option_syntax', + # Always perform optimization. This makes it difficult to use try + # pushes to run a task that would otherwise be optimized, but is a + # compromise to avoid essentially disabling optimization in try. + 'optimize_target_tasks': True, + }, + + 'ash': { + 'target_tasks_method': 'ash_tasks', + 'optimize_target_tasks': True, + }, + + 'cedar': { + 'target_tasks_method': 'cedar_tasks', + 'optimize_target_tasks': True, + }, + + # the default parameters are used for projects that do not match above. + 'default': { + 'target_tasks_method': 'default', + 'optimize_target_tasks': True, + } +} + + +def taskgraph_decision(options): + """ + Run the decision task. This function implements `mach taskgraph decision`, + and is responsible for + + * processing decision task command-line options into parameters + * running task-graph generation exactly the same way the other `mach + taskgraph` commands do + * generating a set of artifacts to memorialize the graph + * calling TaskCluster APIs to create the graph + """ + + parameters = get_decision_parameters(options) + + # create a TaskGraphGenerator instance + target_tasks_method = parameters.get('target_tasks_method', 'all_tasks') + target_tasks_method = get_method(target_tasks_method) + tgg = TaskGraphGenerator( + root_dir=options['root'], + parameters=parameters, + target_tasks_method=target_tasks_method) + + # write out the parameters used to generate this graph + write_artifact('parameters.yml', dict(**parameters)) + + # write out the yml file for action tasks + write_artifact('action.yml', get_action_yml(parameters)) + + # write out the full graph for reference + full_task_json = tgg.full_task_graph.to_json() + write_artifact('full-task-graph.json', full_task_json) + + # this is just a test to check whether the from_json() function is working + _, _ = TaskGraph.from_json(full_task_json) + + # write out the target task set to allow reproducing this as input + write_artifact('target-tasks.json', tgg.target_task_set.tasks.keys()) + + # write out the optimized task graph to describe what will actually happen, + # and the map of labels to taskids + write_artifact('task-graph.json', tgg.optimized_task_graph.to_json()) + write_artifact('label-to-taskid.json', tgg.label_to_taskid) + + # actually create the graph + create_tasks(tgg.optimized_task_graph, tgg.label_to_taskid, parameters) + + +def get_decision_parameters(options): + """ + Load parameters from the command-line options for 'taskgraph decision'. + This also applies per-project parameters, based on the given project. + + """ + parameters = {n: options[n] for n in [ + 'base_repository', + 'head_repository', + 'head_rev', + 'head_ref', + 'message', + 'project', + 'pushlog_id', + 'pushdate', + 'owner', + 'level', + 'triggered_by', + 'target_tasks_method', + ] if n in options} + + # owner must be an email, but sometimes (e.g., for ffxbld) it is not, in which + # case, fake it + if '@' not in parameters['owner']: + parameters['owner'] += '@noreply.mozilla.org' + + # use the pushdate as build_date if given, else use current time + parameters['build_date'] = parameters['pushdate'] or int(time.time()) + # moz_build_date is the build identifier based on build_date + parameters['moz_build_date'] = time.strftime("%Y%m%d%H%M%S", + time.gmtime(parameters['build_date'])) + + project = parameters['project'] + try: + parameters.update(PER_PROJECT_PARAMETERS[project]) + except KeyError: + logger.warning("using default project parameters; add {} to " + "PER_PROJECT_PARAMETERS in {} to customize behavior " + "for this project".format(project, __file__)) + parameters.update(PER_PROJECT_PARAMETERS['default']) + + # `target_tasks_method` has higher precedence than `project` parameters + if options.get('target_tasks_method'): + parameters['target_tasks_method'] = options['target_tasks_method'] + + return Parameters(parameters) + + +def write_artifact(filename, data): + logger.info('writing artifact file `{}`'.format(filename)) + if not os.path.isdir(ARTIFACTS_DIR): + os.mkdir(ARTIFACTS_DIR) + path = os.path.join(ARTIFACTS_DIR, filename) + if filename.endswith('.yml'): + with open(path, 'w') as f: + yaml.safe_dump(data, f, allow_unicode=True, default_flow_style=False) + elif filename.endswith('.json'): + with open(path, 'w') as f: + json.dump(data, f, sort_keys=True, indent=2, separators=(',', ': ')) + else: + raise TypeError("Don't know how to write to {}".format(filename)) + + +def get_action_yml(parameters): + templates = Templates(os.path.join(GECKO, "taskcluster/taskgraph")) + action_parameters = parameters.copy() + action_parameters.update({ + "decision_task_id": "{{decision_task_id}}", + "task_labels": "{{task_labels}}", + "from_now": json_time_from_now, + "now": current_json_time() + }) + return templates.load('action.yml', action_parameters) diff --git a/taskcluster/taskgraph/docker.py b/taskcluster/taskgraph/docker.py new file mode 100644 index 000000000..8159fd5a4 --- /dev/null +++ b/taskcluster/taskgraph/docker.py @@ -0,0 +1,132 @@ +# -*- coding: utf-8 -*- + +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, You can obtain one at http://mozilla.org/MPL/2.0/. + +from __future__ import absolute_import, print_function, unicode_literals + +import json +import os +import subprocess +import tarfile +import tempfile +import urllib2 +import which + +from taskgraph.util import docker + +GECKO = os.path.realpath(os.path.join(__file__, '..', '..', '..')) +IMAGE_DIR = os.path.join(GECKO, 'testing', 'docker') +INDEX_URL = 'https://index.taskcluster.net/v1/task/' + docker.INDEX_PREFIX + '.{}.{}.hash.{}' +ARTIFACT_URL = 'https://queue.taskcluster.net/v1/task/{}/artifacts/{}' + + +def load_image_by_name(image_name): + context_path = os.path.join(GECKO, 'testing', 'docker', image_name) + context_hash = docker.generate_context_hash(GECKO, context_path, image_name) + + image_index_url = INDEX_URL.format('mozilla-central', image_name, context_hash) + print("Fetching", image_index_url) + task = json.load(urllib2.urlopen(image_index_url)) + + return load_image_by_task_id(task['taskId']) + + +def load_image_by_task_id(task_id): + # because we need to read this file twice (and one read is not all the way + # through), it is difficult to stream it. So we download to disk and then + # read it back. + filename = 'temp-docker-image.tar' + + artifact_url = ARTIFACT_URL.format(task_id, 'public/image.tar.zst') + print("Downloading", artifact_url) + tempfilename = 'temp-docker-image.tar.zst' + subprocess.check_call(['curl', '-#', '-L', '-o', tempfilename, artifact_url]) + print("Decompressing") + subprocess.check_call(['zstd', '-d', tempfilename, '-o', filename]) + print("Deleting temporary file") + os.unlink(tempfilename) + + print("Determining image name") + tf = tarfile.open(filename) + repositories = json.load(tf.extractfile('repositories')) + name = repositories.keys()[0] + tag = repositories[name].keys()[0] + name = '{}:{}'.format(name, tag) + print("Image name:", name) + + print("Loading image into docker") + try: + subprocess.check_call(['docker', 'load', '-i', filename]) + except subprocess.CalledProcessError: + print("*** `docker load` failed. You may avoid re-downloading that tarball by fixing the") + print("*** problem and running `docker load < {}`.".format(filename)) + raise + + print("Deleting temporary file") + os.unlink(filename) + + print("The requested docker image is now available as", name) + print("Try: docker run -ti --rm {} bash".format(name)) + + +def build_context(name, outputFile): + """Build a context.tar for image with specified name. + """ + if not name: + raise ValueError('must provide a Docker image name') + if not outputFile: + raise ValueError('must provide a outputFile') + + image_dir = os.path.join(IMAGE_DIR, name) + if not os.path.isdir(image_dir): + raise Exception('image directory does not exist: %s' % image_dir) + + docker.create_context_tar(GECKO, image_dir, outputFile, "") + + +def build_image(name): + """Build a Docker image of specified name. + + Output from image building process will be printed to stdout. + """ + if not name: + raise ValueError('must provide a Docker image name') + + image_dir = os.path.join(IMAGE_DIR, name) + if not os.path.isdir(image_dir): + raise Exception('image directory does not exist: %s' % image_dir) + + tag = docker.docker_image(name, default_version='latest') + + docker_bin = which.which('docker') + + # Verify that Docker is working. + try: + subprocess.check_output([docker_bin, '--version']) + except subprocess.CalledProcessError: + raise Exception('Docker server is unresponsive. Run `docker ps` and ' + 'check that Docker is running') + + # We obtain a context archive and build from that. Going through the + # archive creation is important: it normalizes things like file owners + # and mtimes to increase the chances that image generation is + # deterministic. + fd, context_path = tempfile.mkstemp() + os.close(fd) + try: + docker.create_context_tar(GECKO, image_dir, context_path, name) + docker.build_from_context(docker_bin, context_path, name, tag) + finally: + os.unlink(context_path) + + print('Successfully built %s and tagged with %s' % (name, tag)) + + if tag.endswith(':latest'): + print('*' * 50) + print('WARNING: no VERSION file found in image directory.') + print('Image is not suitable for deploying/pushing.') + print('Create an image suitable for deploying/pushing by creating') + print('a VERSION file in the image directory.') + print('*' * 50) diff --git a/taskcluster/taskgraph/files_changed.py b/taskcluster/taskgraph/files_changed.py new file mode 100644 index 000000000..973a62bb8 --- /dev/null +++ b/taskcluster/taskgraph/files_changed.py @@ -0,0 +1,65 @@ +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, You can obtain one at http://mozilla.org/MPL/2.0/. + +""" +Support for optimizing tasks based on the set of files that have changed. +""" + +from __future__ import absolute_import, print_function, unicode_literals + +import logging +import requests +from redo import retry +from mozpack.path import match as mozpackmatch + +logger = logging.getLogger(__name__) +_cache = {} + + +def get_changed_files(repository, revision): + """ + Get the set of files changed in the push headed by the given revision. + Responses are cached, so multiple calls with the same arguments are OK. + """ + key = repository, revision + if key not in _cache: + url = '%s/json-automationrelevance/%s' % (repository.rstrip('/'), revision) + logger.debug("Querying version control for metadata: %s", url) + + def get_automationrelevance(): + response = requests.get(url, timeout=5) + return response.json() + contents = retry(get_automationrelevance, attempts=2, sleeptime=10) + + logger.debug('{} commits influencing task scheduling:' + .format(len(contents['changesets']))) + changed_files = set() + for c in contents['changesets']: + logger.debug(" {cset} {desc}".format( + cset=c['node'][0:12], + desc=c['desc'].splitlines()[0].encode('ascii', 'ignore'))) + changed_files |= set(c['files']) + + _cache[key] = changed_files + return _cache[key] + + +def check(params, file_patterns): + """Determine whether any of the files changed in the indicated push to + https://hg.mozilla.org match any of the given file patterns.""" + repository = params.get('head_repository') + revision = params.get('head_rev') + if not repository or not revision: + logger.warning("Missing `head_repository` or `head_rev` parameters; " + "assuming all files have changed") + return True + + changed_files = get_changed_files(repository, revision) + + for pattern in file_patterns: + for path in changed_files: + if mozpackmatch(path, pattern): + return True + + return False diff --git a/taskcluster/taskgraph/generator.py b/taskcluster/taskgraph/generator.py new file mode 100644 index 000000000..809ed1f5c --- /dev/null +++ b/taskcluster/taskgraph/generator.py @@ -0,0 +1,218 @@ +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, You can obtain one at http://mozilla.org/MPL/2.0/. + +from __future__ import absolute_import, print_function, unicode_literals +import logging +import os +import yaml + +from .graph import Graph +from .taskgraph import TaskGraph +from .optimize import optimize_task_graph +from .util.python_path import find_object + +logger = logging.getLogger(__name__) + + +class Kind(object): + + def __init__(self, name, path, config): + self.name = name + self.path = path + self.config = config + + def _get_impl_class(self): + # load the class defined by implementation + try: + impl = self.config['implementation'] + except KeyError: + raise KeyError("{!r} does not define implementation".format(self.path)) + return find_object(impl) + + def load_tasks(self, parameters, loaded_tasks): + impl_class = self._get_impl_class() + return impl_class.load_tasks(self.name, self.path, self.config, + parameters, loaded_tasks) + + +class TaskGraphGenerator(object): + """ + The central controller for taskgraph. This handles all phases of graph + generation. The task is generated from all of the kinds defined in + subdirectories of the generator's root directory. + + Access to the results of this generation, as well as intermediate values at + various phases of generation, is available via properties. This encourages + the provision of all generation inputs at instance construction time. + """ + + # Task-graph generation is implemented as a Python generator that yields + # each "phase" of generation. This allows some mach subcommands to short- + # circuit generation of the entire graph by never completing the generator. + + def __init__(self, root_dir, parameters, + target_tasks_method): + """ + @param root_dir: root directory, with subdirectories for each kind + @param parameters: parameters for this task-graph generation + @type parameters: dict + @param target_tasks_method: function to determine the target_task_set; + see `./target_tasks.py`. + @type target_tasks_method: function + """ + + self.root_dir = root_dir + self.parameters = parameters + self.target_tasks_method = target_tasks_method + + # this can be set up until the time the target task set is generated; + # it defaults to parameters['target_tasks'] + self._target_tasks = parameters.get('target_tasks') + + # start the generator + self._run = self._run() + self._run_results = {} + + @property + def full_task_set(self): + """ + The full task set: all tasks defined by any kind (a graph without edges) + + @type: TaskGraph + """ + return self._run_until('full_task_set') + + @property + def full_task_graph(self): + """ + The full task graph: the full task set, with edges representing + dependencies. + + @type: TaskGraph + """ + return self._run_until('full_task_graph') + + @property + def target_task_set(self): + """ + The set of targetted tasks (a graph without edges) + + @type: TaskGraph + """ + return self._run_until('target_task_set') + + @property + def target_task_graph(self): + """ + The set of targetted tasks and all of their dependencies + + @type: TaskGraph + """ + return self._run_until('target_task_graph') + + @property + def optimized_task_graph(self): + """ + The set of targetted tasks and all of their dependencies; tasks that + have been optimized out are either omitted or replaced with a Task + instance containing only a task_id. + + @type: TaskGraph + """ + return self._run_until('optimized_task_graph') + + @property + def label_to_taskid(self): + """ + A dictionary mapping task label to assigned taskId. This property helps + in interpreting `optimized_task_graph`. + + @type: dictionary + """ + return self._run_until('label_to_taskid') + + def _load_kinds(self): + for path in os.listdir(self.root_dir): + path = os.path.join(self.root_dir, path) + if not os.path.isdir(path): + continue + kind_name = os.path.basename(path) + + kind_yml = os.path.join(path, 'kind.yml') + if not os.path.exists(kind_yml): + continue + + logger.debug("loading kind `{}` from `{}`".format(kind_name, path)) + with open(kind_yml) as f: + config = yaml.load(f) + + yield Kind(kind_name, path, config) + + def _run(self): + logger.info("Loading kinds") + # put the kinds into a graph and sort topologically so that kinds are loaded + # in post-order + kinds = {kind.name: kind for kind in self._load_kinds()} + edges = set() + for kind in kinds.itervalues(): + for dep in kind.config.get('kind-dependencies', []): + edges.add((kind.name, dep, 'kind-dependency')) + kind_graph = Graph(set(kinds), edges) + + logger.info("Generating full task set") + all_tasks = {} + for kind_name in kind_graph.visit_postorder(): + logger.debug("Loading tasks for kind {}".format(kind_name)) + kind = kinds[kind_name] + new_tasks = kind.load_tasks(self.parameters, list(all_tasks.values())) + for task in new_tasks: + if task.label in all_tasks: + raise Exception("duplicate tasks with label " + task.label) + all_tasks[task.label] = task + logger.info("Generated {} tasks for kind {}".format(len(new_tasks), kind_name)) + full_task_set = TaskGraph(all_tasks, Graph(set(all_tasks), set())) + yield 'full_task_set', full_task_set + + logger.info("Generating full task graph") + edges = set() + for t in full_task_set: + for dep, depname in t.get_dependencies(full_task_set): + edges.add((t.label, dep, depname)) + + full_task_graph = TaskGraph(all_tasks, + Graph(full_task_set.graph.nodes, edges)) + yield 'full_task_graph', full_task_graph + + logger.info("Generating target task set") + target_tasks = set(self.target_tasks_method(full_task_graph, self.parameters)) + target_task_set = TaskGraph( + {l: all_tasks[l] for l in target_tasks}, + Graph(target_tasks, set())) + yield 'target_task_set', target_task_set + + logger.info("Generating target task graph") + target_graph = full_task_graph.graph.transitive_closure(target_tasks) + target_task_graph = TaskGraph( + {l: all_tasks[l] for l in target_graph.nodes}, + target_graph) + yield 'target_task_graph', target_task_graph + + logger.info("Generating optimized task graph") + do_not_optimize = set() + if not self.parameters.get('optimize_target_tasks', True): + do_not_optimize = target_task_set.graph.nodes + optimized_task_graph, label_to_taskid = optimize_task_graph(target_task_graph, + self.parameters, + do_not_optimize) + yield 'label_to_taskid', label_to_taskid + yield 'optimized_task_graph', optimized_task_graph + + def _run_until(self, name): + while name not in self._run_results: + try: + k, v = self._run.next() + except StopIteration: + raise AttributeError("No such run result {}".format(name)) + self._run_results[k] = v + return self._run_results[name] diff --git a/taskcluster/taskgraph/graph.py b/taskcluster/taskgraph/graph.py new file mode 100644 index 000000000..731341c51 --- /dev/null +++ b/taskcluster/taskgraph/graph.py @@ -0,0 +1,117 @@ +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, You can obtain one at http://mozilla.org/MPL/2.0/. + +from __future__ import absolute_import, print_function, unicode_literals + +import collections + + +class Graph(object): + """ + Generic representation of a directed acyclic graph with labeled edges + connecting the nodes. Graph operations are implemented in a functional + manner, so the data structure is immutable. + + It permits at most one edge of a given name between any set of nodes. The + graph is not checked for cycles, and methods may hang or otherwise fail if + given a cyclic graph. + + The `nodes` and `edges` attributes may be accessed in a read-only fashion. + The `nodes` attribute is a set of node names, while `edges` is a set of + `(left, right, name)` tuples representing an edge named `name` going from + node `left` to node `right.. + """ + + def __init__(self, nodes, edges): + """ + Create a graph. Nodes and edges are both as described in the class + documentation. Both values are used by reference, and should not be + modified after building a graph. + """ + assert isinstance(nodes, set) + assert isinstance(edges, set) + self.nodes = nodes + self.edges = edges + + def __eq__(self, other): + return self.nodes == other.nodes and self.edges == other.edges + + def __repr__(self): + return "<Graph nodes={!r} edges={!r}>".format(self.nodes, self.edges) + + def transitive_closure(self, nodes): + """ + Return the transitive closure of <nodes>: the graph containing all + specified nodes as well as any nodes reachable from them, and any + intervening edges. + """ + assert isinstance(nodes, set) + assert nodes <= self.nodes + + # generate a new graph by expanding along edges until reaching a fixed + # point + new_nodes, new_edges = nodes, set() + nodes, edges = set(), set() + while (new_nodes, new_edges) != (nodes, edges): + nodes, edges = new_nodes, new_edges + add_edges = set((left, right, name) + for (left, right, name) in self.edges + if left in nodes) + add_nodes = set(right for (_, right, _) in add_edges) + new_nodes = nodes | add_nodes + new_edges = edges | add_edges + return Graph(new_nodes, new_edges) + + def visit_postorder(self): + """ + Generate a sequence of nodes in postorder, such that every node is + visited *after* any nodes it links to. + + Behavior is undefined (read: it will hang) if the graph contains a + cycle. + """ + queue = collections.deque(sorted(self.nodes)) + links_by_node = self.links_dict() + seen = set() + while queue: + node = queue.popleft() + if node in seen: + continue + links = links_by_node[node] + if all((n in seen) for n in links): + seen.add(node) + yield node + else: + queue.extend(n for n in links if n not in seen) + queue.append(node) + + def links_dict(self): + """ + Return a dictionary mapping each node to a set of the nodes it links to + (omitting edge names) + """ + links = collections.defaultdict(set) + for left, right, _ in self.edges: + links[left].add(right) + return links + + def named_links_dict(self): + """ + Return a two-level dictionary mapping each node to a dictionary mapping + edge names to labels. + """ + links = collections.defaultdict(dict) + for left, right, name in self.edges: + links[left][name] = right + return links + + def reverse_links_dict(self): + """ + Return a dictionary mapping each node to a set of the nodes linking to + it (omitting edge names) + """ + links = collections.defaultdict(set) + for left, right, _ in self.edges: + links[right].add(left) + return links diff --git a/taskcluster/taskgraph/optimize.py b/taskcluster/taskgraph/optimize.py new file mode 100644 index 000000000..120e6807b --- /dev/null +++ b/taskcluster/taskgraph/optimize.py @@ -0,0 +1,156 @@ +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, You can obtain one at http://mozilla.org/MPL/2.0/. + +from __future__ import absolute_import, print_function, unicode_literals +import logging +import re + +from .graph import Graph +from .taskgraph import TaskGraph +from slugid import nice as slugid + +logger = logging.getLogger(__name__) +TASK_REFERENCE_PATTERN = re.compile('<([^>]+)>') + + +def optimize_task_graph(target_task_graph, params, do_not_optimize, existing_tasks=None): + """ + Perform task optimization, without optimizing tasks named in + do_not_optimize. + """ + named_links_dict = target_task_graph.graph.named_links_dict() + label_to_taskid = {} + + # This proceeds in two phases. First, mark all optimized tasks (those + # which will be removed from the graph) as such, including a replacement + # taskId where applicable. Second, generate a new task graph containing + # only the non-optimized tasks, with all task labels resolved to taskIds + # and with task['dependencies'] populated. + annotate_task_graph(target_task_graph=target_task_graph, + params=params, + do_not_optimize=do_not_optimize, + named_links_dict=named_links_dict, + label_to_taskid=label_to_taskid, + existing_tasks=existing_tasks) + return get_subgraph(target_task_graph, named_links_dict, label_to_taskid), label_to_taskid + + +def resolve_task_references(label, task_def, taskid_for_edge_name): + def repl(match): + key = match.group(1) + try: + return taskid_for_edge_name[key] + except KeyError: + # handle escaping '<' + if key == '<': + return key + raise KeyError("task '{}' has no dependency named '{}'".format(label, key)) + + def recurse(val): + if isinstance(val, list): + return [recurse(v) for v in val] + elif isinstance(val, dict): + if val.keys() == ['task-reference']: + return TASK_REFERENCE_PATTERN.sub(repl, val['task-reference']) + else: + return {k: recurse(v) for k, v in val.iteritems()} + else: + return val + return recurse(task_def) + + +def annotate_task_graph(target_task_graph, params, do_not_optimize, + named_links_dict, label_to_taskid, existing_tasks): + """ + Annotate each task in the graph with .optimized (boolean) and .task_id + (possibly None), following the rules for optimization and calling the task + kinds' `optimize_task` method. + + As a side effect, label_to_taskid is updated with labels for all optimized + tasks that are replaced with existing tasks. + """ + + # set .optimized for all tasks, and .task_id for optimized tasks + # with replacements + for label in target_task_graph.graph.visit_postorder(): + task = target_task_graph.tasks[label] + named_task_dependencies = named_links_dict.get(label, {}) + + # check whether any dependencies have been optimized away + dependencies = [target_task_graph.tasks[l] for l in named_task_dependencies.itervalues()] + for t in dependencies: + if t.optimized and not t.task_id: + raise Exception( + "task {} was optimized away, but {} depends on it".format( + t.label, label)) + + # if this task is blacklisted, don't even consider optimizing + replacement_task_id = None + if label in do_not_optimize: + optimized = False + # Let's check whether this task has been created before + elif existing_tasks is not None and label in existing_tasks: + optimized = True + replacement_task_id = existing_tasks[label] + # otherwise, examine the task itself (which may be an expensive operation) + else: + optimized, replacement_task_id = task.optimize(params) + + task.optimized = optimized + task.task_id = replacement_task_id + if replacement_task_id: + label_to_taskid[label] = replacement_task_id + + if optimized: + if replacement_task_id: + logger.debug("optimizing `{}`, replacing with task `{}`" + .format(label, replacement_task_id)) + else: + logger.debug("optimizing `{}` away".format(label)) + # note: any dependent tasks will fail when they see this + else: + if replacement_task_id: + raise Exception("{}: optimize_task returned False with a taskId".format(label)) + + +def get_subgraph(annotated_task_graph, named_links_dict, label_to_taskid): + """ + Return the subgraph of annotated_task_graph consisting only of + non-optimized tasks and edges between them. + + To avoid losing track of taskIds for tasks optimized away, this method + simultaneously substitutes real taskIds for task labels in the graph, and + populates each task definition's `dependencies` key with the appropriate + taskIds. Task references are resolved in the process. + """ + + # resolve labels to taskIds and populate task['dependencies'] + tasks_by_taskid = {} + for label in annotated_task_graph.graph.visit_postorder(): + task = annotated_task_graph.tasks[label] + if task.optimized: + continue + task.task_id = label_to_taskid[label] = slugid() + named_task_dependencies = { + name: label_to_taskid[label] + for name, label in named_links_dict.get(label, {}).iteritems()} + task.task = resolve_task_references(task.label, task.task, named_task_dependencies) + task.task.setdefault('dependencies', []).extend(named_task_dependencies.itervalues()) + tasks_by_taskid[task.task_id] = task + + # resolve edges to taskIds + edges_by_taskid = ( + (label_to_taskid.get(left), label_to_taskid.get(right), name) + for (left, right, name) in annotated_task_graph.graph.edges + ) + # ..and drop edges that are no longer in the task graph + edges_by_taskid = set( + (left, right, name) + for (left, right, name) in edges_by_taskid + if left in tasks_by_taskid and right in tasks_by_taskid + ) + + return TaskGraph( + tasks_by_taskid, + Graph(set(tasks_by_taskid), edges_by_taskid)) diff --git a/taskcluster/taskgraph/parameters.py b/taskcluster/taskgraph/parameters.py new file mode 100644 index 000000000..51b9e77c7 --- /dev/null +++ b/taskcluster/taskgraph/parameters.py @@ -0,0 +1,72 @@ +# -*- coding: utf-8 -*- + +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, You can obtain one at http://mozilla.org/MPL/2.0/. + +from __future__ import absolute_import, print_function, unicode_literals + +import json +import yaml +from mozbuild.util import ReadOnlyDict + +# Please keep this list sorted and in sync with taskcluster/docs/parameters.rst +PARAMETER_NAMES = set([ + 'base_repository', + 'build_date', + 'head_ref', + 'head_repository', + 'head_rev', + 'level', + 'message', + 'moz_build_date', + 'optimize_target_tasks', + 'owner', + 'project', + 'pushdate', + 'pushlog_id', + 'target_tasks_method', + 'triggered_by', +]) + + +class Parameters(ReadOnlyDict): + """An immutable dictionary with nicer KeyError messages on failure""" + def check(self): + names = set(self) + msg = [] + + missing = PARAMETER_NAMES - names + if missing: + msg.append("missing parameters: " + ", ".join(missing)) + + extra = names - PARAMETER_NAMES + if extra: + msg.append("extra parameters: " + ", ".join(extra)) + + if msg: + raise Exception("; ".join(msg)) + + def __getitem__(self, k): + if k not in PARAMETER_NAMES: + raise KeyError("no such parameter {!r}".format(k)) + try: + return super(Parameters, self).__getitem__(k) + except KeyError: + raise KeyError("taskgraph parameter {!r} not found".format(k)) + + +def load_parameters_file(options): + """ + Load parameters from the --parameters option + """ + filename = options['parameters'] + if not filename: + return Parameters() + with open(filename) as f: + if filename.endswith('.yml'): + return Parameters(**yaml.safe_load(f)) + elif filename.endswith('.json'): + return Parameters(**json.load(f)) + else: + raise TypeError("Parameters file `{}` is not JSON or YAML".format(filename)) diff --git a/taskcluster/taskgraph/target_tasks.py b/taskcluster/taskgraph/target_tasks.py new file mode 100644 index 000000000..d2b3f5a7f --- /dev/null +++ b/taskcluster/taskgraph/target_tasks.py @@ -0,0 +1,121 @@ +# -*- coding: utf-8 -*- + +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, You can obtain one at http://mozilla.org/MPL/2.0/. + +from __future__ import absolute_import, print_function, unicode_literals +from taskgraph import try_option_syntax + +INTEGRATION_PROJECTS = set([ + 'mozilla-inbound', + 'autoland', +]) + +RELEASE_PROJECTS = set([ + 'mozilla-central', + 'mozilla-aurora', + 'mozilla-beta', + 'mozilla-release', + 'mozilla-esr52', +]) + +_target_task_methods = {} + + +def _target_task(name): + def wrap(func): + _target_task_methods[name] = func + return func + return wrap + + +def get_method(method): + """Get a target_task_method to pass to a TaskGraphGenerator.""" + return _target_task_methods[method] + + +@_target_task('try_option_syntax') +def target_tasks_try_option_syntax(full_task_graph, parameters): + """Generate a list of target tasks based on try syntax in + parameters['message'] and, for context, the full task graph.""" + options = try_option_syntax.TryOptionSyntax(parameters['message'], full_task_graph) + target_tasks_labels = [t.label for t in full_task_graph.tasks.itervalues() + if options.task_matches(t.attributes)] + + # If the developer wants test jobs to be rebuilt N times we add that value here + if int(options.trigger_tests) > 1: + for l in target_tasks_labels: + task = full_task_graph[l] + if 'unittest_suite' in task.attributes: + task.attributes['task_duplicates'] = options.trigger_tests + + # Add notifications here as well + if options.notifications: + for task in full_task_graph: + owner = parameters.get('owner') + routes = task.task.setdefault('routes', []) + if options.notifications == 'all': + routes.append("notify.email.{}.on-any".format(owner)) + elif options.notifications == 'failure': + routes.append("notify.email.{}.on-failed".format(owner)) + routes.append("notify.email.{}.on-exception".format(owner)) + + return target_tasks_labels + + +@_target_task('default') +def target_tasks_default(full_task_graph, parameters): + """Target the tasks which have indicated they should be run on this project + via the `run_on_projects` attributes.""" + def filter(task): + run_on_projects = set(t.attributes.get('run_on_projects', [])) + if 'all' in run_on_projects: + return True + project = parameters['project'] + if 'integration' in run_on_projects: + if project in INTEGRATION_PROJECTS: + return True + if 'release' in run_on_projects: + if project in RELEASE_PROJECTS: + return True + return project in run_on_projects + return [l for l, t in full_task_graph.tasks.iteritems() if filter(t)] + + +@_target_task('ash_tasks') +def target_tasks_ash(full_task_graph, parameters): + """Target tasks that only run on the ash branch.""" + def filter(task): + platform = task.attributes.get('build_platform') + # only select platforms + if platform not in ('linux64', 'linux64-asan', 'linux64-pgo'): + return False + # and none of this linux64-asan/debug stuff + if platform == 'linux64-asan' and task.attributes['build_type'] == 'debug': + return False + # no non-et10s tests + if task.attributes.get('unittest_suite') or task.attributes.get('talos_siute'): + if not task.attributes.get('e10s'): + return False + # don't upload symbols + if task.attributes['kind'] == 'upload-symbols': + return False + return True + return [l for l, t in full_task_graph.tasks.iteritems() if filter(t)] + + +@_target_task('cedar_tasks') +def target_tasks_cedar(full_task_graph, parameters): + """Target tasks that only run on the cedar branch.""" + def filter(task): + platform = task.attributes.get('build_platform') + # only select platforms + if platform not in ['linux64']: + return False + if task.attributes.get('unittest_suite'): + if not (task.attributes['unittest_suite'].startswith('mochitest') + or 'xpcshell' in task.attributes['unittest_suite']): + return False + return True + return [l for l, t in full_task_graph.tasks.iteritems() if filter(t)] diff --git a/taskcluster/taskgraph/task/__init__.py b/taskcluster/taskgraph/task/__init__.py new file mode 100644 index 000000000..e69de29bb --- /dev/null +++ b/taskcluster/taskgraph/task/__init__.py diff --git a/taskcluster/taskgraph/task/base.py b/taskcluster/taskgraph/task/base.py new file mode 100644 index 000000000..2d9cbf5d9 --- /dev/null +++ b/taskcluster/taskgraph/task/base.py @@ -0,0 +1,108 @@ +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, You can obtain one at http://mozilla.org/MPL/2.0/. + +from __future__ import absolute_import, print_function, unicode_literals + +import abc + + +class Task(object): + """ + Representation of a task in a TaskGraph. Each Task has, at creation: + + - kind: the name of the task kind + - label; the label for this task + - attributes: a dictionary of attributes for this task (used for filtering) + - task: the task definition (JSON-able dictionary) + + And later, as the task-graph processing proceeds: + + - task_id -- TaskCluster taskId under which this task will be created + - optimized -- true if this task need not be performed + + A kind represents a collection of tasks that share common characteristics. + For example, all build jobs. Each instance of a kind is intialized with a + path from which it draws its task configuration. The instance is free to + store as much local state as it needs. + """ + __metaclass__ = abc.ABCMeta + + def __init__(self, kind, label, attributes, task): + self.kind = kind + self.label = label + self.attributes = attributes + self.task = task + + self.task_id = None + self.optimized = False + + self.attributes['kind'] = kind + + def __eq__(self, other): + return self.kind == other.kind and \ + self.label == other.label and \ + self.attributes == other.attributes and \ + self.task == other.task and \ + self.task_id == other.task_id + + @classmethod + @abc.abstractmethod + def load_tasks(cls, kind, path, config, parameters, loaded_tasks): + """ + Load the tasks for a given kind. + + The `kind` is the name of the kind; the configuration for that kind + named this class. + + The `path` is the path to the configuration directory for the kind. This + can be used to load extra data, templates, etc. + + The `parameters` give details on which to base the task generation. + See `taskcluster/docs/parameters.rst` for details. + + At the time this method is called, all kinds on which this kind depends + (that is, specified in the `kind-dependencies` key in `self.config` + have already loaded their tasks, and those tasks are available in + the list `loaded_tasks`. + + The return value is a list of Task instances. + """ + + @abc.abstractmethod + def get_dependencies(self, taskgraph): + """ + Get the set of task labels this task depends on, by querying the full + task set, given as `taskgraph`. + + Returns a list of (task_label, dependency_name) pairs describing the + dependencies. + """ + + def optimize(self, params): + """ + Determine whether this task can be optimized, and if it can, what taskId + it should be replaced with. + + The return value is a tuple `(optimized, taskId)`. If `optimized` is + true, then the task will be optimized (in other words, not included in + the task graph). If the second argument is a taskid, then any + dependencies on this task will isntead depend on that taskId. It is an + error to return no taskId for a task on which other tasks depend. + + The default never optimizes. + """ + return False, None + + @classmethod + def from_json(cls, task_dict): + """ + Given a data structure as produced by taskgraph.to_json, re-construct + the original Task object. This is used to "resume" the task-graph + generation process, for example in Action tasks. + """ + return cls( + kind=task_dict['attributes']['kind'], + label=task_dict['label'], + attributes=task_dict['attributes'], + task=task_dict['task']) diff --git a/taskcluster/taskgraph/task/docker_image.py b/taskcluster/taskgraph/task/docker_image.py new file mode 100644 index 000000000..fd67c4832 --- /dev/null +++ b/taskcluster/taskgraph/task/docker_image.py @@ -0,0 +1,130 @@ +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, You can obtain one at http://mozilla.org/MPL/2.0/. + +from __future__ import absolute_import, print_function, unicode_literals + +import logging +import json +import os +import urllib2 + +from . import base +from taskgraph.util.docker import ( + docker_image, + generate_context_hash, + INDEX_PREFIX, +) +from taskgraph.util.templates import Templates + +logger = logging.getLogger(__name__) +GECKO = os.path.realpath(os.path.join(__file__, '..', '..', '..', '..')) +ARTIFACT_URL = 'https://queue.taskcluster.net/v1/task/{}/artifacts/{}' +INDEX_URL = 'https://index.taskcluster.net/v1/task/{}' + + +class DockerImageTask(base.Task): + + def __init__(self, *args, **kwargs): + self.index_paths = kwargs.pop('index_paths') + super(DockerImageTask, self).__init__(*args, **kwargs) + + def __eq__(self, other): + return super(DockerImageTask, self).__eq__(other) and \ + self.index_paths == other.index_paths + + @classmethod + def load_tasks(cls, kind, path, config, params, loaded_tasks): + parameters = { + 'pushlog_id': params.get('pushlog_id', 0), + 'pushdate': params['moz_build_date'], + 'pushtime': params['moz_build_date'][8:], + 'year': params['moz_build_date'][0:4], + 'month': params['moz_build_date'][4:6], + 'day': params['moz_build_date'][6:8], + 'project': params['project'], + 'docker_image': docker_image, + 'base_repository': params['base_repository'] or params['head_repository'], + 'head_repository': params['head_repository'], + 'head_ref': params['head_ref'] or params['head_rev'], + 'head_rev': params['head_rev'], + 'owner': params['owner'], + 'level': params['level'], + 'source': '{repo}file/{rev}/taskcluster/ci/docker-image/image.yml' + .format(repo=params['head_repository'], rev=params['head_rev']), + 'index_image_prefix': INDEX_PREFIX, + 'artifact_path': 'public/image.tar.zst', + } + + tasks = [] + templates = Templates(path) + for image_name, image_symbol in config['images'].iteritems(): + context_path = os.path.join('testing', 'docker', image_name) + context_hash = generate_context_hash(GECKO, context_path, image_name) + + image_parameters = dict(parameters) + image_parameters['image_name'] = image_name + image_parameters['context_hash'] = context_hash + + image_task = templates.load('image.yml', image_parameters) + attributes = {'image_name': image_name} + + # unique symbol for different docker image + if 'extra' in image_task['task']: + image_task['task']['extra']['treeherder']['symbol'] = image_symbol + + # As an optimization, if the context hash exists for a high level, that image + # task ID will be used. The reasoning behind this is that eventually everything ends + # up on level 3 at some point if most tasks use this as a common image + # for a given context hash, a worker within Taskcluster does not need to contain + # the same image per branch. + index_paths = ['{}.level-{}.{}.hash.{}'.format( + INDEX_PREFIX, level, image_name, context_hash) + for level in range(int(params['level']), 4)] + + tasks.append(cls(kind, 'build-docker-image-' + image_name, + task=image_task['task'], attributes=attributes, + index_paths=index_paths)) + + return tasks + + def get_dependencies(self, taskgraph): + return [] + + def optimize(self, params): + for index_path in self.index_paths: + try: + url = INDEX_URL.format(index_path) + existing_task = json.load(urllib2.urlopen(url)) + # Only return the task ID if the artifact exists for the indexed + # task. Otherwise, continue on looking at each of the branches. Method + # continues trying other branches in case mozilla-central has an expired + # artifact, but 'project' might not. Only return no task ID if all + # branches have been tried + request = urllib2.Request( + ARTIFACT_URL.format(existing_task['taskId'], 'public/image.tar.zst')) + request.get_method = lambda: 'HEAD' + urllib2.urlopen(request) + + # HEAD success on the artifact is enough + return True, existing_task['taskId'] + except urllib2.HTTPError: + pass + + return False, None + + @classmethod + def from_json(cls, task_dict): + # Generating index_paths for optimization + imgMeta = task_dict['task']['extra']['imageMeta'] + image_name = imgMeta['imageName'] + context_hash = imgMeta['contextHash'] + index_paths = ['{}.level-{}.{}.hash.{}'.format( + INDEX_PREFIX, level, image_name, context_hash) + for level in range(int(imgMeta['level']), 4)] + docker_image_task = cls(kind='docker-image', + label=task_dict['label'], + attributes=task_dict['attributes'], + task=task_dict['task'], + index_paths=index_paths) + return docker_image_task diff --git a/taskcluster/taskgraph/task/post_build.py b/taskcluster/taskgraph/task/post_build.py new file mode 100644 index 000000000..09c76c44a --- /dev/null +++ b/taskcluster/taskgraph/task/post_build.py @@ -0,0 +1,53 @@ +# This Source Code Form is subject to the terms of the Mozilla Public License, +# v. 2.0. If a copy of the MPL was not distributed with this file, You can +# obtain one at http://mozilla.org/MPL/2.0/. + +from __future__ import absolute_import, print_function, unicode_literals + +import copy +import logging + +from . import transform +from ..util.yaml import load_yaml + +logger = logging.getLogger(__name__) + + +class PostBuildTask(transform.TransformTask): + """ + A task implementing a post-build job. These depend on jobs and perform + various followup tasks after a build has completed. + + The `only-for-build-platforms` kind configuration, if specified, will limit + the build platforms for which a post-build task will be created. + + The `job-template' kind configuration points to a yaml file which will + be used to create the input to the transforms. It will have added to it + keys `build-label`, the label for the build task, and `build-platform`, its + platform. + """ + + @classmethod + def get_inputs(cls, kind, path, config, params, loaded_tasks): + if config.get('kind-dependencies', []) != ["build"]: + raise Exception("PostBuildTask kinds must depend on builds") + + only_platforms = config.get('only-for-build-platforms') + prototype = load_yaml(path, config.get('job-template')) + + for task in loaded_tasks: + if task.kind != 'build': + continue + + build_platform = task.attributes.get('build_platform') + build_type = task.attributes.get('build_type') + if not build_platform or not build_type: + continue + platform = "{}/{}".format(build_platform, build_type) + if only_platforms and platform not in only_platforms: + continue + + post_task = copy.deepcopy(prototype) + post_task['build-label'] = task.label + post_task['build-platform'] = platform + yield post_task diff --git a/taskcluster/taskgraph/task/signing.py b/taskcluster/taskgraph/task/signing.py new file mode 100644 index 000000000..a2a9ae3d6 --- /dev/null +++ b/taskcluster/taskgraph/task/signing.py @@ -0,0 +1,64 @@ +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, You can obtain one at http://mozilla.org/MPL/2.0/. + +from __future__ import absolute_import, print_function, unicode_literals + +import logging +import os + +from . import base +from taskgraph.util.templates import Templates + + +logger = logging.getLogger(__name__) +GECKO = os.path.realpath(os.path.join(__file__, '..', '..', '..', '..')) +ARTIFACT_URL = 'https://queue.taskcluster.net/v1/task/{}/artifacts/{}' +INDEX_URL = 'https://index.taskcluster.net/v1/task/{}' + + +class SigningTask(base.Task): + + def __init__(self, kind, name, task, attributes): + self.unsigned_artifact_label = task['unsigned-task']['label'] + super(SigningTask, self).__init__(kind, name, task=task['task'], + attributes=attributes) + + @classmethod + def load_tasks(cls, kind, path, config, params, loaded_tasks): + root = os.path.abspath(path) + + tasks = [] + for filename in config.get('jobs-from', []): + templates = Templates(root) + jobs = templates.load(filename, {}) + + for name, job in jobs.iteritems(): + for artifact in job['unsigned-task']['artifacts']: + url = ARTIFACT_URL.format('<{}>'.format('unsigned-artifact'), artifact) + job['task']['payload']['unsignedArtifacts'].append({ + 'task-reference': url + }) + attributes = job.setdefault('attributes', {}) + attributes.update({'kind': 'signing'}) + tasks.append(cls(kind, name, job, attributes=attributes)) + + return tasks + + def get_dependencies(self, taskgraph): + return [(self.unsigned_artifact_label, 'unsigned-artifact')] + + def optimize(self, params): + return False, None + + @classmethod + def from_json(cls, task_dict): + unsigned_task_label = task_dict['dependencies']['unsigned-artifact'] + task_dict['unsigned-task'] = { + 'label': unsigned_task_label + } + signing_task = cls(kind='build-signing', + name=task_dict['label'], + attributes=task_dict['attributes'], + task=task_dict) + return signing_task diff --git a/taskcluster/taskgraph/task/test.py b/taskcluster/taskgraph/task/test.py new file mode 100644 index 000000000..928f32a5a --- /dev/null +++ b/taskcluster/taskgraph/task/test.py @@ -0,0 +1,112 @@ +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, You can obtain one at http://mozilla.org/MPL/2.0/. + +from __future__ import absolute_import, print_function, unicode_literals + +import copy +import logging + +from . import transform +from ..util.yaml import load_yaml + +logger = logging.getLogger(__name__) + + +class TestTask(transform.TransformTask): + """ + A task implementing a Gecko test. + """ + + @classmethod + def get_inputs(cls, kind, path, config, params, loaded_tasks): + + # the kind on which this one depends + if len(config.get('kind-dependencies', [])) != 1: + raise Exception("TestTask kinds must have exactly one item in kind-dependencies") + dep_kind = config['kind-dependencies'][0] + + # get build tasks, keyed by build platform + builds_by_platform = cls.get_builds_by_platform(dep_kind, loaded_tasks) + + # get the test platforms for those build tasks + test_platforms_cfg = load_yaml(path, 'test-platforms.yml') + test_platforms = cls.get_test_platforms(test_platforms_cfg, builds_by_platform) + + # expand the test sets for each of those platforms + test_sets_cfg = load_yaml(path, 'test-sets.yml') + test_platforms = cls.expand_tests(test_sets_cfg, test_platforms) + + # load the test descriptions + test_descriptions = load_yaml(path, 'tests.yml') + + # generate all tests for all test platforms + for test_platform_name, test_platform in test_platforms.iteritems(): + for test_name in test_platform['test-names']: + test = copy.deepcopy(test_descriptions[test_name]) + test['build-platform'] = test_platform['build-platform'] + test['test-platform'] = test_platform_name + test['build-label'] = test_platform['build-label'] + test['test-name'] = test_name + + logger.debug("Generating tasks for {} test {} on platform {}".format( + kind, test_name, test['test-platform'])) + yield test + + @classmethod + def get_builds_by_platform(cls, dep_kind, loaded_tasks): + """Find the build tasks on which tests will depend, keyed by + platform/type. Returns a dictionary mapping build platform to task + label.""" + builds_by_platform = {} + for task in loaded_tasks: + if task.kind != dep_kind: + continue + + build_platform = task.attributes.get('build_platform') + build_type = task.attributes.get('build_type') + if not build_platform or not build_type: + continue + platform = "{}/{}".format(build_platform, build_type) + if platform in builds_by_platform: + raise Exception("multiple build jobs for " + platform) + builds_by_platform[platform] = task.label + return builds_by_platform + + @classmethod + def get_test_platforms(cls, test_platforms_cfg, builds_by_platform): + """Get the test platforms for which test tasks should be generated, + based on the available build platforms. Returns a dictionary mapping + test platform to {test-set, build-platform, build-label}.""" + test_platforms = {} + for test_platform, cfg in test_platforms_cfg.iteritems(): + build_platform = cfg['build-platform'] + if build_platform not in builds_by_platform: + logger.warning( + "No build task with platform {}; ignoring test platform {}".format( + build_platform, test_platform)) + continue + test_platforms[test_platform] = { + 'test-set': cfg['test-set'], + 'build-platform': build_platform, + 'build-label': builds_by_platform[build_platform], + } + return test_platforms + + @classmethod + def expand_tests(cls, test_sets_cfg, test_platforms): + """Expand the test sets in `test_platforms` out to sets of test names. + Returns a dictionary like `get_test_platforms`, with an additional + `test-names` key for each test platform, containing a set of test + names.""" + rv = {} + for test_platform, cfg in test_platforms.iteritems(): + test_set = cfg['test-set'] + if test_set not in test_sets_cfg: + raise Exception( + "Test set '{}' for test platform {} is not defined".format( + test_set, test_platform)) + test_names = test_sets_cfg[test_set] + rv[test_platform] = cfg.copy() + rv[test_platform]['test-names'] = test_names + return rv diff --git a/taskcluster/taskgraph/task/transform.py b/taskcluster/taskgraph/task/transform.py new file mode 100644 index 000000000..8183254a0 --- /dev/null +++ b/taskcluster/taskgraph/task/transform.py @@ -0,0 +1,109 @@ +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, You can obtain one at http://mozilla.org/MPL/2.0/. + +from __future__ import absolute_import, print_function, unicode_literals + +import logging +import itertools + +from . import base +from .. import files_changed +from ..util.python_path import find_object +from ..util.templates import merge +from ..util.yaml import load_yaml +from ..util.seta import is_low_value_task + +from ..transforms.base import TransformSequence, TransformConfig + +logger = logging.getLogger(__name__) + + +class TransformTask(base.Task): + """ + Tasks of this class are generated by applying transformations to a sequence + of input entities. By default, it gets those inputs from YAML data in the + kind directory, but subclasses may override `get_inputs` to produce them in + some other way. + """ + + @classmethod + def get_inputs(cls, kind, path, config, params, loaded_tasks): + """ + Get the input elements that will be transformed into tasks. The + elements themselves are free-form, and become the input to the first + transform. + + By default, this reads jobs from the `jobs` key, or from yaml files + named by `jobs-from`. The entities are read from mappings, and the + keys to those mappings are added in the `name` key of each entity. + + If there is a `job-defaults` config, then every job is merged with it. + This provides a simple way to set default values for all jobs of a + kind. More complex defaults should be implemented with custom + transforms. + + This method can be overridden in subclasses that need to perform more + complex calculations to generate the list of inputs. + """ + def jobs(): + defaults = config.get('job-defaults') + jobs = config.get('jobs', {}).iteritems() + jobs_from = itertools.chain.from_iterable( + load_yaml(path, filename).iteritems() + for filename in config.get('jobs-from', {})) + for name, job in itertools.chain(jobs, jobs_from): + if defaults: + job = merge(defaults, job) + yield name, job + + for name, job in jobs(): + job['name'] = name + logger.debug("Generating tasks for {} {}".format(kind, name)) + yield job + + @classmethod + def load_tasks(cls, kind, path, config, params, loaded_tasks): + inputs = cls.get_inputs(kind, path, config, params, loaded_tasks) + + transforms = TransformSequence() + for xform_path in config['transforms']: + transform = find_object(xform_path) + transforms.add(transform) + + # perform the transformations + trans_config = TransformConfig(kind, path, config, params) + tasks = [cls(kind, t) for t in transforms(trans_config, inputs)] + return tasks + + def __init__(self, kind, task): + self.dependencies = task['dependencies'] + self.when = task['when'] + super(TransformTask, self).__init__(kind, task['label'], + task['attributes'], task['task']) + + def get_dependencies(self, taskgraph): + return [(label, name) for name, label in self.dependencies.items()] + + def optimize(self, params): + if 'files-changed' in self.when: + changed = files_changed.check( + params, self.when['files-changed']) + if not changed: + logger.debug('no files found matching a pattern in `when.files-changed` for ' + + self.label) + return True, None + + # we would like to return 'False, None' while it's high_value_task + # and we wouldn't optimize it. Otherwise, it will return 'True, None' + if is_low_value_task(self.label, params.get('project')): + # Always optimize away low-value tasks + return True, None + else: + return False, None + + @classmethod + def from_json(cls, task_dict): + # when reading back from JSON, we lose the "when" information + task_dict['when'] = {} + return cls(task_dict['attributes']['kind'], task_dict) diff --git a/taskcluster/taskgraph/taskgraph.py b/taskcluster/taskgraph/taskgraph.py new file mode 100644 index 000000000..7736745ef --- /dev/null +++ b/taskcluster/taskgraph/taskgraph.py @@ -0,0 +1,82 @@ +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, You can obtain one at http://mozilla.org/MPL/2.0/. + +from __future__ import absolute_import, print_function, unicode_literals + +import os + +from .graph import Graph +from .util.python_path import find_object + +TASKCLUSTER_QUEUE_URL = "https://queue.taskcluster.net/v1/task/" +GECKO = os.path.realpath(os.path.join(__file__, '..', '..', '..')) + + +class TaskGraph(object): + """ + Representation of a task graph. + + A task graph is a combination of a Graph and a dictionary of tasks indexed + by label. TaskGraph instances should be treated as immutable. + """ + + def __init__(self, tasks, graph): + assert set(tasks) == graph.nodes + self.tasks = tasks + self.graph = graph + + def to_json(self): + "Return a JSON-able object representing the task graph, as documented" + named_links_dict = self.graph.named_links_dict() + # this dictionary may be keyed by label or by taskid, so let's just call it 'key' + tasks = {} + for key in self.graph.visit_postorder(): + task = self.tasks[key] + implementation = task.__class__.__module__ + ":" + task.__class__.__name__ + task_json = { + 'label': task.label, + 'attributes': task.attributes, + 'dependencies': named_links_dict.get(key, {}), + 'task': task.task, + 'kind_implementation': implementation + } + if task.task_id: + task_json['task_id'] = task.task_id + tasks[key] = task_json + return tasks + + def __getitem__(self, label): + "Get a task by label" + return self.tasks[label] + + def __iter__(self): + "Iterate over tasks in undefined order" + return self.tasks.itervalues() + + def __repr__(self): + return "<TaskGraph graph={!r} tasks={!r}>".format(self.graph, self.tasks) + + def __eq__(self, other): + return self.tasks == other.tasks and self.graph == other.graph + + @classmethod + def from_json(cls, tasks_dict): + """ + This code is used to generate the a TaskGraph using a dictionary + which is representative of the TaskGraph. + """ + tasks = {} + edges = set() + for key, value in tasks_dict.iteritems(): + # We get the implementation from JSON + implementation = value['kind_implementation'] + # Loading the module and creating a Task from a dictionary + task_kind = find_object(implementation) + tasks[key] = task_kind.from_json(value) + if 'task_id' in value: + tasks[key].task_id = value['task_id'] + for depname, dep in value['dependencies'].iteritems(): + edges.add((key, dep, depname)) + task_graph = cls(tasks, Graph(set(tasks), edges)) + return tasks, task_graph diff --git a/taskcluster/taskgraph/test/__init__.py b/taskcluster/taskgraph/test/__init__.py new file mode 100644 index 000000000..e69de29bb --- /dev/null +++ b/taskcluster/taskgraph/test/__init__.py diff --git a/taskcluster/taskgraph/test/automationrelevance.json b/taskcluster/taskgraph/test/automationrelevance.json new file mode 100644 index 000000000..8adfc446d --- /dev/null +++ b/taskcluster/taskgraph/test/automationrelevance.json @@ -0,0 +1,425 @@ +{ + "changesets": [ + { + "author": "James Long <longster@gmail.com>", + "backsoutnodes": [], + "bugs": [ + { + "no": "1300866", + "url": "https://bugzilla.mozilla.org/show_bug.cgi?id=1300866" + } + ], + "date": [ + 1473196655.0, + 14400 + ], + "desc": "Bug 1300866 - expose devtools require to new debugger r=jlast,bgrins", + "extra": { + "branch": "default" + }, + "files": [ + "devtools/client/debugger/new/index.html" + ], + "node": "ae2144aa4356b65c2f8c0de8c9082dcb7e330e24", + "parents": [ + "37c9349b4e8167a61b08b7e119c21ea177b98942" + ], + "perfherderurl": "https://treeherder.mozilla.org/perf.html#/compare?originalProject=mozilla-central&originalRevision=a14f88a9af7a59e677478694bafd9375ac53683e&newProject=mozilla-central&newRevision=ae2144aa4356b65c2f8c0de8c9082dcb7e330e24", + "pushdate": [ + 1473261248, + 0 + ], + "pushhead": "a14f88a9af7a59e677478694bafd9375ac53683e", + "pushid": 30664, + "pushnodes": [ + "ae2144aa4356b65c2f8c0de8c9082dcb7e330e24", + "73a6a267a50a0e1c41e689b265ad3eebe43d7ac6", + "16a1a91f9269ab95dd83eb29dc5d0227665f7d94", + "99c542fa43a72ee863c813b5624048d1b443549b", + "a6b6a93eb41a05e310a11f0172f01ba9b21d3eac", + "541c9086c0f27fba60beecc9bc94543103895c86", + "041a925171e431bf51fb50193ab19d156088c89a", + "a14f88a9af7a59e677478694bafd9375ac53683e" + ], + "pushuser": "cbook@mozilla.com", + "rev": 312890, + "reviewers": [ + { + "name": "jlast", + "revset": "reviewer(jlast)" + }, + { + "name": "bgrins", + "revset": "reviewer(bgrins)" + } + ], + "treeherderrepo": "mozilla-central", + "treeherderrepourl": "https://treeherder.mozilla.org/#/jobs?repo=mozilla-central" + }, + { + "author": "Wes Kocher <wkocher@mozilla.com>", + "backsoutnodes": [], + "bugs": [], + "date": [ + 1473208638.0, + 25200 + ], + "desc": "Merge m-c to fx-team, a=merge", + "extra": { + "branch": "default" + }, + "files": [ + "taskcluster/scripts/builder/build-l10n.sh" + ], + "node": "73a6a267a50a0e1c41e689b265ad3eebe43d7ac6", + "parents": [ + "ae2144aa4356b65c2f8c0de8c9082dcb7e330e24", + "91c2b9d5c1354ca79e5b174591dbb03b32b15bbf" + ], + "perfherderurl": "https://treeherder.mozilla.org/perf.html#/compare?originalProject=mozilla-central&originalRevision=a14f88a9af7a59e677478694bafd9375ac53683e&newProject=mozilla-central&newRevision=ae2144aa4356b65c2f8c0de8c9082dcb7e330e24", + "pushdate": [ + 1473261248, + 0 + ], + "pushhead": "a14f88a9af7a59e677478694bafd9375ac53683e", + "pushid": 30664, + "pushnodes": [ + "ae2144aa4356b65c2f8c0de8c9082dcb7e330e24", + "73a6a267a50a0e1c41e689b265ad3eebe43d7ac6", + "16a1a91f9269ab95dd83eb29dc5d0227665f7d94", + "99c542fa43a72ee863c813b5624048d1b443549b", + "a6b6a93eb41a05e310a11f0172f01ba9b21d3eac", + "541c9086c0f27fba60beecc9bc94543103895c86", + "041a925171e431bf51fb50193ab19d156088c89a", + "a14f88a9af7a59e677478694bafd9375ac53683e" + ], + "pushuser": "cbook@mozilla.com", + "rev": 312891, + "reviewers": [ + { + "name": "merge", + "revset": "reviewer(merge)" + } + ], + "treeherderrepo": "mozilla-central", + "treeherderrepourl": "https://treeherder.mozilla.org/#/jobs?repo=mozilla-central" + }, + { + "author": "Towkir Ahmed <towkir17@gmail.com>", + "backsoutnodes": [], + "bugs": [ + { + "no": "1296648", + "url": "https://bugzilla.mozilla.org/show_bug.cgi?id=1296648" + } + ], + "date": [ + 1472957580.0, + 14400 + ], + "desc": "Bug 1296648 - Fix direction of .ruleview-expander.theme-twisty in RTL locales. r=ntim", + "extra": { + "branch": "default" + }, + "files": [ + "devtools/client/themes/rules.css" + ], + "node": "16a1a91f9269ab95dd83eb29dc5d0227665f7d94", + "parents": [ + "73a6a267a50a0e1c41e689b265ad3eebe43d7ac6" + ], + "perfherderurl": "https://treeherder.mozilla.org/perf.html#/compare?originalProject=mozilla-central&originalRevision=a14f88a9af7a59e677478694bafd9375ac53683e&newProject=mozilla-central&newRevision=ae2144aa4356b65c2f8c0de8c9082dcb7e330e24", + "pushdate": [ + 1473261248, + 0 + ], + "pushhead": "a14f88a9af7a59e677478694bafd9375ac53683e", + "pushid": 30664, + "pushnodes": [ + "ae2144aa4356b65c2f8c0de8c9082dcb7e330e24", + "73a6a267a50a0e1c41e689b265ad3eebe43d7ac6", + "16a1a91f9269ab95dd83eb29dc5d0227665f7d94", + "99c542fa43a72ee863c813b5624048d1b443549b", + "a6b6a93eb41a05e310a11f0172f01ba9b21d3eac", + "541c9086c0f27fba60beecc9bc94543103895c86", + "041a925171e431bf51fb50193ab19d156088c89a", + "a14f88a9af7a59e677478694bafd9375ac53683e" + ], + "pushuser": "cbook@mozilla.com", + "rev": 312892, + "reviewers": [ + { + "name": "ntim", + "revset": "reviewer(ntim)" + } + ], + "treeherderrepo": "mozilla-central", + "treeherderrepourl": "https://treeherder.mozilla.org/#/jobs?repo=mozilla-central" + }, + { + "author": "Oriol <oriol-bugzilla@hotmail.com>", + "backsoutnodes": [], + "bugs": [ + { + "no": "1300336", + "url": "https://bugzilla.mozilla.org/show_bug.cgi?id=1300336" + } + ], + "date": [ + 1472921160.0, + 14400 + ], + "desc": "Bug 1300336 - Allow pseudo-arrays to have a length property. r=fitzgen", + "extra": { + "branch": "default" + }, + "files": [ + "devtools/client/webconsole/test/browser_webconsole_output_06.js", + "devtools/server/actors/object.js" + ], + "node": "99c542fa43a72ee863c813b5624048d1b443549b", + "parents": [ + "16a1a91f9269ab95dd83eb29dc5d0227665f7d94" + ], + "perfherderurl": "https://treeherder.mozilla.org/perf.html#/compare?originalProject=mozilla-central&originalRevision=a14f88a9af7a59e677478694bafd9375ac53683e&newProject=mozilla-central&newRevision=ae2144aa4356b65c2f8c0de8c9082dcb7e330e24", + "pushdate": [ + 1473261248, + 0 + ], + "pushhead": "a14f88a9af7a59e677478694bafd9375ac53683e", + "pushid": 30664, + "pushnodes": [ + "ae2144aa4356b65c2f8c0de8c9082dcb7e330e24", + "73a6a267a50a0e1c41e689b265ad3eebe43d7ac6", + "16a1a91f9269ab95dd83eb29dc5d0227665f7d94", + "99c542fa43a72ee863c813b5624048d1b443549b", + "a6b6a93eb41a05e310a11f0172f01ba9b21d3eac", + "541c9086c0f27fba60beecc9bc94543103895c86", + "041a925171e431bf51fb50193ab19d156088c89a", + "a14f88a9af7a59e677478694bafd9375ac53683e" + ], + "pushuser": "cbook@mozilla.com", + "rev": 312893, + "reviewers": [ + { + "name": "fitzgen", + "revset": "reviewer(fitzgen)" + } + ], + "treeherderrepo": "mozilla-central", + "treeherderrepourl": "https://treeherder.mozilla.org/#/jobs?repo=mozilla-central" + }, + { + "author": "Ruturaj Vartak <ruturaj@gmail.com>", + "backsoutnodes": [], + "bugs": [ + { + "no": "1295010", + "url": "https://bugzilla.mozilla.org/show_bug.cgi?id=1295010" + } + ], + "date": [ + 1472854020.0, + -7200 + ], + "desc": "Bug 1295010 - Don't move the eyedropper to the out of browser window by keyboard navigation. r=pbro\n\nMozReview-Commit-ID: vBwmSxVNXK", + "extra": { + "amend_source": "6885024ef00cfa33d73c59dc03c48ebcda9ccbdd", + "branch": "default", + "histedit_source": "c43167f0a7cbe9f4c733b15da726e5150a9529ba", + "rebase_source": "b74df421630fc46dab6b6cc026bf3e0ae6b4a651" + }, + "files": [ + "devtools/client/inspector/test/browser_inspector_highlighter-eyedropper-events.js", + "devtools/client/inspector/test/head.js", + "devtools/server/actors/highlighters/eye-dropper.js" + ], + "node": "a6b6a93eb41a05e310a11f0172f01ba9b21d3eac", + "parents": [ + "99c542fa43a72ee863c813b5624048d1b443549b" + ], + "perfherderurl": "https://treeherder.mozilla.org/perf.html#/compare?originalProject=mozilla-central&originalRevision=a14f88a9af7a59e677478694bafd9375ac53683e&newProject=mozilla-central&newRevision=ae2144aa4356b65c2f8c0de8c9082dcb7e330e24", + "pushdate": [ + 1473261248, + 0 + ], + "pushhead": "a14f88a9af7a59e677478694bafd9375ac53683e", + "pushid": 30664, + "pushnodes": [ + "ae2144aa4356b65c2f8c0de8c9082dcb7e330e24", + "73a6a267a50a0e1c41e689b265ad3eebe43d7ac6", + "16a1a91f9269ab95dd83eb29dc5d0227665f7d94", + "99c542fa43a72ee863c813b5624048d1b443549b", + "a6b6a93eb41a05e310a11f0172f01ba9b21d3eac", + "541c9086c0f27fba60beecc9bc94543103895c86", + "041a925171e431bf51fb50193ab19d156088c89a", + "a14f88a9af7a59e677478694bafd9375ac53683e" + ], + "pushuser": "cbook@mozilla.com", + "rev": 312894, + "reviewers": [ + { + "name": "pbro", + "revset": "reviewer(pbro)" + } + ], + "treeherderrepo": "mozilla-central", + "treeherderrepourl": "https://treeherder.mozilla.org/#/jobs?repo=mozilla-central" + }, + { + "author": "Matteo Ferretti <mferretti@mozilla.com>", + "backsoutnodes": [], + "bugs": [ + { + "no": "1299154", + "url": "https://bugzilla.mozilla.org/show_bug.cgi?id=1299154" + } + ], + "date": [ + 1472629906.0, + -7200 + ], + "desc": "Bug 1299154 - added Set/GetOverrideDPPX to restorefromHistory; r=mstange\n\nMozReview-Commit-ID: AsyAcG3Igbn\n", + "extra": { + "branch": "default", + "committer": "Matteo Ferretti <mferretti@mozilla.com> 1473236511 -7200" + }, + "files": [ + "docshell/base/nsDocShell.cpp", + "dom/tests/mochitest/general/test_contentViewer_overrideDPPX.html" + ], + "node": "541c9086c0f27fba60beecc9bc94543103895c86", + "parents": [ + "a6b6a93eb41a05e310a11f0172f01ba9b21d3eac" + ], + "perfherderurl": "https://treeherder.mozilla.org/perf.html#/compare?originalProject=mozilla-central&originalRevision=a14f88a9af7a59e677478694bafd9375ac53683e&newProject=mozilla-central&newRevision=ae2144aa4356b65c2f8c0de8c9082dcb7e330e24", + "pushdate": [ + 1473261248, + 0 + ], + "pushhead": "a14f88a9af7a59e677478694bafd9375ac53683e", + "pushid": 30664, + "pushnodes": [ + "ae2144aa4356b65c2f8c0de8c9082dcb7e330e24", + "73a6a267a50a0e1c41e689b265ad3eebe43d7ac6", + "16a1a91f9269ab95dd83eb29dc5d0227665f7d94", + "99c542fa43a72ee863c813b5624048d1b443549b", + "a6b6a93eb41a05e310a11f0172f01ba9b21d3eac", + "541c9086c0f27fba60beecc9bc94543103895c86", + "041a925171e431bf51fb50193ab19d156088c89a", + "a14f88a9af7a59e677478694bafd9375ac53683e" + ], + "pushuser": "cbook@mozilla.com", + "rev": 312895, + "reviewers": [ + { + "name": "mstange", + "revset": "reviewer(mstange)" + } + ], + "treeherderrepo": "mozilla-central", + "treeherderrepourl": "https://treeherder.mozilla.org/#/jobs?repo=mozilla-central" + }, + { + "author": "Patrick Brosset <pbrosset@mozilla.com>", + "backsoutnodes": [], + "bugs": [ + { + "no": "1295010", + "url": "https://bugzilla.mozilla.org/show_bug.cgi?id=1295010" + } + ], + "date": [ + 1473239449.0, + -7200 + ], + "desc": "Bug 1295010 - Removed testActor from highlighterHelper in inspector tests; r=me\n\nMozReview-Commit-ID: GMksl81iGcp", + "extra": { + "branch": "default" + }, + "files": [ + "devtools/client/inspector/test/browser_inspector_highlighter-eyedropper-events.js", + "devtools/client/inspector/test/head.js" + ], + "node": "041a925171e431bf51fb50193ab19d156088c89a", + "parents": [ + "541c9086c0f27fba60beecc9bc94543103895c86" + ], + "perfherderurl": "https://treeherder.mozilla.org/perf.html#/compare?originalProject=mozilla-central&originalRevision=a14f88a9af7a59e677478694bafd9375ac53683e&newProject=mozilla-central&newRevision=ae2144aa4356b65c2f8c0de8c9082dcb7e330e24", + "pushdate": [ + 1473261248, + 0 + ], + "pushhead": "a14f88a9af7a59e677478694bafd9375ac53683e", + "pushid": 30664, + "pushnodes": [ + "ae2144aa4356b65c2f8c0de8c9082dcb7e330e24", + "73a6a267a50a0e1c41e689b265ad3eebe43d7ac6", + "16a1a91f9269ab95dd83eb29dc5d0227665f7d94", + "99c542fa43a72ee863c813b5624048d1b443549b", + "a6b6a93eb41a05e310a11f0172f01ba9b21d3eac", + "541c9086c0f27fba60beecc9bc94543103895c86", + "041a925171e431bf51fb50193ab19d156088c89a", + "a14f88a9af7a59e677478694bafd9375ac53683e" + ], + "pushuser": "cbook@mozilla.com", + "rev": 312896, + "reviewers": [ + { + "name": "me", + "revset": "reviewer(me)" + } + ], + "treeherderrepo": "mozilla-central", + "treeherderrepourl": "https://treeherder.mozilla.org/#/jobs?repo=mozilla-central" + }, + { + "author": "Carsten \"Tomcat\" Book <cbook@mozilla.com>", + "backsoutnodes": [], + "bugs": [], + "date": [ + 1473261233.0, + -7200 + ], + "desc": "merge fx-team to mozilla-central a=merge", + "extra": { + "branch": "default" + }, + "files": [], + "node": "a14f88a9af7a59e677478694bafd9375ac53683e", + "parents": [ + "3d0b41fdd93bd8233745eadb4e0358e385bf2cb9", + "041a925171e431bf51fb50193ab19d156088c89a" + ], + "perfherderurl": "https://treeherder.mozilla.org/perf.html#/compare?originalProject=mozilla-central&originalRevision=a14f88a9af7a59e677478694bafd9375ac53683e&newProject=mozilla-central&newRevision=ae2144aa4356b65c2f8c0de8c9082dcb7e330e24", + "pushdate": [ + 1473261248, + 0 + ], + "pushhead": "a14f88a9af7a59e677478694bafd9375ac53683e", + "pushid": 30664, + "pushnodes": [ + "ae2144aa4356b65c2f8c0de8c9082dcb7e330e24", + "73a6a267a50a0e1c41e689b265ad3eebe43d7ac6", + "16a1a91f9269ab95dd83eb29dc5d0227665f7d94", + "99c542fa43a72ee863c813b5624048d1b443549b", + "a6b6a93eb41a05e310a11f0172f01ba9b21d3eac", + "541c9086c0f27fba60beecc9bc94543103895c86", + "041a925171e431bf51fb50193ab19d156088c89a", + "a14f88a9af7a59e677478694bafd9375ac53683e" + ], + "pushuser": "cbook@mozilla.com", + "rev": 312897, + "reviewers": [ + { + "name": "merge", + "revset": "reviewer(merge)" + } + ], + "treeherderrepo": "mozilla-central", + "treeherderrepourl": "https://treeherder.mozilla.org/#/jobs?repo=mozilla-central" + } + ], + "visible": true +} + diff --git a/taskcluster/taskgraph/test/test_create.py b/taskcluster/taskgraph/test/test_create.py new file mode 100644 index 000000000..b8da3aec0 --- /dev/null +++ b/taskcluster/taskgraph/test/test_create.py @@ -0,0 +1,76 @@ +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, You can obtain one at http://mozilla.org/MPL/2.0/. + +from __future__ import absolute_import, print_function, unicode_literals + +import unittest +import os + +from .. import create +from ..graph import Graph +from ..taskgraph import TaskGraph +from .util import TestTask + +from mozunit import main + + +class TestCreate(unittest.TestCase): + + def setUp(self): + self.old_task_id = os.environ.get('TASK_ID') + if 'TASK_ID' in os.environ: + del os.environ['TASK_ID'] + self.created_tasks = {} + self.old_create_task = create._create_task + create._create_task = self.fake_create_task + + def tearDown(self): + create._create_task = self.old_create_task + if self.old_task_id: + os.environ['TASK_ID'] = self.old_task_id + elif 'TASK_ID' in os.environ: + del os.environ['TASK_ID'] + + def fake_create_task(self, session, task_id, label, task_def): + self.created_tasks[task_id] = task_def + + def test_create_tasks(self): + tasks = { + 'tid-a': TestTask(label='a', task={'payload': 'hello world'}), + 'tid-b': TestTask(label='b', task={'payload': 'hello world'}), + } + label_to_taskid = {'a': 'tid-a', 'b': 'tid-b'} + graph = Graph(nodes={'tid-a', 'tid-b'}, edges={('tid-a', 'tid-b', 'edge')}) + taskgraph = TaskGraph(tasks, graph) + + create.create_tasks(taskgraph, label_to_taskid, {'level': '4'}) + + for tid, task in self.created_tasks.iteritems(): + self.assertEqual(task['payload'], 'hello world') + self.assertEqual(task['schedulerId'], 'gecko-level-4') + # make sure the dependencies exist, at least + for depid in task.get('dependencies', []): + if depid is 'decisiontask': + # Don't look for decisiontask here + continue + self.assertIn(depid, self.created_tasks) + + def test_create_task_without_dependencies(self): + "a task with no dependencies depends on the decision task" + os.environ['TASK_ID'] = 'decisiontask' + tasks = { + 'tid-a': TestTask(label='a', task={'payload': 'hello world'}), + } + label_to_taskid = {'a': 'tid-a'} + graph = Graph(nodes={'tid-a'}, edges=set()) + taskgraph = TaskGraph(tasks, graph) + + create.create_tasks(taskgraph, label_to_taskid, {'level': '4'}) + + for tid, task in self.created_tasks.iteritems(): + self.assertEqual(task.get('dependencies'), [os.environ['TASK_ID']]) + + +if __name__ == '__main__': + main() diff --git a/taskcluster/taskgraph/test/test_decision.py b/taskcluster/taskgraph/test/test_decision.py new file mode 100644 index 000000000..364965206 --- /dev/null +++ b/taskcluster/taskgraph/test/test_decision.py @@ -0,0 +1,78 @@ +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, You can obtain one at http://mozilla.org/MPL/2.0/. + +from __future__ import absolute_import, print_function, unicode_literals + +import os +import json +import yaml +import shutil +import unittest +import tempfile + +from .. import decision +from ..graph import Graph +from ..taskgraph import TaskGraph +from .util import TestTask +from mozunit import main + + +class TestDecision(unittest.TestCase): + + def test_taskgraph_to_json(self): + tasks = { + 'a': TestTask(label='a', attributes={'attr': 'a-task'}), + 'b': TestTask(label='b', task={'task': 'def'}), + } + graph = Graph(nodes=set('ab'), edges={('a', 'b', 'edgelabel')}) + taskgraph = TaskGraph(tasks, graph) + + res = taskgraph.to_json() + + self.assertEqual(res, { + 'a': { + 'label': 'a', + 'attributes': {'attr': 'a-task', 'kind': 'test'}, + 'task': {}, + 'dependencies': {'edgelabel': 'b'}, + 'kind_implementation': 'taskgraph.test.util:TestTask', + }, + 'b': { + 'label': 'b', + 'attributes': {'kind': 'test'}, + 'task': {'task': 'def'}, + 'dependencies': {}, + 'kind_implementation': 'taskgraph.test.util:TestTask', + } + }) + + def test_write_artifact_json(self): + data = [{'some': 'data'}] + tmpdir = tempfile.mkdtemp() + try: + decision.ARTIFACTS_DIR = os.path.join(tmpdir, "artifacts") + decision.write_artifact("artifact.json", data) + with open(os.path.join(decision.ARTIFACTS_DIR, "artifact.json")) as f: + self.assertEqual(json.load(f), data) + finally: + if os.path.exists(tmpdir): + shutil.rmtree(tmpdir) + decision.ARTIFACTS_DIR = 'artifacts' + + def test_write_artifact_yml(self): + data = [{'some': 'data'}] + tmpdir = tempfile.mkdtemp() + try: + decision.ARTIFACTS_DIR = os.path.join(tmpdir, "artifacts") + decision.write_artifact("artifact.yml", data) + with open(os.path.join(decision.ARTIFACTS_DIR, "artifact.yml")) as f: + self.assertEqual(yaml.safe_load(f), data) + finally: + if os.path.exists(tmpdir): + shutil.rmtree(tmpdir) + decision.ARTIFACTS_DIR = 'artifacts' + + +if __name__ == '__main__': + main() diff --git a/taskcluster/taskgraph/test/test_files_changed.py b/taskcluster/taskgraph/test/test_files_changed.py new file mode 100644 index 000000000..0e3366b3c --- /dev/null +++ b/taskcluster/taskgraph/test/test_files_changed.py @@ -0,0 +1,73 @@ +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, You can obtain one at http://mozilla.org/MPL/2.0/. + +from __future__ import absolute_import, print_function, unicode_literals + +import json +import os +import unittest + +from .. import files_changed + +PARAMS = { + 'head_repository': 'https://hg.mozilla.org/mozilla-central', + 'head_rev': 'a14f88a9af7a', +} + +FILES_CHANGED = [ + 'devtools/client/debugger/new/index.html', + 'devtools/client/inspector/test/browser_inspector_highlighter-eyedropper-events.js', + 'devtools/client/inspector/test/head.js', + 'devtools/client/themes/rules.css', + 'devtools/client/webconsole/test/browser_webconsole_output_06.js', + 'devtools/server/actors/highlighters/eye-dropper.js', + 'devtools/server/actors/object.js', + 'docshell/base/nsDocShell.cpp', + 'dom/tests/mochitest/general/test_contentViewer_overrideDPPX.html', + 'taskcluster/scripts/builder/build-l10n.sh', +] + + +class FakeResponse: + + def json(self): + with open(os.path.join(os.path.dirname(__file__), 'automationrelevance.json')) as f: + return json.load(f) + + +class TestGetChangedFiles(unittest.TestCase): + + def setUp(self): + files_changed._cache.clear() + self.old_get = files_changed.requests.get + + def fake_get(url, **kwargs): + return FakeResponse() + files_changed.requests.get = fake_get + + def tearDown(self): + files_changed.requests.get = self.old_get + + def test_get_changed_files(self): + """Get_changed_files correctly gets the list of changed files in a push. + This tests against the production hg.mozilla.org so that it will detect + any changes in the format of the returned data.""" + self.assertEqual( + sorted(files_changed.get_changed_files(PARAMS['head_repository'], PARAMS['head_rev'])), + FILES_CHANGED) + + +class TestCheck(unittest.TestCase): + + def setUp(self): + files_changed._cache[PARAMS['head_repository'], PARAMS['head_rev']] = FILES_CHANGED + + def test_check_no_params(self): + self.assertTrue(files_changed.check({}, ["ignored"])) + + def test_check_no_match(self): + self.assertFalse(files_changed.check(PARAMS, ["nosuch/**"])) + + def test_check_match(self): + self.assertTrue(files_changed.check(PARAMS, ["devtools/**"])) diff --git a/taskcluster/taskgraph/test/test_generator.py b/taskcluster/taskgraph/test/test_generator.py new file mode 100644 index 000000000..f1b466e4d --- /dev/null +++ b/taskcluster/taskgraph/test/test_generator.py @@ -0,0 +1,129 @@ +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, You can obtain one at http://mozilla.org/MPL/2.0/. + +from __future__ import absolute_import, print_function, unicode_literals + +import unittest + +from ..generator import TaskGraphGenerator, Kind +from .. import graph +from ..task import base +from mozunit import main + + +class FakeTask(base.Task): + + def __init__(self, **kwargs): + self.i = kwargs.pop('i') + super(FakeTask, self).__init__(**kwargs) + + @classmethod + def load_tasks(cls, kind, path, config, parameters, loaded_tasks): + return [cls(kind=kind, + label='{}-t-{}'.format(kind, i), + attributes={'tasknum': str(i)}, + task={}, + i=i) + for i in range(3)] + + def get_dependencies(self, full_task_set): + i = self.i + if i > 0: + return [('{}-t-{}'.format(self.kind, i - 1), 'prev')] + else: + return [] + + def optimize(self, params): + return False, None + + +class FakeKind(Kind): + + def _get_impl_class(self): + return FakeTask + + def load_tasks(self, parameters, loaded_tasks): + FakeKind.loaded_kinds.append(self.name) + return super(FakeKind, self).load_tasks(parameters, loaded_tasks) + + +class WithFakeKind(TaskGraphGenerator): + + def _load_kinds(self): + for kind_name, deps in self.parameters['kinds']: + yield FakeKind( + kind_name, '/fake', + {'kind-dependencies': deps} if deps else {}) + + +class TestGenerator(unittest.TestCase): + + def maketgg(self, target_tasks=None, kinds=[('fake', [])]): + FakeKind.loaded_kinds = [] + self.target_tasks = target_tasks or [] + + def target_tasks_method(full_task_graph, parameters): + return self.target_tasks + return WithFakeKind('/root', {'kinds': kinds}, target_tasks_method) + + def test_kind_ordering(self): + "When task kinds depend on each other, they are loaded in postorder" + self.tgg = self.maketgg(kinds=[ + ('fake3', ['fake2', 'fake1']), + ('fake2', ['fake1']), + ('fake1', []), + ]) + self.tgg._run_until('full_task_set') + self.assertEqual(FakeKind.loaded_kinds, ['fake1', 'fake2', 'fake3']) + + def test_full_task_set(self): + "The full_task_set property has all tasks" + self.tgg = self.maketgg() + self.assertEqual(self.tgg.full_task_set.graph, + graph.Graph({'fake-t-0', 'fake-t-1', 'fake-t-2'}, set())) + self.assertEqual(sorted(self.tgg.full_task_set.tasks.keys()), + sorted(['fake-t-0', 'fake-t-1', 'fake-t-2'])) + + def test_full_task_graph(self): + "The full_task_graph property has all tasks, and links" + self.tgg = self.maketgg() + self.assertEqual(self.tgg.full_task_graph.graph, + graph.Graph({'fake-t-0', 'fake-t-1', 'fake-t-2'}, + { + ('fake-t-1', 'fake-t-0', 'prev'), + ('fake-t-2', 'fake-t-1', 'prev'), + })) + self.assertEqual(sorted(self.tgg.full_task_graph.tasks.keys()), + sorted(['fake-t-0', 'fake-t-1', 'fake-t-2'])) + + def test_target_task_set(self): + "The target_task_set property has the targeted tasks" + self.tgg = self.maketgg(['fake-t-1']) + self.assertEqual(self.tgg.target_task_set.graph, + graph.Graph({'fake-t-1'}, set())) + self.assertEqual(self.tgg.target_task_set.tasks.keys(), + ['fake-t-1']) + + def test_target_task_graph(self): + "The target_task_graph property has the targeted tasks and deps" + self.tgg = self.maketgg(['fake-t-1']) + self.assertEqual(self.tgg.target_task_graph.graph, + graph.Graph({'fake-t-0', 'fake-t-1'}, + {('fake-t-1', 'fake-t-0', 'prev')})) + self.assertEqual(sorted(self.tgg.target_task_graph.tasks.keys()), + sorted(['fake-t-0', 'fake-t-1'])) + + def test_optimized_task_graph(self): + "The optimized task graph contains task ids" + self.tgg = self.maketgg(['fake-t-2']) + tid = self.tgg.label_to_taskid + self.assertEqual( + self.tgg.optimized_task_graph.graph, + graph.Graph({tid['fake-t-0'], tid['fake-t-1'], tid['fake-t-2']}, { + (tid['fake-t-1'], tid['fake-t-0'], 'prev'), + (tid['fake-t-2'], tid['fake-t-1'], 'prev'), + })) + +if __name__ == '__main__': + main() diff --git a/taskcluster/taskgraph/test/test_graph.py b/taskcluster/taskgraph/test/test_graph.py new file mode 100644 index 000000000..5c4c950a7 --- /dev/null +++ b/taskcluster/taskgraph/test/test_graph.py @@ -0,0 +1,157 @@ +# -*- coding: utf-8 -*- + +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, You can obtain one at http://mozilla.org/MPL/2.0/. + +from __future__ import absolute_import, print_function, unicode_literals + +import unittest + +from ..graph import Graph +from mozunit import main + + +class TestGraph(unittest.TestCase): + + tree = Graph(set(['a', 'b', 'c', 'd', 'e', 'f', 'g']), { + ('a', 'b', 'L'), + ('a', 'c', 'L'), + ('b', 'd', 'K'), + ('b', 'e', 'K'), + ('c', 'f', 'N'), + ('c', 'g', 'N'), + }) + + linear = Graph(set(['1', '2', '3', '4']), { + ('1', '2', 'L'), + ('2', '3', 'L'), + ('3', '4', 'L'), + }) + + diamonds = Graph(set(['A', 'B', 'C', 'D', 'E', 'F', 'G', 'H', 'I', 'J']), + set(tuple(x) for x in + 'AFL ADL BDL BEL CEL CHL DFL DGL EGL EHL FIL GIL GJL HJL'.split() + )) + + multi_edges = Graph(set(['1', '2', '3', '4']), { + ('2', '1', 'red'), + ('2', '1', 'blue'), + ('3', '1', 'red'), + ('3', '2', 'blue'), + ('3', '2', 'green'), + ('4', '3', 'green'), + }) + + disjoint = Graph(set(['1', '2', '3', '4', 'α', 'β', 'γ']), { + ('2', '1', 'red'), + ('3', '1', 'red'), + ('3', '2', 'green'), + ('4', '3', 'green'), + ('α', 'β', 'πράσινο'), + ('β', 'γ', 'κόκκινο'), + ('α', 'γ', 'μπλε'), + }) + + def test_transitive_closure_empty(self): + "transitive closure of an empty set is an empty graph" + g = Graph(set(['a', 'b', 'c']), {('a', 'b', 'L'), ('a', 'c', 'L')}) + self.assertEqual(g.transitive_closure(set()), + Graph(set(), set())) + + def test_transitive_closure_disjoint(self): + "transitive closure of a disjoint set is a subset" + g = Graph(set(['a', 'b', 'c']), set()) + self.assertEqual(g.transitive_closure(set(['a', 'c'])), + Graph(set(['a', 'c']), set())) + + def test_transitive_closure_trees(self): + "transitive closure of a tree, at two non-root nodes, is the two subtrees" + self.assertEqual(self.tree.transitive_closure(set(['b', 'c'])), + Graph(set(['b', 'c', 'd', 'e', 'f', 'g']), { + ('b', 'd', 'K'), + ('b', 'e', 'K'), + ('c', 'f', 'N'), + ('c', 'g', 'N'), + })) + + def test_transitive_closure_multi_edges(self): + "transitive closure of a tree with multiple edges between nodes keeps those edges" + self.assertEqual(self.multi_edges.transitive_closure(set(['3'])), + Graph(set(['1', '2', '3']), { + ('2', '1', 'red'), + ('2', '1', 'blue'), + ('3', '1', 'red'), + ('3', '2', 'blue'), + ('3', '2', 'green'), + })) + + def test_transitive_closure_disjoint_edges(self): + "transitive closure of a disjoint graph keeps those edges" + self.assertEqual(self.disjoint.transitive_closure(set(['3', 'β'])), + Graph(set(['1', '2', '3', 'β', 'γ']), { + ('2', '1', 'red'), + ('3', '1', 'red'), + ('3', '2', 'green'), + ('β', 'γ', 'κόκκινο'), + })) + + def test_transitive_closure_linear(self): + "transitive closure of a linear graph includes all nodes in the line" + self.assertEqual(self.linear.transitive_closure(set(['1'])), self.linear) + + def test_visit_postorder_empty(self): + "postorder visit of an empty graph is empty" + self.assertEqual(list(Graph(set(), set()).visit_postorder()), []) + + def assert_postorder(self, seq, all_nodes): + seen = set() + for e in seq: + for l, r, n in self.tree.edges: + if l == e: + self.failUnless(r in seen) + seen.add(e) + self.assertEqual(seen, all_nodes) + + def test_visit_postorder_tree(self): + "postorder visit of a tree satisfies invariant" + self.assert_postorder(self.tree.visit_postorder(), self.tree.nodes) + + def test_visit_postorder_diamonds(self): + "postorder visit of a graph full of diamonds satisfies invariant" + self.assert_postorder(self.diamonds.visit_postorder(), self.diamonds.nodes) + + def test_visit_postorder_multi_edges(self): + "postorder visit of a graph with duplicate edges satisfies invariant" + self.assert_postorder(self.multi_edges.visit_postorder(), self.multi_edges.nodes) + + def test_visit_postorder_disjoint(self): + "postorder visit of a disjoint graph satisfies invariant" + self.assert_postorder(self.disjoint.visit_postorder(), self.disjoint.nodes) + + def test_links_dict(self): + "link dict for a graph with multiple edges is correct" + self.assertEqual(self.multi_edges.links_dict(), { + '2': set(['1']), + '3': set(['1', '2']), + '4': set(['3']), + }) + + def test_named_links_dict(self): + "named link dict for a graph with multiple edges is correct" + self.assertEqual(self.multi_edges.named_links_dict(), { + '2': dict(red='1', blue='1'), + '3': dict(red='1', blue='2', green='2'), + '4': dict(green='3'), + }) + + def test_reverse_links_dict(self): + "reverse link dict for a graph with multiple edges is correct" + self.assertEqual(self.multi_edges.reverse_links_dict(), { + '1': set(['2', '3']), + '2': set(['3']), + '3': set(['4']), + }) + +if __name__ == '__main__': + main() diff --git a/taskcluster/taskgraph/test/test_optimize.py b/taskcluster/taskgraph/test/test_optimize.py new file mode 100644 index 000000000..8d2ddf247 --- /dev/null +++ b/taskcluster/taskgraph/test/test_optimize.py @@ -0,0 +1,256 @@ +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, You can obtain one at http://mozilla.org/MPL/2.0/. + +from __future__ import absolute_import, print_function, unicode_literals + +import unittest + +from ..optimize import optimize_task_graph, resolve_task_references +from ..optimize import annotate_task_graph, get_subgraph +from ..taskgraph import TaskGraph +from .. import graph +from .util import TestTask + + +class TestResolveTaskReferences(unittest.TestCase): + + def do(self, input, output): + taskid_for_edge_name = {'edge%d' % n: 'tid%d' % n for n in range(1, 4)} + self.assertEqual(resolve_task_references('subject', input, taskid_for_edge_name), output) + + def test_in_list(self): + "resolve_task_references resolves task references in a list" + self.do({'in-a-list': ['stuff', {'task-reference': '<edge1>'}]}, + {'in-a-list': ['stuff', 'tid1']}) + + def test_in_dict(self): + "resolve_task_references resolves task references in a dict" + self.do({'in-a-dict': {'stuff': {'task-reference': '<edge2>'}}}, + {'in-a-dict': {'stuff': 'tid2'}}) + + def test_multiple(self): + "resolve_task_references resolves multiple references in the same string" + self.do({'multiple': {'task-reference': 'stuff <edge1> stuff <edge2> after'}}, + {'multiple': 'stuff tid1 stuff tid2 after'}) + + def test_embedded(self): + "resolve_task_references resolves ebmedded references" + self.do({'embedded': {'task-reference': 'stuff before <edge3> stuff after'}}, + {'embedded': 'stuff before tid3 stuff after'}) + + def test_escaping(self): + "resolve_task_references resolves escapes in task references" + self.do({'escape': {'task-reference': '<<><edge3>>'}}, + {'escape': '<tid3>'}) + + def test_invalid(self): + "resolve_task_references raises a KeyError on reference to an invalid task" + self.assertRaisesRegexp( + KeyError, + "task 'subject' has no dependency named 'no-such'", + lambda: resolve_task_references('subject', {'task-reference': '<no-such>'}, {}) + ) + + +class OptimizingTask(TestTask): + # the `optimize` method on this class is overridden direclty in the tests + # below. + pass + + +class TestOptimize(unittest.TestCase): + + kind = None + + def make_task(self, label, task_def=None, optimized=None, task_id=None): + task_def = task_def or {'sample': 'task-def'} + task = OptimizingTask(label=label, task=task_def) + task.optimized = optimized + task.task_id = task_id + return task + + def make_graph(self, *tasks_and_edges): + tasks = {t.label: t for t in tasks_and_edges if isinstance(t, OptimizingTask)} + edges = {e for e in tasks_and_edges if not isinstance(e, OptimizingTask)} + return TaskGraph(tasks, graph.Graph(set(tasks), edges)) + + def assert_annotations(self, graph, **annotations): + def repl(task_id): + return 'SLUGID' if task_id and len(task_id) == 22 else task_id + got_annotations = { + t.label: (t.optimized, repl(t.task_id)) for t in graph.tasks.itervalues() + } + self.assertEqual(got_annotations, annotations) + + def test_annotate_task_graph_no_optimize(self): + "annotating marks everything as un-optimized if the kind returns that" + OptimizingTask.optimize = lambda self, params: (False, None) + graph = self.make_graph( + self.make_task('task1'), + self.make_task('task2'), + self.make_task('task3'), + ('task2', 'task1', 'build'), + ('task2', 'task3', 'image'), + ) + annotate_task_graph(graph, {}, set(), graph.graph.named_links_dict(), {}, None) + self.assert_annotations( + graph, + task1=(False, None), + task2=(False, None), + task3=(False, None) + ) + + def test_annotate_task_graph_taskid_without_optimize(self): + "raises exception if kind returns a taskid without optimizing" + OptimizingTask.optimize = lambda self, params: (False, 'some-taskid') + graph = self.make_graph(self.make_task('task1')) + self.assertRaises( + Exception, + lambda: annotate_task_graph(graph, {}, set(), graph.graph.named_links_dict(), {}, None) + ) + + def test_annotate_task_graph_optimize_away_dependency(self): + "raises exception if kind optimizes away a task on which another depends" + OptimizingTask.optimize = \ + lambda self, params: (True, None) if self.label == 'task1' else (False, None) + graph = self.make_graph( + self.make_task('task1'), + self.make_task('task2'), + ('task2', 'task1', 'build'), + ) + self.assertRaises( + Exception, + lambda: annotate_task_graph(graph, {}, set(), graph.graph.named_links_dict(), {}, None) + ) + + def test_annotate_task_graph_do_not_optimize(self): + "annotating marks everything as un-optimized if in do_not_optimize" + OptimizingTask.optimize = lambda self, params: (True, 'taskid') + graph = self.make_graph( + self.make_task('task1'), + self.make_task('task2'), + ('task2', 'task1', 'build'), + ) + label_to_taskid = {} + annotate_task_graph(graph, {}, {'task1', 'task2'}, + graph.graph.named_links_dict(), label_to_taskid, None) + self.assert_annotations( + graph, + task1=(False, None), + task2=(False, None) + ) + self.assertEqual + + def test_annotate_task_graph_nos_do_not_propagate(self): + "a task with a non-optimized dependency can be optimized" + OptimizingTask.optimize = \ + lambda self, params: (False, None) if self.label == 'task1' else (True, 'taskid') + graph = self.make_graph( + self.make_task('task1'), + self.make_task('task2'), + self.make_task('task3'), + ('task2', 'task1', 'build'), + ('task2', 'task3', 'image'), + ) + annotate_task_graph(graph, {}, set(), + graph.graph.named_links_dict(), {}, None) + self.assert_annotations( + graph, + task1=(False, None), + task2=(True, 'taskid'), + task3=(True, 'taskid') + ) + + def test_get_subgraph_single_dep(self): + "when a single dependency is optimized, it is omitted from the graph" + graph = self.make_graph( + self.make_task('task1', optimized=True, task_id='dep1'), + self.make_task('task2', optimized=False), + self.make_task('task3', optimized=False), + ('task2', 'task1', 'build'), + ('task2', 'task3', 'image'), + ) + label_to_taskid = {'task1': 'dep1'} + sub = get_subgraph(graph, graph.graph.named_links_dict(), label_to_taskid) + task2 = label_to_taskid['task2'] + task3 = label_to_taskid['task3'] + self.assertEqual(sub.graph.nodes, {task2, task3}) + self.assertEqual(sub.graph.edges, {(task2, task3, 'image')}) + self.assertEqual(sub.tasks[task2].task_id, task2) + self.assertEqual(sorted(sub.tasks[task2].task['dependencies']), + sorted([task3, 'dep1'])) + self.assertEqual(sub.tasks[task3].task_id, task3) + self.assertEqual(sorted(sub.tasks[task3].task['dependencies']), []) + + def test_get_subgraph_dep_chain(self): + "when a dependency chain is optimized, it is omitted from the graph" + graph = self.make_graph( + self.make_task('task1', optimized=True, task_id='dep1'), + self.make_task('task2', optimized=True, task_id='dep2'), + self.make_task('task3', optimized=False), + ('task2', 'task1', 'build'), + ('task3', 'task2', 'image'), + ) + label_to_taskid = {'task1': 'dep1', 'task2': 'dep2'} + sub = get_subgraph(graph, graph.graph.named_links_dict(), label_to_taskid) + task3 = label_to_taskid['task3'] + self.assertEqual(sub.graph.nodes, {task3}) + self.assertEqual(sub.graph.edges, set()) + self.assertEqual(sub.tasks[task3].task_id, task3) + self.assertEqual(sorted(sub.tasks[task3].task['dependencies']), ['dep2']) + + def test_get_subgraph_opt_away(self): + "when a leaf task is optimized away, it is omitted from the graph" + graph = self.make_graph( + self.make_task('task1', optimized=False), + self.make_task('task2', optimized=True), + ('task2', 'task1', 'build'), + ) + label_to_taskid = {'task2': 'dep2'} + sub = get_subgraph(graph, graph.graph.named_links_dict(), label_to_taskid) + task1 = label_to_taskid['task1'] + self.assertEqual(sub.graph.nodes, {task1}) + self.assertEqual(sub.graph.edges, set()) + self.assertEqual(sub.tasks[task1].task_id, task1) + self.assertEqual(sorted(sub.tasks[task1].task['dependencies']), []) + + def test_get_subgraph_refs_resolved(self): + "get_subgraph resolves task references" + graph = self.make_graph( + self.make_task('task1', optimized=True, task_id='dep1'), + self.make_task( + 'task2', + optimized=False, + task_def={'payload': {'task-reference': 'http://<build>/<test>'}} + ), + ('task2', 'task1', 'build'), + ('task2', 'task3', 'test'), + self.make_task('task3', optimized=False), + ) + label_to_taskid = {'task1': 'dep1'} + sub = get_subgraph(graph, graph.graph.named_links_dict(), label_to_taskid) + task2 = label_to_taskid['task2'] + task3 = label_to_taskid['task3'] + self.assertEqual(sub.graph.nodes, {task2, task3}) + self.assertEqual(sub.graph.edges, {(task2, task3, 'test')}) + self.assertEqual(sub.tasks[task2].task_id, task2) + self.assertEqual(sorted(sub.tasks[task2].task['dependencies']), sorted([task3, 'dep1'])) + self.assertEqual(sub.tasks[task2].task['payload'], 'http://dep1/' + task3) + self.assertEqual(sub.tasks[task3].task_id, task3) + + def test_optimize(self): + "optimize_task_graph annotates and extracts the subgraph from a simple graph" + OptimizingTask.optimize = \ + lambda self, params: (True, 'dep1') if self.label == 'task1' else (False, None) + input = self.make_graph( + self.make_task('task1'), + self.make_task('task2'), + self.make_task('task3'), + ('task2', 'task1', 'build'), + ('task2', 'task3', 'image'), + ) + opt, label_to_taskid = optimize_task_graph(input, {}, set()) + self.assertEqual(opt.graph, graph.Graph( + {label_to_taskid['task2'], label_to_taskid['task3']}, + {(label_to_taskid['task2'], label_to_taskid['task3'], 'image')})) diff --git a/taskcluster/taskgraph/test/test_parameters.py b/taskcluster/taskgraph/test/test_parameters.py new file mode 100644 index 000000000..43d853d7b --- /dev/null +++ b/taskcluster/taskgraph/test/test_parameters.py @@ -0,0 +1,62 @@ +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, You can obtain one at http://mozilla.org/MPL/2.0/. + +from __future__ import absolute_import, print_function, unicode_literals + +import unittest + +from ..parameters import Parameters, load_parameters_file, PARAMETER_NAMES +from mozunit import main, MockedOpen + + +class TestParameters(unittest.TestCase): + + vals = {n: n for n in PARAMETER_NAMES} + + def test_Parameters_immutable(self): + p = Parameters(**self.vals) + + def assign(): + p['head_ref'] = 20 + self.assertRaises(Exception, assign) + + def test_Parameters_missing_KeyError(self): + p = Parameters(**self.vals) + self.assertRaises(KeyError, lambda: p['z']) + + def test_Parameters_invalid_KeyError(self): + """even if the value is present, if it's not a valid property, raise KeyError""" + p = Parameters(xyz=10, **self.vals) + self.assertRaises(KeyError, lambda: p['xyz']) + + def test_Parameters_get(self): + p = Parameters(head_ref=10, level=20) + self.assertEqual(p['head_ref'], 10) + + def test_Parameters_check(self): + p = Parameters(**self.vals) + p.check() # should not raise + + def test_Parameters_check_missing(self): + p = Parameters() + self.assertRaises(Exception, lambda: p.check()) + + def test_Parameters_check_extra(self): + p = Parameters(xyz=10, **self.vals) + self.assertRaises(Exception, lambda: p.check()) + + def test_load_parameters_file_yaml(self): + with MockedOpen({"params.yml": "some: data\n"}): + self.assertEqual( + load_parameters_file({'parameters': 'params.yml'}), + {'some': 'data'}) + + def test_load_parameters_file_json(self): + with MockedOpen({"params.json": '{"some": "data"}'}): + self.assertEqual( + load_parameters_file({'parameters': 'params.json'}), + {'some': 'data'}) + +if __name__ == '__main__': + main() diff --git a/taskcluster/taskgraph/test/test_target_tasks.py b/taskcluster/taskgraph/test/test_target_tasks.py new file mode 100644 index 000000000..035ccefd8 --- /dev/null +++ b/taskcluster/taskgraph/test/test_target_tasks.py @@ -0,0 +1,81 @@ +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, You can obtain one at http://mozilla.org/MPL/2.0/. + +from __future__ import absolute_import, print_function, unicode_literals + +import unittest + +from .. import target_tasks +from .. import try_option_syntax +from ..graph import Graph +from ..taskgraph import TaskGraph +from .util import TestTask +from mozunit import main + + +class FakeTryOptionSyntax(object): + + def __init__(self, message, task_graph): + self.trigger_tests = 0 + self.notifications = None + + def task_matches(self, attributes): + return 'at-at' in attributes + + +class TestTargetTasks(unittest.TestCase): + + def default_matches(self, run_on_projects, project): + method = target_tasks.get_method('default') + graph = TaskGraph(tasks={ + 'a': TestTask(kind='build', label='a', + attributes={'run_on_projects': run_on_projects}), + }, graph=Graph(nodes={'a'}, edges=set())) + parameters = {'project': project} + return 'a' in method(graph, parameters) + + def test_default_all(self): + """run_on_projects=[all] includes release, integration, and other projects""" + self.assertTrue(self.default_matches(['all'], 'mozilla-central')) + self.assertTrue(self.default_matches(['all'], 'mozilla-inbound')) + self.assertTrue(self.default_matches(['all'], 'mozilla-aurora')) + self.assertTrue(self.default_matches(['all'], 'baobab')) + + def test_default_integration(self): + """run_on_projects=[integration] includes integration projects""" + self.assertFalse(self.default_matches(['integration'], 'mozilla-central')) + self.assertTrue(self.default_matches(['integration'], 'mozilla-inbound')) + self.assertFalse(self.default_matches(['integration'], 'baobab')) + + def test_default_relesae(self): + """run_on_projects=[release] includes release projects""" + self.assertTrue(self.default_matches(['release'], 'mozilla-central')) + self.assertFalse(self.default_matches(['release'], 'mozilla-inbound')) + self.assertFalse(self.default_matches(['release'], 'baobab')) + + def test_default_nothing(self): + """run_on_projects=[] includes nothing""" + self.assertFalse(self.default_matches([], 'mozilla-central')) + self.assertFalse(self.default_matches([], 'mozilla-inbound')) + self.assertFalse(self.default_matches([], 'baobab')) + + def test_try_option_syntax(self): + tasks = { + 'a': TestTask(kind=None, label='a'), + 'b': TestTask(kind=None, label='b', attributes={'at-at': 'yep'}), + } + graph = Graph(nodes=set('ab'), edges=set()) + tg = TaskGraph(tasks, graph) + params = {'message': 'try me'} + + orig_TryOptionSyntax = try_option_syntax.TryOptionSyntax + try: + try_option_syntax.TryOptionSyntax = FakeTryOptionSyntax + method = target_tasks.get_method('try_option_syntax') + self.assertEqual(method(tg, params), ['b']) + finally: + try_option_syntax.TryOptionSyntax = orig_TryOptionSyntax + +if __name__ == '__main__': + main() diff --git a/taskcluster/taskgraph/test/test_task_docker_image.py b/taskcluster/taskgraph/test/test_task_docker_image.py new file mode 100644 index 000000000..8f247db3e --- /dev/null +++ b/taskcluster/taskgraph/test/test_task_docker_image.py @@ -0,0 +1,35 @@ +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, You can obtain one at http://mozilla.org/MPL/2.0/. + +from __future__ import absolute_import, print_function, unicode_literals + +import unittest +import os + +from ..task import docker_image +from mozunit import main + + +KIND_PATH = os.path.join(docker_image.GECKO, 'taskcluster', 'ci', 'docker-image') + + +class TestDockerImageKind(unittest.TestCase): + + def setUp(self): + self.task = docker_image.DockerImageTask( + 'docker-image', + KIND_PATH, + {}, + {}, + index_paths=[]) + + def test_get_task_dependencies(self): + # this one's easy! + self.assertEqual(self.task.get_dependencies(None), []) + + # TODO: optimize_task + + +if __name__ == '__main__': + main() diff --git a/taskcluster/taskgraph/test/test_taskgraph.py b/taskcluster/taskgraph/test/test_taskgraph.py new file mode 100644 index 000000000..f8f09bce9 --- /dev/null +++ b/taskcluster/taskgraph/test/test_taskgraph.py @@ -0,0 +1,54 @@ +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, You can obtain one at http://mozilla.org/MPL/2.0/. + +from __future__ import absolute_import, print_function, unicode_literals + +import unittest + +from ..graph import Graph +from ..task.docker_image import DockerImageTask +from ..task.transform import TransformTask +from ..taskgraph import TaskGraph +from mozunit import main +from taskgraph.util.docker import INDEX_PREFIX + + +class TestTargetTasks(unittest.TestCase): + + def test_from_json(self): + task = { + "routes": [], + "extra": { + "imageMeta": { + "contextHash": "<hash>", + "imageName": "<image>", + "level": "1" + } + } + } + index_paths = ["{}.level-{}.<image>.hash.<hash>".format(INDEX_PREFIX, level) + for level in range(1, 4)] + graph = TaskGraph(tasks={ + 'a': TransformTask( + kind='fancy', + task={ + 'label': 'a', + 'attributes': {}, + 'dependencies': {}, + 'when': {}, + 'task': {'task': 'def'}, + }), + 'b': DockerImageTask(kind='docker-image', + label='b', + attributes={}, + task=task, + index_paths=index_paths), + }, graph=Graph(nodes={'a', 'b'}, edges=set())) + + tasks, new_graph = TaskGraph.from_json(graph.to_json()) + self.assertEqual(graph.tasks['a'], new_graph.tasks['a']) + self.assertEqual(graph, new_graph) + +if __name__ == '__main__': + main() diff --git a/taskcluster/taskgraph/test/test_transforms_base.py b/taskcluster/taskgraph/test/test_transforms_base.py new file mode 100644 index 000000000..0a0dfcaf2 --- /dev/null +++ b/taskcluster/taskgraph/test/test_transforms_base.py @@ -0,0 +1,143 @@ +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, You can obtain one at http://mozilla.org/MPL/2.0/. + +from __future__ import absolute_import, print_function, unicode_literals + +import unittest +from mozunit import main +from taskgraph.transforms.base import ( + validate_schema, + get_keyed_by, + TransformSequence +) +from voluptuous import Schema + +schema = Schema({ + 'x': int, + 'y': basestring, +}) + +transforms = TransformSequence() + + +@transforms.add +def trans1(config, tests): + for test in tests: + test['one'] = 1 + yield test + + +@transforms.add +def trans2(config, tests): + for test in tests: + test['two'] = 2 + yield test + + +class TestTransformSequence(unittest.TestCase): + + def test_sequence(self): + tests = [{}, {'two': 1, 'second': True}] + res = list(transforms({}, tests)) + self.assertEqual(res, [ + {u'two': 2, u'one': 1}, + {u'second': True, u'two': 2, u'one': 1}, + ]) + + +class TestValidateSchema(unittest.TestCase): + + def test_valid(self): + validate_schema(schema, {'x': 10, 'y': 'foo'}, "pfx") + + def test_invalid(self): + try: + validate_schema(schema, {'x': 'not-int'}, "pfx") + self.fail("no exception raised") + except Exception, e: + self.failUnless(str(e).startswith("pfx\n")) + + +class TestKeyedBy(unittest.TestCase): + + def test_simple_value(self): + test = { + 'test-name': 'tname', + 'option': 10, + } + self.assertEqual(get_keyed_by(test, 'option', 'x'), 10) + + def test_by_value(self): + test = { + 'test-name': 'tname', + 'option': { + 'by-other-value': { + 'a': 10, + 'b': 20, + }, + }, + 'other-value': 'b', + } + self.assertEqual(get_keyed_by(test, 'option', 'x'), 20) + + def test_by_value_regex(self): + test = { + 'test-name': 'tname', + 'option': { + 'by-test-platform': { + 'macosx64/.*': 10, + 'linux64/debug': 20, + 'default': 5, + }, + }, + 'test-platform': 'macosx64/debug', + } + self.assertEqual(get_keyed_by(test, 'option', 'x'), 10) + + def test_by_value_default(self): + test = { + 'test-name': 'tname', + 'option': { + 'by-other-value': { + 'a': 10, + 'default': 30, + }, + }, + 'other-value': 'xxx', + } + self.assertEqual(get_keyed_by(test, 'option', 'x'), 30) + + def test_by_value_invalid_dict(self): + test = { + 'test-name': 'tname', + 'option': { + 'by-something-else': {}, + 'by-other-value': {}, + }, + } + self.assertRaises(Exception, get_keyed_by, test, 'option', 'x') + + def test_by_value_invalid_no_default(self): + test = { + 'test-name': 'tname', + 'option': { + 'by-other-value': { + 'a': 10, + }, + }, + 'other-value': 'b', + } + self.assertRaises(Exception, get_keyed_by, test, 'option', 'x') + + def test_by_value_invalid_no_by(self): + test = { + 'test-name': 'tname', + 'option': { + 'other-value': {}, + }, + } + self.assertRaises(Exception, get_keyed_by, test, 'option', 'x') + +if __name__ == '__main__': + main() diff --git a/taskcluster/taskgraph/test/test_try_option_syntax.py b/taskcluster/taskgraph/test/test_try_option_syntax.py new file mode 100644 index 000000000..29aa2d5a9 --- /dev/null +++ b/taskcluster/taskgraph/test/test_try_option_syntax.py @@ -0,0 +1,274 @@ +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, You can obtain one at http://mozilla.org/MPL/2.0/. + +from __future__ import absolute_import, print_function, unicode_literals + +import unittest +import itertools + +from ..try_option_syntax import TryOptionSyntax +from ..try_option_syntax import RIDEALONG_BUILDS +from ..graph import Graph +from ..taskgraph import TaskGraph +from .util import TestTask +from mozunit import main + +# an empty graph, for things that don't look at it +empty_graph = TaskGraph({}, Graph(set(), set())) + + +def unittest_task(n, tp): + return (n, TestTask('test', n, { + 'unittest_try_name': n, + 'test_platform': tp, + })) + + +def talos_task(n, tp): + return (n, TestTask('test', n, { + 'talos_try_name': n, + 'test_platform': tp, + })) + +tasks = {k: v for k, v in [ + unittest_task('mochitest-browser-chrome', 'linux'), + unittest_task('mochitest-e10s-browser-chrome', 'linux64'), + unittest_task('mochitest-chrome', 'linux'), + unittest_task('mochitest-webgl', 'linux'), + unittest_task('crashtest-e10s', 'linux'), + unittest_task('gtest', 'linux64'), + talos_task('dromaeojs', 'linux64'), +]} +unittest_tasks = {k: v for k, v in tasks.iteritems() + if 'unittest_try_name' in v.attributes} +talos_tasks = {k: v for k, v in tasks.iteritems() + if 'talos_try_name' in v.attributes} +graph_with_jobs = TaskGraph(tasks, Graph(set(tasks), set())) + + +class TestTryOptionSyntax(unittest.TestCase): + + def test_empty_message(self): + "Given an empty message, it should return an empty value" + tos = TryOptionSyntax('', empty_graph) + self.assertEqual(tos.build_types, []) + self.assertEqual(tos.jobs, []) + self.assertEqual(tos.unittests, []) + self.assertEqual(tos.talos, []) + self.assertEqual(tos.platforms, []) + + def test_message_without_try(self): + "Given a non-try message, it should return an empty value" + tos = TryOptionSyntax('Bug 1234: frobnicte the foo', empty_graph) + self.assertEqual(tos.build_types, []) + self.assertEqual(tos.jobs, []) + self.assertEqual(tos.unittests, []) + self.assertEqual(tos.talos, []) + self.assertEqual(tos.platforms, []) + + def test_unknown_args(self): + "unknown arguments are ignored" + tos = TryOptionSyntax('try: --doubledash -z extra', empty_graph) + # equilvant to "try:".. + self.assertEqual(tos.build_types, []) + self.assertEqual(tos.jobs, None) + + def test_b_do(self): + "-b do should produce both build_types" + tos = TryOptionSyntax('try: -b do', empty_graph) + self.assertEqual(sorted(tos.build_types), ['debug', 'opt']) + + def test_b_d(self): + "-b d should produce build_types=['debug']" + tos = TryOptionSyntax('try: -b d', empty_graph) + self.assertEqual(sorted(tos.build_types), ['debug']) + + def test_b_o(self): + "-b o should produce build_types=['opt']" + tos = TryOptionSyntax('try: -b o', empty_graph) + self.assertEqual(sorted(tos.build_types), ['opt']) + + def test_build_o(self): + "--build o should produce build_types=['opt']" + tos = TryOptionSyntax('try: --build o', empty_graph) + self.assertEqual(sorted(tos.build_types), ['opt']) + + def test_b_dx(self): + "-b dx should produce build_types=['debug'], silently ignoring the x" + tos = TryOptionSyntax('try: -b dx', empty_graph) + self.assertEqual(sorted(tos.build_types), ['debug']) + + def test_j_job(self): + "-j somejob sets jobs=['somejob']" + tos = TryOptionSyntax('try: -j somejob', empty_graph) + self.assertEqual(sorted(tos.jobs), ['somejob']) + + def test_j_jobs(self): + "-j job1,job2 sets jobs=['job1', 'job2']" + tos = TryOptionSyntax('try: -j job1,job2', empty_graph) + self.assertEqual(sorted(tos.jobs), ['job1', 'job2']) + + def test_j_all(self): + "-j all sets jobs=None" + tos = TryOptionSyntax('try: -j all', empty_graph) + self.assertEqual(tos.jobs, None) + + def test_j_twice(self): + "-j job1 -j job2 sets jobs=job1, job2" + tos = TryOptionSyntax('try: -j job1 -j job2', empty_graph) + self.assertEqual(sorted(tos.jobs), sorted(['job1', 'job2'])) + + def test_p_all(self): + "-p all sets platforms=None" + tos = TryOptionSyntax('try: -p all', empty_graph) + self.assertEqual(tos.platforms, None) + + def test_p_linux(self): + "-p linux sets platforms=['linux', 'linux-l10n']" + tos = TryOptionSyntax('try: -p linux', empty_graph) + self.assertEqual(tos.platforms, ['linux', 'linux-l10n']) + + def test_p_linux_win32(self): + "-p linux,win32 sets platforms=['linux', 'linux-l10n', 'win32']" + tos = TryOptionSyntax('try: -p linux,win32', empty_graph) + self.assertEqual(sorted(tos.platforms), ['linux', 'linux-l10n', 'win32']) + + def test_p_expands_ridealongs(self): + "-p linux,linux64 includes the RIDEALONG_BUILDS" + tos = TryOptionSyntax('try: -p linux,linux64', empty_graph) + ridealongs = list(task + for task in itertools.chain.from_iterable( + RIDEALONG_BUILDS.itervalues() + ) + if 'android' not in task) # Don't include android-l10n + self.assertEqual(sorted(tos.platforms), sorted(['linux', 'linux64'] + ridealongs)) + + def test_u_none(self): + "-u none sets unittests=[]" + tos = TryOptionSyntax('try: -u none', graph_with_jobs) + self.assertEqual(sorted(tos.unittests), []) + + def test_u_all(self): + "-u all sets unittests=[..whole list..]" + tos = TryOptionSyntax('try: -u all', graph_with_jobs) + self.assertEqual(sorted(tos.unittests), sorted([{'test': t} for t in unittest_tasks])) + + def test_u_single(self): + "-u mochitest-webgl sets unittests=[mochitest-webgl]" + tos = TryOptionSyntax('try: -u mochitest-webgl', graph_with_jobs) + self.assertEqual(sorted(tos.unittests), sorted([{'test': 'mochitest-webgl'}])) + + def test_u_alias(self): + "-u mochitest-gl sets unittests=[mochitest-webgl]" + tos = TryOptionSyntax('try: -u mochitest-gl', graph_with_jobs) + self.assertEqual(sorted(tos.unittests), sorted([{'test': 'mochitest-webgl'}])) + + def test_u_multi_alias(self): + "-u e10s sets unittests=[all e10s unittests]" + tos = TryOptionSyntax('try: -u e10s', graph_with_jobs) + self.assertEqual(sorted(tos.unittests), sorted([ + {'test': t} for t in unittest_tasks if 'e10s' in t + ])) + + def test_u_commas(self): + "-u mochitest-webgl,gtest sets unittests=both" + tos = TryOptionSyntax('try: -u mochitest-webgl,gtest', graph_with_jobs) + self.assertEqual(sorted(tos.unittests), sorted([ + {'test': 'mochitest-webgl'}, + {'test': 'gtest'}, + ])) + + def test_u_chunks(self): + "-u gtest-3,gtest-4 selects the third and fourth chunk of gtest" + tos = TryOptionSyntax('try: -u gtest-3,gtest-4', graph_with_jobs) + self.assertEqual(sorted(tos.unittests), sorted([ + {'test': 'gtest', 'only_chunks': set('34')}, + ])) + + def test_u_platform(self): + "-u gtest[linux] selects the linux platform for gtest" + tos = TryOptionSyntax('try: -u gtest[linux]', graph_with_jobs) + self.assertEqual(sorted(tos.unittests), sorted([ + {'test': 'gtest', 'platforms': ['linux']}, + ])) + + def test_u_platforms(self): + "-u gtest[linux,win32] selects the linux and win32 platforms for gtest" + tos = TryOptionSyntax('try: -u gtest[linux,win32]', graph_with_jobs) + self.assertEqual(sorted(tos.unittests), sorted([ + {'test': 'gtest', 'platforms': ['linux', 'win32']}, + ])) + + def test_u_platforms_pretty(self): + "-u gtest[Ubuntu] selects the linux, linux64 and linux64-asan platforms for gtest" + tos = TryOptionSyntax('try: -u gtest[Ubuntu]', graph_with_jobs) + self.assertEqual(sorted(tos.unittests), sorted([ + {'test': 'gtest', 'platforms': ['linux', 'linux64', 'linux64-asan']}, + ])) + + def test_u_platforms_negated(self): + "-u gtest[-linux] selects all platforms but linux for gtest" + tos = TryOptionSyntax('try: -u gtest[-linux]', graph_with_jobs) + self.assertEqual(sorted(tos.unittests), sorted([ + {'test': 'gtest', 'platforms': ['linux64']}, + ])) + + def test_u_platforms_negated_pretty(self): + "-u gtest[Ubuntu,-x64] selects just linux for gtest" + tos = TryOptionSyntax('try: -u gtest[Ubuntu,-x64]', graph_with_jobs) + self.assertEqual(sorted(tos.unittests), sorted([ + {'test': 'gtest', 'platforms': ['linux']}, + ])) + + def test_u_chunks_platforms(self): + "-u gtest-1[linux,win32] selects the linux and win32 platforms for chunk 1 of gtest" + tos = TryOptionSyntax('try: -u gtest-1[linux,win32]', graph_with_jobs) + self.assertEqual(sorted(tos.unittests), sorted([ + {'test': 'gtest', 'platforms': ['linux', 'win32'], 'only_chunks': set('1')}, + ])) + + def test_t_none(self): + "-t none sets talos=[]" + tos = TryOptionSyntax('try: -t none', graph_with_jobs) + self.assertEqual(sorted(tos.talos), []) + + def test_t_all(self): + "-t all sets talos=[..whole list..]" + tos = TryOptionSyntax('try: -t all', graph_with_jobs) + self.assertEqual(sorted(tos.talos), sorted([{'test': t} for t in talos_tasks])) + + def test_t_single(self): + "-t mochitest-webgl sets talos=[mochitest-webgl]" + tos = TryOptionSyntax('try: -t mochitest-webgl', graph_with_jobs) + self.assertEqual(sorted(tos.talos), sorted([{'test': 'mochitest-webgl'}])) + + # -t shares an implementation with -u, so it's not tested heavily + + def test_trigger_tests(self): + "--rebuild 10 sets trigger_tests" + tos = TryOptionSyntax('try: --rebuild 10', empty_graph) + self.assertEqual(tos.trigger_tests, 10) + + def test_interactive(self): + "--interactive sets interactive" + tos = TryOptionSyntax('try: --interactive', empty_graph) + self.assertEqual(tos.interactive, True) + + def test_all_email(self): + "--all-emails sets notifications" + tos = TryOptionSyntax('try: --all-emails', empty_graph) + self.assertEqual(tos.notifications, 'all') + + def test_fail_email(self): + "--failure-emails sets notifications" + tos = TryOptionSyntax('try: --failure-emails', empty_graph) + self.assertEqual(tos.notifications, 'failure') + + def test_no_email(self): + "no email settings don't set notifications" + tos = TryOptionSyntax('try:', empty_graph) + self.assertEqual(tos.notifications, None) + +if __name__ == '__main__': + main() diff --git a/taskcluster/taskgraph/test/test_util_attributes.py b/taskcluster/taskgraph/test/test_util_attributes.py new file mode 100644 index 000000000..c3575e917 --- /dev/null +++ b/taskcluster/taskgraph/test/test_util_attributes.py @@ -0,0 +1,45 @@ +# -*- coding: utf-8 -*- + +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, You can obtain one at http://mozilla.org/MPL/2.0/. + +import unittest +from taskgraph.util.attributes import attrmatch + + +class Attrmatch(unittest.TestCase): + + def test_trivial_match(self): + """Given no conditions, anything matches""" + self.assertTrue(attrmatch({})) + + def test_missing_attribute(self): + """If a filtering attribute is not present, no match""" + self.assertFalse(attrmatch({}, someattr=10)) + + def test_literal_attribute(self): + """Literal attributes must match exactly""" + self.assertTrue(attrmatch({'att': 10}, att=10)) + self.assertFalse(attrmatch({'att': 10}, att=20)) + + def test_set_attribute(self): + """Set attributes require set membership""" + self.assertTrue(attrmatch({'att': 10}, att=set([9, 10]))) + self.assertFalse(attrmatch({'att': 10}, att=set([19, 20]))) + + def test_callable_attribute(self): + """Callable attributes are called and any False causes the match to fail""" + self.assertTrue(attrmatch({'att': 10}, att=lambda val: True)) + self.assertFalse(attrmatch({'att': 10}, att=lambda val: False)) + + def even(val): + return val % 2 == 0 + self.assertTrue(attrmatch({'att': 10}, att=even)) + self.assertFalse(attrmatch({'att': 11}, att=even)) + + def test_all_matches_required(self): + """If only one attribute does not match, the result is False""" + self.assertFalse(attrmatch({'a': 1}, a=1, b=2, c=3)) + self.assertFalse(attrmatch({'a': 1, 'b': 2}, a=1, b=2, c=3)) + self.assertTrue(attrmatch({'a': 1, 'b': 2, 'c': 3}, a=1, b=2, c=3)) diff --git a/taskcluster/taskgraph/test/test_util_docker.py b/taskcluster/taskgraph/test/test_util_docker.py new file mode 100644 index 000000000..3bb7fe8f0 --- /dev/null +++ b/taskcluster/taskgraph/test/test_util_docker.py @@ -0,0 +1,194 @@ +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, You can obtain one at http://mozilla.org/MPL/2.0/. + +from __future__ import absolute_import, print_function, unicode_literals + +import os +import shutil +import stat +import tarfile +import tempfile +import unittest + +from ..util import docker +from mozunit import MockedOpen + + +MODE_STANDARD = stat.S_IRUSR | stat.S_IWUSR | stat.S_IRGRP | stat.S_IROTH + + +class TestDocker(unittest.TestCase): + + def test_generate_context_hash(self): + tmpdir = tempfile.mkdtemp() + old_GECKO = docker.GECKO + docker.GECKO = tmpdir + try: + os.makedirs(os.path.join(tmpdir, 'docker', 'my-image')) + p = os.path.join(tmpdir, 'docker', 'my-image', 'Dockerfile') + with open(p, 'w') as f: + f.write("FROM node\nADD a-file\n") + os.chmod(p, MODE_STANDARD) + p = os.path.join(tmpdir, 'docker', 'my-image', 'a-file') + with open(p, 'w') as f: + f.write("data\n") + os.chmod(p, MODE_STANDARD) + self.assertEqual( + docker.generate_context_hash(docker.GECKO, + os.path.join(docker.GECKO, 'docker/my-image'), + 'my-image'), + 'e61e675ce05e8c11424437db3f1004079374c1a5fe6ad6800346cebe137b0797' + ) + finally: + docker.GECKO = old_GECKO + shutil.rmtree(tmpdir) + + def test_docker_image_explicit_registry(self): + files = {} + files["{}/myimage/REGISTRY".format(docker.DOCKER_ROOT)] = "cool-images" + files["{}/myimage/VERSION".format(docker.DOCKER_ROOT)] = "1.2.3" + with MockedOpen(files): + self.assertEqual(docker.docker_image('myimage'), "cool-images/myimage:1.2.3") + + def test_docker_image_default_registry(self): + files = {} + files["{}/REGISTRY".format(docker.DOCKER_ROOT)] = "mozilla" + files["{}/myimage/VERSION".format(docker.DOCKER_ROOT)] = "1.2.3" + with MockedOpen(files): + self.assertEqual(docker.docker_image('myimage'), "mozilla/myimage:1.2.3") + + def test_create_context_tar_basic(self): + tmp = tempfile.mkdtemp() + try: + d = os.path.join(tmp, 'test_image') + os.mkdir(d) + with open(os.path.join(d, 'Dockerfile'), 'a'): + pass + os.chmod(os.path.join(d, 'Dockerfile'), MODE_STANDARD) + + with open(os.path.join(d, 'extra'), 'a'): + pass + os.chmod(os.path.join(d, 'extra'), MODE_STANDARD) + + tp = os.path.join(tmp, 'tar') + h = docker.create_context_tar(tmp, d, tp, 'my_image') + self.assertEqual(h, '2a6d7f1627eba60daf85402418e041d728827d309143c6bc1c6bb3035bde6717') + + # File prefix should be "my_image" + with tarfile.open(tp, 'r:gz') as tf: + self.assertEqual(tf.getnames(), [ + 'my_image/Dockerfile', + 'my_image/extra', + ]) + finally: + shutil.rmtree(tmp) + + def test_create_context_topsrcdir_files(self): + tmp = tempfile.mkdtemp() + try: + d = os.path.join(tmp, 'test-image') + os.mkdir(d) + with open(os.path.join(d, 'Dockerfile'), 'wb') as fh: + fh.write(b'# %include extra/file0\n') + os.chmod(os.path.join(d, 'Dockerfile'), MODE_STANDARD) + + extra = os.path.join(tmp, 'extra') + os.mkdir(extra) + with open(os.path.join(extra, 'file0'), 'a'): + pass + os.chmod(os.path.join(extra, 'file0'), MODE_STANDARD) + + tp = os.path.join(tmp, 'tar') + h = docker.create_context_tar(tmp, d, tp, 'test_image') + self.assertEqual(h, '20faeb7c134f21187b142b5fadba94ae58865dc929c6c293d8cbc0a087269338') + + with tarfile.open(tp, 'r:gz') as tf: + self.assertEqual(tf.getnames(), [ + 'test_image/Dockerfile', + 'test_image/topsrcdir/extra/file0', + ]) + finally: + shutil.rmtree(tmp) + + def test_create_context_absolute_path(self): + tmp = tempfile.mkdtemp() + try: + d = os.path.join(tmp, 'test-image') + os.mkdir(d) + + # Absolute paths in %include syntax are not allowed. + with open(os.path.join(d, 'Dockerfile'), 'wb') as fh: + fh.write(b'# %include /etc/shadow\n') + + with self.assertRaisesRegexp(Exception, 'cannot be absolute'): + docker.create_context_tar(tmp, d, os.path.join(tmp, 'tar'), 'test') + finally: + shutil.rmtree(tmp) + + def test_create_context_outside_topsrcdir(self): + tmp = tempfile.mkdtemp() + try: + d = os.path.join(tmp, 'test-image') + os.mkdir(d) + + with open(os.path.join(d, 'Dockerfile'), 'wb') as fh: + fh.write(b'# %include foo/../../../etc/shadow\n') + + with self.assertRaisesRegexp(Exception, 'path outside topsrcdir'): + docker.create_context_tar(tmp, d, os.path.join(tmp, 'tar'), 'test') + finally: + shutil.rmtree(tmp) + + def test_create_context_missing_extra(self): + tmp = tempfile.mkdtemp() + try: + d = os.path.join(tmp, 'test-image') + os.mkdir(d) + + with open(os.path.join(d, 'Dockerfile'), 'wb') as fh: + fh.write(b'# %include does/not/exist\n') + + with self.assertRaisesRegexp(Exception, 'path does not exist'): + docker.create_context_tar(tmp, d, os.path.join(tmp, 'tar'), 'test') + finally: + shutil.rmtree(tmp) + + def test_create_context_extra_directory(self): + tmp = tempfile.mkdtemp() + try: + d = os.path.join(tmp, 'test-image') + os.mkdir(d) + + with open(os.path.join(d, 'Dockerfile'), 'wb') as fh: + fh.write(b'# %include extra\n') + fh.write(b'# %include file0\n') + os.chmod(os.path.join(d, 'Dockerfile'), MODE_STANDARD) + + extra = os.path.join(tmp, 'extra') + os.mkdir(extra) + for i in range(3): + p = os.path.join(extra, 'file%d' % i) + with open(p, 'wb') as fh: + fh.write(b'file%d' % i) + os.chmod(p, MODE_STANDARD) + + with open(os.path.join(tmp, 'file0'), 'a'): + pass + os.chmod(os.path.join(tmp, 'file0'), MODE_STANDARD) + + tp = os.path.join(tmp, 'tar') + h = docker.create_context_tar(tmp, d, tp, 'my_image') + + self.assertEqual(h, 'e5440513ab46ae4c1d056269e1c6715d5da7d4bd673719d360411e35e5b87205') + + with tarfile.open(tp, 'r:gz') as tf: + self.assertEqual(tf.getnames(), [ + 'my_image/Dockerfile', + 'my_image/topsrcdir/extra/file0', + 'my_image/topsrcdir/extra/file1', + 'my_image/topsrcdir/extra/file2', + 'my_image/topsrcdir/file0', + ]) + finally: + shutil.rmtree(tmp) diff --git a/taskcluster/taskgraph/test/test_util_python_path.py b/taskcluster/taskgraph/test/test_util_python_path.py new file mode 100644 index 000000000..9615d1347 --- /dev/null +++ b/taskcluster/taskgraph/test/test_util_python_path.py @@ -0,0 +1,31 @@ +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, You can obtain one at http://mozilla.org/MPL/2.0/. + +from __future__ import absolute_import, print_function, unicode_literals + +import unittest +from ..util import python_path + + +class TestObject(object): + + testClassProperty = object() + + +class TestPythonPath(unittest.TestCase): + + def test_find_object_no_such_module(self): + """find_object raises ImportError for a nonexistent module""" + self.assertRaises(ImportError, python_path.find_object, "no_such_module:someobj") + + def test_find_object_no_such_object(self): + """find_object raises AttributeError for a nonexistent object""" + self.assertRaises(AttributeError, python_path.find_object, + "taskgraph.test.test_util_python_path:NoSuchObject") + + def test_find_object_exists(self): + """find_object finds an existing object""" + obj = python_path.find_object( + "taskgraph.test.test_util_python_path:TestObject.testClassProperty") + self.assertIs(obj, TestObject.testClassProperty) diff --git a/taskcluster/taskgraph/test/test_util_templates.py b/taskcluster/taskgraph/test/test_util_templates.py new file mode 100755 index 000000000..47f7494a0 --- /dev/null +++ b/taskcluster/taskgraph/test/test_util_templates.py @@ -0,0 +1,232 @@ +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, You can obtain one at http://mozilla.org/MPL/2.0/. + +from __future__ import absolute_import, print_function, unicode_literals + +import unittest +import mozunit +import textwrap +from taskgraph.util.templates import ( + merge_to, + merge, + Templates, + TemplatesException +) + +files = {} +files['/fixtures/circular.yml'] = textwrap.dedent("""\ + $inherits: + from: 'circular_ref.yml' + variables: + woot: 'inherit' + """) + +files['/fixtures/inherit.yml'] = textwrap.dedent("""\ + $inherits: + from: 'templates.yml' + variables: + woot: 'inherit' + """) + +files['/fixtures/extend_child.yml'] = textwrap.dedent("""\ + list: ['1', '2', '3'] + was_list: ['1'] + obj: + level: 1 + deeper: + woot: 'bar' + list: ['baz'] + """) + +files['/fixtures/circular_ref.yml'] = textwrap.dedent("""\ + $inherits: + from: 'circular.yml' + """) + +files['/fixtures/child_pass.yml'] = textwrap.dedent("""\ + values: + - {{a}} + - {{b}} + - {{c}} + """) + +files['/fixtures/inherit_pass.yml'] = textwrap.dedent("""\ + $inherits: + from: 'child_pass.yml' + variables: + a: 'a' + b: 'b' + c: 'c' + """) + +files['/fixtures/deep/2.yml'] = textwrap.dedent("""\ + $inherits: + from: deep/1.yml + + """) + +files['/fixtures/deep/3.yml'] = textwrap.dedent("""\ + $inherits: + from: deep/2.yml + + """) + +files['/fixtures/deep/4.yml'] = textwrap.dedent("""\ + $inherits: + from: deep/3.yml + """) + +files['/fixtures/deep/1.yml'] = textwrap.dedent("""\ + variable: {{value}} + """) + +files['/fixtures/simple.yml'] = textwrap.dedent("""\ + is_simple: true + """) + +files['/fixtures/templates.yml'] = textwrap.dedent("""\ + content: 'content' + variable: '{{woot}}' + """) + +files['/fixtures/extend_parent.yml'] = textwrap.dedent("""\ + $inherits: + from: 'extend_child.yml' + + list: ['4'] + was_list: + replaced: true + obj: + level: 2 + from_parent: true + deeper: + list: ['bar'] + """) + + +class TemplatesTest(unittest.TestCase): + + def setUp(self): + self.mocked_open = mozunit.MockedOpen(files) + self.mocked_open.__enter__() + self.subject = Templates('/fixtures') + + def tearDown(self): + self.mocked_open.__exit__(None, None, None) + + def test_invalid_path(self): + with self.assertRaisesRegexp(TemplatesException, 'must be a directory'): + Templates('/zomg/not/a/dir') + + def test_no_templates(self): + content = self.subject.load('simple.yml', {}) + self.assertEquals(content, { + 'is_simple': True + }) + + def test_with_templates(self): + content = self.subject.load('templates.yml', { + 'woot': 'bar' + }) + + self.assertEquals(content, { + 'content': 'content', + 'variable': 'bar' + }) + + def test_inheritance(self): + ''' + The simple single pass inheritance case. + ''' + content = self.subject.load('inherit.yml', {}) + self.assertEqual(content, { + 'content': 'content', + 'variable': 'inherit' + }) + + def test_inheritance_implicat_pass(self): + ''' + Implicitly pass parameters from the child to the ancestor. + ''' + content = self.subject.load('inherit_pass.yml', { + 'a': 'overriden' + }) + + self.assertEqual(content, {'values': ['overriden', 'b', 'c']}) + + def test_inheritance_circular(self): + ''' + Circular reference handling. + ''' + with self.assertRaisesRegexp(TemplatesException, 'circular'): + self.subject.load('circular.yml', {}) + + def test_deep_inheritance(self): + content = self.subject.load('deep/4.yml', { + 'value': 'myvalue' + }) + self.assertEqual(content, {'variable': 'myvalue'}) + + def test_inheritance_with_simple_extensions(self): + content = self.subject.load('extend_parent.yml', {}) + self.assertEquals(content, { + 'list': ['1', '2', '3', '4'], + 'obj': { + 'from_parent': True, + 'deeper': { + 'woot': 'bar', + 'list': ['baz', 'bar'] + }, + 'level': 2, + }, + 'was_list': {'replaced': True} + }) + + +class MergeTest(unittest.TestCase): + + def test_merge_to_dicts(self): + source = {'a': 1, 'b': 2} + dest = {'b': '20', 'c': 30} + expected = { + 'a': 1, # source only + 'b': 2, # source overrides dest + 'c': 30, # dest only + } + self.assertEqual(merge_to(source, dest), expected) + self.assertEqual(dest, expected) + + def test_merge_to_lists(self): + source = {'x': [3, 4]} + dest = {'x': [1, 2]} + expected = {'x': [1, 2, 3, 4]} # dest first + self.assertEqual(merge_to(source, dest), expected) + self.assertEqual(dest, expected) + + def test_merge_diff_types(self): + source = {'x': [1, 2]} + dest = {'x': 'abc'} + expected = {'x': [1, 2]} # source wins + self.assertEqual(merge_to(source, dest), expected) + self.assertEqual(dest, expected) + + def test_merge(self): + first = {'a': 1, 'b': 2, 'd': 11} + second = {'b': 20, 'c': 30} + third = {'c': 300, 'd': 400} + expected = { + 'a': 1, + 'b': 20, + 'c': 300, + 'd': 400, + } + self.assertEqual(merge(first, second, third), expected) + + # inputs haven't changed.. + self.assertEqual(first, {'a': 1, 'b': 2, 'd': 11}) + self.assertEqual(second, {'b': 20, 'c': 30}) + self.assertEqual(third, {'c': 300, 'd': 400}) + +if __name__ == '__main__': + mozunit.main() diff --git a/taskcluster/taskgraph/test/test_util_time.py b/taskcluster/taskgraph/test/test_util_time.py new file mode 100755 index 000000000..f001c9d9c --- /dev/null +++ b/taskcluster/taskgraph/test/test_util_time.py @@ -0,0 +1,57 @@ +# -*- coding: utf-8 -*- + +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, You can obtain one at http://mozilla.org/MPL/2.0/. + +import unittest +import mozunit +from datetime import datetime +from taskgraph.util.time import ( + InvalidString, + UnknownTimeMeasurement, + value_of, + json_time_from_now +) + + +class FromNowTest(unittest.TestCase): + + def test_invalid_str(self): + with self.assertRaises(InvalidString): + value_of('wtfs') + + def test_missing_unit(self): + with self.assertRaises(InvalidString): + value_of('1') + + def test_missing_unknown_unit(self): + with self.assertRaises(UnknownTimeMeasurement): + value_of('1z') + + def test_value_of(self): + self.assertEqual(value_of('1s').total_seconds(), 1) + self.assertEqual(value_of('1 second').total_seconds(), 1) + self.assertEqual(value_of('1min').total_seconds(), 60) + self.assertEqual(value_of('1h').total_seconds(), 3600) + self.assertEqual(value_of('1d').total_seconds(), 86400) + self.assertEqual(value_of('1mo').total_seconds(), 2592000) + self.assertEqual(value_of('1 month').total_seconds(), 2592000) + self.assertEqual(value_of('1y').total_seconds(), 31536000) + + with self.assertRaises(UnknownTimeMeasurement): + value_of('1m').total_seconds() # ambiguous between minute and month + + def test_json_from_now_utc_now(self): + # Just here to ensure we don't raise. + json_time_from_now('1 years') + + def test_json_from_now(self): + now = datetime(2014, 1, 1) + self.assertEqual(json_time_from_now('1 years', now), + '2015-01-01T00:00:00Z') + self.assertEqual(json_time_from_now('6 days', now), + '2014-01-07T00:00:00Z') + +if __name__ == '__main__': + mozunit.main() diff --git a/taskcluster/taskgraph/test/test_util_treeherder.py b/taskcluster/taskgraph/test/test_util_treeherder.py new file mode 100644 index 000000000..cf7513c00 --- /dev/null +++ b/taskcluster/taskgraph/test/test_util_treeherder.py @@ -0,0 +1,23 @@ +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, You can obtain one at http://mozilla.org/MPL/2.0/. + +from __future__ import absolute_import, print_function, unicode_literals + +import unittest +from taskgraph.util.treeherder import split_symbol, join_symbol + + +class TestSymbols(unittest.TestCase): + + def test_split_no_group(self): + self.assertEqual(split_symbol('xy'), ('?', 'xy')) + + def test_split_with_group(self): + self.assertEqual(split_symbol('ab(xy)'), ('ab', 'xy')) + + def test_join_no_group(self): + self.assertEqual(join_symbol('?', 'xy'), 'xy') + + def test_join_with_group(self): + self.assertEqual(join_symbol('ab', 'xy'), 'ab(xy)') diff --git a/taskcluster/taskgraph/test/test_util_yaml.py b/taskcluster/taskgraph/test/test_util_yaml.py new file mode 100644 index 000000000..d4ff410db --- /dev/null +++ b/taskcluster/taskgraph/test/test_util_yaml.py @@ -0,0 +1,23 @@ +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, You can obtain one at http://mozilla.org/MPL/2.0/. + +from __future__ import absolute_import, print_function, unicode_literals + +import unittest + +from ..util import yaml +from mozunit import MockedOpen + +FOO_YML = """\ +prop: + - val1 +""" + + +class TestYaml(unittest.TestCase): + + def test_load(self): + with MockedOpen({'/dir1/dir2/foo.yml': FOO_YML}): + self.assertEqual(yaml.load_yaml("/dir1/dir2", "foo.yml"), + {'prop': ['val1']}) diff --git a/taskcluster/taskgraph/test/util.py b/taskcluster/taskgraph/test/util.py new file mode 100644 index 000000000..cf9a49ad3 --- /dev/null +++ b/taskcluster/taskgraph/test/util.py @@ -0,0 +1,24 @@ +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, You can obtain one at http://mozilla.org/MPL/2.0/. + +from __future__ import absolute_import, print_function, unicode_literals + +from ..task import base + + +class TestTask(base.Task): + + def __init__(self, kind=None, label=None, attributes=None, task=None): + super(TestTask, self).__init__( + kind or 'test', + label or 'test-label', + attributes or {}, + task or {}) + + @classmethod + def load_tasks(cls, kind, path, config, parameters): + return [] + + def get_dependencies(self, taskgraph): + return [] diff --git a/taskcluster/taskgraph/transforms/__init__.py b/taskcluster/taskgraph/transforms/__init__.py new file mode 100644 index 000000000..e69de29bb --- /dev/null +++ b/taskcluster/taskgraph/transforms/__init__.py diff --git a/taskcluster/taskgraph/transforms/android_stuff.py b/taskcluster/taskgraph/transforms/android_stuff.py new file mode 100644 index 000000000..cb1e0fa5b --- /dev/null +++ b/taskcluster/taskgraph/transforms/android_stuff.py @@ -0,0 +1,46 @@ +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, You can obtain one at http://mozilla.org/MPL/2.0/. +""" +Set dynamic task description properties of the android stuff. Temporary! +""" + +from __future__ import absolute_import, print_function, unicode_literals + +from taskgraph.transforms.base import TransformSequence + +transforms = TransformSequence() + + +@transforms.add +def setup_task(config, tasks): + for task in tasks: + task['label'] = task['name'] + env = task['worker'].setdefault('env', {}) + env.update({ + 'GECKO_BASE_REPOSITORY': config.params['base_repository'], + 'GECKO_HEAD_REF': config.params['head_rev'], + 'GECKO_HEAD_REPOSITORY': config.params['head_repository'], + 'GECKO_HEAD_REV': config.params['head_rev'], + 'MOZ_BUILD_DATE': config.params['moz_build_date'], + 'MOZ_SCM_LEVEL': config.params['level'], + 'MH_BRANCH': config.params['project'], + }) + + task['worker'].setdefault('caches', []).append({ + 'type': 'persistent', + 'name': 'level-{}-{}-tc-vcs'.format( + config.params['level'], config.params['project']), + 'mount-point': "/home/worker/.tc-vcs", + }) + + if int(config.params['level']) > 1: + task['worker'].setdefault('caches', []).append({ + 'type': 'persistent', + 'name': 'level-{}-{}-build-{}-workspace'.format( + config.params['level'], config.params['project'], task['name']), + 'mount-point': "/home/worker/workspace", + }) + + del task['name'] + yield task diff --git a/taskcluster/taskgraph/transforms/base.py b/taskcluster/taskgraph/transforms/base.py new file mode 100644 index 000000000..aab139252 --- /dev/null +++ b/taskcluster/taskgraph/transforms/base.py @@ -0,0 +1,126 @@ +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, You can obtain one at http://mozilla.org/MPL/2.0/. + +from __future__ import absolute_import, print_function, unicode_literals + +import re +import pprint +import voluptuous + + +class TransformConfig(object): + """A container for configuration affecting transforms. The `config` + argument to transforms is an instance of this class, possibly with + additional kind-specific attributes beyond those set here.""" + def __init__(self, kind, path, config, params): + # the name of the current kind + self.kind = kind + + # the path to the kind configuration directory + self.path = path + + # the parsed contents of kind.yml + self.config = config + + # the parameters for this task-graph generation run + self.params = params + + +class TransformSequence(object): + """ + Container for a sequence of transforms. Each transform is represented as a + callable taking (config, items) and returning a generator which will yield + transformed items. The resulting sequence has the same interface. + + This is convenient to use in a file full of transforms, as it provides a + decorator, @transforms.add, that will add the decorated function to the + sequence. + """ + + def __init__(self, transforms=None): + self.transforms = transforms or [] + + def __call__(self, config, items): + for xform in self.transforms: + items = xform(config, items) + if items is None: + raise Exception("Transform {} is not a generator".format(xform)) + return items + + def __repr__(self): + return '\n'.join( + ['TransformSequence(['] + + [repr(x) for x in self.transforms] + + ['])']) + + def add(self, func): + self.transforms.append(func) + return func + + +def validate_schema(schema, obj, msg_prefix): + """ + Validate that object satisfies schema. If not, generate a useful exception + beginning with msg_prefix. + """ + try: + return schema(obj) + except voluptuous.MultipleInvalid as exc: + msg = [msg_prefix] + for error in exc.errors: + msg.append(str(error)) + raise Exception('\n'.join(msg) + '\n' + pprint.pformat(obj)) + + +def get_keyed_by(item, field, item_name, subfield=None): + """ + For values which can either accept a literal value, or be keyed by some + other attribute of the item, perform that lookup. For example, this supports + + chunks: + by-test-platform: + macosx-10.11/debug: 13 + win.*: 6 + default: 12 + + The `item_name` parameter is used to generate useful error messages. + The `subfield` parameter, if specified, allows access to a second level + of the item dictionary: item[field][subfield]. For example, this supports + + mozharness: + config: + by-test-platform: + default: ... + """ + value = item[field] + if not isinstance(value, dict): + return value + if subfield: + value = item[field][subfield] + if not isinstance(value, dict): + return value + + assert len(value) == 1, "Invalid attribute {} in {}".format(field, item_name) + keyed_by = value.keys()[0] + values = value[keyed_by] + if keyed_by.startswith('by-'): + keyed_by = keyed_by[3:] # extract just the keyed-by field name + if item[keyed_by] in values: + return values[item[keyed_by]] + for k in values.keys(): + if re.match(k, item[keyed_by]): + return values[k] + if 'default' in values: + return values['default'] + for k in item[keyed_by], 'default': + if k in values: + return values[k] + else: + raise Exception( + "Neither {} {} nor 'default' found while determining item {} in {}".format( + keyed_by, item[keyed_by], field, item_name)) + else: + raise Exception( + "Invalid attribute {} keyed-by value {} in {}".format( + field, keyed_by, item_name)) diff --git a/taskcluster/taskgraph/transforms/build.py b/taskcluster/taskgraph/transforms/build.py new file mode 100644 index 000000000..3875cbbb1 --- /dev/null +++ b/taskcluster/taskgraph/transforms/build.py @@ -0,0 +1,31 @@ +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, You can obtain one at http://mozilla.org/MPL/2.0/. +""" +Apply some defaults and minor modifications to the jobs defined in the build +kind. +""" + +from __future__ import absolute_import, print_function, unicode_literals + +from taskgraph.transforms.base import TransformSequence + +transforms = TransformSequence() + + +@transforms.add +def set_defaults(config, jobs): + """Set defaults, including those that differ per worker implementation""" + for job in jobs: + job['treeherder'].setdefault('kind', 'build') + job['treeherder'].setdefault('tier', 1) + if job['worker']['implementation'] in ('docker-worker', 'docker-engine'): + job['worker'].setdefault('docker-image', {'in-tree': 'desktop-build'}) + job['worker']['chain-of-trust'] = True + job.setdefault('extra', {}) + job['extra'].setdefault('chainOfTrust', {}) + job['extra']['chainOfTrust'].setdefault('inputs', {}) + job['extra']['chainOfTrust']['inputs']['docker-image'] = { + "task-reference": "<docker-image>" + } + yield job diff --git a/taskcluster/taskgraph/transforms/build_attrs.py b/taskcluster/taskgraph/transforms/build_attrs.py new file mode 100644 index 000000000..56c007614 --- /dev/null +++ b/taskcluster/taskgraph/transforms/build_attrs.py @@ -0,0 +1,33 @@ +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, You can obtain one at http://mozilla.org/MPL/2.0/. +from __future__ import absolute_import, print_function, unicode_literals + +from taskgraph.transforms.base import TransformSequence + +transforms = TransformSequence() + + +@transforms.add +def set_build_attributes(config, jobs): + """ + Set the build_platform and build_type attributes based on the job name. + Although not all jobs using this transform are actual "builds", the try + option syntax treats them as such, and this arranges the attributes + appropriately for that purpose. + """ + for job in jobs: + build_platform, build_type = job['name'].split('/') + + # pgo builds are represented as a different platform, type opt + if build_type == 'pgo': + build_platform = build_platform + '-pgo' + build_type = 'opt' + + attributes = job.setdefault('attributes', {}) + attributes.update({ + 'build_platform': build_platform, + 'build_type': build_type, + }) + + yield job diff --git a/taskcluster/taskgraph/transforms/gecko_v2_whitelist.py b/taskcluster/taskgraph/transforms/gecko_v2_whitelist.py new file mode 100644 index 000000000..3817faa50 --- /dev/null +++ b/taskcluster/taskgraph/transforms/gecko_v2_whitelist.py @@ -0,0 +1,77 @@ +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, You can obtain one at http://mozilla.org/MPL/2.0/. +""" +This file contains a whitelist of gecko.v2 index route job names. The intent +of this whitelist is to raise an alarm when new jobs are added. If those jobs +already run in Buildbot, then it's important that the generated index routes +match (and that only one of Buildbot and TaskCluster be tier-1 at any time). +If the jobs are new and never ran in Buildbot, then their job name can be added +here without any further fuss. + +Once all jobs have been ported from Buildbot, this file can be removed. +""" + +from __future__ import absolute_import, print_function, unicode_literals + +# please keep me in lexical order +JOB_NAME_WHITELIST = set([ + 'android-api-15-debug', + 'android-api-15-gradle-dependencies-opt', + 'android-api-15-gradle-opt', + 'android-api-15-opt', + 'android-api-15-nightly-opt', + 'android-api-15-partner-sample1-opt', + 'android-l10n-opt', + 'android-x86-opt', + 'aries-debug', + 'aries-eng-opt', + 'browser-haz-debug', + 'linux32-l10n-opt', + 'linux64-artifact-opt', + 'linux64-asan-debug', + 'linux64-asan-opt', + 'linux64-ccov-opt', + 'linux64-debug', + 'linux64-jsdcov-opt', + 'linux64-l10n-opt', + 'linux64-opt', + 'linux64-pgo', + 'linux64-st-an-opt', + 'linux64-valgrind-opt', + 'linux-debug', + 'linux-opt', + 'linux-pgo', + 'macosx64-debug', + 'macosx64-opt', + 'macosx64-st-an-opt', + 'nexus-5-l-eng-debug', + 'nexus-5-l-eng-opt', + 'osx-10-10', + 'shell-haz-debug', + 'sm-arm64-sim-debug', + 'sm-arm-sim-debug', + 'sm-asan-opt', + 'sm-compacting-debug', + 'sm-mozjs-sys-debug', + 'sm-msan-opt', + 'sm-nonunified-debug', + 'sm-package-opt', + 'sm-plaindebug-debug', + 'sm-plain-opt', + 'sm-rootanalysis-debug', + 'sm-tsan-opt', + 'win32-debug', + 'win32-opt', + 'win32-pgo', + 'win64-debug', + 'win64-opt', + 'win64-pgo', +]) + +JOB_NAME_WHITELIST_ERROR = """\ +The gecko-v2 job name {} is not in the whitelist in __file__. +If this job runs on Buildbot, please ensure that the job names match between +Buildbot and TaskCluster, then add the job name to the whitelist. If this is a +new job, there is nothing to check -- just add the job to the whitelist. +""" diff --git a/taskcluster/taskgraph/transforms/job/__init__.py b/taskcluster/taskgraph/transforms/job/__init__.py new file mode 100644 index 000000000..a0860c032 --- /dev/null +++ b/taskcluster/taskgraph/transforms/job/__init__.py @@ -0,0 +1,164 @@ +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, You can obtain one at http://mozilla.org/MPL/2.0/. +""" +Convert a job description into a task description. + +Jobs descriptions are similar to task descriptions, but they specify how to run +the job at a higher level, using a "run" field that can be interpreted by +run-using handlers in `taskcluster/taskgraph/transforms/job`. +""" + +from __future__ import absolute_import, print_function, unicode_literals + +import copy +import logging +import os + +from taskgraph.transforms.base import validate_schema, TransformSequence +from taskgraph.transforms.task import task_description_schema +from voluptuous import ( + Optional, + Required, + Schema, + Extra, +) + +logger = logging.getLogger(__name__) + +# Voluptuous uses marker objects as dictionary *keys*, but they are not +# comparable, so we cast all of the keys back to regular strings +task_description_schema = {str(k): v for k, v in task_description_schema.schema.iteritems()} + +# Schema for a build description +job_description_schema = Schema({ + # The name of the job and the job's label. At least one must be specified, + # and the label will be generated from the name if necessary, by prepending + # the kind. + Optional('name'): basestring, + Optional('label'): basestring, + + # the following fields are passed directly through to the task description, + # possibly modified by the run implementation. See + # taskcluster/taskgraph/transforms/task.py for the schema details. + Required('description'): task_description_schema['description'], + Optional('attributes'): task_description_schema['attributes'], + Optional('dependencies'): task_description_schema['dependencies'], + Optional('expires-after'): task_description_schema['expires-after'], + Optional('routes'): task_description_schema['routes'], + Optional('scopes'): task_description_schema['scopes'], + Optional('extra'): task_description_schema['extra'], + Optional('treeherder'): task_description_schema['treeherder'], + Optional('index'): task_description_schema['index'], + Optional('run-on-projects'): task_description_schema['run-on-projects'], + Optional('coalesce-name'): task_description_schema['coalesce-name'], + Optional('worker-type'): task_description_schema['worker-type'], + Required('worker'): task_description_schema['worker'], + Optional('when'): task_description_schema['when'], + + # A description of how to run this job. + 'run': { + # The key to a job implementation in a peer module to this one + 'using': basestring, + + # Any remaining content is verified against that job implementation's + # own schema. + Extra: object, + }, +}) + +transforms = TransformSequence() + + +@transforms.add +def validate(config, jobs): + for job in jobs: + yield validate_schema(job_description_schema, job, + "In job {!r}:".format(job['name'])) + + +@transforms.add +def make_task_description(config, jobs): + """Given a build description, create a task description""" + # import plugin modules first, before iterating over jobs + import_all() + for job in jobs: + if 'label' not in job: + if 'name' not in job: + raise Exception("job has neither a name nor a label") + job['label'] = '{}-{}'.format(config.kind, job['name']) + if job['name']: + del job['name'] + + taskdesc = copy.deepcopy(job) + + # fill in some empty defaults to make run implementations easier + taskdesc.setdefault('attributes', {}) + taskdesc.setdefault('dependencies', {}) + taskdesc.setdefault('routes', []) + taskdesc.setdefault('scopes', []) + taskdesc.setdefault('extra', {}) + + # give the function for job.run.using on this worker implementation a + # chance to set up the task description. + configure_taskdesc_for_run(config, job, taskdesc) + del taskdesc['run'] + + # yield only the task description, discarding the job description + yield taskdesc + +# A registry of all functions decorated with run_job_using +registry = {} + + +def run_job_using(worker_implementation, run_using, schema=None): + """Register the decorated function as able to set up a task description for + jobs with the given worker implementation and `run.using` property. If + `schema` is given, the job's run field will be verified to match it. + + The decorated function should have the signature `using_foo(config, job, + taskdesc) and should modify the task description in-place. The skeleton of + the task description is already set up, but without a payload.""" + def wrap(func): + for_run_using = registry.setdefault(run_using, {}) + if worker_implementation in for_run_using: + raise Exception("run_job_using({!r}, {!r}) already exists: {!r}".format( + run_using, worker_implementation, for_run_using[run_using])) + for_run_using[worker_implementation] = (func, schema) + return func + return wrap + + +def configure_taskdesc_for_run(config, job, taskdesc): + """ + Run the appropriate function for this job against the given task + description. + + This will raise an appropriate error if no function exists, or if the job's + run is not valid according to the schema. + """ + run_using = job['run']['using'] + if run_using not in registry: + raise Exception("no functions for run.using {!r}".format(run_using)) + + worker_implementation = job['worker']['implementation'] + if worker_implementation not in registry[run_using]: + raise Exception("no functions for run.using {!r} on {!r}".format( + run_using, worker_implementation)) + + func, schema = registry[run_using][worker_implementation] + if schema: + job['run'] = validate_schema( + schema, job['run'], + "In job.run using {!r} for job {!r}:".format( + job['run']['using'], job['label'])) + + func(config, job, taskdesc) + + +def import_all(): + """Import all modules that are siblings of this one, triggering the decorator + above in the process.""" + for f in os.listdir(os.path.dirname(__file__)): + if f.endswith('.py') and f not in ('commmon.py', '__init__.py'): + __import__('taskgraph.transforms.job.' + f[:-3]) diff --git a/taskcluster/taskgraph/transforms/job/common.py b/taskcluster/taskgraph/transforms/job/common.py new file mode 100644 index 000000000..59a51d75a --- /dev/null +++ b/taskcluster/taskgraph/transforms/job/common.py @@ -0,0 +1,108 @@ +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, You can obtain one at http://mozilla.org/MPL/2.0/. +""" +Common support for various job types. These functions are all named after the +worker implementation they operate on, and take the same three parameters, for +consistency. +""" + +from __future__ import absolute_import, print_function, unicode_literals + +SECRET_SCOPE = 'secrets:get:project/releng/gecko/{}/level-{}/{}' + + +def docker_worker_add_workspace_cache(config, job, taskdesc): + """Add the workspace cache based on the build platform/type and level, + except on try where workspace caches are not used.""" + if config.params['project'] == 'try': + return + + taskdesc['worker'].setdefault('caches', []).append({ + 'type': 'persistent', + 'name': 'level-{}-{}-build-{}-{}-workspace'.format( + config.params['level'], config.params['project'], + taskdesc['attributes']['build_platform'], + taskdesc['attributes']['build_type'], + ), + 'mount-point': "/home/worker/workspace", + }) + + +def docker_worker_add_tc_vcs_cache(config, job, taskdesc): + taskdesc['worker'].setdefault('caches', []).append({ + 'type': 'persistent', + 'name': 'level-{}-{}-tc-vcs'.format( + config.params['level'], config.params['project']), + 'mount-point': "/home/worker/.tc-vcs", + }) + + +def docker_worker_add_public_artifacts(config, job, taskdesc): + taskdesc['worker'].setdefault('artifacts', []).append({ + 'name': 'public/build', + 'path': '/home/worker/artifacts/', + 'type': 'directory', + }) + + +def docker_worker_add_gecko_vcs_env_vars(config, job, taskdesc): + """Add the GECKO_BASE_* and GECKO_HEAD_* env vars to the worker.""" + env = taskdesc['worker'].setdefault('env', {}) + env.update({ + 'GECKO_BASE_REPOSITORY': config.params['base_repository'], + 'GECKO_HEAD_REF': config.params['head_rev'], + 'GECKO_HEAD_REPOSITORY': config.params['head_repository'], + 'GECKO_HEAD_REV': config.params['head_rev'], + }) + + +def docker_worker_support_vcs_checkout(config, job, taskdesc): + """Update a job/task with parameters to enable a VCS checkout. + + The configuration is intended for tasks using "run-task" and its + VCS checkout behavior. + """ + level = config.params['level'] + + taskdesc['worker'].setdefault('caches', []).append({ + 'type': 'persistent', + # History of versions: + # + # ``level-%s-checkouts`` was initially used and contained a number + # of backwards incompatible changes, such as moving HG_STORE_PATH + # from a separate cache to this cache. + # + # ``v1`` was introduced to provide a clean break from the unversioned + # cache. + 'name': 'level-%s-checkouts-v1' % level, + 'mount-point': '/home/worker/checkouts', + }) + + taskdesc['worker'].setdefault('env', {}).update({ + 'GECKO_BASE_REPOSITORY': config.params['base_repository'], + 'GECKO_HEAD_REPOSITORY': config.params['head_repository'], + 'GECKO_HEAD_REV': config.params['head_rev'], + 'HG_STORE_PATH': '/home/worker/checkouts/hg-store', + }) + + # Give task access to hgfingerprint secret so it can pin the certificate + # for hg.mozilla.org. + taskdesc['scopes'].append('secrets:get:project/taskcluster/gecko/hgfingerprint') + taskdesc['worker']['taskcluster-proxy'] = True + + +def docker_worker_setup_secrets(config, job, taskdesc): + """Set up access to secrets via taskcluster-proxy. The value of + run['secrets'] should be a boolean or a list of secret names that + can be accessed.""" + if not job['run'].get('secrets'): + return + + taskdesc['worker']['taskcluster-proxy'] = True + secrets = job['run']['secrets'] + if secrets is True: + secrets = ['*'] + for sec in secrets: + taskdesc['scopes'].append(SECRET_SCOPE.format( + job['treeherder']['kind'], config.params['level'], sec)) diff --git a/taskcluster/taskgraph/transforms/job/hazard.py b/taskcluster/taskgraph/transforms/job/hazard.py new file mode 100644 index 000000000..c5b500843 --- /dev/null +++ b/taskcluster/taskgraph/transforms/job/hazard.py @@ -0,0 +1,91 @@ +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, You can obtain one at http://mozilla.org/MPL/2.0/. +""" +Support for running hazard jobs via dedicated scripts +""" + +from __future__ import absolute_import, print_function, unicode_literals + +from voluptuous import Schema, Required, Optional, Any + +from taskgraph.transforms.job import run_job_using +from taskgraph.transforms.job.common import ( + docker_worker_add_workspace_cache, + docker_worker_setup_secrets, + docker_worker_add_public_artifacts, + docker_worker_support_vcs_checkout, +) + +haz_run_schema = Schema({ + Required('using'): 'hazard', + + # The command to run within the task image (passed through to the worker) + Required('command'): basestring, + + # The tooltool manifest to use; default in the script is used if omitted + Optional('tooltool-manifest'): basestring, + + # The mozconfig to use; default in the script is used if omitted + Optional('mozconfig'): basestring, + + # The set of secret names to which the task has access; these are prefixed + # with `project/releng/gecko/{treeherder.kind}/level-{level}/`. Setting + # this will enable any worker features required and set the task's scopes + # appropriately. `true` here means ['*'], all secrets. Not supported on + # Windows + Required('secrets', default=False): Any(bool, [basestring]), +}) + + +@run_job_using("docker-worker", "hazard", schema=haz_run_schema) +def docker_worker_hazard(config, job, taskdesc): + run = job['run'] + + worker = taskdesc['worker'] + worker['artifacts'] = [] + worker['caches'] = [] + + docker_worker_add_public_artifacts(config, job, taskdesc) + docker_worker_add_workspace_cache(config, job, taskdesc) + docker_worker_setup_secrets(config, job, taskdesc) + docker_worker_support_vcs_checkout(config, job, taskdesc) + + env = worker['env'] + env.update({ + 'MOZ_BUILD_DATE': config.params['moz_build_date'], + 'MOZ_SCM_LEVEL': config.params['level'], + }) + + # script parameters + if run.get('tooltool-manifest'): + env['TOOLTOOL_MANIFEST'] = run['tooltool-manifest'] + if run.get('mozconfig'): + env['MOZCONFIG'] = run['mozconfig'] + + # tooltool downloads + worker['caches'].append({ + 'type': 'persistent', + 'name': 'tooltool-cache', + 'mount-point': '/home/worker/tooltool-cache', + }) + worker['relengapi-proxy'] = True + taskdesc['scopes'].extend([ + 'docker-worker:relengapi-proxy:tooltool.download.public', + ]) + env['TOOLTOOL_CACHE'] = '/home/worker/tooltool-cache' + env['TOOLTOOL_REPO'] = 'https://github.com/mozilla/build-tooltool' + env['TOOLTOOL_REV'] = 'master' + + # build-haz-linux.sh needs this otherwise it assumes the checkout is in + # the workspace. + env['GECKO_DIR'] = '/home/worker/checkouts/gecko' + + worker['command'] = [ + '/home/worker/bin/run-task', + '--chown-recursive', '/home/worker/tooltool-cache', + '--chown-recursive', '/home/worker/workspace', + '--vcs-checkout', '/home/worker/checkouts/gecko', + '--', + '/bin/bash', '-c', run['command'] + ] diff --git a/taskcluster/taskgraph/transforms/job/mach.py b/taskcluster/taskgraph/transforms/job/mach.py new file mode 100644 index 000000000..8df202dbc --- /dev/null +++ b/taskcluster/taskgraph/transforms/job/mach.py @@ -0,0 +1,30 @@ +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, You can obtain one at http://mozilla.org/MPL/2.0/. +""" +Support for running mach tasks (via run-task) +""" + +from __future__ import absolute_import, print_function, unicode_literals + +from taskgraph.transforms.job import run_job_using +from taskgraph.transforms.job.run_task import docker_worker_run_task +from voluptuous import Schema, Required + +mach_schema = Schema({ + Required('using'): 'mach', + + # The mach command (omitting `./mach`) to run + Required('mach'): basestring, +}) + + +@run_job_using("docker-worker", "mach", schema=mach_schema) +def docker_worker_mach(config, job, taskdesc): + run = job['run'] + + # defer to the run_task implementation + run['command'] = 'cd /home/worker/checkouts/gecko && ./mach ' + run['mach'] + run['checkout'] = True + del run['mach'] + docker_worker_run_task(config, job, taskdesc) diff --git a/taskcluster/taskgraph/transforms/job/mozharness.py b/taskcluster/taskgraph/transforms/job/mozharness.py new file mode 100644 index 000000000..fb3cd00dd --- /dev/null +++ b/taskcluster/taskgraph/transforms/job/mozharness.py @@ -0,0 +1,226 @@ +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, You can obtain one at http://mozilla.org/MPL/2.0/. +""" + +Support for running jobs via mozharness. Ideally, most stuff gets run this +way, and certainly anything using mozharness should use this approach. + +""" + +from __future__ import absolute_import, print_function, unicode_literals + +from voluptuous import Schema, Required, Optional, Any + +from taskgraph.transforms.job import run_job_using +from taskgraph.transforms.job.common import ( + docker_worker_add_workspace_cache, + docker_worker_add_gecko_vcs_env_vars, + docker_worker_setup_secrets, + docker_worker_add_public_artifacts, + docker_worker_support_vcs_checkout, +) + +COALESCE_KEY = 'builds.{project}.{name}' + +mozharness_run_schema = Schema({ + Required('using'): 'mozharness', + + # the mozharness script used to run this task, relative to the testing/ + # directory and using forward slashes even on Windows + Required('script'): basestring, + + # the config files required for the task, relative to + # testing/mozharness/configs and using forward slashes even on Windows + Required('config'): [basestring], + + # any additional actions to pass to the mozharness command; not supported + # on Windows + Optional('actions'): [basestring], + + # any additional options (without leading --) to be passed to mozharness; + # not supported on Windows + Optional('options'): [basestring], + + # --custom-build-variant-cfg value (not supported on Windows) + Optional('custom-build-variant-cfg'): basestring, + + # If not false, tooltool downloads will be enabled via relengAPIProxy + # for either just public files, or all files. Not supported on Windows + Required('tooltool-downloads', default=False): Any( + False, + 'public', + 'internal', + ), + + # The set of secret names to which the task has access; these are prefixed + # with `project/releng/gecko/{treeherder.kind}/level-{level}/`. Setting + # this will enable any worker features required and set the task's scopes + # appropriately. `true` here means ['*'], all secrets. Not supported on + # Windows + Required('secrets', default=False): Any(bool, [basestring]), + + # If true, taskcluster proxy will be enabled; note that it may also be enabled + # automatically e.g., for secrets support. Not supported on Windows. + Required('taskcluster-proxy', default=False): bool, + + # If true, the build scripts will start Xvfb. Not supported on Windows. + Required('need-xvfb', default=False): bool, + + # If false, indicate that builds should skip producing artifacts. Not + # supported on Windows. + Required('keep-artifacts', default=True): bool, + + # If specified, use the in-tree job script specified. + Optional('job-script'): basestring, +}) + + +@run_job_using("docker-worker", "mozharness", schema=mozharness_run_schema) +def mozharness_on_docker_worker_setup(config, job, taskdesc): + run = job['run'] + + worker = taskdesc['worker'] + worker['implementation'] = job['worker']['implementation'] + + # running via mozharness assumes desktop-build (which contains build.sh) + taskdesc['worker']['docker-image'] = {"in-tree": "desktop-build"} + + worker['relengapi-proxy'] = False # but maybe enabled for tooltool below + worker['taskcluster-proxy'] = run.get('taskcluster-proxy') + + docker_worker_add_public_artifacts(config, job, taskdesc) + docker_worker_add_workspace_cache(config, job, taskdesc) + docker_worker_support_vcs_checkout(config, job, taskdesc) + + env = worker.setdefault('env', {}) + env.update({ + 'MOZHARNESS_CONFIG': ' '.join(run['config']), + 'MOZHARNESS_SCRIPT': run['script'], + 'MH_BRANCH': config.params['project'], + 'MH_BUILD_POOL': 'taskcluster', + 'MOZ_BUILD_DATE': config.params['moz_build_date'], + 'MOZ_SCM_LEVEL': config.params['level'], + }) + + if 'actions' in run: + env['MOZHARNESS_ACTIONS'] = ' '.join(run['actions']) + + if 'options' in run: + env['MOZHARNESS_OPTIONS'] = ' '.join(run['options']) + + if 'custom-build-variant-cfg' in run: + env['MH_CUSTOM_BUILD_VARIANT_CFG'] = run['custom-build-variant-cfg'] + + if 'job-script' in run: + env['JOB_SCRIPT'] = run['job-script'] + + # if we're not keeping artifacts, set some env variables to empty values + # that will cause the build process to skip copying the results to the + # artifacts directory. This will have no effect for operations that are + # not builds. + if not run['keep-artifacts']: + env['DIST_TARGET_UPLOADS'] = '' + env['DIST_UPLOADS'] = '' + + # Xvfb + if run['need-xvfb']: + env['NEED_XVFB'] = 'true' + + # tooltool downloads + if run['tooltool-downloads']: + worker['relengapi-proxy'] = True + worker['caches'].append({ + 'type': 'persistent', + 'name': 'tooltool-cache', + 'mount-point': '/home/worker/tooltool-cache', + }) + taskdesc['scopes'].extend([ + 'docker-worker:relengapi-proxy:tooltool.download.public', + ]) + if run['tooltool-downloads'] == 'internal': + taskdesc['scopes'].append( + 'docker-worker:relengapi-proxy:tooltool.download.internal') + env['TOOLTOOL_CACHE'] = '/home/worker/tooltool-cache' + env['TOOLTOOL_REPO'] = 'https://github.com/mozilla/build-tooltool' + env['TOOLTOOL_REV'] = 'master' + + # Retry if mozharness returns TBPL_RETRY + worker['retry-exit-status'] = 4 + + docker_worker_setup_secrets(config, job, taskdesc) + + command = [ + '/home/worker/bin/run-task', + # Various caches/volumes are default owned by root:root. + '--chown-recursive', '/home/worker/workspace', + '--chown-recursive', '/home/worker/tooltool-cache', + '--vcs-checkout', '/home/worker/workspace/build/src', + '--tools-checkout', '/home/worker/workspace/build/tools', + '--', + ] + command.append("/home/worker/workspace/build/src/{}".format( + run.get('job-script', + "taskcluster/scripts/builder/build-linux.sh" + ))) + + worker['command'] = command + + +# We use the generic worker to run tasks on Windows +@run_job_using("generic-worker", "mozharness", schema=mozharness_run_schema) +def mozharness_on_windows(config, job, taskdesc): + run = job['run'] + + # fail if invalid run options are included + invalid = [] + for prop in ['actions', 'custom-build-variant-cfg', + 'tooltool-downloads', 'secrets', 'taskcluster-proxy', + 'need-xvfb']: + if prop in run and run[prop]: + invalid.append(prop) + if not run.get('keep-artifacts', True): + invalid.append('keep-artifacts') + if invalid: + raise Exception("Jobs run using mozharness on Windows do not support properties " + + ', '.join(invalid)) + + worker = taskdesc['worker'] + + worker['artifacts'] = [{ + 'path': r'public\build', + 'type': 'directory', + }] + + docker_worker_add_gecko_vcs_env_vars(config, job, taskdesc) + + env = worker['env'] + env.update({ + 'MOZ_BUILD_DATE': config.params['moz_build_date'], + 'MOZ_SCM_LEVEL': config.params['level'], + 'TOOLTOOL_REPO': 'https://github.com/mozilla/build-tooltool', + 'TOOLTOOL_REV': 'master', + }) + + mh_command = [r'c:\mozilla-build\python\python.exe'] + mh_command.append('\\'.join([r'.\build\src\testing', run['script'].replace('/', '\\')])) + for cfg in run['config']: + mh_command.append('--config ' + cfg.replace('/', '\\')) + mh_command.append('--branch ' + config.params['project']) + mh_command.append(r'--skip-buildbot-actions --work-dir %cd:Z:=z:%\build') + for option in run.get('options', []): + mh_command.append('--' + option) + + hg_command = ['"c:\\Program Files\\Mercurial\\hg.exe"'] + hg_command.append('robustcheckout') + hg_command.extend(['--sharebase', 'y:\\hg-shared']) + hg_command.append('--purge') + hg_command.extend(['--upstream', 'https://hg.mozilla.org/mozilla-unified']) + hg_command.extend(['--revision', env['GECKO_HEAD_REV']]) + hg_command.append(env['GECKO_HEAD_REPOSITORY']) + hg_command.append('.\\build\\src') + + worker['command'] = [ + ' '.join(hg_command), + ' '.join(mh_command) + ] diff --git a/taskcluster/taskgraph/transforms/job/run_task.py b/taskcluster/taskgraph/transforms/job/run_task.py new file mode 100644 index 000000000..296fe43ee --- /dev/null +++ b/taskcluster/taskgraph/transforms/job/run_task.py @@ -0,0 +1,59 @@ +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, You can obtain one at http://mozilla.org/MPL/2.0/. +""" +Support for running jobs that are invoked via the `run-task` script. +""" + +from __future__ import absolute_import, print_function, unicode_literals + +import copy + +from taskgraph.transforms.job import run_job_using +from taskgraph.transforms.job.common import ( + docker_worker_support_vcs_checkout, +) +from voluptuous import Schema, Required, Any + +run_task_schema = Schema({ + Required('using'): 'run-task', + + # if true, add a cache at ~worker/.cache, which is where things like pip + # tend to hide their caches. This cache is never added for level-1 jobs. + Required('cache-dotcache', default=False): bool, + + # if true (the default), perform a checkout in /home/worker/checkouts/gecko + Required('checkout', default=True): bool, + + # The command arguments to pass to the `run-task` script, after the + # checkout arguments. If a list, it will be passed directly; otherwise + # it will be included in a single argument to `bash -cx`. + Required('command'): Any([basestring], basestring), +}) + + +@run_job_using("docker-worker", "run-task", schema=run_task_schema) +def docker_worker_run_task(config, job, taskdesc): + run = job['run'] + + worker = taskdesc['worker'] = copy.deepcopy(job['worker']) + + if run['checkout']: + docker_worker_support_vcs_checkout(config, job, taskdesc) + + if run.get('cache-dotcache') and int(config.params['level']) > 1: + worker['caches'].append({ + 'type': 'persistent', + 'name': 'level-{level}-{project}-dotcache'.format(**config.params), + 'mount-point': '/home/worker/.cache', + }) + + run_command = run['command'] + if isinstance(run_command, basestring): + run_command = ['bash', '-cx', run_command] + command = ['/home/worker/bin/run-task'] + if run['checkout']: + command.append('--vcs-checkout=/home/worker/checkouts/gecko') + command.append('--') + command.extend(run_command) + worker['command'] = command diff --git a/taskcluster/taskgraph/transforms/job/spidermonkey.py b/taskcluster/taskgraph/transforms/job/spidermonkey.py new file mode 100644 index 000000000..d78b78504 --- /dev/null +++ b/taskcluster/taskgraph/transforms/job/spidermonkey.py @@ -0,0 +1,86 @@ +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, You can obtain one at http://mozilla.org/MPL/2.0/. +""" +Support for running spidermonkey jobs via dedicated scripts +""" + +from __future__ import absolute_import, print_function, unicode_literals + +from voluptuous import Schema, Required, Optional, Any + +from taskgraph.transforms.job import run_job_using +from taskgraph.transforms.job.common import ( + docker_worker_add_public_artifacts, + docker_worker_support_vcs_checkout, +) + +sm_run_schema = Schema({ + Required('using'): Any('spidermonkey', 'spidermonkey-package', 'spidermonkey-mozjs-crate'), + + # The SPIDERMONKEY_VARIANT + Required('spidermonkey-variant'): basestring, + + # The tooltool manifest to use; default from sm-tooltool-config.sh is used + # if omitted + Optional('tooltool-manifest'): basestring, +}) + + +@run_job_using("docker-worker", "spidermonkey") +@run_job_using("docker-worker", "spidermonkey-package") +@run_job_using("docker-worker", "spidermonkey-mozjs-crate") +def docker_worker_spidermonkey(config, job, taskdesc, schema=sm_run_schema): + run = job['run'] + + worker = taskdesc['worker'] + worker['artifacts'] = [] + worker['caches'] = [] + + if int(config.params['level']) > 1: + worker['caches'].append({ + 'type': 'persistent', + 'name': 'level-{}-{}-build-spidermonkey-workspace'.format( + config.params['level'], config.params['project']), + 'mount-point': "/home/worker/workspace", + }) + + docker_worker_add_public_artifacts(config, job, taskdesc) + + env = worker['env'] + env.update({ + 'MOZHARNESS_DISABLE': 'true', + 'SPIDERMONKEY_VARIANT': run['spidermonkey-variant'], + 'MOZ_BUILD_DATE': config.params['moz_build_date'], + 'MOZ_SCM_LEVEL': config.params['level'], + }) + + # tooltool downloads; note that this script downloads using the API + # endpoiint directly, rather than via relengapi-proxy + worker['caches'].append({ + 'type': 'persistent', + 'name': 'tooltool-cache', + 'mount-point': '/home/worker/tooltool-cache', + }) + env['TOOLTOOL_CACHE'] = '/home/worker/tooltool-cache' + if run.get('tooltool-manifest'): + env['TOOLTOOL_MANIFEST'] = run['tooltool-manifest'] + + docker_worker_support_vcs_checkout(config, job, taskdesc) + + script = "build-sm.sh" + if run['using'] == 'spidermonkey-package': + script = "build-sm-package.sh" + elif run['using'] == 'spidermonkey-mozjs-crate': + script = "build-sm-mozjs-crate.sh" + + worker['command'] = [ + '/home/worker/bin/run-task', + '--chown-recursive', '/home/worker/workspace', + '--chown-recursive', '/home/worker/tooltool-cache', + '--vcs-checkout', '/home/worker/workspace/build/src', + '--', + '/bin/bash', + '-c', + 'cd /home/worker && workspace/build/src/taskcluster/scripts/builder/%s' % script + ] diff --git a/taskcluster/taskgraph/transforms/job/toolchain.py b/taskcluster/taskgraph/transforms/job/toolchain.py new file mode 100644 index 000000000..d814f7824 --- /dev/null +++ b/taskcluster/taskgraph/transforms/job/toolchain.py @@ -0,0 +1,115 @@ +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, You can obtain one at http://mozilla.org/MPL/2.0/. +""" +Support for running toolchain-building jobs via dedicated scripts +""" + +from __future__ import absolute_import, print_function, unicode_literals + +from voluptuous import Schema, Required + +from taskgraph.transforms.job import run_job_using +from taskgraph.transforms.job.common import ( + docker_worker_add_tc_vcs_cache, + docker_worker_add_gecko_vcs_env_vars +) + +toolchain_run_schema = Schema({ + Required('using'): 'toolchain-script', + + # the script (in taskcluster/scripts/misc) to run + Required('script'): basestring, +}) + + +@run_job_using("docker-worker", "toolchain-script", schema=toolchain_run_schema) +def docker_worker_toolchain(config, job, taskdesc): + run = job['run'] + + worker = taskdesc['worker'] + worker['artifacts'] = [] + worker['caches'] = [] + + worker['artifacts'].append({ + 'name': 'public', + 'path': '/home/worker/workspace/artifacts/', + 'type': 'directory', + }) + + docker_worker_add_tc_vcs_cache(config, job, taskdesc) + docker_worker_add_gecko_vcs_env_vars(config, job, taskdesc) + + env = worker['env'] + env.update({ + 'MOZ_BUILD_DATE': config.params['moz_build_date'], + 'MOZ_SCM_LEVEL': config.params['level'], + 'TOOLS_DISABLE': 'true', + }) + + # tooltool downloads; note that this downloads using the API endpoint directly, + # rather than via relengapi-proxy + worker['caches'].append({ + 'type': 'persistent', + 'name': 'tooltool-cache', + 'mount-point': '/home/worker/tooltool-cache', + }) + env['TOOLTOOL_CACHE'] = '/home/worker/tooltool-cache' + env['TOOLTOOL_REPO'] = 'https://github.com/mozilla/build-tooltool' + env['TOOLTOOL_REV'] = 'master' + + command = ' && '.join([ + "cd /home/worker/", + "./bin/checkout-sources.sh", + "./workspace/build/src/taskcluster/scripts/misc/" + run['script'], + ]) + worker['command'] = ["/bin/bash", "-c", command] + + +@run_job_using("generic-worker", "toolchain-script", schema=toolchain_run_schema) +def windows_toolchain(config, job, taskdesc): + run = job['run'] + + worker = taskdesc['worker'] + + worker['artifacts'] = [{ + 'path': r'public\build', + 'type': 'directory', + }] + + docker_worker_add_gecko_vcs_env_vars(config, job, taskdesc) + + # We fetch LLVM SVN into this. + svn_cache = 'level-{}-toolchain-clang-cl-build-svn'.format(config.params['level']) + worker['mounts'] = [{ + 'cache-name': svn_cache, + 'path': r'llvm-sources', + }] + taskdesc['scopes'].extend([ + 'generic-worker:cache:' + svn_cache, + ]) + + env = worker['env'] + env.update({ + 'MOZ_BUILD_DATE': config.params['moz_build_date'], + 'MOZ_SCM_LEVEL': config.params['level'], + 'TOOLTOOL_REPO': 'https://github.com/mozilla/build-tooltool', + 'TOOLTOOL_REV': 'master', + }) + + hg = r'c:\Program Files\Mercurial\hg.exe' + hg_command = ['"{}"'.format(hg)] + hg_command.append('robustcheckout') + hg_command.extend(['--sharebase', 'y:\\hg-shared']) + hg_command.append('--purge') + hg_command.extend(['--upstream', 'https://hg.mozilla.org/mozilla-unified']) + hg_command.extend(['--revision', '%GECKO_HEAD_REV%']) + hg_command.append('%GECKO_HEAD_REPOSITORY%') + hg_command.append('.\\build\\src') + + bash = r'c:\mozilla-build\msys\bin\bash' + worker['command'] = [ + ' '.join(hg_command), + # do something intelligent. + r'{} -c ./build/src/taskcluster/scripts/misc/{}'.format(bash, run['script']) + ] diff --git a/taskcluster/taskgraph/transforms/l10n.py b/taskcluster/taskgraph/transforms/l10n.py new file mode 100644 index 000000000..42137b558 --- /dev/null +++ b/taskcluster/taskgraph/transforms/l10n.py @@ -0,0 +1,44 @@ +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, You can obtain one at http://mozilla.org/MPL/2.0/. +""" +Do transforms specific to l10n kind +""" + +from __future__ import absolute_import, print_function, unicode_literals + +from taskgraph.transforms.base import TransformSequence + +transforms = TransformSequence() + + +@transforms.add +def mh_config_replace_project(config, jobs): + """ Replaces {project} in mh config entries with the current project """ + # XXXCallek This is a bad pattern but exists to satisfy ease-of-porting for buildbot + for job in jobs: + if not job['run'].get('using') == 'mozharness': + # Nothing to do, not mozharness + yield job + continue + job['run']['config'] = map( + lambda x: x.format(project=config.params['project']), + job['run']['config'] + ) + yield job + + +@transforms.add +def mh_options_replace_project(config, jobs): + """ Replaces {project} in mh option entries with the current project """ + # XXXCallek This is a bad pattern but exists to satisfy ease-of-porting for buildbot + for job in jobs: + if not job['run'].get('using') == 'mozharness': + # Nothing to do, not mozharness + yield job + continue + job['run']['options'] = map( + lambda x: x.format(project=config.params['project']), + job['run']['options'] + ) + yield job diff --git a/taskcluster/taskgraph/transforms/marionette_harness.py b/taskcluster/taskgraph/transforms/marionette_harness.py new file mode 100644 index 000000000..a24db470c --- /dev/null +++ b/taskcluster/taskgraph/transforms/marionette_harness.py @@ -0,0 +1,37 @@ +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, You can obtain one at http://mozilla.org/MPL/2.0/. +""" +Set dynamic task description properties of the marionette-harness task. +""" + +from __future__ import absolute_import, print_function, unicode_literals + +from taskgraph.transforms.base import TransformSequence + +transforms = TransformSequence() + + +@transforms.add +def setup_task(config, tasks): + for task in tasks: + del task['name'] + task['label'] = 'marionette-harness' + env = task['worker'].setdefault('env', {}) + env.update({ + 'GECKO_BASE_REPOSITORY': config.params['base_repository'], + 'GECKO_HEAD_REF': config.params['head_rev'], + 'GECKO_HEAD_REPOSITORY': config.params['head_repository'], + 'GECKO_HEAD_REV': config.params['head_rev'], + 'MOZ_BUILD_DATE': config.params['moz_build_date'], + 'MOZ_SCM_LEVEL': config.params['level'], + }) + + task['worker']['caches'] = [{ + 'type': 'persistent', + 'name': 'level-{}-{}-tc-vcs'.format( + config.params['level'], config.params['project']), + 'mount-point': "/home/worker/.tc-vcs", + }] + + yield task diff --git a/taskcluster/taskgraph/transforms/task.py b/taskcluster/taskgraph/transforms/task.py new file mode 100644 index 000000000..6e371e4ba --- /dev/null +++ b/taskcluster/taskgraph/transforms/task.py @@ -0,0 +1,648 @@ +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, You can obtain one at http://mozilla.org/MPL/2.0/. +""" +These transformations take a task description and turn it into a TaskCluster +task definition (along with attributes, label, etc.). The input to these +transformations is generic to any kind of task, but abstracts away some of the +complexities of worker implementations, scopes, and treeherder annotations. +""" + +from __future__ import absolute_import, print_function, unicode_literals + +import json +import time + +from taskgraph.util.treeherder import split_symbol +from taskgraph.transforms.base import ( + validate_schema, + TransformSequence +) +from voluptuous import Schema, Any, Required, Optional, Extra + +from .gecko_v2_whitelist import JOB_NAME_WHITELIST, JOB_NAME_WHITELIST_ERROR + +# shortcut for a string where task references are allowed +taskref_or_string = Any( + basestring, + {Required('task-reference'): basestring}) + +# A task description is a general description of a TaskCluster task +task_description_schema = Schema({ + # the label for this task + Required('label'): basestring, + + # description of the task (for metadata) + Required('description'): basestring, + + # attributes for this task + Optional('attributes'): {basestring: object}, + + # dependencies of this task, keyed by name; these are passed through + # verbatim and subject to the interpretation of the Task's get_dependencies + # method. + Optional('dependencies'): {basestring: object}, + + # expiration and deadline times, relative to task creation, with units + # (e.g., "14 days"). Defaults are set based on the project. + Optional('expires-after'): basestring, + Optional('deadline-after'): basestring, + + # custom routes for this task; the default treeherder routes will be added + # automatically + Optional('routes'): [basestring], + + # custom scopes for this task; any scopes required for the worker will be + # added automatically + Optional('scopes'): [basestring], + + # custom "task.extra" content + Optional('extra'): {basestring: object}, + + # treeherder-related information; see + # https://schemas.taskcluster.net/taskcluster-treeherder/v1/task-treeherder-config.json + # If not specified, no treeherder extra information or routes will be + # added to the task + Optional('treeherder'): { + # either a bare symbol, or "grp(sym)". + 'symbol': basestring, + + # the job kind + 'kind': Any('build', 'test', 'other'), + + # tier for this task + 'tier': int, + + # task platform, in the form platform/collection, used to set + # treeherder.machine.platform and treeherder.collection or + # treeherder.labels + 'platform': basestring, + + # treeherder environments (defaults to both staging and production) + Required('environments', default=['production', 'staging']): ['production', 'staging'], + }, + + # information for indexing this build so its artifacts can be discovered; + # if omitted, the build will not be indexed. + Optional('index'): { + # the name of the product this build produces + 'product': Any('firefox', 'mobile'), + + # the names to use for this job in the TaskCluster index + 'job-name': Any( + # Assuming the job is named "normally", this is the v2 job name, + # and the v1 and buildbot routes will be determined appropriately. + basestring, + + # otherwise, give separate names for each of the legacy index + # routes; if a name is omitted, no corresponding route will be + # created. + { + # the name as it appears in buildbot routes + Optional('buildbot'): basestring, + Required('gecko-v2'): basestring, + } + ), + + # The rank that the task will receive in the TaskCluster + # index. A newly completed task supercedes the currently + # indexed task iff it has a higher rank. If unspecified, + # 'by-tier' behavior will be used. + 'rank': Any( + # Rank is equal the timestamp of the build_date for tier-1 + # tasks, and zero for non-tier-1. This sorts tier-{2,3} + # builds below tier-1 in the index. + 'by-tier', + + # Rank is given as an integer constant (e.g. zero to make + # sure a task is last in the index). + int, + + # Rank is equal to the timestamp of the build_date. This + # option can be used to override the 'by-tier' behavior + # for non-tier-1 tasks. + 'build_date', + ), + }, + + # The `run_on_projects` attribute, defaulting to "all". This dictates the + # projects on which this task should be included in the target task set. + # See the attributes documentation for details. + Optional('run-on-projects'): [basestring], + + # If the task can be coalesced, this is the name used in the coalesce key + # the project, etc. will be added automatically. Note that try (level 1) + # tasks are never coalesced + Optional('coalesce-name'): basestring, + + # the provisioner-id/worker-type for the task. The following parameters will + # be substituted in this string: + # {level} -- the scm level of this push + 'worker-type': basestring, + + # information specific to the worker implementation that will run this task + 'worker': Any({ + Required('implementation'): Any('docker-worker', 'docker-engine'), + + # For tasks that will run in docker-worker or docker-engine, this is the + # name of the docker image or in-tree docker image to run the task in. If + # in-tree, then a dependency will be created automatically. This is + # generally `desktop-test`, or an image that acts an awful lot like it. + Required('docker-image'): Any( + # a raw Docker image path (repo/image:tag) + basestring, + # an in-tree generated docker image (from `testing/docker/<name>`) + {'in-tree': basestring} + ), + + # worker features that should be enabled + Required('relengapi-proxy', default=False): bool, + Required('chain-of-trust', default=False): bool, + Required('taskcluster-proxy', default=False): bool, + Required('allow-ptrace', default=False): bool, + Required('loopback-video', default=False): bool, + Required('loopback-audio', default=False): bool, + + # caches to set up for the task + Optional('caches'): [{ + # only one type is supported by any of the workers right now + 'type': 'persistent', + + # name of the cache, allowing re-use by subsequent tasks naming the + # same cache + 'name': basestring, + + # location in the task image where the cache will be mounted + 'mount-point': basestring, + }], + + # artifacts to extract from the task image after completion + Optional('artifacts'): [{ + # type of artifact -- simple file, or recursive directory + 'type': Any('file', 'directory'), + + # task image path from which to read artifact + 'path': basestring, + + # name of the produced artifact (root of the names for + # type=directory) + 'name': basestring, + }], + + # environment variables + Required('env', default={}): {basestring: taskref_or_string}, + + # the command to run + 'command': [taskref_or_string], + + # the maximum time to run, in seconds + 'max-run-time': int, + + # the exit status code that indicates the task should be retried + Optional('retry-exit-status'): int, + + }, { + Required('implementation'): 'generic-worker', + + # command is a list of commands to run, sequentially + 'command': [taskref_or_string], + + # artifacts to extract from the task image after completion; note that artifacts + # for the generic worker cannot have names + Optional('artifacts'): [{ + # type of artifact -- simple file, or recursive directory + 'type': Any('file', 'directory'), + + # task image path from which to read artifact + 'path': basestring, + }], + + # directories and/or files to be mounted + Optional('mounts'): [{ + # a unique name for the cache volume + 'cache-name': basestring, + + # task image path for the cache + 'path': basestring, + }], + + # environment variables + Required('env', default={}): {basestring: taskref_or_string}, + + # the maximum time to run, in seconds + 'max-run-time': int, + + # os user groups for test task workers + Optional('os-groups', default=[]): [basestring], + }, { + Required('implementation'): 'buildbot-bridge', + + # see + # https://github.com/mozilla/buildbot-bridge/blob/master/bbb/schemas/payload.yml + 'buildername': basestring, + 'sourcestamp': { + 'branch': basestring, + Optional('revision'): basestring, + Optional('repository'): basestring, + Optional('project'): basestring, + }, + 'properties': { + 'product': basestring, + Extra: basestring, # additional properties are allowed + }, + }, { + 'implementation': 'macosx-engine', + + # A link for an executable to download + Optional('link'): basestring, + + # the command to run + Required('command'): [taskref_or_string], + + # environment variables + Optional('env'): {basestring: taskref_or_string}, + + # artifacts to extract from the task image after completion + Optional('artifacts'): [{ + # type of artifact -- simple file, or recursive directory + Required('type'): Any('file', 'directory'), + + # task image path from which to read artifact + Required('path'): basestring, + + # name of the produced artifact (root of the names for + # type=directory) + Required('name'): basestring, + }], + }), + + # The "when" section contains descriptions of the circumstances + # under which this task can be "optimized", that is, left out of the + # task graph because it is unnecessary. + Optional('when'): Any({ + # This task only needs to be run if a file matching one of the given + # patterns has changed in the push. The patterns use the mozpack + # match function (python/mozbuild/mozpack/path.py). + Optional('files-changed'): [basestring], + }), +}) + +GROUP_NAMES = { + 'tc': 'Executed by TaskCluster', + 'tc-e10s': 'Executed by TaskCluster with e10s', + 'tc-Fxfn-l': 'Firefox functional tests (local) executed by TaskCluster', + 'tc-Fxfn-l-e10s': 'Firefox functional tests (local) executed by TaskCluster with e10s', + 'tc-Fxfn-r': 'Firefox functional tests (remote) executed by TaskCluster', + 'tc-Fxfn-r-e10s': 'Firefox functional tests (remote) executed by TaskCluster with e10s', + 'tc-M': 'Mochitests executed by TaskCluster', + 'tc-M-e10s': 'Mochitests executed by TaskCluster with e10s', + 'tc-R': 'Reftests executed by TaskCluster', + 'tc-R-e10s': 'Reftests executed by TaskCluster with e10s', + 'tc-VP': 'VideoPuppeteer tests executed by TaskCluster', + 'tc-W': 'Web platform tests executed by TaskCluster', + 'tc-W-e10s': 'Web platform tests executed by TaskCluster with e10s', + 'tc-X': 'Xpcshell tests executed by TaskCluster', + 'tc-X-e10s': 'Xpcshell tests executed by TaskCluster with e10s', + 'Aries': 'Aries Device Image', + 'Nexus 5-L': 'Nexus 5-L Device Image', + 'Cc': 'Toolchain builds', + 'SM-tc': 'Spidermonkey builds', +} +UNKNOWN_GROUP_NAME = "Treeherder group {} has no name; add it to " + __file__ + +BUILDBOT_ROUTE_TEMPLATES = [ + "index.buildbot.branches.{project}.{job-name-buildbot}", + "index.buildbot.revisions.{head_rev}.{project}.{job-name-buildbot}", +] + +V2_ROUTE_TEMPLATES = [ + "index.gecko.v2.{project}.latest.{product}.{job-name-gecko-v2}", + "index.gecko.v2.{project}.pushdate.{build_date_long}.{product}.{job-name-gecko-v2}", + "index.gecko.v2.{project}.revision.{head_rev}.{product}.{job-name-gecko-v2}", +] + +# the roots of the treeherder routes, keyed by treeherder environment +TREEHERDER_ROUTE_ROOTS = { + 'production': 'tc-treeherder', + 'staging': 'tc-treeherder-stage', +} + +COALESCE_KEY = 'builds.{project}.{name}' + +# define a collection of payload builders, depending on the worker implementation +payload_builders = {} + + +def payload_builder(name): + def wrap(func): + payload_builders[name] = func + return func + return wrap + + +@payload_builder('docker-worker') +def build_docker_worker_payload(config, task, task_def): + worker = task['worker'] + + image = worker['docker-image'] + if isinstance(image, dict): + docker_image_task = 'build-docker-image-' + image['in-tree'] + task.setdefault('dependencies', {})['docker-image'] = docker_image_task + image = { + "path": "public/image.tar.zst", + "taskId": {"task-reference": "<docker-image>"}, + "type": "task-image", + } + + features = {} + + if worker.get('relengapi-proxy'): + features['relengAPIProxy'] = True + + if worker.get('taskcluster-proxy'): + features['taskclusterProxy'] = True + + if worker.get('allow-ptrace'): + features['allowPtrace'] = True + task_def['scopes'].append('docker-worker:feature:allowPtrace') + + if worker.get('chain-of-trust'): + features['chainOfTrust'] = True + + capabilities = {} + + for lo in 'audio', 'video': + if worker.get('loopback-' + lo): + capitalized = 'loopback' + lo.capitalize() + devices = capabilities.setdefault('devices', {}) + devices[capitalized] = True + task_def['scopes'].append('docker-worker:capability:device:' + capitalized) + + task_def['payload'] = payload = { + 'command': worker['command'], + 'image': image, + 'env': worker['env'], + } + + if 'max-run-time' in worker: + payload['maxRunTime'] = worker['max-run-time'] + + if 'retry-exit-status' in worker: + payload['onExitStatus'] = {'retry': [worker['retry-exit-status']]} + + if 'artifacts' in worker: + artifacts = {} + for artifact in worker['artifacts']: + artifacts[artifact['name']] = { + 'path': artifact['path'], + 'type': artifact['type'], + 'expires': task_def['expires'], # always expire with the task + } + payload['artifacts'] = artifacts + + if 'caches' in worker: + caches = {} + for cache in worker['caches']: + caches[cache['name']] = cache['mount-point'] + task_def['scopes'].append('docker-worker:cache:' + cache['name']) + payload['cache'] = caches + + if features: + payload['features'] = features + if capabilities: + payload['capabilities'] = capabilities + + # coalesce / superseding + if 'coalesce-name' in task and int(config.params['level']) > 1: + key = COALESCE_KEY.format( + project=config.params['project'], + name=task['coalesce-name']) + payload['supersederUrl'] = "https://coalesce.mozilla-releng.net/v1/list/" + key + + +@payload_builder('generic-worker') +def build_generic_worker_payload(config, task, task_def): + worker = task['worker'] + + artifacts = [] + + for artifact in worker['artifacts']: + artifacts.append({ + 'path': artifact['path'], + 'type': artifact['type'], + 'expires': task_def['expires'], # always expire with the task + }) + + mounts = [] + + for mount in worker.get('mounts', []): + mounts.append({ + 'cacheName': mount['cache-name'], + 'directory': mount['path'] + }) + + task_def['payload'] = { + 'command': worker['command'], + 'artifacts': artifacts, + 'env': worker.get('env', {}), + 'mounts': mounts, + 'maxRunTime': worker['max-run-time'], + 'osGroups': worker.get('os-groups', []), + } + + if 'retry-exit-status' in worker: + raise Exception("retry-exit-status not supported in generic-worker") + + +@payload_builder('macosx-engine') +def build_macosx_engine_payload(config, task, task_def): + worker = task['worker'] + artifacts = map(lambda artifact: { + 'name': artifact['name'], + 'path': artifact['path'], + 'type': artifact['type'], + 'expires': task_def['expires'], + }, worker['artifacts']) + + task_def['payload'] = { + 'link': worker['link'], + 'command': worker['command'], + 'env': worker['env'], + 'artifacts': artifacts, + } + +transforms = TransformSequence() + + +@transforms.add +def validate(config, tasks): + for task in tasks: + yield validate_schema( + task_description_schema, task, + "In task {!r}:".format(task.get('label', '?no-label?'))) + + +@transforms.add +def add_index_routes(config, tasks): + for task in tasks: + index = task.get('index') + routes = task.setdefault('routes', []) + + if not index: + yield task + continue + + job_name = index['job-name'] + # unpack the v2 name to v1 and buildbot names + if isinstance(job_name, basestring): + base_name, type_name = job_name.rsplit('-', 1) + job_name = { + 'buildbot': base_name, + 'gecko-v2': '{}-{}'.format(base_name, type_name), + } + + if job_name['gecko-v2'] not in JOB_NAME_WHITELIST: + raise Exception(JOB_NAME_WHITELIST_ERROR.format(job_name['gecko-v2'])) + + subs = config.params.copy() + for n in job_name: + subs['job-name-' + n] = job_name[n] + subs['build_date_long'] = time.strftime("%Y.%m.%d.%Y%m%d%H%M%S", + time.gmtime(config.params['build_date'])) + subs['product'] = index['product'] + + if 'buildbot' in job_name: + for tpl in BUILDBOT_ROUTE_TEMPLATES: + routes.append(tpl.format(**subs)) + if 'gecko-v2' in job_name: + for tpl in V2_ROUTE_TEMPLATES: + routes.append(tpl.format(**subs)) + + # The default behavior is to rank tasks according to their tier + extra_index = task.setdefault('extra', {}).setdefault('index', {}) + rank = index.get('rank', 'by-tier') + + if rank == 'by-tier': + # rank is zero for non-tier-1 tasks and based on pushid for others; + # this sorts tier-{2,3} builds below tier-1 in the index + tier = task.get('treeherder', {}).get('tier', 3) + extra_index['rank'] = 0 if tier > 1 else int(config.params['build_date']) + elif rank == 'build_date': + extra_index['rank'] = int(config.params['build_date']) + else: + extra_index['rank'] = rank + + del task['index'] + yield task + + +@transforms.add +def build_task(config, tasks): + for task in tasks: + worker_type = task['worker-type'].format(level=str(config.params['level'])) + provisioner_id, worker_type = worker_type.split('/', 1) + + routes = task.get('routes', []) + scopes = task.get('scopes', []) + + # set up extra + extra = task.get('extra', {}) + task_th = task.get('treeherder') + if task_th: + extra['treeherderEnv'] = task_th['environments'] + + treeherder = extra.setdefault('treeherder', {}) + + machine_platform, collection = task_th['platform'].split('/', 1) + treeherder['machine'] = {'platform': machine_platform} + treeherder['collection'] = {collection: True} + + groupSymbol, symbol = split_symbol(task_th['symbol']) + if groupSymbol != '?': + treeherder['groupSymbol'] = groupSymbol + if groupSymbol not in GROUP_NAMES: + raise Exception(UNKNOWN_GROUP_NAME.format(groupSymbol)) + treeherder['groupName'] = GROUP_NAMES[groupSymbol] + treeherder['symbol'] = symbol + treeherder['jobKind'] = task_th['kind'] + treeherder['tier'] = task_th['tier'] + + routes.extend([ + '{}.v2.{}.{}.{}'.format(TREEHERDER_ROUTE_ROOTS[env], + config.params['project'], + config.params['head_rev'], + config.params['pushlog_id']) + for env in task_th['environments'] + ]) + + if 'expires-after' not in task: + task['expires-after'] = '28 days' if config.params['project'] == 'try' else '1 year' + + if 'deadline-after' not in task: + task['deadline-after'] = '1 day' + + if 'coalesce-name' in task and int(config.params['level']) > 1: + key = COALESCE_KEY.format( + project=config.params['project'], + name=task['coalesce-name']) + routes.append('coalesce.v1.' + key) + + task_def = { + 'provisionerId': provisioner_id, + 'workerType': worker_type, + 'routes': routes, + 'created': {'relative-datestamp': '0 seconds'}, + 'deadline': {'relative-datestamp': task['deadline-after']}, + 'expires': {'relative-datestamp': task['expires-after']}, + 'scopes': scopes, + 'metadata': { + 'description': task['description'], + 'name': task['label'], + 'owner': config.params['owner'], + 'source': '{}/file/{}/{}'.format( + config.params['head_repository'], + config.params['head_rev'], + config.path), + }, + 'extra': extra, + 'tags': {'createdForUser': config.params['owner']}, + } + + # add the payload and adjust anything else as required (e.g., scopes) + payload_builders[task['worker']['implementation']](config, task, task_def) + + attributes = task.get('attributes', {}) + attributes['run_on_projects'] = task.get('run-on-projects', ['all']) + + yield { + 'label': task['label'], + 'task': task_def, + 'dependencies': task.get('dependencies', {}), + 'attributes': attributes, + 'when': task.get('when', {}), + } + + +# Check that the v2 route templates match those used by Mozharness. This can +# go away once Mozharness builds are no longer performed in Buildbot, and the +# Mozharness code referencing routes.json is deleted. +def check_v2_routes(): + with open("testing/mozharness/configs/routes.json", "rb") as f: + routes_json = json.load(f) + + # we only deal with the 'routes' key here + routes = routes_json['routes'] + + # we use different variables than mozharness + for mh, tg in [ + ('{index}', 'index'), + ('{build_product}', '{product}'), + ('{build_name}-{build_type}', '{job-name-gecko-v2}'), + ('{year}.{month}.{day}.{pushdate}', '{build_date_long}')]: + routes = [r.replace(mh, tg) for r in routes] + + if sorted(routes) != sorted(V2_ROUTE_TEMPLATES): + raise Exception("V2_ROUTE_TEMPLATES does not match Mozharness's routes.json: " + "%s vs %s" % (V2_ROUTE_TEMPLATES, routes)) + +check_v2_routes() diff --git a/taskcluster/taskgraph/transforms/tests/__init__.py b/taskcluster/taskgraph/transforms/tests/__init__.py new file mode 100644 index 000000000..e69de29bb --- /dev/null +++ b/taskcluster/taskgraph/transforms/tests/__init__.py diff --git a/taskcluster/taskgraph/transforms/tests/all_kinds.py b/taskcluster/taskgraph/transforms/tests/all_kinds.py new file mode 100644 index 000000000..f2aa1f841 --- /dev/null +++ b/taskcluster/taskgraph/transforms/tests/all_kinds.py @@ -0,0 +1,137 @@ +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, You can obtain one at http://mozilla.org/MPL/2.0/. +""" +Changes here apply to all tests, regardless of kind. + +This is a great place for: + + * Applying rules based on platform, project, etc. that should span kinds +""" + +from __future__ import absolute_import, print_function, unicode_literals + +from taskgraph.util.treeherder import split_symbol, join_symbol +from taskgraph.transforms.base import TransformSequence, get_keyed_by + +import copy + + +transforms = TransformSequence() + + +@transforms.add +def set_worker_implementation(config, tests): + """Set the worker implementation based on the test platform.""" + for test in tests: + if test['test-platform'].startswith('win'): + test['worker-implementation'] = 'generic-worker' + elif test['test-platform'].startswith('macosx'): + test['worker-implementation'] = 'macosx-engine' + else: + test['worker-implementation'] = 'docker-worker' + yield test + + +@transforms.add +def set_tier(config, tests): + """Set the tier based on policy for all test descriptions that do not + specify a tier otherwise.""" + for test in tests: + # only override if not set for the test + if 'tier' not in test: + if test['test-platform'] in ['linux64/debug', + 'linux64-asan/opt', + 'android-4.3-arm7-api-15/debug', + 'android-x86/opt']: + test['tier'] = 1 + else: + test['tier'] = 2 + yield test + + +@transforms.add +def set_expires_after(config, tests): + """Try jobs expire after 2 weeks; everything else lasts 1 year. This helps + keep storage costs low.""" + for test in tests: + if 'expires-after' not in test: + if config.params['project'] == 'try': + test['expires-after'] = "14 days" + else: + test['expires-after'] = "1 year" + yield test + + +@transforms.add +def set_download_symbols(config, tests): + """In general, we download symbols immediately for debug builds, but only + on demand for everything else. ASAN builds shouldn't download + symbols since they don't product symbol zips see bug 1283879""" + for test in tests: + if test['test-platform'].split('/')[-1] == 'debug': + test['mozharness']['download-symbols'] = True + elif test['build-platform'] == 'linux64-asan/opt': + if 'download-symbols' in test['mozharness']: + del test['mozharness']['download-symbols'] + else: + test['mozharness']['download-symbols'] = 'ondemand' + yield test + + +@transforms.add +def resolve_keyed_by(config, tests): + """Resolve fields that can be keyed by platform, etc.""" + fields = [ + 'instance-size', + 'max-run-time', + 'chunks', + 'e10s', + 'suite', + 'run-on-projects', + ] + for test in tests: + for field in fields: + test[field] = get_keyed_by(item=test, field=field, item_name=test['test-name']) + test['mozharness']['config'] = get_keyed_by(item=test, + field='mozharness', + subfield='config', + item_name=test['test-name']) + test['mozharness']['extra-options'] = get_keyed_by(item=test, + field='mozharness', + subfield='extra-options', + item_name=test['test-name']) + yield test + + +@transforms.add +def split_chunks(config, tests): + """Based on the 'chunks' key, split tests up into chunks by duplicating + them and assigning 'this-chunk' appropriately and updating the treeherder + symbol.""" + for test in tests: + if test['chunks'] == 1: + test['this-chunk'] = 1 + yield test + continue + + for this_chunk in range(1, test['chunks'] + 1): + # copy the test and update with the chunk number + chunked = copy.deepcopy(test) + chunked['this-chunk'] = this_chunk + + # add the chunk number to the TH symbol + group, symbol = split_symbol(chunked['treeherder-symbol']) + symbol += str(this_chunk) + chunked['treeherder-symbol'] = join_symbol(group, symbol) + + yield chunked + + +@transforms.add +def set_retry_exit_status(config, tests): + """Set the retry exit status to TBPL_RETRY, the value returned by mozharness + scripts to indicate a transient failure that should be retried.""" + for test in tests: + test['retry-exit-status'] = 4 + yield test diff --git a/taskcluster/taskgraph/transforms/tests/android_test.py b/taskcluster/taskgraph/transforms/tests/android_test.py new file mode 100644 index 000000000..7c13b16f5 --- /dev/null +++ b/taskcluster/taskgraph/transforms/tests/android_test.py @@ -0,0 +1,42 @@ +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, You can obtain one at http://mozilla.org/MPL/2.0/. +""" +These transforms are specific to the android-test kind, and apply defaults to +the test descriptions appropriate to that kind. + +Both the input to and output from these transforms must conform to +`taskgraph.transforms.tests.test:test_schema`. +""" + +from __future__ import absolute_import, print_function, unicode_literals +from taskgraph.transforms.base import TransformSequence + +transforms = TransformSequence() + + +@transforms.add +def set_defaults(config, tests): + for test in tests: + # all Android test tasks download internal objects from tooltool + test['mozharness']['tooltool-downloads'] = True + test['mozharness']['build-artifact-name'] = 'public/build/target.apk' + test['mozharness']['actions'] = ['get-secrets'] + yield test + + +@transforms.add +def set_treeherder_machine_platform(config, tests): + """Set the appropriate task.extra.treeherder.machine.platform""" + # The build names for these build platforms have partially evolved over the + # years.. This is temporary until we can clean up the handling of + # platforms + translation = { + 'android-api-15/debug': 'android-4-3-armv7-api15/debug', + 'android-api-15/opt': 'android-4-3-armv7-api15/opt', + 'android-x86/opt': 'android-4-2-x86/opt', + } + for test in tests: + build_platform = test['build-platform'] + test['treeherder-machine-platform'] = translation.get(build_platform, build_platform) + yield test diff --git a/taskcluster/taskgraph/transforms/tests/desktop_test.py b/taskcluster/taskgraph/transforms/tests/desktop_test.py new file mode 100644 index 000000000..44a907903 --- /dev/null +++ b/taskcluster/taskgraph/transforms/tests/desktop_test.py @@ -0,0 +1,118 @@ +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, You can obtain one at http://mozilla.org/MPL/2.0/. +""" +These transforms are specific to the desktop-test kind, and apply defaults to +the test descriptions appropriate to that kind. + +Both the input to and output from these transforms must conform to +`taskgraph.transforms.tests.test:test_schema`. +""" + +from __future__ import absolute_import, print_function, unicode_literals +from taskgraph.transforms.base import TransformSequence, get_keyed_by +from taskgraph.util.treeherder import split_symbol, join_symbol + +import copy + +transforms = TransformSequence() + + +@transforms.add +def set_defaults(config, tests): + for test in tests: + build_platform = test['build-platform'] + if build_platform.startswith('macosx'): + target = 'target.dmg' + else: + target = 'target.tar.bz2' + test['mozharness']['build-artifact-name'] = 'public/build/' + target + # all desktop tests want to run the bits that require node + test['mozharness']['set-moz-node-path'] = True + yield test + + +@transforms.add +def set_treeherder_machine_platform(config, tests): + """Set the appropriate task.extra.treeherder.machine.platform""" + # Linux64 build platforms for asan and pgo are specified differently to + # treeherder. This is temporary until we can clean up the handling of + # platforms + translation = { + 'linux64-asan/opt': 'linux64/asan', + 'linux64-pgo/opt': 'linux64/pgo', + 'macosx64/debug': 'osx-10-10/debug', + 'macosx64/opt': 'osx-10-10/opt', + } + for test in tests: + build_platform = test['build-platform'] + test_platform = test['test-platform'] + test['treeherder-machine-platform'] = translation.get(build_platform, test_platform) + yield test + + +@transforms.add +def set_asan_docker_image(config, tests): + """Set the appropriate task.extra.treeherder.docker-image""" + # Linux64-asan has many leaks with running mochitest-media jobs + # on Ubuntu 16.04, please remove this when bug 1289209 is resolved + for test in tests: + if test['suite'] == 'mochitest/mochitest-media' and \ + test['build-platform'] == 'linux64-asan/opt': + test['docker-image'] = {"in-tree": "desktop-test"} + yield test + + +@transforms.add +def split_e10s(config, tests): + for test in tests: + e10s = get_keyed_by(item=test, field='e10s', + item_name=test['test-name']) + test.setdefault('attributes', {}) + test['e10s'] = False + test['attributes']['e10s'] = False + + if e10s == 'both': + yield test + test = copy.deepcopy(test) + e10s = True + if e10s: + test['test-name'] += '-e10s' + test['e10s'] = True + test['attributes']['e10s'] = True + group, symbol = split_symbol(test['treeherder-symbol']) + if group != '?': + group += '-e10s' + test['treeherder-symbol'] = join_symbol(group, symbol) + test['mozharness'].setdefault('extra-options', []).append('--e10s') + yield test + + +@transforms.add +def allow_software_gl_layers(config, tests): + for test in tests: + + # since this value defaults to true, but is not applicable on windows, + # it's overriden for that platform here. + allow = not test['test-platform'].startswith('win') \ + and get_keyed_by(item=test, field='allow-software-gl-layers', + item_name=test['test-name']) + if allow: + assert test['instance-size'] != 'legacy',\ + 'Software GL layers on a legacy instance is disallowed (bug 1296086).' + + # This should be set always once bug 1296086 is resolved. + test['mozharness'].setdefault('extra-options', [])\ + .append("--allow-software-gl-layers") + + yield test + + +@transforms.add +def add_os_groups(config, tests): + for test in tests: + if test['test-platform'].startswith('win'): + groups = get_keyed_by(item=test, field='os-groups', item_name=test['test-name']) + if groups: + test['os-groups'] = groups + yield test diff --git a/taskcluster/taskgraph/transforms/tests/make_task_description.py b/taskcluster/taskgraph/transforms/tests/make_task_description.py new file mode 100644 index 000000000..fc3f94893 --- /dev/null +++ b/taskcluster/taskgraph/transforms/tests/make_task_description.py @@ -0,0 +1,445 @@ +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, You can obtain one at http://mozilla.org/MPL/2.0/. +""" +These transforms construct a task description to run the given test, based on a +test description. The implementation here is shared among all test kinds, but +contains specific support for how we run tests in Gecko (via mozharness, +invoked in particular ways). + +This is a good place to translate a test-description option such as +`single-core: true` to the implementation of that option in a task description +(worker options, mozharness commandline, environment variables, etc.) + +The test description should be fully formed by the time it reaches these +transforms, and these transforms should not embody any specific knowledge about +what should run where. this is the wrong place for special-casing platforms, +for example - use `all_tests.py` instead. +""" + +from __future__ import absolute_import, print_function, unicode_literals + +from taskgraph.transforms.base import TransformSequence +from taskgraph.transforms.job.common import ( + docker_worker_support_vcs_checkout, +) + +import logging +import os.path + +ARTIFACT_URL = 'https://queue.taskcluster.net/v1/task/{}/artifacts/{}' +WORKER_TYPE = { + # default worker types keyed by instance-size + 'large': 'aws-provisioner-v1/gecko-t-linux-large', + 'xlarge': 'aws-provisioner-v1/gecko-t-linux-xlarge', + 'legacy': 'aws-provisioner-v1/gecko-t-linux-medium', + 'default': 'aws-provisioner-v1/gecko-t-linux-large', + # windows worker types keyed by test-platform + 'windows7-32-vm': 'aws-provisioner-v1/gecko-t-win7-32', + 'windows7-32': 'aws-provisioner-v1/gecko-t-win7-32-gpu', + 'windows10-64-vm': 'aws-provisioner-v1/gecko-t-win10-64', + 'windows10-64': 'aws-provisioner-v1/gecko-t-win10-64-gpu' +} + +ARTIFACTS = [ + # (artifact name prefix, in-image path) + ("public/logs/", "build/upload/logs/"), + ("public/test", "artifacts/"), + ("public/test_info/", "build/blobber_upload_dir/"), +] + +logger = logging.getLogger(__name__) + +transforms = TransformSequence() + + +@transforms.add +def make_task_description(config, tests): + """Convert *test* descriptions to *task* descriptions (input to + taskgraph.transforms.task)""" + + for test in tests: + label = '{}-{}-{}'.format(config.kind, test['test-platform'], test['test-name']) + if test['chunks'] > 1: + label += '-{}'.format(test['this-chunk']) + + build_label = test['build-label'] + + unittest_try_name = test.get('unittest-try-name', test['test-name']) + + attr_build_platform, attr_build_type = test['build-platform'].split('/', 1) + + suite = test['suite'] + if '/' in suite: + suite, flavor = suite.split('/', 1) + else: + flavor = suite + + attributes = test.get('attributes', {}) + attributes.update({ + 'build_platform': attr_build_platform, + 'build_type': attr_build_type, + # only keep the first portion of the test platform + 'test_platform': test['test-platform'].split('/')[0], + 'test_chunk': str(test['this-chunk']), + 'unittest_suite': suite, + 'unittest_flavor': flavor, + 'unittest_try_name': unittest_try_name, + }) + + taskdesc = {} + taskdesc['label'] = label + taskdesc['description'] = test['description'] + taskdesc['attributes'] = attributes + taskdesc['dependencies'] = {'build': build_label} + taskdesc['deadline-after'] = '1 day' + taskdesc['expires-after'] = test['expires-after'] + taskdesc['routes'] = [] + taskdesc['run-on-projects'] = test.get('run-on-projects', ['all']) + taskdesc['scopes'] = [] + taskdesc['extra'] = { + 'chunks': { + 'current': test['this-chunk'], + 'total': test['chunks'], + }, + 'suite': { + 'name': suite, + 'flavor': flavor, + }, + } + taskdesc['treeherder'] = { + 'symbol': test['treeherder-symbol'], + 'kind': 'test', + 'tier': test['tier'], + 'platform': test.get('treeherder-machine-platform', test['build-platform']), + } + + # the remainder (the worker-type and worker) differs depending on the + # worker implementation + worker_setup_functions[test['worker-implementation']](config, test, taskdesc) + + # yield only the task description, discarding the test description + yield taskdesc + + +worker_setup_functions = {} + + +def worker_setup_function(name): + def wrap(func): + worker_setup_functions[name] = func + return func + return wrap + + +@worker_setup_function("docker-engine") +@worker_setup_function("docker-worker") +def docker_worker_setup(config, test, taskdesc): + + artifacts = [ + # (artifact name prefix, in-image path) + ("public/logs/", "/home/worker/workspace/build/upload/logs/"), + ("public/test", "/home/worker/artifacts/"), + ("public/test_info/", "/home/worker/workspace/build/blobber_upload_dir/"), + ] + mozharness = test['mozharness'] + + installer_url = ARTIFACT_URL.format('<build>', mozharness['build-artifact-name']) + test_packages_url = ARTIFACT_URL.format('<build>', + 'public/build/target.test_packages.json') + mozharness_url = ARTIFACT_URL.format('<build>', + 'public/build/mozharness.zip') + + taskdesc['worker-type'] = WORKER_TYPE[test['instance-size']] + + worker = taskdesc['worker'] = {} + worker['implementation'] = test['worker-implementation'] + worker['docker-image'] = test['docker-image'] + + worker['allow-ptrace'] = True # required for all tests, for crashreporter + worker['relengapi-proxy'] = False # but maybe enabled for tooltool below + worker['loopback-video'] = test['loopback-video'] + worker['loopback-audio'] = test['loopback-audio'] + worker['max-run-time'] = test['max-run-time'] + worker['retry-exit-status'] = test['retry-exit-status'] + + worker['artifacts'] = [{ + 'name': prefix, + 'path': os.path.join('/home/worker/workspace', path), + 'type': 'directory', + } for (prefix, path) in artifacts] + + worker['caches'] = [{ + 'type': 'persistent', + 'name': 'level-{}-{}-test-workspace'.format( + config.params['level'], config.params['project']), + 'mount-point': "/home/worker/workspace", + }] + + env = worker['env'] = { + 'MOZHARNESS_CONFIG': ' '.join(mozharness['config']), + 'MOZHARNESS_SCRIPT': mozharness['script'], + 'MOZILLA_BUILD_URL': {'task-reference': installer_url}, + 'NEED_PULSEAUDIO': 'true', + 'NEED_WINDOW_MANAGER': 'true', + } + + if mozharness['set-moz-node-path']: + env['MOZ_NODE_PATH'] = '/usr/local/bin/node' + + if 'actions' in mozharness: + env['MOZHARNESS_ACTIONS'] = ' '.join(mozharness['actions']) + + if config.params['project'] == 'try': + env['TRY_COMMIT_MSG'] = config.params['message'] + + # handle some of the mozharness-specific options + + if mozharness['tooltool-downloads']: + worker['relengapi-proxy'] = True + worker['caches'].append({ + 'type': 'persistent', + 'name': 'tooltool-cache', + 'mount-point': '/home/worker/tooltool-cache', + }) + taskdesc['scopes'].extend([ + 'docker-worker:relengapi-proxy:tooltool.download.internal', + 'docker-worker:relengapi-proxy:tooltool.download.public', + ]) + + # assemble the command line + command = [ + '/home/worker/bin/run-task', + # The workspace cache/volume is default owned by root:root. + '--chown', '/home/worker/workspace', + ] + + # Support vcs checkouts regardless of whether the task runs from + # source or not in case it is needed on an interactive loaner. + docker_worker_support_vcs_checkout(config, test, taskdesc) + + # If we have a source checkout, run mozharness from it instead of + # downloading a zip file with the same content. + if test['checkout']: + command.extend(['--vcs-checkout', '/home/worker/checkouts/gecko']) + env['MOZHARNESS_PATH'] = '/home/worker/checkouts/gecko/testing/mozharness' + else: + env['MOZHARNESS_URL'] = {'task-reference': mozharness_url} + + command.extend([ + '--', + '/home/worker/bin/test-linux.sh', + ]) + + if mozharness.get('no-read-buildbot-config'): + command.append("--no-read-buildbot-config") + command.extend([ + {"task-reference": "--installer-url=" + installer_url}, + {"task-reference": "--test-packages-url=" + test_packages_url}, + ]) + command.extend(mozharness.get('extra-options', [])) + + # TODO: remove the need for run['chunked'] + if mozharness.get('chunked') or test['chunks'] > 1: + # Implement mozharness['chunking-args'], modifying command in place + if mozharness['chunking-args'] == 'this-chunk': + command.append('--total-chunk={}'.format(test['chunks'])) + command.append('--this-chunk={}'.format(test['this-chunk'])) + elif mozharness['chunking-args'] == 'test-suite-suffix': + suffix = mozharness['chunk-suffix'].replace('<CHUNK>', str(test['this-chunk'])) + for i, c in enumerate(command): + if isinstance(c, basestring) and c.startswith('--test-suite'): + command[i] += suffix + + if 'download-symbols' in mozharness: + download_symbols = mozharness['download-symbols'] + download_symbols = {True: 'true', False: 'false'}.get(download_symbols, download_symbols) + command.append('--download-symbols=' + download_symbols) + + worker['command'] = command + + +def normpath(path): + return path.replace('/', '\\') + + +def get_firefox_version(): + with open('browser/config/version.txt', 'r') as f: + return f.readline().strip() + + +@worker_setup_function('generic-worker') +def generic_worker_setup(config, test, taskdesc): + artifacts = [ + { + 'path': 'public\\logs\\localconfig.json', + 'type': 'file' + }, + { + 'path': 'public\\logs\\log_critical.log', + 'type': 'file' + }, + { + 'path': 'public\\logs\\log_error.log', + 'type': 'file' + }, + { + 'path': 'public\\logs\\log_fatal.log', + 'type': 'file' + }, + { + 'path': 'public\\logs\\log_info.log', + 'type': 'file' + }, + { + 'path': 'public\\logs\\log_raw.log', + 'type': 'file' + }, + { + 'path': 'public\\logs\\log_warning.log', + 'type': 'file' + }, + { + 'path': 'public\\test_info', + 'type': 'directory' + } + ] + mozharness = test['mozharness'] + + build_platform = taskdesc['attributes']['build_platform'] + test_platform = test['test-platform'].split('/')[0] + + target = 'firefox-{}.en-US.{}'.format(get_firefox_version(), build_platform) + + installer_url = ARTIFACT_URL.format( + '<build>', 'public/build/{}.zip'.format(target)) + test_packages_url = ARTIFACT_URL.format( + '<build>', 'public/build/{}.test_packages.json'.format(target)) + mozharness_url = ARTIFACT_URL.format( + '<build>', 'public/build/mozharness.zip') + + taskdesc['worker-type'] = WORKER_TYPE[test_platform] + + taskdesc['scopes'].extend( + ['generic-worker:os-group:{}'.format(group) for group in test['os-groups']]) + + worker = taskdesc['worker'] = {} + worker['os-groups'] = test['os-groups'] + worker['implementation'] = test['worker-implementation'] + worker['max-run-time'] = test['max-run-time'] + worker['artifacts'] = artifacts + + env = worker['env'] = { + # Bug 1306989 + 'APPDATA': '%cd%\\AppData\\Roaming', + 'LOCALAPPDATA': '%cd%\\AppData\\Local', + 'TEMP': '%cd%\\AppData\\Local\\Temp', + 'TMP': '%cd%\\AppData\\Local\\Temp', + 'USERPROFILE': '%cd%', + } + + # assemble the command line + mh_command = [ + 'c:\\mozilla-build\\python\\python.exe', + '-u', + 'mozharness\\scripts\\' + normpath(mozharness['script']) + ] + for mh_config in mozharness['config']: + mh_command.extend(['--cfg', 'mozharness\\configs\\' + normpath(mh_config)]) + mh_command.extend(mozharness.get('extra-options', [])) + if mozharness.get('no-read-buildbot-config'): + mh_command.append('--no-read-buildbot-config') + mh_command.extend(['--installer-url', installer_url]) + mh_command.extend(['--test-packages-url', test_packages_url]) + if mozharness.get('download-symbols'): + if isinstance(mozharness['download-symbols'], basestring): + mh_command.extend(['--download-symbols', mozharness['download-symbols']]) + else: + mh_command.extend(['--download-symbols', 'true']) + + # TODO: remove the need for run['chunked'] + if mozharness.get('chunked') or test['chunks'] > 1: + # Implement mozharness['chunking-args'], modifying command in place + if mozharness['chunking-args'] == 'this-chunk': + mh_command.append('--total-chunk={}'.format(test['chunks'])) + mh_command.append('--this-chunk={}'.format(test['this-chunk'])) + elif mozharness['chunking-args'] == 'test-suite-suffix': + suffix = mozharness['chunk-suffix'].replace('<CHUNK>', str(test['this-chunk'])) + for i, c in enumerate(mh_command): + if isinstance(c, basestring) and c.startswith('--test-suite'): + mh_command[i] += suffix + + worker['command'] = [ + 'mkdir {} {}'.format(env['APPDATA'], env['TMP']), + {'task-reference': 'c:\\mozilla-build\\wget\\wget.exe {}'.format(mozharness_url)}, + 'c:\\mozilla-build\\info-zip\\unzip.exe mozharness.zip', + {'task-reference': ' '.join(mh_command)}, + 'xcopy build\\blobber_upload_dir public\\test_info /e /i', + 'copy /y logs\\*.* public\\logs\\' + ] + + +@worker_setup_function("macosx-engine") +def macosx_engine_setup(config, test, taskdesc): + mozharness = test['mozharness'] + + installer_url = ARTIFACT_URL.format('<build>', mozharness['build-artifact-name']) + test_packages_url = ARTIFACT_URL.format('<build>', + 'public/build/target.test_packages.json') + mozharness_url = ARTIFACT_URL.format('<build>', + 'public/build/mozharness.zip') + + # for now we have only 10.10 machines + taskdesc['worker-type'] = 'tc-worker-provisioner/gecko-t-osx-10-10' + + worker = taskdesc['worker'] = {} + worker['implementation'] = test['worker-implementation'] + + worker['artifacts'] = [{ + 'name': prefix.rstrip('/'), + 'path': path.rstrip('/'), + 'type': 'directory', + } for (prefix, path) in ARTIFACTS] + + worker['env'] = { + 'GECKO_HEAD_REPOSITORY': config.params['head_repository'], + 'GECKO_HEAD_REV': config.params['head_rev'], + 'MOZHARNESS_CONFIG': ' '.join(mozharness['config']), + 'MOZHARNESS_SCRIPT': mozharness['script'], + 'MOZHARNESS_URL': {'task-reference': mozharness_url}, + 'MOZILLA_BUILD_URL': {'task-reference': installer_url}, + } + + # assemble the command line + + worker['link'] = '{}/raw-file/{}/taskcluster/scripts/tester/test-macosx.sh'.format( + config.params['head_repository'], config.params['head_rev'] + ) + + command = worker['command'] = ["./test-macosx.sh"] + if mozharness.get('no-read-buildbot-config'): + command.append("--no-read-buildbot-config") + command.extend([ + {"task-reference": "--installer-url=" + installer_url}, + {"task-reference": "--test-packages-url=" + test_packages_url}, + ]) + if mozharness.get('include-blob-upload-branch'): + command.append('--blob-upload-branch=' + config.params['project']) + command.extend(mozharness.get('extra-options', [])) + + # TODO: remove the need for run['chunked'] + if mozharness.get('chunked') or test['chunks'] > 1: + # Implement mozharness['chunking-args'], modifying command in place + if mozharness['chunking-args'] == 'this-chunk': + command.append('--total-chunk={}'.format(test['chunks'])) + command.append('--this-chunk={}'.format(test['this-chunk'])) + elif mozharness['chunking-args'] == 'test-suite-suffix': + suffix = mozharness['chunk-suffix'].replace('<CHUNK>', str(test['this-chunk'])) + for i, c in enumerate(command): + if isinstance(c, basestring) and c.startswith('--test-suite'): + command[i] += suffix + + if 'download-symbols' in mozharness: + download_symbols = mozharness['download-symbols'] + download_symbols = {True: 'true', False: 'false'}.get(download_symbols, download_symbols) + command.append('--download-symbols=' + download_symbols) diff --git a/taskcluster/taskgraph/transforms/tests/test_description.py b/taskcluster/taskgraph/transforms/tests/test_description.py new file mode 100644 index 000000000..1365919fe --- /dev/null +++ b/taskcluster/taskgraph/transforms/tests/test_description.py @@ -0,0 +1,235 @@ +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, You can obtain one at http://mozilla.org/MPL/2.0/. +""" +This file defines the schema for tests -- the things in `tests.yml`. It should +be run both before and after the kind-specific transforms, to ensure that the +transforms do not generate invalid tests. +""" + +from __future__ import absolute_import, print_function, unicode_literals + +from taskgraph.transforms.base import validate_schema +from voluptuous import ( + Any, + Optional, + Required, + Schema, +) + + +# Schema for a test description +# +# *****WARNING***** +# +# This is a great place for baffling cruft to accumulate, and that makes +# everyone move more slowly. Be considerate of your fellow hackers! +# See the warnings in taskcluster/docs/how-tos.rst +# +# *****WARNING***** +test_description_schema = Schema({ + # description of the suite, for the task metadata + 'description': basestring, + + # test suite name, or <suite>/<flavor> + Required('suite'): Any( + basestring, + {'by-test-platform': {basestring: basestring}}, + ), + + # the name by which this test suite is addressed in try syntax; defaults to + # the test-name + Optional('unittest-try-name'): basestring, + + # the symbol, or group(symbol), under which this task should appear in + # treeherder. + 'treeherder-symbol': basestring, + + # the value to place in task.extra.treeherder.machine.platform; ideally + # this is the same as build-platform, and that is the default, but in + # practice it's not always a match. + Optional('treeherder-machine-platform'): basestring, + + # attributes to appear in the resulting task (later transforms will add the + # common attributes) + Optional('attributes'): {basestring: object}, + + # The `run_on_projects` attribute, defaulting to "all". This dictates the + # projects on which this task should be included in the target task set. + # See the attributes documentation for details. + Optional('run-on-projects', default=['all']): Any( + [basestring], + {'by-test-platform': {basestring: [basestring]}}, + ), + + # the sheriffing tier for this task (default: set based on test platform) + Optional('tier'): int, + + # number of chunks to create for this task. This can be keyed by test + # platform by passing a dictionary in the `by-test-platform` key. If the + # test platform is not found, the key 'default' will be tried. + Required('chunks', default=1): Any( + int, + {'by-test-platform': {basestring: int}}, + ), + + # the time (with unit) after which this task is deleted; default depends on + # the branch (see below) + Optional('expires-after'): basestring, + + # Whether to run this task with e10s (desktop-test only). If false, run + # without e10s; if true, run with e10s; if 'both', run one task with and + # one task without e10s. E10s tasks have "-e10s" appended to the test name + # and treeherder group. + Required('e10s', default='both'): Any( + bool, 'both', + {'by-test-platform': {basestring: Any(bool, 'both')}}, + ), + + # The EC2 instance size to run these tests on. + Required('instance-size', default='default'): Any( + Any('default', 'large', 'xlarge', 'legacy'), + {'by-test-platform': {basestring: Any('default', 'large', 'xlarge', 'legacy')}}, + ), + + # Whether the task requires loopback audio or video (whatever that may mean + # on the platform) + Required('loopback-audio', default=False): bool, + Required('loopback-video', default=False): bool, + + # Whether the test can run using a software GL implementation on Linux + # using the GL compositor. May not be used with "legacy" sized instances + # due to poor LLVMPipe performance (bug 1296086). + Optional('allow-software-gl-layers', default=True): bool, + + # The worker implementation for this test, as dictated by policy and by the + # test platform. + Optional('worker-implementation'): Any( + 'docker-worker', + 'macosx-engine', + 'generic-worker', + # coming soon: + 'docker-engine', + 'buildbot-bridge', + ), + + # For tasks that will run in docker-worker or docker-engine, this is the + # name of the docker image or in-tree docker image to run the task in. If + # in-tree, then a dependency will be created automatically. This is + # generally `desktop-test`, or an image that acts an awful lot like it. + Required('docker-image', default={'in-tree': 'desktop-test'}): Any( + # a raw Docker image path (repo/image:tag) + basestring, + # an in-tree generated docker image (from `testing/docker/<name>`) + {'in-tree': basestring} + ), + + # seconds of runtime after which the task will be killed. Like 'chunks', + # this can be keyed by test pltaform. + Required('max-run-time', default=3600): Any( + int, + {'by-test-platform': {basestring: int}}, + ), + + # the exit status code that indicates the task should be retried + Optional('retry-exit-status'): int, + + # Whether to perform a gecko checkout. + Required('checkout', default=False): bool, + + # What to run + Required('mozharness'): Any({ + # the mozharness script used to run this task + Required('script'): basestring, + + # the config files required for the task + Required('config'): Any( + [basestring], + {'by-test-platform': {basestring: [basestring]}}, + ), + + # any additional actions to pass to the mozharness command + Optional('actions'): [basestring], + + # additional command-line options for mozharness, beyond those + # automatically added + Required('extra-options', default=[]): Any( + [basestring], + {'by-test-platform': {basestring: [basestring]}}, + ), + + # the artifact name (including path) to test on the build task; this is + # generally set in a per-kind transformation + Optional('build-artifact-name'): basestring, + + # If true, tooltool downloads will be enabled via relengAPIProxy. + Required('tooltool-downloads', default=False): bool, + + # This mozharness script also runs in Buildbot and tries to read a + # buildbot config file, so tell it not to do so in TaskCluster + Required('no-read-buildbot-config', default=False): bool, + + # Add --blob-upload-branch=<project> mozharness parameter + Optional('include-blob-upload-branch'): bool, + + # The setting for --download-symbols (if omitted, the option will not + # be passed to mozharness) + Optional('download-symbols'): Any(True, 'ondemand'), + + # If set, then MOZ_NODE_PATH=/usr/local/bin/node is included in the + # environment. This is more than just a helpful path setting -- it + # causes xpcshell tests to start additional servers, and runs + # additional tests. + Required('set-moz-node-path', default=False): bool, + + # If true, include chunking information in the command even if the number + # of chunks is 1 + Required('chunked', default=False): bool, + + # The chunking argument format to use + Required('chunking-args', default='this-chunk'): Any( + # Use the usual --this-chunk/--total-chunk arguments + 'this-chunk', + # Use --test-suite=<suite>-<chunk-suffix>; see chunk-suffix, below + 'test-suite-suffix', + ), + + # the string to append to the `--test-suite` arugment when + # chunking-args = test-suite-suffix; "<CHUNK>" in this string will + # be replaced with the chunk number. + Optional('chunk-suffix'): basestring, + }), + + # The current chunk; this is filled in by `all_kinds.py` + Optional('this-chunk'): int, + + # os user groups for test task workers; required scopes, will be + # added automatically + Optional('os-groups', default=[]): Any( + [basestring], + # todo: create a dedicated elevated worker group and name here + {'by-test-platform': {basestring: [basestring]}}, + ), + + # -- values supplied by the task-generation infrastructure + + # the platform of the build this task is testing + 'build-platform': basestring, + + # the label of the build task generating the materials to test + 'build-label': basestring, + + # the platform on which the tests will run + 'test-platform': basestring, + + # the name of the test (the key in tests.yml) + 'test-name': basestring, + +}, required=True) + + +# TODO: can we have validate and validate_full for before and after? +def validate(config, tests): + for test in tests: + yield validate_schema(test_description_schema, test, + "In test {!r}:".format(test['test-name'])) diff --git a/taskcluster/taskgraph/transforms/upload_symbols.py b/taskcluster/taskgraph/transforms/upload_symbols.py new file mode 100644 index 000000000..9b4884a97 --- /dev/null +++ b/taskcluster/taskgraph/transforms/upload_symbols.py @@ -0,0 +1,36 @@ +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, You can obtain one at http://mozilla.org/MPL/2.0/. +""" +Transform the upload-symbols task description template, + taskcluster/ci/upload-symbols/job-template.yml +into an actual task description. +""" + +from __future__ import absolute_import, print_function, unicode_literals + +from taskgraph.transforms.base import TransformSequence + + +transforms = TransformSequence() + + +@transforms.add +def fill_template(config, tasks): + for task in tasks: + # Fill out the dynamic fields in the task description + task['label'] = task['build-label'] + '-upload-symbols' + task['dependencies'] = {'build': task['build-label']} + task['worker']['env']['GECKO_HEAD_REPOSITORY'] = config.params['head_repository'] + task['worker']['env']['GECKO_HEAD_REV'] = config.params['head_rev'] + + build_platform, build_type = task['build-platform'].split('/') + attributes = task.setdefault('attributes', {}) + attributes['build_platform'] = build_platform + attributes['build_type'] = build_type + + # clear out the stuff that's not part of a task description + del task['build-label'] + del task['build-platform'] + + yield task diff --git a/taskcluster/taskgraph/try_option_syntax.py b/taskcluster/taskgraph/try_option_syntax.py new file mode 100644 index 000000000..b5988db98 --- /dev/null +++ b/taskcluster/taskgraph/try_option_syntax.py @@ -0,0 +1,559 @@ +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, You can obtain one at http://mozilla.org/MPL/2.0/. + +from __future__ import absolute_import, print_function, unicode_literals + +import argparse +import copy +import logging +import re +import shlex + +logger = logging.getLogger(__name__) + +TRY_DELIMITER = 'try:' + +# The build type aliases are very cryptic and only used in try flags these are +# mappings from the single char alias to a longer more recognizable form. +BUILD_TYPE_ALIASES = { + 'o': 'opt', + 'd': 'debug' +} + +# consider anything in this whitelist of kinds to be governed by -b/-p +BUILD_KINDS = set([ + 'build', + 'artifact-build', + 'hazard', + 'l10n', + 'upload-symbols', + 'valgrind', + 'static-analysis', + 'spidermonkey', +]) + +# anything in this list is governed by -j +JOB_KINDS = set([ + 'source-check', + 'toolchain', + 'marionette-harness', + 'android-stuff', +]) + + +# mapping from shortcut name (usable with -u) to a boolean function identifying +# matching test names +def alias_prefix(prefix): + return lambda name: name.startswith(prefix) + + +def alias_contains(infix): + return lambda name: infix in name + + +def alias_matches(pattern): + pattern = re.compile(pattern) + return lambda name: pattern.match(name) + +UNITTEST_ALIASES = { + # Aliases specify shorthands that can be used in try syntax. The shorthand + # is the dictionary key, with the value representing a pattern for matching + # unittest_try_names. + # + # Note that alias expansion is performed in the absence of any chunk + # prefixes. For example, the first example above would replace "foo-7" + # with "foobar-7". Note that a few aliases allowed chunks to be specified + # without a leading `-`, for example 'mochitest-dt1'. That's no longer + # supported. + 'cppunit': alias_prefix('cppunit'), + 'crashtest': alias_prefix('crashtest'), + 'crashtest-e10s': alias_prefix('crashtest-e10s'), + 'e10s': alias_contains('e10s'), + 'external-media-tests': alias_prefix('external-media-tests'), + 'firefox-ui-functional': alias_prefix('firefox-ui-functional'), + 'firefox-ui-functional-e10s': alias_prefix('firefox-ui-functional-e10s'), + 'gaia-js-integration': alias_contains('gaia-js-integration'), + 'gtest': alias_prefix('gtest'), + 'jittest': alias_prefix('jittest'), + 'jittests': alias_prefix('jittest'), + 'jsreftest': alias_prefix('jsreftest'), + 'jsreftest-e10s': alias_prefix('jsreftest-e10s'), + 'marionette': alias_prefix('marionette'), + 'marionette-e10s': alias_prefix('marionette-e10s'), + 'mochitest': alias_prefix('mochitest'), + 'mochitests': alias_prefix('mochitest'), + 'mochitest-e10s': alias_prefix('mochitest-e10s'), + 'mochitests-e10s': alias_prefix('mochitest-e10s'), + 'mochitest-debug': alias_prefix('mochitest-debug-'), + 'mochitest-a11y': alias_contains('mochitest-a11y'), + 'mochitest-bc': alias_prefix('mochitest-browser-chrome'), + 'mochitest-e10s-bc': alias_prefix('mochitest-e10s-browser-chrome'), + 'mochitest-browser-chrome': alias_prefix('mochitest-browser-chrome'), + 'mochitest-e10s-browser-chrome': alias_prefix('mochitest-e10s-browser-chrome'), + 'mochitest-chrome': alias_contains('mochitest-chrome'), + 'mochitest-dt': alias_prefix('mochitest-devtools-chrome'), + 'mochitest-e10s-dt': alias_prefix('mochitest-e10s-devtools-chrome'), + 'mochitest-gl': alias_prefix('mochitest-webgl'), + 'mochitest-gl-e10s': alias_prefix('mochitest-webgl-e10s'), + 'mochitest-gpu': alias_prefix('mochitest-gpu'), + 'mochitest-gpu-e10s': alias_prefix('mochitest-gpu-e10s'), + 'mochitest-clipboard': alias_prefix('mochitest-clipboard'), + 'mochitest-clipboard-e10s': alias_prefix('mochitest-clipboard-e10s'), + 'mochitest-jetpack': alias_prefix('mochitest-jetpack'), + 'mochitest-media': alias_prefix('mochitest-media'), + 'mochitest-media-e10s': alias_prefix('mochitest-media-e10s'), + 'mochitest-vg': alias_prefix('mochitest-valgrind'), + 'reftest': alias_matches(r'^(plain-)?reftest.*$'), + 'reftest-no-accel': alias_matches(r'^(plain-)?reftest-no-accel.*$'), + 'reftests': alias_matches(r'^(plain-)?reftest.*$'), + 'reftests-e10s': alias_matches(r'^(plain-)?reftest-e10s.*$'), + 'robocop': alias_prefix('robocop'), + 'web-platform-test': alias_prefix('web-platform-tests'), + 'web-platform-tests': alias_prefix('web-platform-tests'), + 'web-platform-tests-e10s': alias_prefix('web-platform-tests-e10s'), + 'web-platform-tests-reftests': alias_prefix('web-platform-tests-reftests'), + 'web-platform-tests-reftests-e10s': alias_prefix('web-platform-tests-reftests-e10s'), + 'xpcshell': alias_prefix('xpcshell'), +} + +# unittest platforms can be specified by substring of the "pretty name", which +# is basically the old Buildbot builder name. This dict has {pretty name, +# [test_platforms]} translations, This includes only the most commonly-used +# substrings. This is intended only for backward-compatibility. New test +# platforms should have their `test_platform` spelled out fully in try syntax. +UNITTEST_PLATFORM_PRETTY_NAMES = { + 'Ubuntu': ['linux', 'linux64', 'linux64-asan'], + 'x64': ['linux64', 'linux64-asan'], + 'Android 4.3': ['android-4.3-arm7-api-15'], + # other commonly-used substrings for platforms not yet supported with + # in-tree taskgraphs: + # '10.10': [..TODO..], + # '10.10.5': [..TODO..], + # '10.6': [..TODO..], + # '10.8': [..TODO..], + # 'Android 2.3 API9': [..TODO..], + # 'Windows 7': [..TODO..], + # 'Windows 7 VM': [..TODO..], + # 'Windows 8': [..TODO..], + # 'Windows XP': [..TODO..], + # 'win32': [..TODO..], + # 'win64': [..TODO..], +} + +# We have a few platforms for which we want to do some "extra" builds, or at +# least build-ish things. Sort of. Anyway, these other things are implemented +# as different "platforms". These do *not* automatically ride along with "-p +# all" +RIDEALONG_BUILDS = { + 'android-api-15': [ + 'android-api-15-l10n', + ], + 'linux': [ + 'linux-l10n', + ], + 'linux64': [ + 'linux64-l10n', + 'sm-plain', + 'sm-nonunified', + 'sm-arm-sim', + 'sm-arm64-sim', + 'sm-compacting', + 'sm-rootanalysis', + 'sm-package', + 'sm-tsan', + 'sm-asan', + 'sm-mozjs-sys', + 'sm-msan', + ], +} + +TEST_CHUNK_SUFFIX = re.compile('(.*)-([0-9]+)$') + + +class TryOptionSyntax(object): + + def __init__(self, message, full_task_graph): + """ + Parse a "try syntax" formatted commit message. This is the old "-b do -p + win32 -u all" format. Aliases are applied to map short names to full + names. + + The resulting object has attributes: + + - build_types: a list containing zero or more of 'opt' and 'debug' + - platforms: a list of selected platform names, or None for all + - unittests: a list of tests, of the form given below, or None for all + - jobs: a list of requested job names, or None for all + - trigger_tests: the number of times tests should be triggered (--rebuild) + - interactive: true if --interactive + - notifications: either None if no notifications or one of 'all' or 'failure' + + Note that -t is currently completely ignored. + + The unittests and talos lists contain dictionaries of the form: + + { + 'test': '<suite name>', + 'platforms': [..platform names..], # to limit to only certain platforms + 'only_chunks': set([..chunk numbers..]), # to limit only to certain chunks + } + """ + self.jobs = [] + self.build_types = [] + self.platforms = [] + self.unittests = [] + self.talos = [] + self.trigger_tests = 0 + self.interactive = False + self.notifications = None + + # shlex used to ensure we split correctly when giving values to argparse. + parts = shlex.split(self.escape_whitespace_in_brackets(message)) + try_idx = None + for idx, part in enumerate(parts): + if part == TRY_DELIMITER: + try_idx = idx + break + + if try_idx is None: + return + + # Argument parser based on try flag flags + parser = argparse.ArgumentParser() + parser.add_argument('-b', '--build', dest='build_types') + parser.add_argument('-p', '--platform', nargs='?', + dest='platforms', const='all', default='all') + parser.add_argument('-u', '--unittests', nargs='?', + dest='unittests', const='all', default='all') + parser.add_argument('-t', '--talos', nargs='?', dest='talos', const='all', default='all') + parser.add_argument('-i', '--interactive', + dest='interactive', action='store_true', default=False) + parser.add_argument('-e', '--all-emails', + dest='notifications', action='store_const', const='all') + parser.add_argument('-f', '--failure-emails', + dest='notifications', action='store_const', const='failure') + parser.add_argument('-j', '--job', dest='jobs', action='append') + # In order to run test jobs multiple times + parser.add_argument('--rebuild', dest='trigger_tests', type=int, default=1) + args, _ = parser.parse_known_args(parts[try_idx:]) + + self.jobs = self.parse_jobs(args.jobs) + self.build_types = self.parse_build_types(args.build_types) + self.platforms = self.parse_platforms(args.platforms) + self.unittests = self.parse_test_option( + "unittest_try_name", args.unittests, full_task_graph) + self.talos = self.parse_test_option("talos_try_name", args.talos, full_task_graph) + self.trigger_tests = args.trigger_tests + self.interactive = args.interactive + self.notifications = args.notifications + + def parse_jobs(self, jobs_arg): + if not jobs_arg or jobs_arg == ['all']: + return None + expanded = [] + for job in jobs_arg: + expanded.extend(j.strip() for j in job.split(',')) + return expanded + + def parse_build_types(self, build_types_arg): + if build_types_arg is None: + build_types_arg = [] + build_types = filter(None, [BUILD_TYPE_ALIASES.get(build_type) for + build_type in build_types_arg]) + return build_types + + def parse_platforms(self, platform_arg): + if platform_arg == 'all': + return None + + results = [] + for build in platform_arg.split(','): + results.append(build) + if build in RIDEALONG_BUILDS: + results.extend(RIDEALONG_BUILDS[build]) + logger.info("platform %s triggers ridealong builds %s" % + (build, ', '.join(RIDEALONG_BUILDS[build]))) + + return results + + def parse_test_option(self, attr_name, test_arg, full_task_graph): + ''' + + Parse a unittest (-u) or talos (-t) option, in the context of a full + task graph containing available `unittest_try_name` or `talos_try_name` + attributes. There are three cases: + + - test_arg is == 'none' (meaning an empty list) + - test_arg is == 'all' (meaning use the list of jobs for that job type) + - test_arg is comma string which needs to be parsed + ''' + + # Empty job list case... + if test_arg is None or test_arg == 'none': + return [] + + all_platforms = set(t.attributes['test_platform'] + for t in full_task_graph.tasks.itervalues() + if 'test_platform' in t.attributes) + + tests = self.parse_test_opts(test_arg, all_platforms) + + if not tests: + return [] + + all_tests = set(t.attributes[attr_name] + for t in full_task_graph.tasks.itervalues() + if attr_name in t.attributes) + + # Special case where tests is 'all' and must be expanded + if tests[0]['test'] == 'all': + results = [] + all_entry = tests[0] + for test in all_tests: + entry = {'test': test} + # If there are platform restrictions copy them across the list. + if 'platforms' in all_entry: + entry['platforms'] = list(all_entry['platforms']) + results.append(entry) + return self.parse_test_chunks(all_tests, results) + else: + return self.parse_test_chunks(all_tests, tests) + + def parse_test_opts(self, input_str, all_platforms): + ''' + Parse `testspec,testspec,..`, where each testspec is a test name + optionally followed by a list of test platforms or negated platforms in + `[]`. + + No brackets indicates that tests should run on all platforms for which + builds are available. If testspecs are provided, then each is treated, + from left to right, as an instruction to include or (if negated) + exclude a set of test platforms. A single spec may expand to multiple + test platforms via UNITTEST_PLATFORM_PRETTY_NAMES. If the first test + spec is negated, processing begins with the full set of available test + platforms; otherwise, processing begins with an empty set of test + platforms. + ''' + + # Final results which we will return. + tests = [] + + cur_test = {} + token = '' + in_platforms = False + + def normalize_platforms(): + if 'platforms' not in cur_test: + return + # if the first spec is a negation, start with all platforms + if cur_test['platforms'][0][0] == '-': + platforms = all_platforms.copy() + else: + platforms = [] + for platform in cur_test['platforms']: + if platform[0] == '-': + platforms = [p for p in platforms if p != platform[1:]] + else: + platforms.append(platform) + cur_test['platforms'] = platforms + + def add_test(value): + normalize_platforms() + cur_test['test'] = value.strip() + tests.insert(0, cur_test) + + def add_platform(value): + platform = value.strip() + if platform[0] == '-': + negated = True + platform = platform[1:] + else: + negated = False + platforms = UNITTEST_PLATFORM_PRETTY_NAMES.get(platform, [platform]) + if negated: + platforms = ["-" + p for p in platforms] + cur_test['platforms'] = platforms + cur_test.get('platforms', []) + + # This might be somewhat confusing but we parse the string _backwards_ so + # there is no ambiguity over what state we are in. + for char in reversed(input_str): + + # , indicates exiting a state + if char == ',': + + # Exit a particular platform. + if in_platforms: + add_platform(token) + + # Exit a particular test. + else: + add_test(token) + cur_test = {} + + # Token must always be reset after we exit a state + token = '' + elif char == '[': + # Exiting platform state entering test state. + add_platform(token) + token = '' + in_platforms = False + elif char == ']': + # Entering platform state. + in_platforms = True + else: + # Accumulator. + token = char + token + + # Handle any left over tokens. + if token: + add_test(token) + + return tests + + def handle_alias(self, test, all_tests): + ''' + Expand a test if its name refers to an alias, returning a list of test + dictionaries cloned from the first (to maintain any metadata). + ''' + if test['test'] not in UNITTEST_ALIASES: + return [test] + + alias = UNITTEST_ALIASES[test['test']] + + def mktest(name): + newtest = copy.deepcopy(test) + newtest['test'] = name + return newtest + + def exprmatch(alias): + return [t for t in all_tests if alias(t)] + + return [mktest(t) for t in exprmatch(alias)] + + def parse_test_chunks(self, all_tests, tests): + ''' + Test flags may include parameters to narrow down the number of chunks in a + given push. We don't model 1 chunk = 1 job in taskcluster so we must check + each test flag to see if it is actually specifying a chunk. + ''' + results = [] + seen_chunks = {} + for test in tests: + matches = TEST_CHUNK_SUFFIX.match(test['test']) + if matches: + name = matches.group(1) + chunk = matches.group(2) + if name in seen_chunks: + seen_chunks[name].add(chunk) + else: + seen_chunks[name] = {chunk} + test['test'] = name + test['only_chunks'] = seen_chunks[name] + results.append(test) + else: + results.extend(self.handle_alias(test, all_tests)) + + # uniquify the results over the test names + results = {test['test']: test for test in results}.values() + return results + + def find_all_attribute_suffixes(self, graph, prefix): + rv = set() + for t in graph.tasks.itervalues(): + for a in t.attributes: + if a.startswith(prefix): + rv.add(a[len(prefix):]) + return sorted(rv) + + def escape_whitespace_in_brackets(self, input_str): + ''' + In tests you may restrict them by platform [] inside of the brackets + whitespace may occur this is typically invalid shell syntax so we escape it + with backslash sequences . + ''' + result = "" + in_brackets = False + for char in input_str: + if char == '[': + in_brackets = True + result += char + continue + + if char == ']': + in_brackets = False + result += char + continue + + if char == ' ' and in_brackets: + result += '\ ' + continue + + result += char + + return result + + def task_matches(self, attributes): + attr = attributes.get + + def check_run_on_projects(): + return set(['try', 'all']) & set(attr('run_on_projects', [])) + + def match_test(try_spec, attr_name): + if attr('build_type') not in self.build_types: + return False + if self.platforms is not None: + if attr('build_platform') not in self.platforms: + return False + else: + if not check_run_on_projects(): + return False + if try_spec is None: + return True + # TODO: optimize this search a bit + for test in try_spec: + if attr(attr_name) == test['test']: + break + else: + return False + if 'platforms' in test and attr('test_platform') not in test['platforms']: + return False + if 'only_chunks' in test and attr('test_chunk') not in test['only_chunks']: + return False + return True + + if attr('kind') in ('desktop-test', 'android-test'): + return match_test(self.unittests, 'unittest_try_name') + elif attr('kind') in JOB_KINDS: + if self.jobs is None: + return True + if attr('build_platform') in self.jobs: + return True + elif attr('kind') in BUILD_KINDS: + if attr('build_type') not in self.build_types: + return False + elif self.platforms is None: + # for "-p all", look for try in the 'run_on_projects' attribute + return check_run_on_projects() + else: + if attr('build_platform') not in self.platforms: + return False + return True + else: + return False + + def __str__(self): + def none_for_all(list): + if list is None: + return '<all>' + return ', '.join(str(e) for e in list) + + return "\n".join([ + "build_types: " + ", ".join(self.build_types), + "platforms: " + none_for_all(self.platforms), + "unittests: " + none_for_all(self.unittests), + "jobs: " + none_for_all(self.jobs), + "trigger_tests: " + str(self.trigger_tests), + "interactive: " + str(self.interactive), + "notifications: " + self.notifications, + ]) diff --git a/taskcluster/taskgraph/util/__init__.py b/taskcluster/taskgraph/util/__init__.py new file mode 100644 index 000000000..e69de29bb --- /dev/null +++ b/taskcluster/taskgraph/util/__init__.py diff --git a/taskcluster/taskgraph/util/attributes.py b/taskcluster/taskgraph/util/attributes.py new file mode 100644 index 000000000..b44a3364f --- /dev/null +++ b/taskcluster/taskgraph/util/attributes.py @@ -0,0 +1,26 @@ +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, You can obtain one at http://mozilla.org/MPL/2.0/. + + +def attrmatch(attributes, **kwargs): + """Determine whether the given set of task attributes matches. The + conditions are given as keyword arguments, where each keyword names an + attribute. The keyword value can be a literal, a set, or a callable. A + literal must match the attribute exactly. Given a set, the attribute value + must be in the set. A callable is called with the attribute value. If an + attribute is specified as a keyword argument but not present in the + attributes, the result is False.""" + for kwkey, kwval in kwargs.iteritems(): + if kwkey not in attributes: + return False + attval = attributes[kwkey] + if isinstance(kwval, set): + if attval not in kwval: + return False + elif callable(kwval): + if not kwval(attval): + return False + elif kwval != attributes[kwkey]: + return False + return True diff --git a/taskcluster/taskgraph/util/docker.py b/taskcluster/taskgraph/util/docker.py new file mode 100644 index 000000000..df97e57bc --- /dev/null +++ b/taskcluster/taskgraph/util/docker.py @@ -0,0 +1,160 @@ +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, You can obtain one at http://mozilla.org/MPL/2.0/. + +from __future__ import absolute_import, print_function, unicode_literals + +import hashlib +import os +import shutil +import subprocess +import tarfile +import tempfile + +from mozpack.archive import ( + create_tar_gz_from_files, +) + + +GECKO = os.path.realpath(os.path.join(__file__, '..', '..', '..', '..')) +DOCKER_ROOT = os.path.join(GECKO, 'testing', 'docker') +INDEX_PREFIX = 'docker.images.v2' +ARTIFACT_URL = 'https://queue.taskcluster.net/v1/task/{}/artifacts/{}' + + +def docker_image(name, default_version=None): + '''Determine the docker image name, including repository and tag, from an + in-tree docker file.''' + try: + with open(os.path.join(DOCKER_ROOT, name, 'REGISTRY')) as f: + registry = f.read().strip() + except IOError: + with open(os.path.join(DOCKER_ROOT, 'REGISTRY')) as f: + registry = f.read().strip() + + try: + with open(os.path.join(DOCKER_ROOT, name, 'VERSION')) as f: + version = f.read().strip() + except IOError: + if not default_version: + raise + + version = default_version + + return '{}/{}:{}'.format(registry, name, version) + + +def generate_context_hash(topsrcdir, image_path, image_name): + """Generates a sha256 hash for context directory used to build an image.""" + + # It is a bit unfortunate we have to create a temp file here - it would + # be nicer to use an in-memory buffer. + fd, p = tempfile.mkstemp() + os.close(fd) + try: + return create_context_tar(topsrcdir, image_path, p, image_name) + finally: + os.unlink(p) + + +def create_context_tar(topsrcdir, context_dir, out_path, prefix): + """Create a context tarball. + + A directory ``context_dir`` containing a Dockerfile will be assembled into + a gzipped tar file at ``out_path``. Files inside the archive will be + prefixed by directory ``prefix``. + + We also scan the source Dockerfile for special syntax that influences + context generation. + + If a line in the Dockerfile has the form ``# %include <path>``, + the relative path specified on that line will be matched against + files in the source repository and added to the context under the + path ``topsrcdir/``. If an entry is a directory, we add all files + under that directory. + + Returns the SHA-256 hex digest of the created archive. + """ + archive_files = {} + + for root, dirs, files in os.walk(context_dir): + for f in files: + source_path = os.path.join(root, f) + rel = source_path[len(context_dir) + 1:] + archive_path = os.path.join(prefix, rel) + archive_files[archive_path] = source_path + + # Parse Dockerfile for special syntax of extra files to include. + with open(os.path.join(context_dir, 'Dockerfile'), 'rb') as fh: + for line in fh: + line = line.rstrip() + if not line.startswith('# %include'): + continue + + p = line[len('# %include '):].strip() + if os.path.isabs(p): + raise Exception('extra include path cannot be absolute: %s' % p) + + fs_path = os.path.normpath(os.path.join(topsrcdir, p)) + # Check for filesystem traversal exploits. + if not fs_path.startswith(topsrcdir): + raise Exception('extra include path outside topsrcdir: %s' % p) + + if not os.path.exists(fs_path): + raise Exception('extra include path does not exist: %s' % p) + + if os.path.isdir(fs_path): + for root, dirs, files in os.walk(fs_path): + for f in files: + source_path = os.path.join(root, f) + archive_path = os.path.join(prefix, 'topsrcdir', p, f) + archive_files[archive_path] = source_path + else: + archive_path = os.path.join(prefix, 'topsrcdir', p) + archive_files[archive_path] = fs_path + + with open(out_path, 'wb') as fh: + create_tar_gz_from_files(fh, archive_files, '%s.tar.gz' % prefix) + + h = hashlib.sha256() + with open(out_path, 'rb') as fh: + while True: + data = fh.read(32768) + if not data: + break + h.update(data) + return h.hexdigest() + + +def build_from_context(docker_bin, context_path, prefix, tag=None): + """Build a Docker image from a context archive. + + Given the path to a `docker` binary, a image build tar.gz (produced with + ``create_context_tar()``, a prefix in that context containing files, and + an optional ``tag`` for the produced image, build that Docker image. + """ + d = tempfile.mkdtemp() + try: + with tarfile.open(context_path, 'r:gz') as tf: + tf.extractall(d) + + # If we wanted to do post-processing of the Dockerfile, this is + # where we'd do it. + + args = [ + docker_bin, + 'build', + # Use --no-cache so we always get the latest package updates. + '--no-cache', + ] + + if tag: + args.extend(['-t', tag]) + + args.append('.') + + res = subprocess.call(args, cwd=os.path.join(d, prefix)) + if res: + raise Exception('error building image') + finally: + shutil.rmtree(d) diff --git a/taskcluster/taskgraph/util/python_path.py b/taskcluster/taskgraph/util/python_path.py new file mode 100644 index 000000000..b14223ca6 --- /dev/null +++ b/taskcluster/taskgraph/util/python_path.py @@ -0,0 +1,27 @@ +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, You can obtain one at http://mozilla.org/MPL/2.0/. + +from __future__ import absolute_import, print_function, unicode_literals + + +def find_object(path): + """ + Find a Python object given a path of the form <modulepath>:<objectpath>. + Conceptually equivalent to + + def find_object(modulepath, objectpath): + import <modulepath> as mod + return mod.<objectpath> + """ + if path.count(':') != 1: + raise ValueError( + 'python path {!r} does not have the form "module:object"'.format(path)) + + modulepath, objectpath = path.split(':') + obj = __import__(modulepath) + for a in modulepath.split('.')[1:]: + obj = getattr(obj, a) + for a in objectpath.split('.'): + obj = getattr(obj, a) + return obj diff --git a/taskcluster/taskgraph/util/seta.py b/taskcluster/taskgraph/util/seta.py new file mode 100644 index 000000000..a0cd30675 --- /dev/null +++ b/taskcluster/taskgraph/util/seta.py @@ -0,0 +1,85 @@ +import json +import logging +import requests +from redo import retry +from requests import exceptions + +logger = logging.getLogger(__name__) +headers = { + 'User-Agent': 'TaskCluster' +} + +# It's a list of project name which SETA is useful on +SETA_PROJECTS = ['mozilla-inbound', 'autoland'] +SETA_ENDPOINT = "https://seta.herokuapp.com/data/setadetails/?branch=%s" + + +class SETA(object): + """ + Interface to the SETA service, which defines low-value tasks that can be optimized out + of the taskgraph. + """ + def __init__(self): + # cached low value tasks, by project + self.low_value_tasks = {} + + def query_low_value_tasks(self, project): + # Request the set of low value tasks from the SETA service. Low value tasks will be + # optimized out of the task graph. + if project not in SETA_PROJECTS: + logger.debug("SETA is not enabled for project `{}`".format(project)) + return [] + + logger.debug("Querying SETA service for low-value tasks on {}".format(project)) + low_value_tasks = [] + + url = SETA_ENDPOINT % project + # Try to fetch the SETA data twice, falling back to an empty list of low value tasks. + # There are 10 seconds between each try. + try: + logger.debug("Retrieving low-value jobs list from SETA") + response = retry(requests.get, attempts=2, sleeptime=10, + args=(url, ), + kwargs={'timeout': 5, 'headers': headers}) + task_list = json.loads(response.content).get('jobtypes', '') + if len(task_list) > 0: + low_value_tasks = task_list.values()[0] + + # Bug 1315145, disable SETA for tier-1 platforms until backfill is implemented. + low_value_tasks = [x for x in low_value_tasks if x.find('debug') == -1] + low_value_tasks = [x for x in low_value_tasks if x.find('asan') == -1] + + # In the event of request times out, requests will raise a TimeoutError. + except exceptions.Timeout: + logger.warning("SETA server is timeout, we will treat all test tasks as high value.") + + # In the event of a network problem (e.g. DNS failure, refused connection, etc), + # requests will raise a ConnectionError. + except exceptions.ConnectionError: + logger.warning("SETA server is timeout, we will treat all test tasks as high value.") + + # In the event of the rare invalid HTTP response(e.g 404, 401), + # requests will raise an HTTPError exception + except exceptions.HTTPError: + logger.warning("We got bad Http response from ouija," + " we will treat all test tasks as high value.") + + # We just print the error out as a debug message if we failed to catch the exception above + except exceptions.RequestException as error: + logger.warning(error) + + # When we get invalid JSON (i.e. 500 error), it results in a ValueError (bug 1313426) + except ValueError as error: + logger.warning("Invalid JSON, possible server error: {}".format(error)) + + return low_value_tasks + + def is_low_value_task(self, label, project): + # cache the low value tasks per project to avoid repeated SETA server queries + if project not in self.low_value_tasks: + self.low_value_tasks[project] = self.query_low_value_tasks(project) + return label in self.low_value_tasks[project] + +# create a single instance of this class, and expose its `is_low_value_task` +# bound method as a module-level function +is_low_value_task = SETA().is_low_value_task diff --git a/taskcluster/taskgraph/util/templates.py b/taskcluster/taskgraph/util/templates.py new file mode 100644 index 000000000..97620fa75 --- /dev/null +++ b/taskcluster/taskgraph/util/templates.py @@ -0,0 +1,155 @@ +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, You can obtain one at http://mozilla.org/MPL/2.0/. + +from __future__ import absolute_import, print_function, unicode_literals + +import os + +import pystache +import yaml +import copy + +# Key used in template inheritance... +INHERITS_KEY = '$inherits' + + +def merge_to(source, dest): + ''' + Merge dict and arrays (override scalar values) + + Keys from source override keys from dest, and elements from lists in source + are appended to lists in dest. + + :param dict source: to copy from + :param dict dest: to copy to (modified in place) + ''' + + for key, value in source.items(): + # Override mismatching or empty types + if type(value) != type(dest.get(key)): # noqa + dest[key] = source[key] + continue + + # Merge dict + if isinstance(value, dict): + merge_to(value, dest[key]) + continue + + if isinstance(value, list): + dest[key] = dest[key] + source[key] + continue + + dest[key] = source[key] + + return dest + + +def merge(*objects): + ''' + Merge the given objects, using the semantics described for merge_to, with + objects later in the list taking precedence. From an inheritance + perspective, "parents" should be listed before "children". + + Returns the result without modifying any arguments. + ''' + if len(objects) == 1: + return copy.deepcopy(objects[0]) + return merge_to(objects[-1], merge(*objects[:-1])) + + +class TemplatesException(Exception): + pass + + +class Templates(): + ''' + The taskcluster integration makes heavy use of yaml to describe tasks this + class handles the loading/rendering. + ''' + + def __init__(self, root): + ''' + Initialize the template render. + + :param str root: Root path where to load yaml files. + ''' + if not root: + raise TemplatesException('Root is required') + + if not os.path.isdir(root): + raise TemplatesException('Root must be a directory') + + self.root = root + + def _inherits(self, path, obj, properties, seen): + blueprint = obj.pop(INHERITS_KEY) + seen.add(path) + + # Resolve the path here so we can detect circular references. + template = self.resolve_path(blueprint.get('from')) + variables = blueprint.get('variables', {}) + + # Passed parameters override anything in the task itself. + for key in properties: + variables[key] = properties[key] + + if not template: + msg = '"{}" inheritance template missing'.format(path) + raise TemplatesException(msg) + + if template in seen: + msg = 'Error while handling "{}" in "{}" circular template' + \ + 'inheritance seen \n {}' + raise TemplatesException(msg.format(path, template, seen)) + + try: + out = self.load(template, variables, seen) + except TemplatesException as e: + msg = 'Error expanding parent ("{}") of "{}" original error {}' + raise TemplatesException(msg.format(template, path, str(e))) + + # Anything left in obj is merged into final results (and overrides) + return merge_to(obj, out) + + def render(self, path, content, parameters, seen): + ''' + Renders a given yaml string. + + :param str path: used to prevent infinite recursion in inheritance. + :param str content: Of yaml file. + :param dict parameters: For mustache templates. + :param set seen: Seen files (used for inheritance) + ''' + content = pystache.render(content, parameters) + result = yaml.load(content) + + # In addition to the usual template logic done by mustache we also + # handle special '$inherit' dict keys. + if isinstance(result, dict) and INHERITS_KEY in result: + return self._inherits(path, result, parameters, seen) + + return result + + def resolve_path(self, path): + return os.path.join(self.root, path) + + def load(self, path, parameters=None, seen=None): + ''' + Load an render the given yaml path. + + :param str path: Location of yaml file to load (relative to root). + :param dict parameters: To template yaml file with. + ''' + seen = seen or set() + + if not path: + raise TemplatesException('path is required') + + path = self.resolve_path(path) + + if not os.path.isfile(path): + raise TemplatesException('"{}" is not a file'.format(path)) + + content = open(path).read() + return self.render(path, content, parameters, seen) diff --git a/taskcluster/taskgraph/util/time.py b/taskcluster/taskgraph/util/time.py new file mode 100644 index 000000000..160aaa70c --- /dev/null +++ b/taskcluster/taskgraph/util/time.py @@ -0,0 +1,114 @@ +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, You can obtain one at http://mozilla.org/MPL/2.0/. + +# Python port of the ms.js node module this is not a direct port some things are +# more complicated or less precise and we lean on time delta here. + +import re +import datetime + +PATTERN = re.compile( + '((?:\d+)?\.?\d+) *([a-z]+)' +) + + +def seconds(value): + return datetime.timedelta(seconds=int(value)) + + +def minutes(value): + return datetime.timedelta(minutes=int(value)) + + +def hours(value): + return datetime.timedelta(hours=int(value)) + + +def days(value): + return datetime.timedelta(days=int(value)) + + +def months(value): + # See warning in years(), below + return datetime.timedelta(days=int(value) * 30) + + +def years(value): + # Warning here "years" are vague don't use this for really sensitive date + # computation the idea is to give you a absolute amount of time in the + # future which is not the same thing as "precisely on this date next year" + return datetime.timedelta(days=int(value) * 365) + +ALIASES = {} +ALIASES['seconds'] = ALIASES['second'] = ALIASES['s'] = seconds +ALIASES['minutes'] = ALIASES['minute'] = ALIASES['min'] = minutes +ALIASES['hours'] = ALIASES['hour'] = ALIASES['h'] = hours +ALIASES['days'] = ALIASES['day'] = ALIASES['d'] = days +ALIASES['months'] = ALIASES['month'] = ALIASES['mo'] = months +ALIASES['years'] = ALIASES['year'] = ALIASES['y'] = years + + +class InvalidString(Exception): + pass + + +class UnknownTimeMeasurement(Exception): + pass + + +def value_of(input_str): + ''' + Convert a string to a json date in the future + :param str input_str: (ex: 1d, 2d, 6years, 2 seconds) + :returns: Unit given in seconds + ''' + + matches = PATTERN.search(input_str) + + if matches is None or len(matches.groups()) < 2: + raise InvalidString("'{}' is invalid string".format(input_str)) + + value, unit = matches.groups() + + if unit not in ALIASES: + raise UnknownTimeMeasurement( + '{} is not a valid time measure use one of {}'.format( + unit, + sorted(ALIASES.keys()) + ) + ) + + return ALIASES[unit](value) + + +def json_time_from_now(input_str, now=None, datetime_format=False): + ''' + :param str input_str: Input string (see value of) + :param datetime now: Optionally set the definition of `now` + :param boolean datetime_format: Set `True` to get a `datetime` output + :returns: JSON string representation of time in future. + ''' + + if now is None: + now = datetime.datetime.utcnow() + + time = now + value_of(input_str) + + if datetime_format is True: + return time + else: + # Sorta a big hack but the json schema validator for date does not like the + # ISO dates until 'Z' (for timezone) is added... + return time.isoformat() + 'Z' + + +def current_json_time(datetime_format=False): + ''' + :param boolean datetime_format: Set `True` to get a `datetime` output + :returns: JSON string representation of the current time. + ''' + if datetime_format is True: + return datetime.datetime.utcnow() + else: + return datetime.datetime.utcnow().isoformat() + 'Z' diff --git a/taskcluster/taskgraph/util/treeherder.py b/taskcluster/taskgraph/util/treeherder.py new file mode 100644 index 000000000..e66db582f --- /dev/null +++ b/taskcluster/taskgraph/util/treeherder.py @@ -0,0 +1,24 @@ +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, You can obtain one at http://mozilla.org/MPL/2.0/. + +from __future__ import absolute_import, print_function, unicode_literals +import re + + +def split_symbol(treeherder_symbol): + """Split a symbol expressed as grp(sym) into its two parts. If no group is + given, the returned group is '?'""" + groupSymbol = '?' + symbol = treeherder_symbol + if '(' in symbol: + groupSymbol, symbol = re.match(r'([^(]*)\(([^)]*)\)', symbol).groups() + return groupSymbol, symbol + + +def join_symbol(group, symbol): + """Perform the reverse of split_symbol, combining the given group and + symbol. If the group is '?', then it is omitted.""" + if group == '?': + return symbol + return '{}({})'.format(group, symbol) diff --git a/taskcluster/taskgraph/util/yaml.py b/taskcluster/taskgraph/util/yaml.py new file mode 100644 index 000000000..4e541b775 --- /dev/null +++ b/taskcluster/taskgraph/util/yaml.py @@ -0,0 +1,16 @@ +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, You can obtain one at http://mozilla.org/MPL/2.0/. + +from __future__ import absolute_import, print_function, unicode_literals + +import os +import yaml + + +def load_yaml(path, name): + """Convenience function to load a YAML file in the given path. This is + useful for loading kind configuration files from the kind path.""" + filename = os.path.join(path, name) + with open(filename, "rb") as f: + return yaml.load(f) |