From 5f8de423f190bbb79a62f804151bc24824fa32d8 Mon Sep 17 00:00:00 2001 From: "Matt A. Tobin" Date: Fri, 2 Feb 2018 04:16:08 -0500 Subject: Add m-esr52 at 52.6.0 --- python/mozbuild/TODO | 3 + python/mozbuild/dumbmake/__init__.py | 0 python/mozbuild/dumbmake/dumbmake.py | 122 ++ python/mozbuild/dumbmake/test/__init__.py | 0 python/mozbuild/dumbmake/test/test_dumbmake.py | 106 + python/mozbuild/mozbuild/__init__.py | 0 python/mozbuild/mozbuild/action/__init__.py | 0 python/mozbuild/mozbuild/action/buildlist.py | 52 + python/mozbuild/mozbuild/action/cl.py | 124 ++ python/mozbuild/mozbuild/action/dump_env.py | 10 + python/mozbuild/mozbuild/action/explode_aar.py | 72 + python/mozbuild/mozbuild/action/file_generate.py | 108 + .../mozbuild/action/generate_browsersearch.py | 131 ++ .../mozbuild/action/generate_searchjson.py | 23 + .../mozbuild/action/generate_suggestedsites.py | 147 ++ .../mozbuild/action/generate_symbols_file.py | 91 + python/mozbuild/mozbuild/action/jar_maker.py | 17 + python/mozbuild/mozbuild/action/make_dmg.py | 37 + .../mozbuild/action/output_searchplugins_list.py | 21 + .../mozbuild/mozbuild/action/package_fennec_apk.py | 150 ++ python/mozbuild/mozbuild/action/preprocessor.py | 18 + .../mozbuild/action/process_define_files.py | 94 + .../mozbuild/action/process_install_manifest.py | 120 + python/mozbuild/mozbuild/action/test_archive.py | 565 +++++ python/mozbuild/mozbuild/action/webidl.py | 19 + python/mozbuild/mozbuild/action/xpccheck.py | 83 + python/mozbuild/mozbuild/action/xpidl-process.py | 94 + python/mozbuild/mozbuild/action/zip.py | 39 + python/mozbuild/mozbuild/android_version_code.py | 167 ++ python/mozbuild/mozbuild/artifacts.py | 1089 ++++++++++ python/mozbuild/mozbuild/backend/__init__.py | 26 + .../mozbuild/mozbuild/backend/android_eclipse.py | 267 +++ python/mozbuild/mozbuild/backend/base.py | 317 +++ python/mozbuild/mozbuild/backend/common.py | 567 +++++ .../mozbuild/mozbuild/backend/configenvironment.py | 199 ++ python/mozbuild/mozbuild/backend/cpp_eclipse.py | 698 ++++++ python/mozbuild/mozbuild/backend/fastermake.py | 165 ++ python/mozbuild/mozbuild/backend/mach_commands.py | 132 ++ python/mozbuild/mozbuild/backend/recursivemake.py | 1513 +++++++++++++ .../backend/templates/android_eclipse/.classpath | 10 + .../com.android.ide.eclipse.adt.ApkBuilder.launch | 8 + ...droid.ide.eclipse.adt.PreCompilerBuilder.launch | 8 + ...d.ide.eclipse.adt.ResourceManagerBuilder.launch | 8 + .../org.eclipse.jdt.core.javabuilder.launch | 8 + .../templates/android_eclipse/AndroidManifest.xml | 11 + .../backend/templates/android_eclipse/gen/tmp | 1 + .../backend/templates/android_eclipse/lint.xml | 5 + .../templates/android_eclipse/project.properties | 14 + .../.not_an_android_resource | 5 + python/mozbuild/mozbuild/backend/tup.py | 344 +++ python/mozbuild/mozbuild/backend/visualstudio.py | 582 +++++ python/mozbuild/mozbuild/base.py | 850 ++++++++ python/mozbuild/mozbuild/codecoverage/__init__.py | 0 .../mozbuild/mozbuild/codecoverage/chrome_map.py | 105 + python/mozbuild/mozbuild/codecoverage/packager.py | 43 + python/mozbuild/mozbuild/compilation/__init__.py | 0 .../mozbuild/mozbuild/compilation/codecomplete.py | 63 + python/mozbuild/mozbuild/compilation/database.py | 252 +++ python/mozbuild/mozbuild/compilation/util.py | 54 + python/mozbuild/mozbuild/compilation/warnings.py | 376 ++++ python/mozbuild/mozbuild/config_status.py | 182 ++ python/mozbuild/mozbuild/configure/__init__.py | 935 ++++++++ .../mozbuild/configure/check_debug_ranges.py | 62 + python/mozbuild/mozbuild/configure/constants.py | 103 + python/mozbuild/mozbuild/configure/help.py | 45 + python/mozbuild/mozbuild/configure/libstdcxx.py | 81 + python/mozbuild/mozbuild/configure/lint.py | 78 + python/mozbuild/mozbuild/configure/lint_util.py | 52 + python/mozbuild/mozbuild/configure/options.py | 485 +++++ python/mozbuild/mozbuild/configure/util.py | 226 ++ python/mozbuild/mozbuild/controller/__init__.py | 0 python/mozbuild/mozbuild/controller/building.py | 680 ++++++ python/mozbuild/mozbuild/controller/clobber.py | 237 ++ python/mozbuild/mozbuild/doctor.py | 293 +++ python/mozbuild/mozbuild/dotproperties.py | 83 + python/mozbuild/mozbuild/frontend/__init__.py | 0 python/mozbuild/mozbuild/frontend/context.py | 2292 ++++++++++++++++++++ python/mozbuild/mozbuild/frontend/data.py | 1113 ++++++++++ python/mozbuild/mozbuild/frontend/emitter.py | 1416 ++++++++++++ python/mozbuild/mozbuild/frontend/gyp_reader.py | 248 +++ python/mozbuild/mozbuild/frontend/mach_commands.py | 218 ++ python/mozbuild/mozbuild/frontend/reader.py | 1408 ++++++++++++ python/mozbuild/mozbuild/frontend/sandbox.py | 308 +++ python/mozbuild/mozbuild/html_build_viewer.py | 120 + python/mozbuild/mozbuild/jar.py | 597 +++++ .../mozbuild/locale/en-US/LC_MESSAGES/mozbuild.mo | Bin 0 -> 301 bytes .../mozbuild/locale/en-US/LC_MESSAGES/mozbuild.po | 8 + python/mozbuild/mozbuild/mach_commands.py | 1603 ++++++++++++++ python/mozbuild/mozbuild/makeutil.py | 186 ++ python/mozbuild/mozbuild/milestone.py | 75 + python/mozbuild/mozbuild/mozconfig.py | 485 +++++ python/mozbuild/mozbuild/mozconfig_loader | 80 + python/mozbuild/mozbuild/mozinfo.py | 160 ++ python/mozbuild/mozbuild/preprocessor.py | 805 +++++++ python/mozbuild/mozbuild/pythonutil.py | 25 + .../resources/html-build-viewer/index.html | 475 ++++ python/mozbuild/mozbuild/shellutil.py | 209 ++ python/mozbuild/mozbuild/sphinx.py | 200 ++ python/mozbuild/mozbuild/test/__init__.py | 0 .../test/action/data/invalid/region.properties | 12 + .../data/package_fennec_apk/assets/asset.txt | 1 + .../action/data/package_fennec_apk/classes.dex | 1 + .../test/action/data/package_fennec_apk/input1.ap_ | Bin 0 -> 503 bytes .../data/package_fennec_apk/input1/res/res.txt | 1 + .../data/package_fennec_apk/input1/resources.arsc | 1 + .../test/action/data/package_fennec_apk/input2.apk | Bin 0 -> 1649 bytes .../package_fennec_apk/input2/assets/asset.txt | 1 + .../data/package_fennec_apk/input2/assets/omni.ja | 1 + .../data/package_fennec_apk/input2/classes.dex | 1 + .../data/package_fennec_apk/input2/lib/lib.txt | 1 + .../data/package_fennec_apk/input2/res/res.txt | 1 + .../data/package_fennec_apk/input2/resources.arsc | 1 + .../data/package_fennec_apk/input2/root_file.txt | 1 + .../action/data/package_fennec_apk/lib/lib.txt | 1 + .../test/action/data/package_fennec_apk/omni.ja | 1 + .../action/data/package_fennec_apk/root_file.txt | 1 + .../test/action/data/valid-zh-CN/region.properties | 37 + .../mozbuild/test/action/test_buildlist.py | 89 + .../test/action/test_generate_browsersearch.py | 55 + .../test/action/test_package_fennec_apk.py | 70 + python/mozbuild/mozbuild/test/backend/__init__.py | 0 python/mozbuild/mozbuild/test/backend/common.py | 156 ++ .../library1/resources/values/strings.xml | 1 + .../data/android_eclipse/main1/AndroidManifest.xml | 1 + .../data/android_eclipse/main2/AndroidManifest.xml | 1 + .../data/android_eclipse/main2/assets/dummy.txt | 1 + .../backend/data/android_eclipse/main2/extra.jar | 1 + .../android_eclipse/main2/res/values/strings.xml | 1 + .../data/android_eclipse/main3/AndroidManifest.xml | 1 + .../backend/data/android_eclipse/main3/a/A.java | 1 + .../backend/data/android_eclipse/main3/b/B.java | 1 + .../backend/data/android_eclipse/main3/c/C.java | 1 + .../test/backend/data/android_eclipse/main4 | 1 + .../test/backend/data/android_eclipse/moz.build | 37 + .../backend/data/android_eclipse/subdir/moz.build | 13 + .../subdir/submain/AndroidManifest.xml | 1 + .../backend/data/binary-components/bar/moz.build | 2 + .../backend/data/binary-components/foo/moz.build | 1 + .../test/backend/data/binary-components/moz.build | 10 + .../test/backend/data/branding-files/bar.ico | 0 .../test/backend/data/branding-files/foo.ico | 0 .../test/backend/data/branding-files/moz.build | 12 + .../test/backend/data/branding-files/sub/quux.png | 0 .../mozbuild/test/backend/data/build/app/moz.build | 54 + .../mozbuild/test/backend/data/build/bar.ini | 1 + .../mozbuild/test/backend/data/build/bar.js | 2 + .../mozbuild/test/backend/data/build/bar.jsm | 1 + .../mozbuild/test/backend/data/build/baz.ini | 2 + .../mozbuild/test/backend/data/build/baz.jsm | 2 + .../test/backend/data/build/components.manifest | 2 + .../mozbuild/test/backend/data/build/foo.css | 2 + .../mozbuild/test/backend/data/build/foo.ini | 1 + .../mozbuild/test/backend/data/build/foo.js | 1 + .../mozbuild/test/backend/data/build/foo.jsm | 1 + .../mozbuild/test/backend/data/build/jar.mn | 11 + .../mozbuild/test/backend/data/build/moz.build | 68 + .../mozbuild/test/backend/data/build/prefs.js | 1 + .../mozbuild/test/backend/data/build/qux.ini | 5 + .../mozbuild/test/backend/data/build/qux.jsm | 5 + .../mozbuild/test/backend/data/build/resource | 1 + .../mozbuild/test/backend/data/build/resource2 | 1 + .../mozbuild/test/backend/data/build/subdir/bar.js | 1 + .../mozbuild/test/backend/data/defines/moz.build | 14 + .../test/backend/data/dist-files/install.rdf | 0 .../mozbuild/test/backend/data/dist-files/main.js | 0 .../test/backend/data/dist-files/moz.build | 8 + .../test/backend/data/exports-generated/dom1.h | 0 .../test/backend/data/exports-generated/foo.h | 0 .../test/backend/data/exports-generated/gfx.h | 0 .../test/backend/data/exports-generated/moz.build | 12 + .../test/backend/data/exports-generated/mozilla1.h | 0 .../mozbuild/test/backend/data/exports/dom1.h | 0 .../mozbuild/test/backend/data/exports/dom2.h | 0 .../mozbuild/test/backend/data/exports/foo.h | 0 .../mozbuild/test/backend/data/exports/gfx.h | 0 .../mozbuild/test/backend/data/exports/moz.build | 8 + .../mozbuild/test/backend/data/exports/mozilla1.h | 0 .../mozbuild/test/backend/data/exports/mozilla2.h | 0 .../mozbuild/test/backend/data/exports/pprio.h | 0 .../test/backend/data/final_target/both/moz.build | 6 + .../data/final_target/dist-subdir/moz.build | 5 + .../data/final_target/final-target/moz.build | 5 + .../test/backend/data/final_target/moz.build | 5 + .../backend/data/final_target/xpi-name/moz.build | 5 + .../test/backend/data/generated-files/foo-data | 0 .../backend/data/generated-files/generate-bar.py | 0 .../backend/data/generated-files/generate-foo.py | 0 .../test/backend/data/generated-files/moz.build | 12 + .../test/backend/data/generated_includes/moz.build | 5 + .../test/backend/data/host-defines/moz.build | 14 + .../data/install_substitute_config_files/moz.build | 6 + .../install_substitute_config_files/sub/foo.h.in | 1 + .../install_substitute_config_files/sub/moz.build | 7 + .../test/backend/data/ipdl_sources/bar/moz.build | 10 + .../test/backend/data/ipdl_sources/foo/moz.build | 10 + .../test/backend/data/ipdl_sources/moz.build | 10 + .../test/backend/data/jar-manifests/moz.build | 8 + .../bar/baz/dummy_file_for_nonempty_directory | 0 .../foo/dummy_file_for_nonempty_directory | 0 .../test/backend/data/local_includes/moz.build | 5 + .../test/backend/data/resources/bar.res.in | 0 .../test/backend/data/resources/cursor.cur | 0 .../test/backend/data/resources/desktop1.ttf | 0 .../test/backend/data/resources/desktop2.ttf | 0 .../test/backend/data/resources/extra.manifest | 0 .../mozbuild/test/backend/data/resources/font1.ttf | 0 .../mozbuild/test/backend/data/resources/font2.ttf | 0 .../mozbuild/test/backend/data/resources/foo.res | 0 .../test/backend/data/resources/mobile.ttf | 0 .../mozbuild/test/backend/data/resources/moz.build | 9 + .../test/backend/data/resources/test.manifest | 0 .../mozbuild/test/backend/data/sdk-files/bar.ico | 0 .../mozbuild/test/backend/data/sdk-files/foo.ico | 0 .../mozbuild/test/backend/data/sdk-files/moz.build | 11 + .../test/backend/data/sdk-files/sub/quux.png | 0 .../mozbuild/test/backend/data/sources/bar.c | 0 .../mozbuild/test/backend/data/sources/bar.cpp | 0 .../mozbuild/test/backend/data/sources/bar.mm | 0 .../mozbuild/test/backend/data/sources/bar.s | 0 .../mozbuild/test/backend/data/sources/baz.S | 0 .../mozbuild/test/backend/data/sources/foo.S | 0 .../mozbuild/test/backend/data/sources/foo.asm | 0 .../mozbuild/test/backend/data/sources/foo.c | 0 .../mozbuild/test/backend/data/sources/foo.cpp | 0 .../mozbuild/test/backend/data/sources/foo.mm | 0 .../mozbuild/test/backend/data/sources/moz.build | 21 + .../mozbuild/test/backend/data/stub0/Makefile.in | 4 + .../test/backend/data/stub0/dir1/Makefile.in | 7 + .../test/backend/data/stub0/dir1/moz.build | 5 + .../test/backend/data/stub0/dir2/moz.build | 4 + .../test/backend/data/stub0/dir3/Makefile.in | 7 + .../test/backend/data/stub0/dir3/moz.build | 4 + .../mozbuild/test/backend/data/stub0/moz.build | 7 + .../data/substitute_config_files/Makefile.in | 0 .../backend/data/substitute_config_files/foo.in | 1 + .../backend/data/substitute_config_files/moz.build | 5 + .../child/another-file.sjs | 0 .../test-manifest-shared-support/child/browser.ini | 6 + .../child/data/one.txt | 0 .../child/data/two.txt | 0 .../test-manifest-shared-support/child/test_sub.js | 0 .../test-manifest-shared-support/mochitest.ini | 8 + .../data/test-manifest-shared-support/moz.build | 5 + .../test-manifest-shared-support/support-file.txt | 0 .../data/test-manifest-shared-support/test_foo.js | 0 .../mochitest1.ini | 4 + .../mochitest2.ini | 4 + .../moz.build | 7 + .../test_bar.js | 0 .../test_foo.js | 0 .../instrumentation.ini | 1 + .../test-manifests-package-tests/mochitest.ini | 1 + .../data/test-manifests-package-tests/mochitest.js | 0 .../data/test-manifests-package-tests/moz.build | 10 + .../test-manifests-package-tests/not_packaged.java | 0 .../data/test-manifests-written/dir1/test_bar.js | 0 .../data/test-manifests-written/dir1/xpcshell.ini | 3 + .../data/test-manifests-written/mochitest.ini | 3 + .../data/test-manifests-written/mochitest.js | 0 .../backend/data/test-manifests-written/moz.build | 9 + .../data/test-manifests-written/xpcshell.ini | 4 + .../data/test-manifests-written/xpcshell.js | 0 .../mozbuild/test/backend/data/test_config/file.in | 3 + .../test/backend/data/test_config/moz.build | 3 + .../backend/data/variable_passthru/Makefile.in | 0 .../test/backend/data/variable_passthru/moz.build | 23 + .../test/backend/data/variable_passthru/test1.c | 0 .../test/backend/data/variable_passthru/test1.cpp | 0 .../test/backend/data/variable_passthru/test1.mm | 0 .../test/backend/data/variable_passthru/test2.c | 0 .../test/backend/data/variable_passthru/test2.cpp | 0 .../test/backend/data/variable_passthru/test2.mm | 0 .../test/backend/data/visual-studio/dir1/bar.cpp | 0 .../test/backend/data/visual-studio/dir1/foo.cpp | 0 .../test/backend/data/visual-studio/dir1/moz.build | 9 + .../test/backend/data/visual-studio/moz.build | 7 + .../data/xpidl/config/makefiles/xpidl/Makefile.in | 0 .../mozbuild/test/backend/data/xpidl/moz.build | 6 + .../mozbuild/test/backend/test_android_eclipse.py | 153 ++ .../mozbuild/mozbuild/test/backend/test_build.py | 233 ++ .../test/backend/test_configenvironment.py | 63 + .../mozbuild/test/backend/test_recursivemake.py | 942 ++++++++ .../mozbuild/test/backend/test_visualstudio.py | 64 + python/mozbuild/mozbuild/test/common.py | 50 + .../mozbuild/mozbuild/test/compilation/__init__.py | 0 .../mozbuild/test/compilation/test_warnings.py | 241 ++ python/mozbuild/mozbuild/test/configure/common.py | 279 +++ .../test/configure/data/decorators.configure | 44 + .../mozbuild/test/configure/data/empty_mozconfig | 0 .../mozbuild/test/configure/data/extra.configure | 13 + .../test/configure/data/imply_option/imm.configure | 32 + .../configure/data/imply_option/infer.configure | 24 + .../configure/data/imply_option/infer_ko.configure | 31 + .../configure/data/imply_option/negative.configure | 34 + .../configure/data/imply_option/simple.configure | 24 + .../configure/data/imply_option/values.configure | 24 + .../test/configure/data/included.configure | 53 + .../mozbuild/test/configure/data/moz.configure | 174 ++ .../test/configure/data/set_config.configure | 43 + .../test/configure/data/set_define.configure | 43 + .../test/configure/data/subprocess.configure | 23 + python/mozbuild/mozbuild/test/configure/lint.py | 65 + .../test/configure/test_checks_configure.py | 940 ++++++++ .../mozbuild/test/configure/test_compile_checks.py | 403 ++++ .../mozbuild/test/configure/test_configure.py | 1273 +++++++++++ .../mozbuild/mozbuild/test/configure/test_lint.py | 132 ++ .../mozbuild/test/configure/test_moz_configure.py | 93 + .../mozbuild/test/configure/test_options.py | 852 ++++++++ .../test/configure/test_toolchain_configure.py | 1271 +++++++++++ .../test/configure/test_toolchain_helpers.py | 437 ++++ .../test/configure/test_toolkit_moz_configure.py | 67 + .../mozbuild/mozbuild/test/configure/test_util.py | 558 +++++ .../mozbuild/mozbuild/test/controller/__init__.py | 0 .../mozbuild/test/controller/test_ccachestats.py | 208 ++ .../mozbuild/test/controller/test_clobber.py | 213 ++ python/mozbuild/mozbuild/test/data/Makefile | 0 python/mozbuild/mozbuild/test/data/bad.properties | 12 + .../mozbuild/mozbuild/test/data/test-dir/Makefile | 0 .../mozbuild/test/data/test-dir/with/Makefile | 0 .../test/data/test-dir/with/without/with/Makefile | 0 .../test/data/test-dir/without/with/Makefile | 0 .../mozbuild/mozbuild/test/data/valid.properties | 11 + python/mozbuild/mozbuild/test/frontend/__init__.py | 0 .../test/frontend/data/android-res-dirs/dir1/foo | 0 .../test/frontend/data/android-res-dirs/moz.build | 9 + .../frontend/data/binary-components/bar/moz.build | 2 + .../frontend/data/binary-components/foo/moz.build | 1 + .../test/frontend/data/binary-components/moz.build | 10 + .../test/frontend/data/branding-files/bar.ico | 0 .../test/frontend/data/branding-files/baz.png | 0 .../test/frontend/data/branding-files/foo.xpm | 0 .../test/frontend/data/branding-files/moz.build | 13 + .../test/frontend/data/branding-files/quux.icns | 0 .../data/config-file-substitution/moz.build | 6 + .../crate-dependency-path-resolution/Cargo.toml | 18 + .../crate-dependency-path-resolution/moz.build | 18 + .../shallow/Cargo.toml | 6 + .../the/depths/Cargo.toml | 9 + .../mozbuild/test/frontend/data/defines/moz.build | 14 + .../frontend/data/dist-files-missing/install.rdf | 0 .../frontend/data/dist-files-missing/moz.build | 8 + .../test/frontend/data/dist-files/install.rdf | 0 .../mozbuild/test/frontend/data/dist-files/main.js | 0 .../test/frontend/data/dist-files/moz.build | 8 + .../test/frontend/data/exports-generated/foo.h | 0 .../test/frontend/data/exports-generated/moz.build | 8 + .../frontend/data/exports-generated/mozilla1.h | 0 .../frontend/data/exports-missing-generated/foo.h | 0 .../data/exports-missing-generated/moz.build | 5 + .../test/frontend/data/exports-missing/foo.h | 0 .../test/frontend/data/exports-missing/moz.build | 6 + .../test/frontend/data/exports-missing/mozilla1.h | 0 .../mozbuild/test/frontend/data/exports/bar.h | 0 .../mozbuild/test/frontend/data/exports/baz.h | 0 .../mozbuild/test/frontend/data/exports/dom1.h | 0 .../mozbuild/test/frontend/data/exports/dom2.h | 0 .../mozbuild/test/frontend/data/exports/dom3.h | 0 .../mozbuild/test/frontend/data/exports/foo.h | 0 .../mozbuild/test/frontend/data/exports/gfx.h | 0 .../mozbuild/test/frontend/data/exports/mem.h | 0 .../mozbuild/test/frontend/data/exports/mem2.h | 0 .../mozbuild/test/frontend/data/exports/moz.build | 13 + .../mozbuild/test/frontend/data/exports/mozilla1.h | 0 .../mozbuild/test/frontend/data/exports/mozilla2.h | 0 .../mozbuild/test/frontend/data/exports/pprio.h | 0 .../mozbuild/test/frontend/data/exports/pprthred.h | 0 .../bug_component/bad-assignment/moz.build | 2 + .../bug_component/different-matchers/moz.build | 4 + .../data/files-info/bug_component/final/moz.build | 3 + .../bug_component/final/subcomponent/moz.build | 2 + .../data/files-info/bug_component/moz.build | 2 + .../data/files-info/bug_component/simple/moz.build | 2 + .../data/files-info/bug_component/static/moz.build | 5 + .../test/frontend/data/files-info/moz.build | 0 .../data/files-test-metadata/default/module.js | 0 .../data/files-test-metadata/default/moz.build | 6 + .../default/tests/reftests/reftest-stylo.list | 2 + .../default/tests/reftests/reftest.list | 1 + .../default/tests/reftests/test1-ref.html | 0 .../default/tests/reftests/test1.html | 0 .../default/tests/xpcshell/test_default_mod.js | 0 .../default/tests/xpcshell/xpcshell.ini | 1 + .../frontend/data/files-test-metadata/moz.build | 4 + .../data/files-test-metadata/simple/base.cpp | 0 .../files-test-metadata/simple/browser/browser.ini | 1 + .../files-test-metadata/simple/browser/test_mod.js | 0 .../data/files-test-metadata/simple/moz.build | 22 + .../data/files-test-metadata/simple/src/module.jsm | 0 .../data/files-test-metadata/simple/src/moz.build | 3 + .../files-test-metadata/simple/tests/mochitest.ini | 2 + .../files-test-metadata/simple/tests/moz.build | 1 + .../simple/tests/test_general.html | 0 .../simple/tests/test_specific.html | 0 .../data/files-test-metadata/tagged/moz.build | 15 + .../data/files-test-metadata/tagged/src/bar.jsm | 0 .../tagged/src/submodule/foo.js | 0 .../files-test-metadata/tagged/tests/mochitest.ini | 3 + .../files-test-metadata/tagged/tests/test_bar.js | 0 .../tagged/tests/test_simple.html | 0 .../tagged/tests/test_specific.html | 0 .../files-test-metadata/tagged/tests/xpcshell.ini | 1 + .../final-target-pp-files-non-srcdir/moz.build | 7 + .../data/generated-files-absolute-script/moz.build | 9 + .../data/generated-files-absolute-script/script.py | 0 .../data/generated-files-method-names/moz.build | 13 + .../data/generated-files-method-names/script.py | 0 .../data/generated-files-no-inputs/moz.build | 9 + .../data/generated-files-no-inputs/script.py | 0 .../generated-files-no-python-script/moz.build | 8 + .../generated-files-no-python-script/script.rb | 0 .../data/generated-files-no-script/moz.build | 8 + .../test/frontend/data/generated-files/moz.build | 5 + .../test/frontend/data/generated-sources/a.cpp | 0 .../test/frontend/data/generated-sources/b.cc | 0 .../test/frontend/data/generated-sources/c.cxx | 0 .../test/frontend/data/generated-sources/d.c | 0 .../test/frontend/data/generated-sources/e.m | 0 .../test/frontend/data/generated-sources/f.mm | 0 .../test/frontend/data/generated-sources/g.S | 0 .../test/frontend/data/generated-sources/h.s | 0 .../test/frontend/data/generated-sources/i.asm | 0 .../test/frontend/data/generated-sources/moz.build | 37 + .../frontend/data/generated_includes/moz.build | 5 + .../test/frontend/data/host-defines/moz.build | 14 + .../mozbuild/test/frontend/data/host-sources/a.cpp | 0 .../mozbuild/test/frontend/data/host-sources/b.cc | 0 .../mozbuild/test/frontend/data/host-sources/c.cxx | 0 .../mozbuild/test/frontend/data/host-sources/d.c | 0 .../mozbuild/test/frontend/data/host-sources/e.mm | 0 .../mozbuild/test/frontend/data/host-sources/f.mm | 0 .../test/frontend/data/host-sources/moz.build | 25 + .../frontend/data/include-basic/included.build | 4 + .../test/frontend/data/include-basic/moz.build | 7 + .../data/include-file-stack/included-1.build | 4 + .../data/include-file-stack/included-2.build | 4 + .../frontend/data/include-file-stack/moz.build | 5 + .../test/frontend/data/include-missing/moz.build | 5 + .../data/include-outside-topsrcdir/relative.build | 4 + .../include-relative-from-child/child/child.build | 4 + .../include-relative-from-child/child/child2.build | 4 + .../child/grandchild/grandchild.build | 4 + .../data/include-relative-from-child/parent.build | 4 + .../data/include-topsrcdir-relative/moz.build | 5 + .../data/include-topsrcdir-relative/sibling.build | 4 + .../data/inheriting-variables/bar/moz.build | 5 + .../data/inheriting-variables/foo/baz/moz.build | 7 + .../data/inheriting-variables/foo/moz.build | 7 + .../frontend/data/inheriting-variables/moz.build | 10 + .../test/frontend/data/ipdl_sources/bar/moz.build | 10 + .../test/frontend/data/ipdl_sources/foo/moz.build | 10 + .../test/frontend/data/ipdl_sources/moz.build | 10 + .../data/jar-manifests-multiple-files/moz.build | 8 + .../test/frontend/data/jar-manifests/moz.build | 7 + .../frontend/data/library-defines/liba/moz.build | 5 + .../frontend/data/library-defines/libb/moz.build | 7 + .../frontend/data/library-defines/libc/moz.build | 5 + .../frontend/data/library-defines/libd/moz.build | 5 + .../test/frontend/data/library-defines/moz.build | 9 + .../bar/baz/dummy_file_for_nonempty_directory | 0 .../foo/dummy_file_for_nonempty_directory | 0 .../test/frontend/data/local_includes/moz.build | 5 + .../frontend/data/missing-local-includes/moz.build | 5 + .../data/multiple-rust-libraries/moz.build | 27 + .../data/multiple-rust-libraries/rust1/Cargo.toml | 15 + .../data/multiple-rust-libraries/rust1/moz.build | 4 + .../data/multiple-rust-libraries/rust2/Cargo.toml | 15 + .../data/multiple-rust-libraries/rust2/moz.build | 4 + .../mozbuild/test/frontend/data/program/moz.build | 15 + .../frontend/data/reader-error-bad-dir/moz.build | 5 + .../frontend/data/reader-error-basic/moz.build | 5 + .../data/reader-error-empty-list/moz.build | 5 + .../data/reader-error-error-func/moz.build | 6 + .../data/reader-error-included-from/child.build | 4 + .../data/reader-error-included-from/moz.build | 5 + .../data/reader-error-missing-include/moz.build | 5 + .../data/reader-error-outside-topsrcdir/moz.build | 5 + .../reader-error-read-unknown-global/moz.build | 5 + .../data/reader-error-repeated-dir/moz.build | 7 + .../data/reader-error-script-error/moz.build | 5 + .../frontend/data/reader-error-syntax/moz.build | 5 + .../data/reader-error-write-bad-value/moz.build | 5 + .../reader-error-write-unknown-global/moz.build | 7 + .../reader-relevant-mozbuild/d1/every-level/a/file | 0 .../d1/every-level/a/moz.build | 0 .../reader-relevant-mozbuild/d1/every-level/b/file | 0 .../d1/every-level/b/moz.build | 0 .../d1/every-level/moz.build | 0 .../data/reader-relevant-mozbuild/d1/file1 | 0 .../data/reader-relevant-mozbuild/d1/file2 | 0 .../data/reader-relevant-mozbuild/d1/moz.build | 0 .../d1/no-intermediate-moz-build/child/file | 0 .../d1/no-intermediate-moz-build/child/moz.build | 0 .../d1/parent-is-far/dir1/dir2/dir3/file | 0 .../d1/parent-is-far/moz.build | 0 .../data/reader-relevant-mozbuild/d2/dir1/file | 0 .../reader-relevant-mozbuild/d2/dir1/moz.build | 0 .../data/reader-relevant-mozbuild/d2/dir2/file | 0 .../reader-relevant-mozbuild/d2/dir2/moz.build | 0 .../data/reader-relevant-mozbuild/d2/moz.build | 0 .../frontend/data/reader-relevant-mozbuild/file | 0 .../data/reader-relevant-mozbuild/moz.build | 0 .../data/rust-library-dash-folding/Cargo.toml | 15 + .../data/rust-library-dash-folding/moz.build | 18 + .../rust-library-invalid-crate-type/Cargo.toml | 15 + .../data/rust-library-invalid-crate-type/moz.build | 18 + .../data/rust-library-name-mismatch/Cargo.toml | 12 + .../data/rust-library-name-mismatch/moz.build | 18 + .../data/rust-library-no-cargo-toml/moz.build | 18 + .../data/rust-library-no-lib-section/Cargo.toml | 12 + .../data/rust-library-no-lib-section/moz.build | 18 + .../rust-library-no-profile-section/Cargo.toml | 12 + .../data/rust-library-no-profile-section/moz.build | 18 + .../data/rust-library-non-abort-panic/Cargo.toml | 14 + .../data/rust-library-non-abort-panic/moz.build | 18 + .../mozbuild/test/frontend/data/sdk-files/bar.ico | 0 .../mozbuild/test/frontend/data/sdk-files/baz.png | 0 .../mozbuild/test/frontend/data/sdk-files/foo.xpm | 0 .../test/frontend/data/sdk-files/moz.build | 12 + .../test/frontend/data/sdk-files/quux.icns | 0 .../mozbuild/test/frontend/data/sources-just-c/d.c | 0 .../mozbuild/test/frontend/data/sources-just-c/e.m | 0 .../mozbuild/test/frontend/data/sources-just-c/g.S | 0 .../mozbuild/test/frontend/data/sources-just-c/h.s | 0 .../test/frontend/data/sources-just-c/i.asm | 0 .../test/frontend/data/sources-just-c/moz.build | 27 + .../mozbuild/test/frontend/data/sources/a.cpp | 0 .../mozbuild/test/frontend/data/sources/b.cc | 0 .../mozbuild/test/frontend/data/sources/c.cxx | 0 .../mozbuild/test/frontend/data/sources/d.c | 0 .../mozbuild/test/frontend/data/sources/e.m | 0 .../mozbuild/test/frontend/data/sources/f.mm | 0 .../mozbuild/test/frontend/data/sources/g.S | 0 .../mozbuild/test/frontend/data/sources/h.s | 0 .../mozbuild/test/frontend/data/sources/i.asm | 0 .../mozbuild/test/frontend/data/sources/moz.build | 37 + .../frontend/data/templates/templates.mozbuild | 21 + .../data/test-harness-files-root/moz.build | 4 + .../frontend/data/test-harness-files/mochitest.ini | 1 + .../frontend/data/test-harness-files/mochitest.py | 1 + .../frontend/data/test-harness-files/moz.build | 7 + .../frontend/data/test-harness-files/runtests.py | 1 + .../test/frontend/data/test-harness-files/utils.py | 1 + .../data/test-install-shared-lib/moz.build | 12 + .../data/test-linkables-cxx-link/moz.build | 11 + .../data/test-linkables-cxx-link/one/foo.cpp | 0 .../data/test-linkables-cxx-link/one/moz.build | 9 + .../data/test-linkables-cxx-link/three/moz.build | 5 + .../data/test-linkables-cxx-link/two/foo.c | 0 .../data/test-linkables-cxx-link/two/moz.build | 9 + .../absolute-support.ini | 4 + .../data/test-manifest-absolute-support/foo.txt | 1 + .../data/test-manifest-absolute-support/moz.build | 4 + .../test-manifest-absolute-support/test_file.js | 0 .../test/frontend/data/test-manifest-dupes/bar.js | 0 .../test/frontend/data/test-manifest-dupes/foo.js | 0 .../data/test-manifest-dupes/mochitest.ini | 7 + .../frontend/data/test-manifest-dupes/moz.build | 4 + .../frontend/data/test-manifest-dupes/test_baz.js | 0 .../included-reftest.list | 1 + .../data/test-manifest-emitted-includes/moz.build | 1 + .../reftest-stylo.list | 3 + .../test-manifest-emitted-includes/reftest.list | 2 + .../frontend/data/test-manifest-empty/empty.ini | 2 + .../frontend/data/test-manifest-empty/moz.build | 4 + .../test_inactive.html | 0 .../data/test-manifest-install-includes/common.ini | 1 + .../test-manifest-install-includes/mochitest.ini | 4 + .../data/test-manifest-install-includes/moz.build | 4 + .../test-manifest-install-includes/test_foo.html | 1 + .../data/test-manifest-install-subdir/moz.build | 4 + .../data/test-manifest-install-subdir/subdir.ini | 5 + .../test-manifest-install-subdir/test_foo.html | 1 + .../data/test-manifest-just-support/foo.txt | 1 + .../test-manifest-just-support/just-support.ini | 2 + .../data/test-manifest-just-support/moz.build | 4 + .../a11y-support/dir1/bar | 0 .../test-manifest-keys-extracted/a11y-support/foo | 0 .../data/test-manifest-keys-extracted/a11y.ini | 4 + .../data/test-manifest-keys-extracted/browser.ini | 4 + .../data/test-manifest-keys-extracted/chrome.ini | 4 + .../test-manifest-keys-extracted/crashtest.list | 1 + .../data/test-manifest-keys-extracted/metro.ini | 3 + .../test-manifest-keys-extracted/mochitest.ini | 5 + .../data/test-manifest-keys-extracted/moz.build | 12 + .../reftest-stylo.list | 2 + .../data/test-manifest-keys-extracted/reftest.list | 1 + .../data/test-manifest-keys-extracted/test_a11y.js | 0 .../test-manifest-keys-extracted/test_browser.js | 0 .../test-manifest-keys-extracted/test_chrome.js | 0 .../data/test-manifest-keys-extracted/test_foo.py | 0 .../test-manifest-keys-extracted/test_metro.js | 0 .../test-manifest-keys-extracted/test_mochitest.js | 0 .../test-manifest-keys-extracted/test_xpcshell.js | 0 .../data/test-manifest-keys-extracted/xpcshell.ini | 6 + .../data/test-manifest-missing-manifest/moz.build | 4 + .../moz.build | 4 + .../xpcshell.ini | 4 + .../test-manifest-missing-test-file/mochitest.ini | 1 + .../data/test-manifest-missing-test-file/moz.build | 4 + .../child/mochitest.ini | 4 + .../child/test_foo.js | 0 .../moz.build | 4 + .../support-file.txt | 0 .../child/another-file.sjs | 0 .../test-manifest-shared-missing/child/browser.ini | 6 + .../child/data/one.txt | 0 .../child/data/two.txt | 0 .../test-manifest-shared-missing/child/test_sub.js | 0 .../test-manifest-shared-missing/mochitest.ini | 9 + .../data/test-manifest-shared-missing/moz.build | 5 + .../test-manifest-shared-missing/support-file.txt | 0 .../data/test-manifest-shared-missing/test_foo.js | 0 .../child/another-file.sjs | 0 .../test-manifest-shared-support/child/browser.ini | 6 + .../child/data/one.txt | 0 .../child/data/two.txt | 0 .../test-manifest-shared-support/child/test_sub.js | 0 .../test-manifest-shared-support/mochitest.ini | 8 + .../data/test-manifest-shared-support/moz.build | 5 + .../test-manifest-shared-support/support-file.txt | 0 .../data/test-manifest-shared-support/test_foo.js | 0 .../test-manifest-unmatched-generated/moz.build | 4 + .../test-manifest-unmatched-generated/test.ini | 4 + .../test-manifest-unmatched-generated/test_foo | 0 .../data/test-python-unit-test-missing/moz.build | 4 + .../moz.build | 10 + .../frontend/data/test-symbols-file-objdir/foo.py | 0 .../data/test-symbols-file-objdir/moz.build | 13 + .../frontend/data/test-symbols-file/foo.symbols | 1 + .../test/frontend/data/test-symbols-file/moz.build | 10 + .../frontend/data/traversal-all-vars/moz.build | 6 + .../data/traversal-all-vars/parallel/moz.build | 0 .../data/traversal-all-vars/regular/moz.build | 0 .../data/traversal-all-vars/test/moz.build | 0 .../data/traversal-outside-topsrcdir/moz.build | 5 + .../data/traversal-relative-dirs/bar/moz.build | 0 .../data/traversal-relative-dirs/foo/moz.build | 5 + .../data/traversal-relative-dirs/moz.build | 5 + .../data/traversal-repeated-dirs/bar/moz.build | 5 + .../data/traversal-repeated-dirs/foo/moz.build | 5 + .../data/traversal-repeated-dirs/moz.build | 5 + .../frontend/data/traversal-simple/bar/moz.build | 0 .../data/traversal-simple/foo/biz/moz.build | 0 .../frontend/data/traversal-simple/foo/moz.build | 2 + .../test/frontend/data/traversal-simple/moz.build | 5 + .../data/unified-sources-non-unified/bar.cxx | 0 .../frontend/data/unified-sources-non-unified/c1.c | 0 .../frontend/data/unified-sources-non-unified/c2.c | 0 .../data/unified-sources-non-unified/foo.cpp | 0 .../data/unified-sources-non-unified/moz.build | 28 + .../data/unified-sources-non-unified/objc1.mm | 0 .../data/unified-sources-non-unified/objc2.mm | 0 .../data/unified-sources-non-unified/quux.cc | 0 .../test/frontend/data/unified-sources/bar.cxx | 0 .../test/frontend/data/unified-sources/c1.c | 0 .../test/frontend/data/unified-sources/c2.c | 0 .../test/frontend/data/unified-sources/foo.cpp | 0 .../test/frontend/data/unified-sources/moz.build | 28 + .../test/frontend/data/unified-sources/objc1.mm | 0 .../test/frontend/data/unified-sources/objc2.mm | 0 .../test/frontend/data/unified-sources/quux.cc | 0 .../mozbuild/test/frontend/data/use-yasm/moz.build | 5 + .../test/frontend/data/variable-passthru/bans.S | 0 .../test/frontend/data/variable-passthru/moz.build | 25 + .../test/frontend/data/variable-passthru/test1.c | 0 .../test/frontend/data/variable-passthru/test1.cpp | 0 .../test/frontend/data/variable-passthru/test1.mm | 0 .../test/frontend/data/variable-passthru/test2.c | 0 .../test/frontend/data/variable-passthru/test2.cpp | 0 .../test/frontend/data/variable-passthru/test2.mm | 0 .../data/xpidl-module-no-sources/moz.build | 5 + .../mozbuild/test/frontend/test_context.py | 721 ++++++ .../mozbuild/test/frontend/test_emitter.py | 1172 ++++++++++ .../mozbuild/test/frontend/test_namespaces.py | 207 ++ .../mozbuild/mozbuild/test/frontend/test_reader.py | 485 +++++ .../mozbuild/test/frontend/test_sandbox.py | 534 +++++ .../mozbuild/test/test_android_version_code.py | 63 + python/mozbuild/mozbuild/test/test_base.py | 410 ++++ python/mozbuild/mozbuild/test/test_containers.py | 224 ++ .../mozbuild/mozbuild/test/test_dotproperties.py | 178 ++ python/mozbuild/mozbuild/test/test_expression.py | 82 + python/mozbuild/mozbuild/test/test_jarmaker.py | 367 ++++ python/mozbuild/mozbuild/test/test_line_endings.py | 46 + python/mozbuild/mozbuild/test/test_makeutil.py | 165 ++ python/mozbuild/mozbuild/test/test_mozconfig.py | 489 +++++ python/mozbuild/mozbuild/test/test_mozinfo.py | 278 +++ python/mozbuild/mozbuild/test/test_preprocessor.py | 646 ++++++ python/mozbuild/mozbuild/test/test_pythonutil.py | 23 + python/mozbuild/mozbuild/test/test_testing.py | 332 +++ python/mozbuild/mozbuild/test/test_util.py | 924 ++++++++ python/mozbuild/mozbuild/testing.py | 535 +++++ python/mozbuild/mozbuild/util.py | 1264 +++++++++++ python/mozbuild/mozbuild/vendor_rust.py | 86 + python/mozbuild/mozbuild/virtualenv.py | 568 +++++ python/mozbuild/mozpack/__init__.py | 0 python/mozbuild/mozpack/archive.py | 107 + python/mozbuild/mozpack/chrome/__init__.py | 0 python/mozbuild/mozpack/chrome/flags.py | 258 +++ python/mozbuild/mozpack/chrome/manifest.py | 368 ++++ python/mozbuild/mozpack/copier.py | 568 +++++ python/mozbuild/mozpack/dmg.py | 121 ++ python/mozbuild/mozpack/errors.py | 139 ++ python/mozbuild/mozpack/executables.py | 124 ++ python/mozbuild/mozpack/files.py | 1106 ++++++++++ python/mozbuild/mozpack/hg.py | 95 + python/mozbuild/mozpack/manifests.py | 419 ++++ python/mozbuild/mozpack/mozjar.py | 816 +++++++ python/mozbuild/mozpack/packager/__init__.py | 408 ++++ python/mozbuild/mozpack/packager/formats.py | 324 +++ python/mozbuild/mozpack/packager/l10n.py | 259 +++ python/mozbuild/mozpack/packager/unpack.py | 202 ++ python/mozbuild/mozpack/path.py | 136 ++ python/mozbuild/mozpack/test/__init__.py | 0 python/mozbuild/mozpack/test/data/test_data | 1 + .../mozpack/test/support/minify_js_verify.py | 17 + python/mozbuild/mozpack/test/test_archive.py | 190 ++ python/mozbuild/mozpack/test/test_chrome_flags.py | 148 ++ .../mozbuild/mozpack/test/test_chrome_manifest.py | 149 ++ python/mozbuild/mozpack/test/test_copier.py | 529 +++++ python/mozbuild/mozpack/test/test_errors.py | 93 + python/mozbuild/mozpack/test/test_files.py | 1160 ++++++++++ python/mozbuild/mozpack/test/test_manifests.py | 375 ++++ python/mozbuild/mozpack/test/test_mozjar.py | 342 +++ python/mozbuild/mozpack/test/test_packager.py | 490 +++++ .../mozbuild/mozpack/test/test_packager_formats.py | 428 ++++ python/mozbuild/mozpack/test/test_packager_l10n.py | 126 ++ .../mozbuild/mozpack/test/test_packager_unpack.py | 65 + python/mozbuild/mozpack/test/test_path.py | 143 ++ python/mozbuild/mozpack/test/test_unify.py | 199 ++ python/mozbuild/mozpack/unify.py | 231 ++ python/mozbuild/setup.py | 29 + 731 files changed, 57244 insertions(+) create mode 100644 python/mozbuild/TODO create mode 100644 python/mozbuild/dumbmake/__init__.py create mode 100644 python/mozbuild/dumbmake/dumbmake.py create mode 100644 python/mozbuild/dumbmake/test/__init__.py create mode 100644 python/mozbuild/dumbmake/test/test_dumbmake.py create mode 100644 python/mozbuild/mozbuild/__init__.py create mode 100644 python/mozbuild/mozbuild/action/__init__.py create mode 100644 python/mozbuild/mozbuild/action/buildlist.py create mode 100644 python/mozbuild/mozbuild/action/cl.py create mode 100644 python/mozbuild/mozbuild/action/dump_env.py create mode 100644 python/mozbuild/mozbuild/action/explode_aar.py create mode 100644 python/mozbuild/mozbuild/action/file_generate.py create mode 100644 python/mozbuild/mozbuild/action/generate_browsersearch.py create mode 100644 python/mozbuild/mozbuild/action/generate_searchjson.py create mode 100644 python/mozbuild/mozbuild/action/generate_suggestedsites.py create mode 100644 python/mozbuild/mozbuild/action/generate_symbols_file.py create mode 100644 python/mozbuild/mozbuild/action/jar_maker.py create mode 100644 python/mozbuild/mozbuild/action/make_dmg.py create mode 100644 python/mozbuild/mozbuild/action/output_searchplugins_list.py create mode 100644 python/mozbuild/mozbuild/action/package_fennec_apk.py create mode 100644 python/mozbuild/mozbuild/action/preprocessor.py create mode 100644 python/mozbuild/mozbuild/action/process_define_files.py create mode 100644 python/mozbuild/mozbuild/action/process_install_manifest.py create mode 100644 python/mozbuild/mozbuild/action/test_archive.py create mode 100644 python/mozbuild/mozbuild/action/webidl.py create mode 100644 python/mozbuild/mozbuild/action/xpccheck.py create mode 100755 python/mozbuild/mozbuild/action/xpidl-process.py create mode 100644 python/mozbuild/mozbuild/action/zip.py create mode 100644 python/mozbuild/mozbuild/android_version_code.py create mode 100644 python/mozbuild/mozbuild/artifacts.py create mode 100644 python/mozbuild/mozbuild/backend/__init__.py create mode 100644 python/mozbuild/mozbuild/backend/android_eclipse.py create mode 100644 python/mozbuild/mozbuild/backend/base.py create mode 100644 python/mozbuild/mozbuild/backend/common.py create mode 100644 python/mozbuild/mozbuild/backend/configenvironment.py create mode 100644 python/mozbuild/mozbuild/backend/cpp_eclipse.py create mode 100644 python/mozbuild/mozbuild/backend/fastermake.py create mode 100644 python/mozbuild/mozbuild/backend/mach_commands.py create mode 100644 python/mozbuild/mozbuild/backend/recursivemake.py create mode 100644 python/mozbuild/mozbuild/backend/templates/android_eclipse/.classpath create mode 100644 python/mozbuild/mozbuild/backend/templates/android_eclipse/.externalToolBuilders/com.android.ide.eclipse.adt.ApkBuilder.launch create mode 100644 python/mozbuild/mozbuild/backend/templates/android_eclipse/.externalToolBuilders/com.android.ide.eclipse.adt.PreCompilerBuilder.launch create mode 100644 python/mozbuild/mozbuild/backend/templates/android_eclipse/.externalToolBuilders/com.android.ide.eclipse.adt.ResourceManagerBuilder.launch create mode 100644 python/mozbuild/mozbuild/backend/templates/android_eclipse/.externalToolBuilders/org.eclipse.jdt.core.javabuilder.launch create mode 100644 python/mozbuild/mozbuild/backend/templates/android_eclipse/AndroidManifest.xml create mode 100644 python/mozbuild/mozbuild/backend/templates/android_eclipse/gen/tmp create mode 100644 python/mozbuild/mozbuild/backend/templates/android_eclipse/lint.xml create mode 100644 python/mozbuild/mozbuild/backend/templates/android_eclipse/project.properties create mode 100644 python/mozbuild/mozbuild/backend/templates/android_eclipse_empty_resource_directory/.not_an_android_resource create mode 100644 python/mozbuild/mozbuild/backend/tup.py create mode 100644 python/mozbuild/mozbuild/backend/visualstudio.py create mode 100644 python/mozbuild/mozbuild/base.py create mode 100644 python/mozbuild/mozbuild/codecoverage/__init__.py create mode 100644 python/mozbuild/mozbuild/codecoverage/chrome_map.py create mode 100644 python/mozbuild/mozbuild/codecoverage/packager.py create mode 100644 python/mozbuild/mozbuild/compilation/__init__.py create mode 100644 python/mozbuild/mozbuild/compilation/codecomplete.py create mode 100644 python/mozbuild/mozbuild/compilation/database.py create mode 100644 python/mozbuild/mozbuild/compilation/util.py create mode 100644 python/mozbuild/mozbuild/compilation/warnings.py create mode 100644 python/mozbuild/mozbuild/config_status.py create mode 100644 python/mozbuild/mozbuild/configure/__init__.py create mode 100644 python/mozbuild/mozbuild/configure/check_debug_ranges.py create mode 100644 python/mozbuild/mozbuild/configure/constants.py create mode 100644 python/mozbuild/mozbuild/configure/help.py create mode 100644 python/mozbuild/mozbuild/configure/libstdcxx.py create mode 100644 python/mozbuild/mozbuild/configure/lint.py create mode 100644 python/mozbuild/mozbuild/configure/lint_util.py create mode 100644 python/mozbuild/mozbuild/configure/options.py create mode 100644 python/mozbuild/mozbuild/configure/util.py create mode 100644 python/mozbuild/mozbuild/controller/__init__.py create mode 100644 python/mozbuild/mozbuild/controller/building.py create mode 100644 python/mozbuild/mozbuild/controller/clobber.py create mode 100644 python/mozbuild/mozbuild/doctor.py create mode 100644 python/mozbuild/mozbuild/dotproperties.py create mode 100644 python/mozbuild/mozbuild/frontend/__init__.py create mode 100644 python/mozbuild/mozbuild/frontend/context.py create mode 100644 python/mozbuild/mozbuild/frontend/data.py create mode 100644 python/mozbuild/mozbuild/frontend/emitter.py create mode 100644 python/mozbuild/mozbuild/frontend/gyp_reader.py create mode 100644 python/mozbuild/mozbuild/frontend/mach_commands.py create mode 100644 python/mozbuild/mozbuild/frontend/reader.py create mode 100644 python/mozbuild/mozbuild/frontend/sandbox.py create mode 100644 python/mozbuild/mozbuild/html_build_viewer.py create mode 100644 python/mozbuild/mozbuild/jar.py create mode 100644 python/mozbuild/mozbuild/locale/en-US/LC_MESSAGES/mozbuild.mo create mode 100644 python/mozbuild/mozbuild/locale/en-US/LC_MESSAGES/mozbuild.po create mode 100644 python/mozbuild/mozbuild/mach_commands.py create mode 100644 python/mozbuild/mozbuild/makeutil.py create mode 100644 python/mozbuild/mozbuild/milestone.py create mode 100644 python/mozbuild/mozbuild/mozconfig.py create mode 100755 python/mozbuild/mozbuild/mozconfig_loader create mode 100755 python/mozbuild/mozbuild/mozinfo.py create mode 100644 python/mozbuild/mozbuild/preprocessor.py create mode 100644 python/mozbuild/mozbuild/pythonutil.py create mode 100644 python/mozbuild/mozbuild/resources/html-build-viewer/index.html create mode 100644 python/mozbuild/mozbuild/shellutil.py create mode 100644 python/mozbuild/mozbuild/sphinx.py create mode 100644 python/mozbuild/mozbuild/test/__init__.py create mode 100644 python/mozbuild/mozbuild/test/action/data/invalid/region.properties create mode 100644 python/mozbuild/mozbuild/test/action/data/package_fennec_apk/assets/asset.txt create mode 100644 python/mozbuild/mozbuild/test/action/data/package_fennec_apk/classes.dex create mode 100644 python/mozbuild/mozbuild/test/action/data/package_fennec_apk/input1.ap_ create mode 100644 python/mozbuild/mozbuild/test/action/data/package_fennec_apk/input1/res/res.txt create mode 100644 python/mozbuild/mozbuild/test/action/data/package_fennec_apk/input1/resources.arsc create mode 100644 python/mozbuild/mozbuild/test/action/data/package_fennec_apk/input2.apk create mode 100644 python/mozbuild/mozbuild/test/action/data/package_fennec_apk/input2/assets/asset.txt create mode 100644 python/mozbuild/mozbuild/test/action/data/package_fennec_apk/input2/assets/omni.ja create mode 100644 python/mozbuild/mozbuild/test/action/data/package_fennec_apk/input2/classes.dex create mode 100644 python/mozbuild/mozbuild/test/action/data/package_fennec_apk/input2/lib/lib.txt create mode 100644 python/mozbuild/mozbuild/test/action/data/package_fennec_apk/input2/res/res.txt create mode 100644 python/mozbuild/mozbuild/test/action/data/package_fennec_apk/input2/resources.arsc create mode 100644 python/mozbuild/mozbuild/test/action/data/package_fennec_apk/input2/root_file.txt create mode 100644 python/mozbuild/mozbuild/test/action/data/package_fennec_apk/lib/lib.txt create mode 100644 python/mozbuild/mozbuild/test/action/data/package_fennec_apk/omni.ja create mode 100644 python/mozbuild/mozbuild/test/action/data/package_fennec_apk/root_file.txt create mode 100644 python/mozbuild/mozbuild/test/action/data/valid-zh-CN/region.properties create mode 100644 python/mozbuild/mozbuild/test/action/test_buildlist.py create mode 100644 python/mozbuild/mozbuild/test/action/test_generate_browsersearch.py create mode 100644 python/mozbuild/mozbuild/test/action/test_package_fennec_apk.py create mode 100644 python/mozbuild/mozbuild/test/backend/__init__.py create mode 100644 python/mozbuild/mozbuild/test/backend/common.py create mode 100644 python/mozbuild/mozbuild/test/backend/data/android_eclipse/library1/resources/values/strings.xml create mode 100644 python/mozbuild/mozbuild/test/backend/data/android_eclipse/main1/AndroidManifest.xml create mode 100644 python/mozbuild/mozbuild/test/backend/data/android_eclipse/main2/AndroidManifest.xml create mode 100644 python/mozbuild/mozbuild/test/backend/data/android_eclipse/main2/assets/dummy.txt create mode 100644 python/mozbuild/mozbuild/test/backend/data/android_eclipse/main2/extra.jar create mode 100644 python/mozbuild/mozbuild/test/backend/data/android_eclipse/main2/res/values/strings.xml create mode 100644 python/mozbuild/mozbuild/test/backend/data/android_eclipse/main3/AndroidManifest.xml create mode 100644 python/mozbuild/mozbuild/test/backend/data/android_eclipse/main3/a/A.java create mode 100644 python/mozbuild/mozbuild/test/backend/data/android_eclipse/main3/b/B.java create mode 100644 python/mozbuild/mozbuild/test/backend/data/android_eclipse/main3/c/C.java create mode 100644 python/mozbuild/mozbuild/test/backend/data/android_eclipse/main4 create mode 100644 python/mozbuild/mozbuild/test/backend/data/android_eclipse/moz.build create mode 100644 python/mozbuild/mozbuild/test/backend/data/android_eclipse/subdir/moz.build create mode 100644 python/mozbuild/mozbuild/test/backend/data/android_eclipse/subdir/submain/AndroidManifest.xml create mode 100644 python/mozbuild/mozbuild/test/backend/data/binary-components/bar/moz.build create mode 100644 python/mozbuild/mozbuild/test/backend/data/binary-components/foo/moz.build create mode 100644 python/mozbuild/mozbuild/test/backend/data/binary-components/moz.build create mode 100644 python/mozbuild/mozbuild/test/backend/data/branding-files/bar.ico create mode 100644 python/mozbuild/mozbuild/test/backend/data/branding-files/foo.ico create mode 100644 python/mozbuild/mozbuild/test/backend/data/branding-files/moz.build create mode 100644 python/mozbuild/mozbuild/test/backend/data/branding-files/sub/quux.png create mode 100644 python/mozbuild/mozbuild/test/backend/data/build/app/moz.build create mode 100644 python/mozbuild/mozbuild/test/backend/data/build/bar.ini create mode 100644 python/mozbuild/mozbuild/test/backend/data/build/bar.js create mode 100644 python/mozbuild/mozbuild/test/backend/data/build/bar.jsm create mode 100644 python/mozbuild/mozbuild/test/backend/data/build/baz.ini create mode 100644 python/mozbuild/mozbuild/test/backend/data/build/baz.jsm create mode 100644 python/mozbuild/mozbuild/test/backend/data/build/components.manifest create mode 100644 python/mozbuild/mozbuild/test/backend/data/build/foo.css create mode 100644 python/mozbuild/mozbuild/test/backend/data/build/foo.ini create mode 100644 python/mozbuild/mozbuild/test/backend/data/build/foo.js create mode 100644 python/mozbuild/mozbuild/test/backend/data/build/foo.jsm create mode 100644 python/mozbuild/mozbuild/test/backend/data/build/jar.mn create mode 100644 python/mozbuild/mozbuild/test/backend/data/build/moz.build create mode 100644 python/mozbuild/mozbuild/test/backend/data/build/prefs.js create mode 100644 python/mozbuild/mozbuild/test/backend/data/build/qux.ini create mode 100644 python/mozbuild/mozbuild/test/backend/data/build/qux.jsm create mode 100644 python/mozbuild/mozbuild/test/backend/data/build/resource create mode 100644 python/mozbuild/mozbuild/test/backend/data/build/resource2 create mode 100644 python/mozbuild/mozbuild/test/backend/data/build/subdir/bar.js create mode 100644 python/mozbuild/mozbuild/test/backend/data/defines/moz.build create mode 100644 python/mozbuild/mozbuild/test/backend/data/dist-files/install.rdf create mode 100644 python/mozbuild/mozbuild/test/backend/data/dist-files/main.js create mode 100644 python/mozbuild/mozbuild/test/backend/data/dist-files/moz.build create mode 100644 python/mozbuild/mozbuild/test/backend/data/exports-generated/dom1.h create mode 100644 python/mozbuild/mozbuild/test/backend/data/exports-generated/foo.h create mode 100644 python/mozbuild/mozbuild/test/backend/data/exports-generated/gfx.h create mode 100644 python/mozbuild/mozbuild/test/backend/data/exports-generated/moz.build create mode 100644 python/mozbuild/mozbuild/test/backend/data/exports-generated/mozilla1.h create mode 100644 python/mozbuild/mozbuild/test/backend/data/exports/dom1.h create mode 100644 python/mozbuild/mozbuild/test/backend/data/exports/dom2.h create mode 100644 python/mozbuild/mozbuild/test/backend/data/exports/foo.h create mode 100644 python/mozbuild/mozbuild/test/backend/data/exports/gfx.h create mode 100644 python/mozbuild/mozbuild/test/backend/data/exports/moz.build create mode 100644 python/mozbuild/mozbuild/test/backend/data/exports/mozilla1.h create mode 100644 python/mozbuild/mozbuild/test/backend/data/exports/mozilla2.h create mode 100644 python/mozbuild/mozbuild/test/backend/data/exports/pprio.h create mode 100644 python/mozbuild/mozbuild/test/backend/data/final_target/both/moz.build create mode 100644 python/mozbuild/mozbuild/test/backend/data/final_target/dist-subdir/moz.build create mode 100644 python/mozbuild/mozbuild/test/backend/data/final_target/final-target/moz.build create mode 100644 python/mozbuild/mozbuild/test/backend/data/final_target/moz.build create mode 100644 python/mozbuild/mozbuild/test/backend/data/final_target/xpi-name/moz.build create mode 100644 python/mozbuild/mozbuild/test/backend/data/generated-files/foo-data create mode 100644 python/mozbuild/mozbuild/test/backend/data/generated-files/generate-bar.py create mode 100644 python/mozbuild/mozbuild/test/backend/data/generated-files/generate-foo.py create mode 100644 python/mozbuild/mozbuild/test/backend/data/generated-files/moz.build create mode 100644 python/mozbuild/mozbuild/test/backend/data/generated_includes/moz.build create mode 100644 python/mozbuild/mozbuild/test/backend/data/host-defines/moz.build create mode 100644 python/mozbuild/mozbuild/test/backend/data/install_substitute_config_files/moz.build create mode 100644 python/mozbuild/mozbuild/test/backend/data/install_substitute_config_files/sub/foo.h.in create mode 100644 python/mozbuild/mozbuild/test/backend/data/install_substitute_config_files/sub/moz.build create mode 100644 python/mozbuild/mozbuild/test/backend/data/ipdl_sources/bar/moz.build create mode 100644 python/mozbuild/mozbuild/test/backend/data/ipdl_sources/foo/moz.build create mode 100644 python/mozbuild/mozbuild/test/backend/data/ipdl_sources/moz.build create mode 100644 python/mozbuild/mozbuild/test/backend/data/jar-manifests/moz.build create mode 100644 python/mozbuild/mozbuild/test/backend/data/local_includes/bar/baz/dummy_file_for_nonempty_directory create mode 100644 python/mozbuild/mozbuild/test/backend/data/local_includes/foo/dummy_file_for_nonempty_directory create mode 100644 python/mozbuild/mozbuild/test/backend/data/local_includes/moz.build create mode 100644 python/mozbuild/mozbuild/test/backend/data/resources/bar.res.in create mode 100644 python/mozbuild/mozbuild/test/backend/data/resources/cursor.cur create mode 100644 python/mozbuild/mozbuild/test/backend/data/resources/desktop1.ttf create mode 100644 python/mozbuild/mozbuild/test/backend/data/resources/desktop2.ttf create mode 100644 python/mozbuild/mozbuild/test/backend/data/resources/extra.manifest create mode 100644 python/mozbuild/mozbuild/test/backend/data/resources/font1.ttf create mode 100644 python/mozbuild/mozbuild/test/backend/data/resources/font2.ttf create mode 100644 python/mozbuild/mozbuild/test/backend/data/resources/foo.res create mode 100644 python/mozbuild/mozbuild/test/backend/data/resources/mobile.ttf create mode 100644 python/mozbuild/mozbuild/test/backend/data/resources/moz.build create mode 100644 python/mozbuild/mozbuild/test/backend/data/resources/test.manifest create mode 100644 python/mozbuild/mozbuild/test/backend/data/sdk-files/bar.ico create mode 100644 python/mozbuild/mozbuild/test/backend/data/sdk-files/foo.ico create mode 100644 python/mozbuild/mozbuild/test/backend/data/sdk-files/moz.build create mode 100644 python/mozbuild/mozbuild/test/backend/data/sdk-files/sub/quux.png create mode 100644 python/mozbuild/mozbuild/test/backend/data/sources/bar.c create mode 100644 python/mozbuild/mozbuild/test/backend/data/sources/bar.cpp create mode 100644 python/mozbuild/mozbuild/test/backend/data/sources/bar.mm create mode 100644 python/mozbuild/mozbuild/test/backend/data/sources/bar.s create mode 100644 python/mozbuild/mozbuild/test/backend/data/sources/baz.S create mode 100644 python/mozbuild/mozbuild/test/backend/data/sources/foo.S create mode 100644 python/mozbuild/mozbuild/test/backend/data/sources/foo.asm create mode 100644 python/mozbuild/mozbuild/test/backend/data/sources/foo.c create mode 100644 python/mozbuild/mozbuild/test/backend/data/sources/foo.cpp create mode 100644 python/mozbuild/mozbuild/test/backend/data/sources/foo.mm create mode 100644 python/mozbuild/mozbuild/test/backend/data/sources/moz.build create mode 100644 python/mozbuild/mozbuild/test/backend/data/stub0/Makefile.in create mode 100644 python/mozbuild/mozbuild/test/backend/data/stub0/dir1/Makefile.in create mode 100644 python/mozbuild/mozbuild/test/backend/data/stub0/dir1/moz.build create mode 100644 python/mozbuild/mozbuild/test/backend/data/stub0/dir2/moz.build create mode 100644 python/mozbuild/mozbuild/test/backend/data/stub0/dir3/Makefile.in create mode 100644 python/mozbuild/mozbuild/test/backend/data/stub0/dir3/moz.build create mode 100644 python/mozbuild/mozbuild/test/backend/data/stub0/moz.build create mode 100644 python/mozbuild/mozbuild/test/backend/data/substitute_config_files/Makefile.in create mode 100644 python/mozbuild/mozbuild/test/backend/data/substitute_config_files/foo.in create mode 100644 python/mozbuild/mozbuild/test/backend/data/substitute_config_files/moz.build create mode 100644 python/mozbuild/mozbuild/test/backend/data/test-manifest-shared-support/child/another-file.sjs create mode 100644 python/mozbuild/mozbuild/test/backend/data/test-manifest-shared-support/child/browser.ini create mode 100644 python/mozbuild/mozbuild/test/backend/data/test-manifest-shared-support/child/data/one.txt create mode 100644 python/mozbuild/mozbuild/test/backend/data/test-manifest-shared-support/child/data/two.txt create mode 100644 python/mozbuild/mozbuild/test/backend/data/test-manifest-shared-support/child/test_sub.js create mode 100644 python/mozbuild/mozbuild/test/backend/data/test-manifest-shared-support/mochitest.ini create mode 100644 python/mozbuild/mozbuild/test/backend/data/test-manifest-shared-support/moz.build create mode 100644 python/mozbuild/mozbuild/test/backend/data/test-manifest-shared-support/support-file.txt create mode 100644 python/mozbuild/mozbuild/test/backend/data/test-manifest-shared-support/test_foo.js create mode 100644 python/mozbuild/mozbuild/test/backend/data/test-manifests-duplicate-support-files/mochitest1.ini create mode 100644 python/mozbuild/mozbuild/test/backend/data/test-manifests-duplicate-support-files/mochitest2.ini create mode 100644 python/mozbuild/mozbuild/test/backend/data/test-manifests-duplicate-support-files/moz.build create mode 100644 python/mozbuild/mozbuild/test/backend/data/test-manifests-duplicate-support-files/test_bar.js create mode 100644 python/mozbuild/mozbuild/test/backend/data/test-manifests-duplicate-support-files/test_foo.js create mode 100644 python/mozbuild/mozbuild/test/backend/data/test-manifests-package-tests/instrumentation.ini create mode 100644 python/mozbuild/mozbuild/test/backend/data/test-manifests-package-tests/mochitest.ini create mode 100644 python/mozbuild/mozbuild/test/backend/data/test-manifests-package-tests/mochitest.js create mode 100644 python/mozbuild/mozbuild/test/backend/data/test-manifests-package-tests/moz.build create mode 100644 python/mozbuild/mozbuild/test/backend/data/test-manifests-package-tests/not_packaged.java create mode 100644 python/mozbuild/mozbuild/test/backend/data/test-manifests-written/dir1/test_bar.js create mode 100644 python/mozbuild/mozbuild/test/backend/data/test-manifests-written/dir1/xpcshell.ini create mode 100644 python/mozbuild/mozbuild/test/backend/data/test-manifests-written/mochitest.ini create mode 100644 python/mozbuild/mozbuild/test/backend/data/test-manifests-written/mochitest.js create mode 100644 python/mozbuild/mozbuild/test/backend/data/test-manifests-written/moz.build create mode 100644 python/mozbuild/mozbuild/test/backend/data/test-manifests-written/xpcshell.ini create mode 100644 python/mozbuild/mozbuild/test/backend/data/test-manifests-written/xpcshell.js create mode 100644 python/mozbuild/mozbuild/test/backend/data/test_config/file.in create mode 100644 python/mozbuild/mozbuild/test/backend/data/test_config/moz.build create mode 100644 python/mozbuild/mozbuild/test/backend/data/variable_passthru/Makefile.in create mode 100644 python/mozbuild/mozbuild/test/backend/data/variable_passthru/moz.build create mode 100644 python/mozbuild/mozbuild/test/backend/data/variable_passthru/test1.c create mode 100644 python/mozbuild/mozbuild/test/backend/data/variable_passthru/test1.cpp create mode 100644 python/mozbuild/mozbuild/test/backend/data/variable_passthru/test1.mm create mode 100644 python/mozbuild/mozbuild/test/backend/data/variable_passthru/test2.c create mode 100644 python/mozbuild/mozbuild/test/backend/data/variable_passthru/test2.cpp create mode 100644 python/mozbuild/mozbuild/test/backend/data/variable_passthru/test2.mm create mode 100644 python/mozbuild/mozbuild/test/backend/data/visual-studio/dir1/bar.cpp create mode 100644 python/mozbuild/mozbuild/test/backend/data/visual-studio/dir1/foo.cpp create mode 100644 python/mozbuild/mozbuild/test/backend/data/visual-studio/dir1/moz.build create mode 100644 python/mozbuild/mozbuild/test/backend/data/visual-studio/moz.build create mode 100644 python/mozbuild/mozbuild/test/backend/data/xpidl/config/makefiles/xpidl/Makefile.in create mode 100644 python/mozbuild/mozbuild/test/backend/data/xpidl/moz.build create mode 100644 python/mozbuild/mozbuild/test/backend/test_android_eclipse.py create mode 100644 python/mozbuild/mozbuild/test/backend/test_build.py create mode 100644 python/mozbuild/mozbuild/test/backend/test_configenvironment.py create mode 100644 python/mozbuild/mozbuild/test/backend/test_recursivemake.py create mode 100644 python/mozbuild/mozbuild/test/backend/test_visualstudio.py create mode 100644 python/mozbuild/mozbuild/test/common.py create mode 100644 python/mozbuild/mozbuild/test/compilation/__init__.py create mode 100644 python/mozbuild/mozbuild/test/compilation/test_warnings.py create mode 100644 python/mozbuild/mozbuild/test/configure/common.py create mode 100644 python/mozbuild/mozbuild/test/configure/data/decorators.configure create mode 100644 python/mozbuild/mozbuild/test/configure/data/empty_mozconfig create mode 100644 python/mozbuild/mozbuild/test/configure/data/extra.configure create mode 100644 python/mozbuild/mozbuild/test/configure/data/imply_option/imm.configure create mode 100644 python/mozbuild/mozbuild/test/configure/data/imply_option/infer.configure create mode 100644 python/mozbuild/mozbuild/test/configure/data/imply_option/infer_ko.configure create mode 100644 python/mozbuild/mozbuild/test/configure/data/imply_option/negative.configure create mode 100644 python/mozbuild/mozbuild/test/configure/data/imply_option/simple.configure create mode 100644 python/mozbuild/mozbuild/test/configure/data/imply_option/values.configure create mode 100644 python/mozbuild/mozbuild/test/configure/data/included.configure create mode 100644 python/mozbuild/mozbuild/test/configure/data/moz.configure create mode 100644 python/mozbuild/mozbuild/test/configure/data/set_config.configure create mode 100644 python/mozbuild/mozbuild/test/configure/data/set_define.configure create mode 100644 python/mozbuild/mozbuild/test/configure/data/subprocess.configure create mode 100644 python/mozbuild/mozbuild/test/configure/lint.py create mode 100644 python/mozbuild/mozbuild/test/configure/test_checks_configure.py create mode 100644 python/mozbuild/mozbuild/test/configure/test_compile_checks.py create mode 100644 python/mozbuild/mozbuild/test/configure/test_configure.py create mode 100644 python/mozbuild/mozbuild/test/configure/test_lint.py create mode 100644 python/mozbuild/mozbuild/test/configure/test_moz_configure.py create mode 100644 python/mozbuild/mozbuild/test/configure/test_options.py create mode 100644 python/mozbuild/mozbuild/test/configure/test_toolchain_configure.py create mode 100644 python/mozbuild/mozbuild/test/configure/test_toolchain_helpers.py create mode 100644 python/mozbuild/mozbuild/test/configure/test_toolkit_moz_configure.py create mode 100644 python/mozbuild/mozbuild/test/configure/test_util.py create mode 100644 python/mozbuild/mozbuild/test/controller/__init__.py create mode 100644 python/mozbuild/mozbuild/test/controller/test_ccachestats.py create mode 100644 python/mozbuild/mozbuild/test/controller/test_clobber.py create mode 100644 python/mozbuild/mozbuild/test/data/Makefile create mode 100644 python/mozbuild/mozbuild/test/data/bad.properties create mode 100644 python/mozbuild/mozbuild/test/data/test-dir/Makefile create mode 100644 python/mozbuild/mozbuild/test/data/test-dir/with/Makefile create mode 100644 python/mozbuild/mozbuild/test/data/test-dir/with/without/with/Makefile create mode 100644 python/mozbuild/mozbuild/test/data/test-dir/without/with/Makefile create mode 100644 python/mozbuild/mozbuild/test/data/valid.properties create mode 100644 python/mozbuild/mozbuild/test/frontend/__init__.py create mode 100644 python/mozbuild/mozbuild/test/frontend/data/android-res-dirs/dir1/foo create mode 100644 python/mozbuild/mozbuild/test/frontend/data/android-res-dirs/moz.build create mode 100644 python/mozbuild/mozbuild/test/frontend/data/binary-components/bar/moz.build create mode 100644 python/mozbuild/mozbuild/test/frontend/data/binary-components/foo/moz.build create mode 100644 python/mozbuild/mozbuild/test/frontend/data/binary-components/moz.build create mode 100644 python/mozbuild/mozbuild/test/frontend/data/branding-files/bar.ico create mode 100644 python/mozbuild/mozbuild/test/frontend/data/branding-files/baz.png create mode 100644 python/mozbuild/mozbuild/test/frontend/data/branding-files/foo.xpm create mode 100644 python/mozbuild/mozbuild/test/frontend/data/branding-files/moz.build create mode 100644 python/mozbuild/mozbuild/test/frontend/data/branding-files/quux.icns create mode 100644 python/mozbuild/mozbuild/test/frontend/data/config-file-substitution/moz.build create mode 100644 python/mozbuild/mozbuild/test/frontend/data/crate-dependency-path-resolution/Cargo.toml create mode 100644 python/mozbuild/mozbuild/test/frontend/data/crate-dependency-path-resolution/moz.build create mode 100644 python/mozbuild/mozbuild/test/frontend/data/crate-dependency-path-resolution/shallow/Cargo.toml create mode 100644 python/mozbuild/mozbuild/test/frontend/data/crate-dependency-path-resolution/the/depths/Cargo.toml create mode 100644 python/mozbuild/mozbuild/test/frontend/data/defines/moz.build create mode 100644 python/mozbuild/mozbuild/test/frontend/data/dist-files-missing/install.rdf create mode 100644 python/mozbuild/mozbuild/test/frontend/data/dist-files-missing/moz.build create mode 100644 python/mozbuild/mozbuild/test/frontend/data/dist-files/install.rdf create mode 100644 python/mozbuild/mozbuild/test/frontend/data/dist-files/main.js create mode 100644 python/mozbuild/mozbuild/test/frontend/data/dist-files/moz.build create mode 100644 python/mozbuild/mozbuild/test/frontend/data/exports-generated/foo.h create mode 100644 python/mozbuild/mozbuild/test/frontend/data/exports-generated/moz.build create mode 100644 python/mozbuild/mozbuild/test/frontend/data/exports-generated/mozilla1.h create mode 100644 python/mozbuild/mozbuild/test/frontend/data/exports-missing-generated/foo.h create mode 100644 python/mozbuild/mozbuild/test/frontend/data/exports-missing-generated/moz.build create mode 100644 python/mozbuild/mozbuild/test/frontend/data/exports-missing/foo.h create mode 100644 python/mozbuild/mozbuild/test/frontend/data/exports-missing/moz.build create mode 100644 python/mozbuild/mozbuild/test/frontend/data/exports-missing/mozilla1.h create mode 100644 python/mozbuild/mozbuild/test/frontend/data/exports/bar.h create mode 100644 python/mozbuild/mozbuild/test/frontend/data/exports/baz.h create mode 100644 python/mozbuild/mozbuild/test/frontend/data/exports/dom1.h create mode 100644 python/mozbuild/mozbuild/test/frontend/data/exports/dom2.h create mode 100644 python/mozbuild/mozbuild/test/frontend/data/exports/dom3.h create mode 100644 python/mozbuild/mozbuild/test/frontend/data/exports/foo.h create mode 100644 python/mozbuild/mozbuild/test/frontend/data/exports/gfx.h create mode 100644 python/mozbuild/mozbuild/test/frontend/data/exports/mem.h create mode 100644 python/mozbuild/mozbuild/test/frontend/data/exports/mem2.h create mode 100644 python/mozbuild/mozbuild/test/frontend/data/exports/moz.build create mode 100644 python/mozbuild/mozbuild/test/frontend/data/exports/mozilla1.h create mode 100644 python/mozbuild/mozbuild/test/frontend/data/exports/mozilla2.h create mode 100644 python/mozbuild/mozbuild/test/frontend/data/exports/pprio.h create mode 100644 python/mozbuild/mozbuild/test/frontend/data/exports/pprthred.h create mode 100644 python/mozbuild/mozbuild/test/frontend/data/files-info/bug_component/bad-assignment/moz.build create mode 100644 python/mozbuild/mozbuild/test/frontend/data/files-info/bug_component/different-matchers/moz.build create mode 100644 python/mozbuild/mozbuild/test/frontend/data/files-info/bug_component/final/moz.build create mode 100644 python/mozbuild/mozbuild/test/frontend/data/files-info/bug_component/final/subcomponent/moz.build create mode 100644 python/mozbuild/mozbuild/test/frontend/data/files-info/bug_component/moz.build create mode 100644 python/mozbuild/mozbuild/test/frontend/data/files-info/bug_component/simple/moz.build create mode 100644 python/mozbuild/mozbuild/test/frontend/data/files-info/bug_component/static/moz.build create mode 100644 python/mozbuild/mozbuild/test/frontend/data/files-info/moz.build create mode 100644 python/mozbuild/mozbuild/test/frontend/data/files-test-metadata/default/module.js create mode 100644 python/mozbuild/mozbuild/test/frontend/data/files-test-metadata/default/moz.build create mode 100644 python/mozbuild/mozbuild/test/frontend/data/files-test-metadata/default/tests/reftests/reftest-stylo.list create mode 100644 python/mozbuild/mozbuild/test/frontend/data/files-test-metadata/default/tests/reftests/reftest.list create mode 100644 python/mozbuild/mozbuild/test/frontend/data/files-test-metadata/default/tests/reftests/test1-ref.html create mode 100644 python/mozbuild/mozbuild/test/frontend/data/files-test-metadata/default/tests/reftests/test1.html create mode 100644 python/mozbuild/mozbuild/test/frontend/data/files-test-metadata/default/tests/xpcshell/test_default_mod.js create mode 100644 python/mozbuild/mozbuild/test/frontend/data/files-test-metadata/default/tests/xpcshell/xpcshell.ini create mode 100644 python/mozbuild/mozbuild/test/frontend/data/files-test-metadata/moz.build create mode 100644 python/mozbuild/mozbuild/test/frontend/data/files-test-metadata/simple/base.cpp create mode 100644 python/mozbuild/mozbuild/test/frontend/data/files-test-metadata/simple/browser/browser.ini create mode 100644 python/mozbuild/mozbuild/test/frontend/data/files-test-metadata/simple/browser/test_mod.js create mode 100644 python/mozbuild/mozbuild/test/frontend/data/files-test-metadata/simple/moz.build create mode 100644 python/mozbuild/mozbuild/test/frontend/data/files-test-metadata/simple/src/module.jsm create mode 100644 python/mozbuild/mozbuild/test/frontend/data/files-test-metadata/simple/src/moz.build create mode 100644 python/mozbuild/mozbuild/test/frontend/data/files-test-metadata/simple/tests/mochitest.ini create mode 100644 python/mozbuild/mozbuild/test/frontend/data/files-test-metadata/simple/tests/moz.build create mode 100644 python/mozbuild/mozbuild/test/frontend/data/files-test-metadata/simple/tests/test_general.html create mode 100644 python/mozbuild/mozbuild/test/frontend/data/files-test-metadata/simple/tests/test_specific.html create mode 100644 python/mozbuild/mozbuild/test/frontend/data/files-test-metadata/tagged/moz.build create mode 100644 python/mozbuild/mozbuild/test/frontend/data/files-test-metadata/tagged/src/bar.jsm create mode 100644 python/mozbuild/mozbuild/test/frontend/data/files-test-metadata/tagged/src/submodule/foo.js create mode 100644 python/mozbuild/mozbuild/test/frontend/data/files-test-metadata/tagged/tests/mochitest.ini create mode 100644 python/mozbuild/mozbuild/test/frontend/data/files-test-metadata/tagged/tests/test_bar.js create mode 100644 python/mozbuild/mozbuild/test/frontend/data/files-test-metadata/tagged/tests/test_simple.html create mode 100644 python/mozbuild/mozbuild/test/frontend/data/files-test-metadata/tagged/tests/test_specific.html create mode 100644 python/mozbuild/mozbuild/test/frontend/data/files-test-metadata/tagged/tests/xpcshell.ini create mode 100644 python/mozbuild/mozbuild/test/frontend/data/final-target-pp-files-non-srcdir/moz.build create mode 100644 python/mozbuild/mozbuild/test/frontend/data/generated-files-absolute-script/moz.build create mode 100644 python/mozbuild/mozbuild/test/frontend/data/generated-files-absolute-script/script.py create mode 100644 python/mozbuild/mozbuild/test/frontend/data/generated-files-method-names/moz.build create mode 100644 python/mozbuild/mozbuild/test/frontend/data/generated-files-method-names/script.py create mode 100644 python/mozbuild/mozbuild/test/frontend/data/generated-files-no-inputs/moz.build create mode 100644 python/mozbuild/mozbuild/test/frontend/data/generated-files-no-inputs/script.py create mode 100644 python/mozbuild/mozbuild/test/frontend/data/generated-files-no-python-script/moz.build create mode 100644 python/mozbuild/mozbuild/test/frontend/data/generated-files-no-python-script/script.rb create mode 100644 python/mozbuild/mozbuild/test/frontend/data/generated-files-no-script/moz.build create mode 100644 python/mozbuild/mozbuild/test/frontend/data/generated-files/moz.build create mode 100644 python/mozbuild/mozbuild/test/frontend/data/generated-sources/a.cpp create mode 100644 python/mozbuild/mozbuild/test/frontend/data/generated-sources/b.cc create mode 100644 python/mozbuild/mozbuild/test/frontend/data/generated-sources/c.cxx create mode 100644 python/mozbuild/mozbuild/test/frontend/data/generated-sources/d.c create mode 100644 python/mozbuild/mozbuild/test/frontend/data/generated-sources/e.m create mode 100644 python/mozbuild/mozbuild/test/frontend/data/generated-sources/f.mm create mode 100644 python/mozbuild/mozbuild/test/frontend/data/generated-sources/g.S create mode 100644 python/mozbuild/mozbuild/test/frontend/data/generated-sources/h.s create mode 100644 python/mozbuild/mozbuild/test/frontend/data/generated-sources/i.asm create mode 100644 python/mozbuild/mozbuild/test/frontend/data/generated-sources/moz.build create mode 100644 python/mozbuild/mozbuild/test/frontend/data/generated_includes/moz.build create mode 100644 python/mozbuild/mozbuild/test/frontend/data/host-defines/moz.build create mode 100644 python/mozbuild/mozbuild/test/frontend/data/host-sources/a.cpp create mode 100644 python/mozbuild/mozbuild/test/frontend/data/host-sources/b.cc create mode 100644 python/mozbuild/mozbuild/test/frontend/data/host-sources/c.cxx create mode 100644 python/mozbuild/mozbuild/test/frontend/data/host-sources/d.c create mode 100644 python/mozbuild/mozbuild/test/frontend/data/host-sources/e.mm create mode 100644 python/mozbuild/mozbuild/test/frontend/data/host-sources/f.mm create mode 100644 python/mozbuild/mozbuild/test/frontend/data/host-sources/moz.build create mode 100644 python/mozbuild/mozbuild/test/frontend/data/include-basic/included.build create mode 100644 python/mozbuild/mozbuild/test/frontend/data/include-basic/moz.build create mode 100644 python/mozbuild/mozbuild/test/frontend/data/include-file-stack/included-1.build create mode 100644 python/mozbuild/mozbuild/test/frontend/data/include-file-stack/included-2.build create mode 100644 python/mozbuild/mozbuild/test/frontend/data/include-file-stack/moz.build create mode 100644 python/mozbuild/mozbuild/test/frontend/data/include-missing/moz.build create mode 100644 python/mozbuild/mozbuild/test/frontend/data/include-outside-topsrcdir/relative.build create mode 100644 python/mozbuild/mozbuild/test/frontend/data/include-relative-from-child/child/child.build create mode 100644 python/mozbuild/mozbuild/test/frontend/data/include-relative-from-child/child/child2.build create mode 100644 python/mozbuild/mozbuild/test/frontend/data/include-relative-from-child/child/grandchild/grandchild.build create mode 100644 python/mozbuild/mozbuild/test/frontend/data/include-relative-from-child/parent.build create mode 100644 python/mozbuild/mozbuild/test/frontend/data/include-topsrcdir-relative/moz.build create mode 100644 python/mozbuild/mozbuild/test/frontend/data/include-topsrcdir-relative/sibling.build create mode 100644 python/mozbuild/mozbuild/test/frontend/data/inheriting-variables/bar/moz.build create mode 100644 python/mozbuild/mozbuild/test/frontend/data/inheriting-variables/foo/baz/moz.build create mode 100644 python/mozbuild/mozbuild/test/frontend/data/inheriting-variables/foo/moz.build create mode 100644 python/mozbuild/mozbuild/test/frontend/data/inheriting-variables/moz.build create mode 100644 python/mozbuild/mozbuild/test/frontend/data/ipdl_sources/bar/moz.build create mode 100644 python/mozbuild/mozbuild/test/frontend/data/ipdl_sources/foo/moz.build create mode 100644 python/mozbuild/mozbuild/test/frontend/data/ipdl_sources/moz.build create mode 100644 python/mozbuild/mozbuild/test/frontend/data/jar-manifests-multiple-files/moz.build create mode 100644 python/mozbuild/mozbuild/test/frontend/data/jar-manifests/moz.build create mode 100644 python/mozbuild/mozbuild/test/frontend/data/library-defines/liba/moz.build create mode 100644 python/mozbuild/mozbuild/test/frontend/data/library-defines/libb/moz.build create mode 100644 python/mozbuild/mozbuild/test/frontend/data/library-defines/libc/moz.build create mode 100644 python/mozbuild/mozbuild/test/frontend/data/library-defines/libd/moz.build create mode 100644 python/mozbuild/mozbuild/test/frontend/data/library-defines/moz.build create mode 100644 python/mozbuild/mozbuild/test/frontend/data/local_includes/bar/baz/dummy_file_for_nonempty_directory create mode 100644 python/mozbuild/mozbuild/test/frontend/data/local_includes/foo/dummy_file_for_nonempty_directory create mode 100644 python/mozbuild/mozbuild/test/frontend/data/local_includes/moz.build create mode 100644 python/mozbuild/mozbuild/test/frontend/data/missing-local-includes/moz.build create mode 100644 python/mozbuild/mozbuild/test/frontend/data/multiple-rust-libraries/moz.build create mode 100644 python/mozbuild/mozbuild/test/frontend/data/multiple-rust-libraries/rust1/Cargo.toml create mode 100644 python/mozbuild/mozbuild/test/frontend/data/multiple-rust-libraries/rust1/moz.build create mode 100644 python/mozbuild/mozbuild/test/frontend/data/multiple-rust-libraries/rust2/Cargo.toml create mode 100644 python/mozbuild/mozbuild/test/frontend/data/multiple-rust-libraries/rust2/moz.build create mode 100644 python/mozbuild/mozbuild/test/frontend/data/program/moz.build create mode 100644 python/mozbuild/mozbuild/test/frontend/data/reader-error-bad-dir/moz.build create mode 100644 python/mozbuild/mozbuild/test/frontend/data/reader-error-basic/moz.build create mode 100644 python/mozbuild/mozbuild/test/frontend/data/reader-error-empty-list/moz.build create mode 100644 python/mozbuild/mozbuild/test/frontend/data/reader-error-error-func/moz.build create mode 100644 python/mozbuild/mozbuild/test/frontend/data/reader-error-included-from/child.build create mode 100644 python/mozbuild/mozbuild/test/frontend/data/reader-error-included-from/moz.build create mode 100644 python/mozbuild/mozbuild/test/frontend/data/reader-error-missing-include/moz.build create mode 100644 python/mozbuild/mozbuild/test/frontend/data/reader-error-outside-topsrcdir/moz.build create mode 100644 python/mozbuild/mozbuild/test/frontend/data/reader-error-read-unknown-global/moz.build create mode 100644 python/mozbuild/mozbuild/test/frontend/data/reader-error-repeated-dir/moz.build create mode 100644 python/mozbuild/mozbuild/test/frontend/data/reader-error-script-error/moz.build create mode 100644 python/mozbuild/mozbuild/test/frontend/data/reader-error-syntax/moz.build create mode 100644 python/mozbuild/mozbuild/test/frontend/data/reader-error-write-bad-value/moz.build create mode 100644 python/mozbuild/mozbuild/test/frontend/data/reader-error-write-unknown-global/moz.build create mode 100644 python/mozbuild/mozbuild/test/frontend/data/reader-relevant-mozbuild/d1/every-level/a/file create mode 100644 python/mozbuild/mozbuild/test/frontend/data/reader-relevant-mozbuild/d1/every-level/a/moz.build create mode 100644 python/mozbuild/mozbuild/test/frontend/data/reader-relevant-mozbuild/d1/every-level/b/file create mode 100644 python/mozbuild/mozbuild/test/frontend/data/reader-relevant-mozbuild/d1/every-level/b/moz.build create mode 100644 python/mozbuild/mozbuild/test/frontend/data/reader-relevant-mozbuild/d1/every-level/moz.build create mode 100644 python/mozbuild/mozbuild/test/frontend/data/reader-relevant-mozbuild/d1/file1 create mode 100644 python/mozbuild/mozbuild/test/frontend/data/reader-relevant-mozbuild/d1/file2 create mode 100644 python/mozbuild/mozbuild/test/frontend/data/reader-relevant-mozbuild/d1/moz.build create mode 100644 python/mozbuild/mozbuild/test/frontend/data/reader-relevant-mozbuild/d1/no-intermediate-moz-build/child/file create mode 100644 python/mozbuild/mozbuild/test/frontend/data/reader-relevant-mozbuild/d1/no-intermediate-moz-build/child/moz.build create mode 100644 python/mozbuild/mozbuild/test/frontend/data/reader-relevant-mozbuild/d1/parent-is-far/dir1/dir2/dir3/file create mode 100644 python/mozbuild/mozbuild/test/frontend/data/reader-relevant-mozbuild/d1/parent-is-far/moz.build create mode 100644 python/mozbuild/mozbuild/test/frontend/data/reader-relevant-mozbuild/d2/dir1/file create mode 100644 python/mozbuild/mozbuild/test/frontend/data/reader-relevant-mozbuild/d2/dir1/moz.build create mode 100644 python/mozbuild/mozbuild/test/frontend/data/reader-relevant-mozbuild/d2/dir2/file create mode 100644 python/mozbuild/mozbuild/test/frontend/data/reader-relevant-mozbuild/d2/dir2/moz.build create mode 100644 python/mozbuild/mozbuild/test/frontend/data/reader-relevant-mozbuild/d2/moz.build create mode 100644 python/mozbuild/mozbuild/test/frontend/data/reader-relevant-mozbuild/file create mode 100644 python/mozbuild/mozbuild/test/frontend/data/reader-relevant-mozbuild/moz.build create mode 100644 python/mozbuild/mozbuild/test/frontend/data/rust-library-dash-folding/Cargo.toml create mode 100644 python/mozbuild/mozbuild/test/frontend/data/rust-library-dash-folding/moz.build create mode 100644 python/mozbuild/mozbuild/test/frontend/data/rust-library-invalid-crate-type/Cargo.toml create mode 100644 python/mozbuild/mozbuild/test/frontend/data/rust-library-invalid-crate-type/moz.build create mode 100644 python/mozbuild/mozbuild/test/frontend/data/rust-library-name-mismatch/Cargo.toml create mode 100644 python/mozbuild/mozbuild/test/frontend/data/rust-library-name-mismatch/moz.build create mode 100644 python/mozbuild/mozbuild/test/frontend/data/rust-library-no-cargo-toml/moz.build create mode 100644 python/mozbuild/mozbuild/test/frontend/data/rust-library-no-lib-section/Cargo.toml create mode 100644 python/mozbuild/mozbuild/test/frontend/data/rust-library-no-lib-section/moz.build create mode 100644 python/mozbuild/mozbuild/test/frontend/data/rust-library-no-profile-section/Cargo.toml create mode 100644 python/mozbuild/mozbuild/test/frontend/data/rust-library-no-profile-section/moz.build create mode 100644 python/mozbuild/mozbuild/test/frontend/data/rust-library-non-abort-panic/Cargo.toml create mode 100644 python/mozbuild/mozbuild/test/frontend/data/rust-library-non-abort-panic/moz.build create mode 100644 python/mozbuild/mozbuild/test/frontend/data/sdk-files/bar.ico create mode 100644 python/mozbuild/mozbuild/test/frontend/data/sdk-files/baz.png create mode 100644 python/mozbuild/mozbuild/test/frontend/data/sdk-files/foo.xpm create mode 100644 python/mozbuild/mozbuild/test/frontend/data/sdk-files/moz.build create mode 100644 python/mozbuild/mozbuild/test/frontend/data/sdk-files/quux.icns create mode 100644 python/mozbuild/mozbuild/test/frontend/data/sources-just-c/d.c create mode 100644 python/mozbuild/mozbuild/test/frontend/data/sources-just-c/e.m create mode 100644 python/mozbuild/mozbuild/test/frontend/data/sources-just-c/g.S create mode 100644 python/mozbuild/mozbuild/test/frontend/data/sources-just-c/h.s create mode 100644 python/mozbuild/mozbuild/test/frontend/data/sources-just-c/i.asm create mode 100644 python/mozbuild/mozbuild/test/frontend/data/sources-just-c/moz.build create mode 100644 python/mozbuild/mozbuild/test/frontend/data/sources/a.cpp create mode 100644 python/mozbuild/mozbuild/test/frontend/data/sources/b.cc create mode 100644 python/mozbuild/mozbuild/test/frontend/data/sources/c.cxx create mode 100644 python/mozbuild/mozbuild/test/frontend/data/sources/d.c create mode 100644 python/mozbuild/mozbuild/test/frontend/data/sources/e.m create mode 100644 python/mozbuild/mozbuild/test/frontend/data/sources/f.mm create mode 100644 python/mozbuild/mozbuild/test/frontend/data/sources/g.S create mode 100644 python/mozbuild/mozbuild/test/frontend/data/sources/h.s create mode 100644 python/mozbuild/mozbuild/test/frontend/data/sources/i.asm create mode 100644 python/mozbuild/mozbuild/test/frontend/data/sources/moz.build create mode 100644 python/mozbuild/mozbuild/test/frontend/data/templates/templates.mozbuild create mode 100644 python/mozbuild/mozbuild/test/frontend/data/test-harness-files-root/moz.build create mode 100644 python/mozbuild/mozbuild/test/frontend/data/test-harness-files/mochitest.ini create mode 100644 python/mozbuild/mozbuild/test/frontend/data/test-harness-files/mochitest.py create mode 100644 python/mozbuild/mozbuild/test/frontend/data/test-harness-files/moz.build create mode 100644 python/mozbuild/mozbuild/test/frontend/data/test-harness-files/runtests.py create mode 100644 python/mozbuild/mozbuild/test/frontend/data/test-harness-files/utils.py create mode 100644 python/mozbuild/mozbuild/test/frontend/data/test-install-shared-lib/moz.build create mode 100644 python/mozbuild/mozbuild/test/frontend/data/test-linkables-cxx-link/moz.build create mode 100644 python/mozbuild/mozbuild/test/frontend/data/test-linkables-cxx-link/one/foo.cpp create mode 100644 python/mozbuild/mozbuild/test/frontend/data/test-linkables-cxx-link/one/moz.build create mode 100644 python/mozbuild/mozbuild/test/frontend/data/test-linkables-cxx-link/three/moz.build create mode 100644 python/mozbuild/mozbuild/test/frontend/data/test-linkables-cxx-link/two/foo.c create mode 100644 python/mozbuild/mozbuild/test/frontend/data/test-linkables-cxx-link/two/moz.build create mode 100644 python/mozbuild/mozbuild/test/frontend/data/test-manifest-absolute-support/absolute-support.ini create mode 100644 python/mozbuild/mozbuild/test/frontend/data/test-manifest-absolute-support/foo.txt create mode 100644 python/mozbuild/mozbuild/test/frontend/data/test-manifest-absolute-support/moz.build create mode 100644 python/mozbuild/mozbuild/test/frontend/data/test-manifest-absolute-support/test_file.js create mode 100644 python/mozbuild/mozbuild/test/frontend/data/test-manifest-dupes/bar.js create mode 100644 python/mozbuild/mozbuild/test/frontend/data/test-manifest-dupes/foo.js create mode 100644 python/mozbuild/mozbuild/test/frontend/data/test-manifest-dupes/mochitest.ini create mode 100644 python/mozbuild/mozbuild/test/frontend/data/test-manifest-dupes/moz.build create mode 100644 python/mozbuild/mozbuild/test/frontend/data/test-manifest-dupes/test_baz.js create mode 100644 python/mozbuild/mozbuild/test/frontend/data/test-manifest-emitted-includes/included-reftest.list create mode 100644 python/mozbuild/mozbuild/test/frontend/data/test-manifest-emitted-includes/moz.build create mode 100644 python/mozbuild/mozbuild/test/frontend/data/test-manifest-emitted-includes/reftest-stylo.list create mode 100644 python/mozbuild/mozbuild/test/frontend/data/test-manifest-emitted-includes/reftest.list create mode 100644 python/mozbuild/mozbuild/test/frontend/data/test-manifest-empty/empty.ini create mode 100644 python/mozbuild/mozbuild/test/frontend/data/test-manifest-empty/moz.build create mode 100644 python/mozbuild/mozbuild/test/frontend/data/test-manifest-inactive-ignored/test_inactive.html create mode 100644 python/mozbuild/mozbuild/test/frontend/data/test-manifest-install-includes/common.ini create mode 100644 python/mozbuild/mozbuild/test/frontend/data/test-manifest-install-includes/mochitest.ini create mode 100644 python/mozbuild/mozbuild/test/frontend/data/test-manifest-install-includes/moz.build create mode 100644 python/mozbuild/mozbuild/test/frontend/data/test-manifest-install-includes/test_foo.html create mode 100644 python/mozbuild/mozbuild/test/frontend/data/test-manifest-install-subdir/moz.build create mode 100644 python/mozbuild/mozbuild/test/frontend/data/test-manifest-install-subdir/subdir.ini create mode 100644 python/mozbuild/mozbuild/test/frontend/data/test-manifest-install-subdir/test_foo.html create mode 100644 python/mozbuild/mozbuild/test/frontend/data/test-manifest-just-support/foo.txt create mode 100644 python/mozbuild/mozbuild/test/frontend/data/test-manifest-just-support/just-support.ini create mode 100644 python/mozbuild/mozbuild/test/frontend/data/test-manifest-just-support/moz.build create mode 100644 python/mozbuild/mozbuild/test/frontend/data/test-manifest-keys-extracted/a11y-support/dir1/bar create mode 100644 python/mozbuild/mozbuild/test/frontend/data/test-manifest-keys-extracted/a11y-support/foo create mode 100644 python/mozbuild/mozbuild/test/frontend/data/test-manifest-keys-extracted/a11y.ini create mode 100644 python/mozbuild/mozbuild/test/frontend/data/test-manifest-keys-extracted/browser.ini create mode 100644 python/mozbuild/mozbuild/test/frontend/data/test-manifest-keys-extracted/chrome.ini create mode 100644 python/mozbuild/mozbuild/test/frontend/data/test-manifest-keys-extracted/crashtest.list create mode 100644 python/mozbuild/mozbuild/test/frontend/data/test-manifest-keys-extracted/metro.ini create mode 100644 python/mozbuild/mozbuild/test/frontend/data/test-manifest-keys-extracted/mochitest.ini create mode 100644 python/mozbuild/mozbuild/test/frontend/data/test-manifest-keys-extracted/moz.build create mode 100644 python/mozbuild/mozbuild/test/frontend/data/test-manifest-keys-extracted/reftest-stylo.list create mode 100644 python/mozbuild/mozbuild/test/frontend/data/test-manifest-keys-extracted/reftest.list create mode 100644 python/mozbuild/mozbuild/test/frontend/data/test-manifest-keys-extracted/test_a11y.js create mode 100644 python/mozbuild/mozbuild/test/frontend/data/test-manifest-keys-extracted/test_browser.js create mode 100644 python/mozbuild/mozbuild/test/frontend/data/test-manifest-keys-extracted/test_chrome.js create mode 100644 python/mozbuild/mozbuild/test/frontend/data/test-manifest-keys-extracted/test_foo.py create mode 100644 python/mozbuild/mozbuild/test/frontend/data/test-manifest-keys-extracted/test_metro.js create mode 100644 python/mozbuild/mozbuild/test/frontend/data/test-manifest-keys-extracted/test_mochitest.js create mode 100644 python/mozbuild/mozbuild/test/frontend/data/test-manifest-keys-extracted/test_xpcshell.js create mode 100644 python/mozbuild/mozbuild/test/frontend/data/test-manifest-keys-extracted/xpcshell.ini create mode 100644 python/mozbuild/mozbuild/test/frontend/data/test-manifest-missing-manifest/moz.build create mode 100644 python/mozbuild/mozbuild/test/frontend/data/test-manifest-missing-test-file-unfiltered/moz.build create mode 100644 python/mozbuild/mozbuild/test/frontend/data/test-manifest-missing-test-file-unfiltered/xpcshell.ini create mode 100644 python/mozbuild/mozbuild/test/frontend/data/test-manifest-missing-test-file/mochitest.ini create mode 100644 python/mozbuild/mozbuild/test/frontend/data/test-manifest-missing-test-file/moz.build create mode 100644 python/mozbuild/mozbuild/test/frontend/data/test-manifest-parent-support-files-dir/child/mochitest.ini create mode 100644 python/mozbuild/mozbuild/test/frontend/data/test-manifest-parent-support-files-dir/child/test_foo.js create mode 100644 python/mozbuild/mozbuild/test/frontend/data/test-manifest-parent-support-files-dir/moz.build create mode 100644 python/mozbuild/mozbuild/test/frontend/data/test-manifest-parent-support-files-dir/support-file.txt create mode 100644 python/mozbuild/mozbuild/test/frontend/data/test-manifest-shared-missing/child/another-file.sjs create mode 100644 python/mozbuild/mozbuild/test/frontend/data/test-manifest-shared-missing/child/browser.ini create mode 100644 python/mozbuild/mozbuild/test/frontend/data/test-manifest-shared-missing/child/data/one.txt create mode 100644 python/mozbuild/mozbuild/test/frontend/data/test-manifest-shared-missing/child/data/two.txt create mode 100644 python/mozbuild/mozbuild/test/frontend/data/test-manifest-shared-missing/child/test_sub.js create mode 100644 python/mozbuild/mozbuild/test/frontend/data/test-manifest-shared-missing/mochitest.ini create mode 100644 python/mozbuild/mozbuild/test/frontend/data/test-manifest-shared-missing/moz.build create mode 100644 python/mozbuild/mozbuild/test/frontend/data/test-manifest-shared-missing/support-file.txt create mode 100644 python/mozbuild/mozbuild/test/frontend/data/test-manifest-shared-missing/test_foo.js create mode 100644 python/mozbuild/mozbuild/test/frontend/data/test-manifest-shared-support/child/another-file.sjs create mode 100644 python/mozbuild/mozbuild/test/frontend/data/test-manifest-shared-support/child/browser.ini create mode 100644 python/mozbuild/mozbuild/test/frontend/data/test-manifest-shared-support/child/data/one.txt create mode 100644 python/mozbuild/mozbuild/test/frontend/data/test-manifest-shared-support/child/data/two.txt create mode 100644 python/mozbuild/mozbuild/test/frontend/data/test-manifest-shared-support/child/test_sub.js create mode 100644 python/mozbuild/mozbuild/test/frontend/data/test-manifest-shared-support/mochitest.ini create mode 100644 python/mozbuild/mozbuild/test/frontend/data/test-manifest-shared-support/moz.build create mode 100644 python/mozbuild/mozbuild/test/frontend/data/test-manifest-shared-support/support-file.txt create mode 100644 python/mozbuild/mozbuild/test/frontend/data/test-manifest-shared-support/test_foo.js create mode 100644 python/mozbuild/mozbuild/test/frontend/data/test-manifest-unmatched-generated/moz.build create mode 100644 python/mozbuild/mozbuild/test/frontend/data/test-manifest-unmatched-generated/test.ini create mode 100644 python/mozbuild/mozbuild/test/frontend/data/test-manifest-unmatched-generated/test_foo create mode 100644 python/mozbuild/mozbuild/test/frontend/data/test-python-unit-test-missing/moz.build create mode 100644 python/mozbuild/mozbuild/test/frontend/data/test-symbols-file-objdir-missing-generated/moz.build create mode 100644 python/mozbuild/mozbuild/test/frontend/data/test-symbols-file-objdir/foo.py create mode 100644 python/mozbuild/mozbuild/test/frontend/data/test-symbols-file-objdir/moz.build create mode 100644 python/mozbuild/mozbuild/test/frontend/data/test-symbols-file/foo.symbols create mode 100644 python/mozbuild/mozbuild/test/frontend/data/test-symbols-file/moz.build create mode 100644 python/mozbuild/mozbuild/test/frontend/data/traversal-all-vars/moz.build create mode 100644 python/mozbuild/mozbuild/test/frontend/data/traversal-all-vars/parallel/moz.build create mode 100644 python/mozbuild/mozbuild/test/frontend/data/traversal-all-vars/regular/moz.build create mode 100644 python/mozbuild/mozbuild/test/frontend/data/traversal-all-vars/test/moz.build create mode 100644 python/mozbuild/mozbuild/test/frontend/data/traversal-outside-topsrcdir/moz.build create mode 100644 python/mozbuild/mozbuild/test/frontend/data/traversal-relative-dirs/bar/moz.build create mode 100644 python/mozbuild/mozbuild/test/frontend/data/traversal-relative-dirs/foo/moz.build create mode 100644 python/mozbuild/mozbuild/test/frontend/data/traversal-relative-dirs/moz.build create mode 100644 python/mozbuild/mozbuild/test/frontend/data/traversal-repeated-dirs/bar/moz.build create mode 100644 python/mozbuild/mozbuild/test/frontend/data/traversal-repeated-dirs/foo/moz.build create mode 100644 python/mozbuild/mozbuild/test/frontend/data/traversal-repeated-dirs/moz.build create mode 100644 python/mozbuild/mozbuild/test/frontend/data/traversal-simple/bar/moz.build create mode 100644 python/mozbuild/mozbuild/test/frontend/data/traversal-simple/foo/biz/moz.build create mode 100644 python/mozbuild/mozbuild/test/frontend/data/traversal-simple/foo/moz.build create mode 100644 python/mozbuild/mozbuild/test/frontend/data/traversal-simple/moz.build create mode 100644 python/mozbuild/mozbuild/test/frontend/data/unified-sources-non-unified/bar.cxx create mode 100644 python/mozbuild/mozbuild/test/frontend/data/unified-sources-non-unified/c1.c create mode 100644 python/mozbuild/mozbuild/test/frontend/data/unified-sources-non-unified/c2.c create mode 100644 python/mozbuild/mozbuild/test/frontend/data/unified-sources-non-unified/foo.cpp create mode 100644 python/mozbuild/mozbuild/test/frontend/data/unified-sources-non-unified/moz.build create mode 100644 python/mozbuild/mozbuild/test/frontend/data/unified-sources-non-unified/objc1.mm create mode 100644 python/mozbuild/mozbuild/test/frontend/data/unified-sources-non-unified/objc2.mm create mode 100644 python/mozbuild/mozbuild/test/frontend/data/unified-sources-non-unified/quux.cc create mode 100644 python/mozbuild/mozbuild/test/frontend/data/unified-sources/bar.cxx create mode 100644 python/mozbuild/mozbuild/test/frontend/data/unified-sources/c1.c create mode 100644 python/mozbuild/mozbuild/test/frontend/data/unified-sources/c2.c create mode 100644 python/mozbuild/mozbuild/test/frontend/data/unified-sources/foo.cpp create mode 100644 python/mozbuild/mozbuild/test/frontend/data/unified-sources/moz.build create mode 100644 python/mozbuild/mozbuild/test/frontend/data/unified-sources/objc1.mm create mode 100644 python/mozbuild/mozbuild/test/frontend/data/unified-sources/objc2.mm create mode 100644 python/mozbuild/mozbuild/test/frontend/data/unified-sources/quux.cc create mode 100644 python/mozbuild/mozbuild/test/frontend/data/use-yasm/moz.build create mode 100644 python/mozbuild/mozbuild/test/frontend/data/variable-passthru/bans.S create mode 100644 python/mozbuild/mozbuild/test/frontend/data/variable-passthru/moz.build create mode 100644 python/mozbuild/mozbuild/test/frontend/data/variable-passthru/test1.c create mode 100644 python/mozbuild/mozbuild/test/frontend/data/variable-passthru/test1.cpp create mode 100644 python/mozbuild/mozbuild/test/frontend/data/variable-passthru/test1.mm create mode 100644 python/mozbuild/mozbuild/test/frontend/data/variable-passthru/test2.c create mode 100644 python/mozbuild/mozbuild/test/frontend/data/variable-passthru/test2.cpp create mode 100644 python/mozbuild/mozbuild/test/frontend/data/variable-passthru/test2.mm create mode 100644 python/mozbuild/mozbuild/test/frontend/data/xpidl-module-no-sources/moz.build create mode 100644 python/mozbuild/mozbuild/test/frontend/test_context.py create mode 100644 python/mozbuild/mozbuild/test/frontend/test_emitter.py create mode 100644 python/mozbuild/mozbuild/test/frontend/test_namespaces.py create mode 100644 python/mozbuild/mozbuild/test/frontend/test_reader.py create mode 100644 python/mozbuild/mozbuild/test/frontend/test_sandbox.py create mode 100644 python/mozbuild/mozbuild/test/test_android_version_code.py create mode 100644 python/mozbuild/mozbuild/test/test_base.py create mode 100644 python/mozbuild/mozbuild/test/test_containers.py create mode 100644 python/mozbuild/mozbuild/test/test_dotproperties.py create mode 100644 python/mozbuild/mozbuild/test/test_expression.py create mode 100644 python/mozbuild/mozbuild/test/test_jarmaker.py create mode 100644 python/mozbuild/mozbuild/test/test_line_endings.py create mode 100644 python/mozbuild/mozbuild/test/test_makeutil.py create mode 100644 python/mozbuild/mozbuild/test/test_mozconfig.py create mode 100755 python/mozbuild/mozbuild/test/test_mozinfo.py create mode 100644 python/mozbuild/mozbuild/test/test_preprocessor.py create mode 100644 python/mozbuild/mozbuild/test/test_pythonutil.py create mode 100644 python/mozbuild/mozbuild/test/test_testing.py create mode 100644 python/mozbuild/mozbuild/test/test_util.py create mode 100644 python/mozbuild/mozbuild/testing.py create mode 100644 python/mozbuild/mozbuild/util.py create mode 100644 python/mozbuild/mozbuild/vendor_rust.py create mode 100644 python/mozbuild/mozbuild/virtualenv.py create mode 100644 python/mozbuild/mozpack/__init__.py create mode 100644 python/mozbuild/mozpack/archive.py create mode 100644 python/mozbuild/mozpack/chrome/__init__.py create mode 100644 python/mozbuild/mozpack/chrome/flags.py create mode 100644 python/mozbuild/mozpack/chrome/manifest.py create mode 100644 python/mozbuild/mozpack/copier.py create mode 100644 python/mozbuild/mozpack/dmg.py create mode 100644 python/mozbuild/mozpack/errors.py create mode 100644 python/mozbuild/mozpack/executables.py create mode 100644 python/mozbuild/mozpack/files.py create mode 100644 python/mozbuild/mozpack/hg.py create mode 100644 python/mozbuild/mozpack/manifests.py create mode 100644 python/mozbuild/mozpack/mozjar.py create mode 100644 python/mozbuild/mozpack/packager/__init__.py create mode 100644 python/mozbuild/mozpack/packager/formats.py create mode 100644 python/mozbuild/mozpack/packager/l10n.py create mode 100644 python/mozbuild/mozpack/packager/unpack.py create mode 100644 python/mozbuild/mozpack/path.py create mode 100644 python/mozbuild/mozpack/test/__init__.py create mode 100644 python/mozbuild/mozpack/test/data/test_data create mode 100644 python/mozbuild/mozpack/test/support/minify_js_verify.py create mode 100644 python/mozbuild/mozpack/test/test_archive.py create mode 100644 python/mozbuild/mozpack/test/test_chrome_flags.py create mode 100644 python/mozbuild/mozpack/test/test_chrome_manifest.py create mode 100644 python/mozbuild/mozpack/test/test_copier.py create mode 100644 python/mozbuild/mozpack/test/test_errors.py create mode 100644 python/mozbuild/mozpack/test/test_files.py create mode 100644 python/mozbuild/mozpack/test/test_manifests.py create mode 100644 python/mozbuild/mozpack/test/test_mozjar.py create mode 100644 python/mozbuild/mozpack/test/test_packager.py create mode 100644 python/mozbuild/mozpack/test/test_packager_formats.py create mode 100644 python/mozbuild/mozpack/test/test_packager_l10n.py create mode 100644 python/mozbuild/mozpack/test/test_packager_unpack.py create mode 100644 python/mozbuild/mozpack/test/test_path.py create mode 100644 python/mozbuild/mozpack/test/test_unify.py create mode 100644 python/mozbuild/mozpack/unify.py create mode 100644 python/mozbuild/setup.py (limited to 'python/mozbuild') diff --git a/python/mozbuild/TODO b/python/mozbuild/TODO new file mode 100644 index 000000000..4f519f9dd --- /dev/null +++ b/python/mozbuild/TODO @@ -0,0 +1,3 @@ +dom/imptests Makefile.in's are autogenerated. See +dom/imptests/writeMakefile.py and bug 782651. We will need to update +writeMakefile.py to produce mozbuild files. diff --git a/python/mozbuild/dumbmake/__init__.py b/python/mozbuild/dumbmake/__init__.py new file mode 100644 index 000000000..e69de29bb diff --git a/python/mozbuild/dumbmake/dumbmake.py b/python/mozbuild/dumbmake/dumbmake.py new file mode 100644 index 000000000..5457c8b0a --- /dev/null +++ b/python/mozbuild/dumbmake/dumbmake.py @@ -0,0 +1,122 @@ +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, You can obtain one at http://mozilla.org/MPL/2.0/. + +from __future__ import absolute_import, unicode_literals + +from collections import OrderedDict +from itertools import groupby +from operator import itemgetter +from os.path import dirname + +WHITESPACE_CHARACTERS = ' \t' + +def indentation(line): + """Number of whitespace (tab and space) characters at start of |line|.""" + i = 0 + while i < len(line): + if line[i] not in WHITESPACE_CHARACTERS: + break + i += 1 + return i + +def dependency_map(lines): + """Return a dictionary with keys that are targets and values that + are ordered lists of targets that should also be built. + + This implementation is O(n^2), but lovely and simple! We walk the + targets in the list, and for each target we walk backwards + collecting its dependencies. To make the walking easier, we + reverse the list so that we are always walking forwards. + + """ + pairs = [(indentation(line), line.strip()) for line in lines] + pairs.reverse() + + deps = {} + + for i, (indent, target) in enumerate(pairs): + if not deps.has_key(target): + deps[target] = [] + + for j in range(i+1, len(pairs)): + ind, tar = pairs[j] + if ind < indent: + indent = ind + if tar not in deps[target]: + deps[target].append(tar) + + return deps + +def all_dependencies(*targets, **kwargs): + """Return a list containing all the dependencies of |targets|. + + The relative order of targets is maintained if possible. + + """ + dm = kwargs.pop('dependency_map', None) + if dm is None: + dm = dependency_map(targets) + + all_targets = OrderedDict() # Used as an ordered set. + + for target in targets: + if target in dm: + for dependency in dm[target]: + # Move element back in the ordered set. + if dependency in all_targets: + del all_targets[dependency] + all_targets[dependency] = True + + return all_targets.keys() + +def get_components(path): + """Take a path and return all the components of the path.""" + paths = [path] + while True: + parent = dirname(paths[-1]) + if parent == "": + break + paths.append(parent) + + paths.reverse() + return paths + +def add_extra_dependencies(target_pairs, dependency_map): + """Take a list [(make_dir, make_target)] and expand (make_dir, None) + entries with extra make dependencies from |dependency_map|. + + Returns an iterator of pairs (make_dir, make_target). + + """ + all_targets = OrderedDict() # Used as an ordered set. + make_dirs = OrderedDict() # Used as an ordered set. + + for make_target, group in groupby(target_pairs, itemgetter(1)): + # Return non-simple directory targets untouched. + if make_target is not None: + for pair in group: + # Generate dependencies for all components of a path. + # Given path a/b/c, examine a, a/b, and a/b/c in that order. + paths = get_components(pair[1]) + # For each component of a path, find and add all dependencies + # to the final target list. + for target in paths: + if target not in all_targets: + yield pair[0], target + all_targets[target] = True + continue + + # Add extra dumbmake dependencies to simple directory targets. + for make_dir, _ in group: + if make_dir not in make_dirs: + yield make_dir, None + make_dirs[make_dir] = True + + all_components = [] + for make_dir in make_dirs.iterkeys(): + all_components.extend(get_components(make_dir)) + + for i in all_dependencies(*all_components, dependency_map=dependency_map): + if i not in make_dirs: + yield i, None diff --git a/python/mozbuild/dumbmake/test/__init__.py b/python/mozbuild/dumbmake/test/__init__.py new file mode 100644 index 000000000..e69de29bb diff --git a/python/mozbuild/dumbmake/test/test_dumbmake.py b/python/mozbuild/dumbmake/test/test_dumbmake.py new file mode 100644 index 000000000..1172117aa --- /dev/null +++ b/python/mozbuild/dumbmake/test/test_dumbmake.py @@ -0,0 +1,106 @@ +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this file, +# You can obtain one at http://mozilla.org/MPL/2.0/. +from __future__ import unicode_literals + +import unittest + +from mozunit import ( + main, +) + +from dumbmake.dumbmake import ( + add_extra_dependencies, + all_dependencies, + dependency_map, + indentation, +) + +class TestDumbmake(unittest.TestCase): + def test_indentation(self): + self.assertEqual(indentation(""), 0) + self.assertEqual(indentation("x"), 0) + self.assertEqual(indentation(" x"), 1) + self.assertEqual(indentation("\tx"), 1) + self.assertEqual(indentation(" \tx"), 2) + self.assertEqual(indentation("\t x"), 2) + self.assertEqual(indentation(" x "), 1) + self.assertEqual(indentation("\tx\t"), 1) + self.assertEqual(indentation(" x"), 2) + self.assertEqual(indentation(" x"), 4) + + def test_dependency_map(self): + self.assertEqual(dependency_map([]), {}) + self.assertEqual(dependency_map(["a"]), {"a": []}) + self.assertEqual(dependency_map(["a", "b"]), {"a": [], "b": []}) + self.assertEqual(dependency_map(["a", "b", "c"]), {"a": [], "b": [], "c": []}) + # indentation + self.assertEqual(dependency_map(["a", "\tb", "a", "\tc"]), {"a": [], "b": ["a"], "c": ["a"]}) + self.assertEqual(dependency_map(["a", "\tb", "\t\tc"]), {"a": [], "b": ["a"], "c": ["b", "a"]}) + self.assertEqual(dependency_map(["a", "\tb", "\t\tc", "\td", "\te", "f"]), {"a": [], "b": ["a"], "c": ["b", "a"], "d": ["a"], "e": ["a"], "f": []}) + # irregular indentation + self.assertEqual(dependency_map(["\ta", "b"]), {"a": [], "b": []}) + self.assertEqual(dependency_map(["a", "\t\t\tb", "\t\tc"]), {"a": [], "b": ["a"], "c": ["a"]}) + self.assertEqual(dependency_map(["a", "\t\tb", "\t\t\tc", "\t\td", "\te", "f"]), {"a": [], "b": ["a"], "c": ["b", "a"], "d": ["a"], "e": ["a"], "f": []}) + # repetitions + self.assertEqual(dependency_map(["a", "\tb", "a", "\tb"]), {"a": [], "b": ["a"]}) + self.assertEqual(dependency_map(["a", "\tb", "\t\tc", "b", "\td", "\t\te"]), {"a": [], "b": ["a"], "d": ["b"], "e": ["d", "b"], "c": ["b", "a"]}) + # cycles are okay + self.assertEqual(dependency_map(["a", "\tb", "\t\ta"]), {"a": ["b", "a"], "b": ["a"]}) + + def test_all_dependencies(self): + dm = {"a": [], "b": ["a"], "c": ["b", "a"], "d": ["a"], "e": ["a"], "f": []} + self.assertEqual(all_dependencies("a", dependency_map=dm), []) + self.assertEqual(all_dependencies("b", dependency_map=dm), ["a"]) + self.assertEqual(all_dependencies("c", "a", "b", dependency_map=dm), ["b", "a"]) + self.assertEqual(all_dependencies("d", dependency_map=dm), ["a"]) + self.assertEqual(all_dependencies("d", "f", "c", dependency_map=dm), ["b", "a"]) + self.assertEqual(all_dependencies("a", "b", dependency_map=dm), ["a"]) + self.assertEqual(all_dependencies("b", "b", dependency_map=dm), ["a"]) + + def test_missing_entry(self): + # a depends on b, which is missing + dm = {"a": ["b"]} + self.assertEqual(all_dependencies("a", dependency_map=dm), ["b"]) + self.assertEqual(all_dependencies("a", "b", dependency_map=dm), ["b"]) + self.assertEqual(all_dependencies("b", dependency_map=dm), []) + + def test_two_dependencies(self): + dm = {"a": ["c"], "b": ["c"], "c": []} + # suppose a and b both depend on c. Then we want to build a and b before c... + self.assertEqual(all_dependencies("a", "b", dependency_map=dm), ["c"]) + # ... but relative order is preserved. + self.assertEqual(all_dependencies("b", "a", dependency_map=dm), ["c"]) + + def test_nested_dependencies(self): + # a depends on b depends on c depends on d + dm = {"a": ["b", "c", "d"], "b": ["c", "d"], "c": ["d"]} + self.assertEqual(all_dependencies("b", "a", dependency_map=dm), ["b", "c", "d"]) + self.assertEqual(all_dependencies("c", "a", dependency_map=dm), ["b", "c", "d"]) + + def test_add_extra_dependencies(self): + # a depends on b depends on c depends on d + dm = {"a": ["b", "c", "d"], "b": ["c", "d"], "c": ["d"]} + # Edge cases. + self.assertEqual(list(add_extra_dependencies([], dependency_map=dm)), + []) + self.assertEqual(list(add_extra_dependencies([(None, "package")], dependency_map=dm)), + [(None, "package")]) + # Easy expansion. + self.assertEqual(list(add_extra_dependencies([("b", None)], dependency_map=dm)), + [("b", None), ("c", None), ("d", None)]) + # Expansion with two groups -- each group is handled independently. + self.assertEqual(list(add_extra_dependencies([("b", None), + (None, "package"), + ("c", None)], dependency_map=dm)), + [("b", None), (None, "package"), + ("c", None), ("d", None)]) + # Two groups, no duplicate dependencies in each group. + self.assertEqual(list(add_extra_dependencies([("a", None), ("b", None), + (None, "package"), (None, "install"), + ("c", None), ("d", None)], dependency_map=dm)), + [("a", None), ("b", None), (None, "package"), + (None, "install"), ("c", None), ("d", None)]) + +if __name__ == '__main__': + main() diff --git a/python/mozbuild/mozbuild/__init__.py b/python/mozbuild/mozbuild/__init__.py new file mode 100644 index 000000000..e69de29bb diff --git a/python/mozbuild/mozbuild/action/__init__.py b/python/mozbuild/mozbuild/action/__init__.py new file mode 100644 index 000000000..e69de29bb diff --git a/python/mozbuild/mozbuild/action/buildlist.py b/python/mozbuild/mozbuild/action/buildlist.py new file mode 100644 index 000000000..9d601d69a --- /dev/null +++ b/python/mozbuild/mozbuild/action/buildlist.py @@ -0,0 +1,52 @@ +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, You can obtain one at http://mozilla.org/MPL/2.0/. + +'''A generic script to add entries to a file +if the entry does not already exist. + +Usage: buildlist.py [ ...] +''' +from __future__ import absolute_import, print_function + +import sys +import os + +from mozbuild.util import ( + ensureParentDir, + lock_file, +) + +def addEntriesToListFile(listFile, entries): + """Given a file |listFile| containing one entry per line, + add each entry in |entries| to the file, unless it is already + present.""" + ensureParentDir(listFile) + lock = lock_file(listFile + ".lck") + try: + if os.path.exists(listFile): + f = open(listFile) + existing = set(x.strip() for x in f.readlines()) + f.close() + else: + existing = set() + for e in entries: + if e not in existing: + existing.add(e) + with open(listFile, 'wb') as f: + f.write("\n".join(sorted(existing))+"\n") + finally: + lock = None + + +def main(args): + if len(args) < 2: + print("Usage: buildlist.py [ ...]", + file=sys.stderr) + return 1 + + return addEntriesToListFile(args[0], args[1:]) + + +if __name__ == '__main__': + sys.exit(main(sys.argv[1:])) diff --git a/python/mozbuild/mozbuild/action/cl.py b/python/mozbuild/mozbuild/action/cl.py new file mode 100644 index 000000000..1840d7d85 --- /dev/null +++ b/python/mozbuild/mozbuild/action/cl.py @@ -0,0 +1,124 @@ +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, You can obtain one at http://mozilla.org/MPL/2.0/. + +from __future__ import absolute_import + +import ctypes +import os +import sys + +from mozprocess.processhandler import ProcessHandlerMixin +from mozbuild.makeutil import Makefile + +CL_INCLUDES_PREFIX = os.environ.get("CL_INCLUDES_PREFIX", "Note: including file:") + +GetShortPathName = ctypes.windll.kernel32.GetShortPathNameW +GetLongPathName = ctypes.windll.kernel32.GetLongPathNameW + + +# cl.exe likes to print inconsistent paths in the showIncludes output +# (some lowercased, some not, with different directions of slashes), +# and we need the original file case for make/pymake to be happy. +# As this is slow and needs to be called a lot of times, use a cache +# to speed things up. +_normcase_cache = {} + +def normcase(path): + # Get*PathName want paths with backslashes + path = path.replace('/', os.sep) + dir = os.path.dirname(path) + # name is fortunately always going to have the right case, + # so we can use a cache for the directory part only. + name = os.path.basename(path) + if dir in _normcase_cache: + result = _normcase_cache[dir] + else: + path = ctypes.create_unicode_buffer(dir) + length = GetShortPathName(path, None, 0) + shortpath = ctypes.create_unicode_buffer(length) + GetShortPathName(path, shortpath, length) + length = GetLongPathName(shortpath, None, 0) + if length > len(path): + path = ctypes.create_unicode_buffer(length) + GetLongPathName(shortpath, path, length) + result = _normcase_cache[dir] = path.value + return os.path.join(result, name) + + +def InvokeClWithDependencyGeneration(cmdline): + target = "" + # Figure out what the target is + for arg in cmdline: + if arg.startswith("-Fo"): + target = arg[3:] + break + + if target is None: + print >>sys.stderr, "No target set" + return 1 + + # Assume the source file is the last argument + source = cmdline[-1] + assert not source.startswith('-') + + # The deps target lives here + depstarget = os.path.basename(target) + ".pp" + + cmdline += ['-showIncludes'] + + mk = Makefile() + rule = mk.create_rule([target]) + rule.add_dependencies([normcase(source)]) + + def on_line(line): + # cl -showIncludes prefixes every header with "Note: including file:" + # and an indentation corresponding to the depth (which we don't need) + if line.startswith(CL_INCLUDES_PREFIX): + dep = line[len(CL_INCLUDES_PREFIX):].strip() + # We can't handle pathes with spaces properly in mddepend.pl, but + # we can assume that anything in a path with spaces is a system + # header and throw it away. + dep = normcase(dep) + if ' ' not in dep: + rule.add_dependencies([dep]) + else: + # Make sure we preserve the relevant output from cl. mozprocess + # swallows the newline delimiter, so we need to re-add it. + sys.stdout.write(line) + sys.stdout.write('\n') + + # We need to ignore children because MSVC can fire up a background process + # during compilation. This process is cleaned up on its own. If we kill it, + # we can run into weird compilation issues. + p = ProcessHandlerMixin(cmdline, processOutputLine=[on_line], + ignore_children=True) + p.run() + p.processOutput() + ret = p.wait() + + if ret != 0 or target == "": + # p.wait() returns a long. Somehow sys.exit(long(0)) is like + # sys.exit(1). Don't ask why. + return int(ret) + + depsdir = os.path.normpath(os.path.join(os.curdir, ".deps")) + depstarget = os.path.join(depsdir, depstarget) + if not os.path.isdir(depsdir): + try: + os.makedirs(depsdir) + except OSError: + pass # This suppresses the error we get when the dir exists, at the + # cost of masking failure to create the directory. We'll just + # die on the next line though, so it's not that much of a loss. + + with open(depstarget, "w") as f: + mk.dump(f) + + return 0 + +def main(args): + return InvokeClWithDependencyGeneration(args) + +if __name__ == "__main__": + sys.exit(main(sys.argv[1:])) diff --git a/python/mozbuild/mozbuild/action/dump_env.py b/python/mozbuild/mozbuild/action/dump_env.py new file mode 100644 index 000000000..a6fa19f3a --- /dev/null +++ b/python/mozbuild/mozbuild/action/dump_env.py @@ -0,0 +1,10 @@ +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, You can obtain one at http://mozilla.org/MPL/2.0/. + +# We invoke a Python program to dump our environment in order to get +# native paths printed on Windows so that these paths can be incorporated +# into Python configure's environment. +import os +for key, value in os.environ.items(): + print('%s=%s' % (key, value)) diff --git a/python/mozbuild/mozbuild/action/explode_aar.py b/python/mozbuild/mozbuild/action/explode_aar.py new file mode 100644 index 000000000..fcaf594c1 --- /dev/null +++ b/python/mozbuild/mozbuild/action/explode_aar.py @@ -0,0 +1,72 @@ +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, You can obtain one at http://mozilla.org/MPL/2.0/. + +from __future__ import absolute_import, print_function, unicode_literals + +import argparse +import errno +import os +import shutil +import sys +import zipfile + +from mozpack.files import FileFinder +import mozpack.path as mozpath +from mozbuild.util import ensureParentDir + +def explode(aar, destdir): + # Take just the support-v4-22.2.1 part. + name, _ = os.path.splitext(os.path.basename(aar)) + + destdir = mozpath.join(destdir, name) + if os.path.exists(destdir): + # We always want to start fresh. + shutil.rmtree(destdir) + ensureParentDir(destdir) + with zipfile.ZipFile(aar) as zf: + zf.extractall(destdir) + + # classes.jar is always present. However, multiple JAR files with the same + # name confuses our staged Proguard process in + # mobile/android/base/Makefile.in, so we make the names unique here. + classes_jar = mozpath.join(destdir, name + '-classes.jar') + os.rename(mozpath.join(destdir, 'classes.jar'), classes_jar) + + # Embedded JAR libraries are optional. + finder = FileFinder(mozpath.join(destdir, 'libs'), find_executables=False) + for p, _ in finder.find('*.jar'): + jar = mozpath.join(finder.base, name + '-' + p) + os.rename(mozpath.join(finder.base, p), jar) + + # Frequently assets/ is present but empty. Protect against meaningless + # changes to the AAR files by deleting empty assets/ directories. + assets = mozpath.join(destdir, 'assets') + try: + os.rmdir(assets) + except OSError, e: + if e.errno in (errno.ENOTEMPTY, errno.ENOENT): + pass + else: + raise + + return True + + +def main(argv): + parser = argparse.ArgumentParser( + description='Explode Android AAR file.') + + parser.add_argument('--destdir', required=True, help='Destination directory.') + parser.add_argument('aars', nargs='+', help='Path to AAR file(s).') + + args = parser.parse_args(argv) + + for aar in args.aars: + if not explode(aar, args.destdir): + return 1 + return 0 + + +if __name__ == '__main__': + sys.exit(main(sys.argv[1:])) diff --git a/python/mozbuild/mozbuild/action/file_generate.py b/python/mozbuild/mozbuild/action/file_generate.py new file mode 100644 index 000000000..3bdbc264b --- /dev/null +++ b/python/mozbuild/mozbuild/action/file_generate.py @@ -0,0 +1,108 @@ +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, You can obtain one at http://mozilla.org/MPL/2.0/. + +# Given a Python script and arguments describing the output file, and +# the arguments that can be used to generate the output file, call the +# script's |main| method with appropriate arguments. + +from __future__ import absolute_import, print_function + +import argparse +import imp +import os +import sys +import traceback + +from mozbuild.pythonutil import iter_modules_in_path +from mozbuild.makeutil import Makefile +from mozbuild.util import FileAvoidWrite +import buildconfig + + +def main(argv): + parser = argparse.ArgumentParser('Generate a file from a Python script', + add_help=False) + parser.add_argument('python_script', metavar='python-script', type=str, + help='The Python script to run') + parser.add_argument('method_name', metavar='method-name', type=str, + help='The method of the script to invoke') + parser.add_argument('output_file', metavar='output-file', type=str, + help='The file to generate') + parser.add_argument('dep_file', metavar='dep-file', type=str, + help='File to write any additional make dependencies to') + parser.add_argument('additional_arguments', metavar='arg', + nargs=argparse.REMAINDER, + help="Additional arguments to the script's main() method") + + args = parser.parse_args(argv) + + script = args.python_script + # Permit the script to import modules from the same directory in which it + # resides. The justification for doing this is that if we were invoking + # the script as: + # + # python script arg1... + # + # then importing modules from the script's directory would come for free. + # Since we're invoking the script in a roundabout way, we provide this + # bit of convenience. + sys.path.append(os.path.dirname(script)) + with open(script, 'r') as fh: + module = imp.load_module('script', fh, script, + ('.py', 'r', imp.PY_SOURCE)) + method = args.method_name + if not hasattr(module, method): + print('Error: script "{0}" is missing a {1} method'.format(script, method), + file=sys.stderr) + return 1 + + ret = 1 + try: + with FileAvoidWrite(args.output_file) as output: + ret = module.__dict__[method](output, *args.additional_arguments) + # The following values indicate a statement of success: + # - a set() (see below) + # - 0 + # - False + # - None + # + # Everything else is an error (so scripts can conveniently |return + # 1| or similar). If a set is returned, the elements of the set + # indicate additional dependencies that will be listed in the deps + # file. Python module imports are automatically included as + # dependencies. + if isinstance(ret, set): + deps = ret + # The script succeeded, so reset |ret| to indicate that. + ret = None + else: + deps = set() + + # Only write out the dependencies if the script was successful + if not ret: + # Add dependencies on any python modules that were imported by + # the script. + deps |= set(iter_modules_in_path(buildconfig.topsrcdir, + buildconfig.topobjdir)) + mk = Makefile() + mk.create_rule([args.output_file]).add_dependencies(deps) + with FileAvoidWrite(args.dep_file) as dep_file: + mk.dump(dep_file) + # Even when our file's contents haven't changed, we want to update + # the file's mtime so make knows this target isn't still older than + # whatever prerequisite caused it to be built this time around. + try: + os.utime(args.output_file, None) + except: + print('Error processing file "{0}"'.format(args.output_file), + file=sys.stderr) + traceback.print_exc() + except IOError as e: + print('Error opening file "{0}"'.format(e.filename), file=sys.stderr) + traceback.print_exc() + return 1 + return ret + +if __name__ == '__main__': + sys.exit(main(sys.argv[1:])) diff --git a/python/mozbuild/mozbuild/action/generate_browsersearch.py b/python/mozbuild/mozbuild/action/generate_browsersearch.py new file mode 100644 index 000000000..231abe9be --- /dev/null +++ b/python/mozbuild/mozbuild/action/generate_browsersearch.py @@ -0,0 +1,131 @@ +# -*- Mode: python; indent-tabs-mode: nil; tab-width: 40 -*- +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, You can obtain one at http://mozilla.org/MPL/2.0/. + +''' +Script to generate the browsersearch.json file for Fennec. + +This script follows these steps: + +1. Read the region.properties file in all the given source directories (see +srcdir option). Merge all properties into a single dict accounting for the +priority of source directories. + +2. Read the default search plugin from 'browser.search.defaultenginename'. + +3. Read the list of search plugins from the 'browser.search.order.INDEX' +properties with values identifying particular search plugins by name. + +4. Read each region-specific default search plugin from each property named like +'browser.search.defaultenginename.REGION'. + +5. Read the list of region-specific search plugins from the +'browser.search.order.REGION.INDEX' properties with values identifying +particular search plugins by name. Here, REGION is derived from a REGION for +which we have seen a region-specific default plugin. + +6. Generate a JSON representation of the above information, and write the result +to browsersearch.json in the locale-specific raw resource directory +e.g. raw/browsersearch.json, raw-pt-rBR/browsersearch.json. +''' + +from __future__ import ( + absolute_import, + print_function, + unicode_literals, +) + +import argparse +import codecs +import json +import sys +import os + +from mozbuild.dotproperties import ( + DotProperties, +) +from mozbuild.util import ( + FileAvoidWrite, +) +import mozpack.path as mozpath + + +def merge_properties(filename, srcdirs): + """Merges properties from the given file in the given source directories.""" + properties = DotProperties() + for srcdir in srcdirs: + path = mozpath.join(srcdir, filename) + try: + properties.update(path) + except IOError: + # Ignore non-existing files + continue + return properties + + +def main(args): + parser = argparse.ArgumentParser() + parser.add_argument('--verbose', '-v', default=False, action='store_true', + help='be verbose') + parser.add_argument('--silent', '-s', default=False, action='store_true', + help='be silent') + parser.add_argument('--srcdir', metavar='SRCDIR', + action='append', required=True, + help='directories to read inputs from, in order of priority') + parser.add_argument('output', metavar='OUTPUT', + help='output') + opts = parser.parse_args(args) + + # Use reversed order so that the first srcdir has higher priority to override keys. + properties = merge_properties('region.properties', reversed(opts.srcdir)) + + # Default, not region-specific. + default = properties.get('browser.search.defaultenginename') + engines = properties.get_list('browser.search.order') + + writer = codecs.getwriter('utf-8')(sys.stdout) + if opts.verbose: + print('Read {len} engines: {engines}'.format(len=len(engines), engines=engines), file=writer) + print("Default engine is '{default}'.".format(default=default), file=writer) + + browsersearch = {} + browsersearch['default'] = default + browsersearch['engines'] = engines + + # This gets defaults, yes; but it also gets the list of regions known. + regions = properties.get_dict('browser.search.defaultenginename') + + browsersearch['regions'] = {} + for region in regions.keys(): + region_default = regions[region] + region_engines = properties.get_list('browser.search.order.{region}'.format(region=region)) + + if opts.verbose: + print("Region '{region}': Read {len} engines: {region_engines}".format( + len=len(region_engines), region=region, region_engines=region_engines), file=writer) + print("Region '{region}': Default engine is '{region_default}'.".format( + region=region, region_default=region_default), file=writer) + + browsersearch['regions'][region] = { + 'default': region_default, + 'engines': region_engines, + } + + # FileAvoidWrite creates its parent directories. + output = os.path.abspath(opts.output) + fh = FileAvoidWrite(output) + json.dump(browsersearch, fh) + existed, updated = fh.close() + + if not opts.silent: + if updated: + print('{output} updated'.format(output=output)) + else: + print('{output} already up-to-date'.format(output=output)) + + return 0 + + +if __name__ == '__main__': + sys.exit(main(sys.argv[1:])) diff --git a/python/mozbuild/mozbuild/action/generate_searchjson.py b/python/mozbuild/mozbuild/action/generate_searchjson.py new file mode 100644 index 000000000..765a3550a --- /dev/null +++ b/python/mozbuild/mozbuild/action/generate_searchjson.py @@ -0,0 +1,23 @@ +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, You can obtain one at http://mozilla.org/MPL/2.0/. + +import sys +import json + +engines = [] + +locale = sys.argv[2] +output_file = sys.argv[3] + +output = open(output_file, 'w') + +with open(sys.argv[1]) as f: + searchinfo = json.load(f) + +if locale in searchinfo["locales"]: + output.write(json.dumps(searchinfo["locales"][locale])) +else: + output.write(json.dumps(searchinfo["default"])) + +output.close(); diff --git a/python/mozbuild/mozbuild/action/generate_suggestedsites.py b/python/mozbuild/mozbuild/action/generate_suggestedsites.py new file mode 100644 index 000000000..96d824cc2 --- /dev/null +++ b/python/mozbuild/mozbuild/action/generate_suggestedsites.py @@ -0,0 +1,147 @@ +# -*- Mode: python; indent-tabs-mode: nil; tab-width: 40 -*- +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, You can obtain one at http://mozilla.org/MPL/2.0/. + +''' Script to generate the suggestedsites.json file for Fennec. + +This script follows these steps: + +1. Read the region.properties file in all the given source directories +(see srcdir option). Merge all properties into a single dict accounting for +the priority of source directories. + +2. Read the list of sites from the list 'browser.suggestedsites.list.INDEX' and +'browser.suggestedsites.restricted.list.INDEX' properties with value of these keys +being an identifier for each suggested site e.g. browser.suggestedsites.list.0=mozilla, +browser.suggestedsites.list.1=fxmarketplace. + +3. For each site identifier defined by the list keys, look for matching branches +containing the respective properties i.e. url, title, etc. For example, +for a 'mozilla' identifier, we'll look for keys like: +browser.suggestedsites.mozilla.url, browser.suggestedsites.mozilla.title, etc. + +4. Generate a JSON representation of each site, join them in a JSON array, and +write the result to suggestedsites.json on the locale-specific raw resource +directory e.g. raw/suggestedsites.json, raw-pt-rBR/suggestedsites.json. +''' + +from __future__ import absolute_import, print_function + +import argparse +import copy +import json +import sys +import os + +from mozbuild.dotproperties import ( + DotProperties, +) +from mozbuild.util import ( + FileAvoidWrite, +) +from mozpack.files import ( + FileFinder, +) +import mozpack.path as mozpath + + +def merge_properties(filename, srcdirs): + """Merges properties from the given file in the given source directories.""" + properties = DotProperties() + for srcdir in srcdirs: + path = mozpath.join(srcdir, filename) + try: + properties.update(path) + except IOError: + # Ignore non-existing files + continue + return properties + + +def main(args): + parser = argparse.ArgumentParser() + parser.add_argument('--verbose', '-v', default=False, action='store_true', + help='be verbose') + parser.add_argument('--silent', '-s', default=False, action='store_true', + help='be silent') + parser.add_argument('--android-package-name', metavar='NAME', + required=True, + help='Android package name') + parser.add_argument('--resources', metavar='RESOURCES', + default=None, + help='optional Android resource directory to find drawables in') + parser.add_argument('--srcdir', metavar='SRCDIR', + action='append', required=True, + help='directories to read inputs from, in order of priority') + parser.add_argument('output', metavar='OUTPUT', + help='output') + opts = parser.parse_args(args) + + # Use reversed order so that the first srcdir has higher priority to override keys. + properties = merge_properties('region.properties', reversed(opts.srcdir)) + + # Keep these two in sync. + image_url_template = 'android.resource://%s/drawable/suggestedsites_{name}' % opts.android_package_name + drawables_template = 'drawable*/suggestedsites_{name}.*' + + # Load properties corresponding to each site name and define their + # respective image URL. + sites = [] + + def add_names(names, defaults={}): + for name in names: + site = copy.deepcopy(defaults) + site.update(properties.get_dict('browser.suggestedsites.{name}'.format(name=name), required_keys=('title', 'url', 'bgcolor'))) + site['imageurl'] = image_url_template.format(name=name) + sites.append(site) + + # Now check for existence of an appropriately named drawable. If none + # exists, throw. This stops a locale discovering, at runtime, that the + # corresponding drawable was not added to en-US. + if not opts.resources: + continue + resources = os.path.abspath(opts.resources) + finder = FileFinder(resources) + matches = [p for p, _ in finder.find(drawables_template.format(name=name))] + if not matches: + raise Exception("Could not find drawable in '{resources}' for '{name}'" + .format(resources=resources, name=name)) + else: + if opts.verbose: + print("Found {len} drawables in '{resources}' for '{name}': {matches}" + .format(len=len(matches), resources=resources, name=name, matches=matches)) + + # We want the lists to be ordered for reproducibility. Each list has a + # "default" JSON list item which will be extended by the properties read. + lists = [ + ('browser.suggestedsites.list', {}), + ('browser.suggestedsites.restricted.list', {'restricted': True}), + ] + if opts.verbose: + print('Reading {len} suggested site lists: {lists}'.format(len=len(lists), lists=[list_name for list_name, _ in lists])) + + for (list_name, list_item_defaults) in lists: + names = properties.get_list(list_name) + if opts.verbose: + print('Reading {len} suggested sites from {list}: {names}'.format(len=len(names), list=list_name, names=names)) + add_names(names, list_item_defaults) + + + # FileAvoidWrite creates its parent directories. + output = os.path.abspath(opts.output) + fh = FileAvoidWrite(output) + json.dump(sites, fh) + existed, updated = fh.close() + + if not opts.silent: + if updated: + print('{output} updated'.format(output=output)) + else: + print('{output} already up-to-date'.format(output=output)) + + return 0 + + +if __name__ == '__main__': + sys.exit(main(sys.argv[1:])) diff --git a/python/mozbuild/mozbuild/action/generate_symbols_file.py b/python/mozbuild/mozbuild/action/generate_symbols_file.py new file mode 100644 index 000000000..ff6136bb1 --- /dev/null +++ b/python/mozbuild/mozbuild/action/generate_symbols_file.py @@ -0,0 +1,91 @@ +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, You can obtain one at http://mozilla.org/MPL/2.0/. + +from __future__ import absolute_import, print_function, unicode_literals + +import argparse +import buildconfig +import os +from StringIO import StringIO +from mozbuild.preprocessor import Preprocessor +from mozbuild.util import DefinesAction + + +def generate_symbols_file(output, *args): + ''' ''' + parser = argparse.ArgumentParser() + parser.add_argument('input') + parser.add_argument('-D', action=DefinesAction) + parser.add_argument('-U', action='append', default=[]) + args = parser.parse_args(args) + input = os.path.abspath(args.input) + + pp = Preprocessor() + pp.context.update(buildconfig.defines) + if args.D: + pp.context.update(args.D) + for undefine in args.U: + if undefine in pp.context: + del pp.context[undefine] + # Hack until MOZ_DEBUG_FLAGS are simply part of buildconfig.defines + if buildconfig.substs['MOZ_DEBUG']: + pp.context['DEBUG'] = '1' + # Ensure @DATA@ works as expected (see the Windows section further below) + if buildconfig.substs['OS_TARGET'] == 'WINNT': + pp.context['DATA'] = 'DATA' + else: + pp.context['DATA'] = '' + pp.out = StringIO() + pp.do_filter('substitution') + pp.do_include(input) + + symbols = [s.strip() for s in pp.out.getvalue().splitlines() if s.strip()] + + if buildconfig.substs['OS_TARGET'] == 'WINNT': + # A def file is generated for MSVC link.exe that looks like the + # following: + # LIBRARY library.dll + # EXPORTS + # symbol1 + # symbol2 + # ... + # + # link.exe however requires special markers for data symbols, so in + # that case the symbols look like: + # data_symbol1 DATA + # data_symbol2 DATA + # ... + # + # In the input file, this is just annotated with the following syntax: + # data_symbol1 @DATA@ + # data_symbol2 @DATA@ + # ... + # The DATA variable is "simply" expanded by the preprocessor, to + # nothing on non-Windows, such that we only get the symbol name on + # those platforms, and to DATA on Windows, so that the "DATA" part + # is, in fact, part of the symbol name as far as the symbols variable + # is concerned. + libname, ext = os.path.splitext(os.path.basename(output.name)) + assert ext == '.def' + output.write('LIBRARY %s\nEXPORTS\n %s\n' + % (libname, '\n '.join(symbols))) + elif buildconfig.substs['GCC_USE_GNU_LD']: + # A linker version script is generated for GNU LD that looks like the + # following: + # { + # global: + # symbol1; + # symbol2; + # ... + # local: + # *; + # }; + output.write('{\nglobal:\n %s;\nlocal:\n *;\n};' + % ';\n '.join(symbols)) + elif buildconfig.substs['OS_TARGET'] == 'Darwin': + # A list of symbols is generated for Apple ld that simply lists all + # symbols, with an underscore prefix. + output.write(''.join('_%s\n' % s for s in symbols)) + + return set(pp.includes) diff --git a/python/mozbuild/mozbuild/action/jar_maker.py b/python/mozbuild/mozbuild/action/jar_maker.py new file mode 100644 index 000000000..3e3c3c83e --- /dev/null +++ b/python/mozbuild/mozbuild/action/jar_maker.py @@ -0,0 +1,17 @@ +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, You can obtain one at http://mozilla.org/MPL/2.0/. + +from __future__ import absolute_import + +import sys + +import mozbuild.jar + + +def main(args): + return mozbuild.jar.main(args) + + +if __name__ == '__main__': + sys.exit(main(sys.argv[1:])) diff --git a/python/mozbuild/mozbuild/action/make_dmg.py b/python/mozbuild/mozbuild/action/make_dmg.py new file mode 100644 index 000000000..8d77bf374 --- /dev/null +++ b/python/mozbuild/mozbuild/action/make_dmg.py @@ -0,0 +1,37 @@ +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, You can obtain one at http://mozilla.org/MPL/2.0/. + +from __future__ import print_function + +from mozbuild.base import MozbuildObject +from mozpack import dmg + +import os +import sys + + +def make_dmg(source_directory, output_dmg): + build = MozbuildObject.from_environment() + extra_files = [ + (os.path.join(build.distdir, 'branding', 'dsstore'), '.DS_Store'), + (os.path.join(build.distdir, 'branding', 'background.png'), + '.background/background.png'), + (os.path.join(build.distdir, 'branding', 'disk.icns'), + '.VolumeIcon.icns'), + ] + volume_name = build.substs['MOZ_APP_DISPLAYNAME'] + dmg.create_dmg(source_directory, output_dmg, volume_name, extra_files) + + +def main(args): + if len(args) != 2: + print('Usage: make_dmg.py ', + file=sys.stderr) + return 1 + make_dmg(args[0], args[1]) + return 0 + + +if __name__ == '__main__': + sys.exit(main(sys.argv[1:])) diff --git a/python/mozbuild/mozbuild/action/output_searchplugins_list.py b/python/mozbuild/mozbuild/action/output_searchplugins_list.py new file mode 100644 index 000000000..c20e2c732 --- /dev/null +++ b/python/mozbuild/mozbuild/action/output_searchplugins_list.py @@ -0,0 +1,21 @@ +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, You can obtain one at http://mozilla.org/MPL/2.0/. + +import sys +import json + +engines = [] + +locale = sys.argv[2] + +with open(sys.argv[1]) as f: + searchinfo = json.load(f) + +if locale in searchinfo["locales"]: + for region in searchinfo["locales"][locale]: + engines = list(set(engines)|set(searchinfo["locales"][locale][region]["visibleDefaultEngines"])) +else: + engines = searchinfo["default"]["visibleDefaultEngines"] + +print '\n'.join(engines) diff --git a/python/mozbuild/mozbuild/action/package_fennec_apk.py b/python/mozbuild/mozbuild/action/package_fennec_apk.py new file mode 100644 index 000000000..ecd5a9af3 --- /dev/null +++ b/python/mozbuild/mozbuild/action/package_fennec_apk.py @@ -0,0 +1,150 @@ +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, You can obtain one at http://mozilla.org/MPL/2.0/. + +''' +Script to produce an Android package (.apk) for Fennec. +''' + +from __future__ import absolute_import, print_function + +import argparse +import buildconfig +import os +import subprocess +import sys + +from mozpack.copier import Jarrer +from mozpack.files import ( + DeflatedFile, + File, + FileFinder, +) +from mozpack.mozjar import JarReader +import mozpack.path as mozpath + + +def package_fennec_apk(inputs=[], omni_ja=None, classes_dex=None, + lib_dirs=[], + assets_dirs=[], + features_dirs=[], + root_files=[], + verbose=False): + jarrer = Jarrer(optimize=False) + + # First, take input files. The contents of the later files overwrites the + # content of earlier files. + for input in inputs: + jar = JarReader(input) + for file in jar: + path = file.filename + if jarrer.contains(path): + jarrer.remove(path) + jarrer.add(path, DeflatedFile(file), compress=file.compressed) + + def add(path, file, compress=None): + abspath = os.path.abspath(file.path) + if verbose: + print('Packaging %s from %s' % (path, file.path)) + if not os.path.exists(abspath): + raise ValueError('File %s not found (looked for %s)' % \ + (file.path, abspath)) + if jarrer.contains(path): + jarrer.remove(path) + jarrer.add(path, file, compress=compress) + + for features_dir in features_dirs: + finder = FileFinder(features_dir, find_executables=False) + for p, f in finder.find('**'): + add(mozpath.join('assets', 'features', p), f, False) + + for assets_dir in assets_dirs: + finder = FileFinder(assets_dir, find_executables=False) + for p, f in finder.find('**'): + compress = None # Take default from Jarrer. + if p.endswith('.so'): + # Asset libraries are special. + if f.open().read(5)[1:] == '7zXZ': + print('%s is already compressed' % p) + # We need to store (rather than deflate) compressed libraries + # (even if we don't compress them ourselves). + compress = False + elif buildconfig.substs.get('XZ'): + cmd = [buildconfig.substs.get('XZ'), '-zkf', + mozpath.join(finder.base, p)] + + bcj = None + if buildconfig.substs.get('MOZ_THUMB2'): + bcj = '--armthumb' + elif buildconfig.substs.get('CPU_ARCH') == 'arm': + bcj = '--arm' + elif buildconfig.substs.get('CPU_ARCH') == 'x86': + bcj = '--x86' + + if bcj: + cmd.extend([bcj, '--lzma2']) + print('xz-compressing %s with %s' % (p, ' '.join(cmd))) + subprocess.check_output(cmd) + os.rename(f.path + '.xz', f.path) + compress = False + + add(mozpath.join('assets', p), f, compress=compress) + + for lib_dir in lib_dirs: + finder = FileFinder(lib_dir, find_executables=False) + for p, f in finder.find('**'): + add(mozpath.join('lib', p), f) + + for root_file in root_files: + add(os.path.basename(root_file), File(root_file)) + + if omni_ja: + add(mozpath.join('assets', 'omni.ja'), File(omni_ja), compress=False) + + if classes_dex: + add('classes.dex', File(classes_dex)) + + return jarrer + + +def main(args): + parser = argparse.ArgumentParser() + parser.add_argument('--verbose', '-v', default=False, action='store_true', + help='be verbose') + parser.add_argument('--inputs', nargs='+', + help='Input skeleton AP_ or APK file(s).') + parser.add_argument('-o', '--output', + help='Output APK file.') + parser.add_argument('--omnijar', default=None, + help='Optional omni.ja to pack into APK file.') + parser.add_argument('--classes-dex', default=None, + help='Optional classes.dex to pack into APK file.') + parser.add_argument('--lib-dirs', nargs='*', default=[], + help='Optional lib/ dirs to pack into APK file.') + parser.add_argument('--assets-dirs', nargs='*', default=[], + help='Optional assets/ dirs to pack into APK file.') + parser.add_argument('--features-dirs', nargs='*', default=[], + help='Optional features/ dirs to pack into APK file.') + parser.add_argument('--root-files', nargs='*', default=[], + help='Optional files to pack into APK file root.') + args = parser.parse_args(args) + + if buildconfig.substs.get('OMNIJAR_NAME') != 'assets/omni.ja': + raise ValueError("Don't know how package Fennec APKs when " + " OMNIJAR_NAME is not 'assets/omni.jar'.") + + jarrer = package_fennec_apk(inputs=args.inputs, + omni_ja=args.omnijar, + classes_dex=args.classes_dex, + lib_dirs=args.lib_dirs, + assets_dirs=args.assets_dirs, + features_dirs=args.features_dirs, + root_files=args.root_files, + verbose=args.verbose) + jarrer.copy(args.output) + + return 0 + + +if __name__ == '__main__': + sys.exit(main(sys.argv[1:])) diff --git a/python/mozbuild/mozbuild/action/preprocessor.py b/python/mozbuild/mozbuild/action/preprocessor.py new file mode 100644 index 000000000..e5a4d576b --- /dev/null +++ b/python/mozbuild/mozbuild/action/preprocessor.py @@ -0,0 +1,18 @@ +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, You can obtain one at http://mozilla.org/MPL/2.0/. + +from __future__ import absolute_import + +import sys + +from mozbuild.preprocessor import Preprocessor + + +def main(args): + pp = Preprocessor() + pp.handleCommandLine(args, True) + + +if __name__ == "__main__": + main(sys.argv[1:]) diff --git a/python/mozbuild/mozbuild/action/process_define_files.py b/python/mozbuild/mozbuild/action/process_define_files.py new file mode 100644 index 000000000..f6d0c1695 --- /dev/null +++ b/python/mozbuild/mozbuild/action/process_define_files.py @@ -0,0 +1,94 @@ +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, You can obtain one at http://mozilla.org/MPL/2.0/. + +from __future__ import absolute_import, print_function, unicode_literals + +import argparse +import os +import re +import sys +from buildconfig import topobjdir +from mozbuild.backend.configenvironment import ConfigEnvironment +from mozbuild.util import FileAvoidWrite +import mozpack.path as mozpath + + +def process_define_file(output, input): + '''Creates the given config header. A config header is generated by + taking the corresponding source file and replacing some #define/#undef + occurences: + "#undef NAME" is turned into "#define NAME VALUE" + "#define NAME" is unchanged + "#define NAME ORIGINAL_VALUE" is turned into "#define NAME VALUE" + "#undef UNKNOWN_NAME" is turned into "/* #undef UNKNOWN_NAME */" + Whitespaces are preserved. + + As a special rule, "#undef ALLDEFINES" is turned into "#define NAME + VALUE" for all the defined variables. + ''' + + path = os.path.abspath(input) + + config = ConfigEnvironment.from_config_status( + mozpath.join(topobjdir, 'config.status')) + + if mozpath.basedir(path, + [mozpath.join(config.topsrcdir, 'js/src')]) and \ + not config.substs.get('JS_STANDALONE'): + config = ConfigEnvironment.from_config_status( + mozpath.join(topobjdir, 'js', 'src', 'config.status')) + + with open(path, 'rU') as input: + r = re.compile('^\s*#\s*(?P[a-z]+)(?:\s+(?P\S+)(?:\s+(?P\S+))?)?', re.U) + for l in input: + m = r.match(l) + if m: + cmd = m.group('cmd') + name = m.group('name') + value = m.group('value') + if name: + if name == 'ALLDEFINES': + if cmd == 'define': + raise Exception( + '`#define ALLDEFINES` is not allowed in a ' + 'CONFIGURE_DEFINE_FILE') + defines = '\n'.join(sorted( + '#define %s %s' % (name, val) + for name, val in config.defines.iteritems() + if name not in config.non_global_defines)) + l = l[:m.start('cmd') - 1] \ + + defines + l[m.end('name'):] + elif name in config.defines: + if cmd == 'define' and value: + l = l[:m.start('value')] \ + + str(config.defines[name]) \ + + l[m.end('value'):] + elif cmd == 'undef': + l = l[:m.start('cmd')] \ + + 'define' \ + + l[m.end('cmd'):m.end('name')] \ + + ' ' \ + + str(config.defines[name]) \ + + l[m.end('name'):] + elif cmd == 'undef': + l = '/* ' + l[:m.end('name')] + ' */' + l[m.end('name'):] + + output.write(l) + + return {path, config.source} + + +def main(argv): + parser = argparse.ArgumentParser( + description='Process define files.') + + parser.add_argument('input', help='Input define file.') + + args = parser.parse_args(argv) + + return process_define_file(sys.stdout, args.input) + + +if __name__ == '__main__': + sys.exit(main(sys.argv[1:])) diff --git a/python/mozbuild/mozbuild/action/process_install_manifest.py b/python/mozbuild/mozbuild/action/process_install_manifest.py new file mode 100644 index 000000000..e19fe4eda --- /dev/null +++ b/python/mozbuild/mozbuild/action/process_install_manifest.py @@ -0,0 +1,120 @@ +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, You can obtain one at http://mozilla.org/MPL/2.0/. + +from __future__ import absolute_import, print_function, unicode_literals + +import argparse +import os +import sys +import time + +from mozpack.copier import ( + FileCopier, + FileRegistry, +) +from mozpack.files import ( + BaseFile, + FileFinder, +) +from mozpack.manifests import ( + InstallManifest, + InstallManifestNoSymlinks, +) +from mozbuild.util import DefinesAction + + +COMPLETE = 'Elapsed: {elapsed:.2f}s; From {dest}: Kept {existing} existing; ' \ + 'Added/updated {updated}; ' \ + 'Removed {rm_files} files and {rm_dirs} directories.' + + +def process_manifest(destdir, paths, track=None, + remove_unaccounted=True, + remove_all_directory_symlinks=True, + remove_empty_directories=True, + no_symlinks=False, + defines={}): + + if track: + if os.path.exists(track): + # We use the same format as install manifests for the tracking + # data. + manifest = InstallManifest(path=track) + remove_unaccounted = FileRegistry() + dummy_file = BaseFile() + + finder = FileFinder(destdir, find_executables=False, + find_dotfiles=True) + for dest in manifest._dests: + for p, f in finder.find(dest): + remove_unaccounted.add(p, dummy_file) + + else: + # If tracking is enabled and there is no file, we don't want to + # be removing anything. + remove_unaccounted=False + remove_empty_directories=False + remove_all_directory_symlinks=False + + manifest_cls = InstallManifestNoSymlinks if no_symlinks else InstallManifest + manifest = manifest_cls() + for path in paths: + manifest |= manifest_cls(path=path) + + copier = FileCopier() + manifest.populate_registry(copier, defines_override=defines) + result = copier.copy(destdir, + remove_unaccounted=remove_unaccounted, + remove_all_directory_symlinks=remove_all_directory_symlinks, + remove_empty_directories=remove_empty_directories) + + if track: + manifest.write(path=track) + + return result + + +def main(argv): + parser = argparse.ArgumentParser( + description='Process install manifest files.') + + parser.add_argument('destdir', help='Destination directory.') + parser.add_argument('manifests', nargs='+', help='Path to manifest file(s).') + parser.add_argument('--no-remove', action='store_true', + help='Do not remove unaccounted files from destination.') + parser.add_argument('--no-remove-all-directory-symlinks', action='store_true', + help='Do not remove all directory symlinks from destination.') + parser.add_argument('--no-remove-empty-directories', action='store_true', + help='Do not remove empty directories from destination.') + parser.add_argument('--no-symlinks', action='store_true', + help='Do not install symbolic links. Always copy files') + parser.add_argument('--track', metavar="PATH", + help='Use installed files tracking information from the given path.') + parser.add_argument('-D', action=DefinesAction, + dest='defines', metavar="VAR[=VAL]", + help='Define a variable to override what is specified in the manifest') + + args = parser.parse_args(argv) + + start = time.time() + + result = process_manifest(args.destdir, args.manifests, + track=args.track, remove_unaccounted=not args.no_remove, + remove_all_directory_symlinks=not args.no_remove_all_directory_symlinks, + remove_empty_directories=not args.no_remove_empty_directories, + no_symlinks=args.no_symlinks, + defines=args.defines) + + elapsed = time.time() - start + + print(COMPLETE.format( + elapsed=elapsed, + dest=args.destdir, + existing=result.existing_files_count, + updated=result.updated_files_count, + rm_files=result.removed_files_count, + rm_dirs=result.removed_directories_count)) + +if __name__ == '__main__': + main(sys.argv[1:]) diff --git a/python/mozbuild/mozbuild/action/test_archive.py b/python/mozbuild/mozbuild/action/test_archive.py new file mode 100644 index 000000000..8ec4dd2a9 --- /dev/null +++ b/python/mozbuild/mozbuild/action/test_archive.py @@ -0,0 +1,565 @@ +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, You can obtain one at http://mozilla.org/MPL/2.0/. + +# This action is used to produce test archives. +# +# Ideally, the data in this file should be defined in moz.build files. +# It is defined inline because this was easiest to make test archive +# generation faster. + +from __future__ import absolute_import, print_function, unicode_literals + +import argparse +import itertools +import os +import sys +import time + +from manifestparser import TestManifest +from reftest import ReftestManifest + +from mozbuild.util import ensureParentDir +from mozpack.files import FileFinder +from mozpack.mozjar import JarWriter +import mozpack.path as mozpath + +import buildconfig + +STAGE = mozpath.join(buildconfig.topobjdir, 'dist', 'test-stage') + +TEST_HARNESS_BINS = [ + 'BadCertServer', + 'GenerateOCSPResponse', + 'OCSPStaplingServer', + 'SmokeDMD', + 'certutil', + 'crashinject', + 'fileid', + 'minidumpwriter', + 'pk12util', + 'screenshot', + 'screentopng', + 'ssltunnel', + 'xpcshell', +] + +# The fileid utility depends on mozglue. See bug 1069556. +TEST_HARNESS_DLLS = [ + 'crashinjectdll', + 'mozglue' +] + +TEST_PLUGIN_DLLS = [ + 'npctrltest', + 'npsecondtest', + 'npswftest', + 'nptest', + 'nptestjava', + 'npthirdtest', +] + +TEST_PLUGIN_DIRS = [ + 'JavaTest.plugin/**', + 'SecondTest.plugin/**', + 'Test.plugin/**', + 'ThirdTest.plugin/**', + 'npctrltest.plugin/**', + 'npswftest.plugin/**', +] + +GMP_TEST_PLUGIN_DIRS = [ + 'gmp-clearkey/**', + 'gmp-fake/**', + 'gmp-fakeopenh264/**', +] + + +ARCHIVE_FILES = { + 'common': [ + { + 'source': STAGE, + 'base': '', + 'pattern': '**', + 'ignore': [ + 'cppunittest/**', + 'gtest/**', + 'mochitest/**', + 'reftest/**', + 'talos/**', + 'web-platform/**', + 'xpcshell/**', + ], + }, + { + 'source': buildconfig.topobjdir, + 'base': '_tests', + 'pattern': 'modules/**', + }, + { + 'source': buildconfig.topsrcdir, + 'base': 'testing/marionette', + 'patterns': [ + 'client/**', + 'harness/**', + 'puppeteer/**', + 'mach_test_package_commands.py', + ], + 'dest': 'marionette', + 'ignore': [ + 'client/docs', + 'harness/marionette_harness/tests', + 'puppeteer/firefox/docs', + ], + }, + { + 'source': buildconfig.topsrcdir, + 'base': '', + 'manifests': [ + 'testing/marionette/harness/marionette_harness/tests/unit-tests.ini', + 'testing/marionette/harness/marionette_harness/tests/webapi-tests.ini', + ], + # We also need the manifests and harness_unit tests + 'pattern': 'testing/marionette/harness/marionette_harness/tests/**', + 'dest': 'marionette/tests', + }, + { + 'source': buildconfig.topobjdir, + 'base': '_tests', + 'pattern': 'mozbase/**', + }, + { + 'source': buildconfig.topsrcdir, + 'base': 'testing', + 'pattern': 'firefox-ui/**', + }, + { + 'source': buildconfig.topsrcdir, + 'base': 'dom/media/test/external', + 'pattern': '**', + 'dest': 'external-media-tests', + }, + { + 'source': buildconfig.topsrcdir, + 'base': 'js/src', + 'pattern': 'jit-test/**', + 'dest': 'jit-test', + }, + { + 'source': buildconfig.topsrcdir, + 'base': 'js/src/tests', + 'pattern': 'ecma_6/**', + 'dest': 'jit-test/tests', + }, + { + 'source': buildconfig.topsrcdir, + 'base': 'js/src/tests', + 'pattern': 'js1_8_5/**', + 'dest': 'jit-test/tests', + }, + { + 'source': buildconfig.topsrcdir, + 'base': 'js/src/tests', + 'pattern': 'lib/**', + 'dest': 'jit-test/tests', + }, + { + 'source': buildconfig.topsrcdir, + 'base': 'js/src', + 'pattern': 'jsapi.h', + 'dest': 'jit-test', + }, + { + 'source': buildconfig.topsrcdir, + 'base': 'testing', + 'pattern': 'tps/**', + }, + { + 'source': buildconfig.topsrcdir, + 'base': 'services/sync/', + 'pattern': 'tps/**', + }, + { + 'source': buildconfig.topsrcdir, + 'base': 'services/sync/tests/tps', + 'pattern': '**', + 'dest': 'tps/tests', + }, + { + 'source': buildconfig.topsrcdir, + 'base': 'testing/web-platform/tests/tools/wptserve', + 'pattern': '**', + 'dest': 'tools/wptserve', + }, + { + 'source': buildconfig.topobjdir, + 'base': '', + 'pattern': 'mozinfo.json', + }, + { + 'source': buildconfig.topobjdir, + 'base': 'dist/bin', + 'patterns': [ + '%s%s' % (f, buildconfig.substs['BIN_SUFFIX']) + for f in TEST_HARNESS_BINS + ] + [ + '%s%s%s' % (buildconfig.substs['DLL_PREFIX'], f, buildconfig.substs['DLL_SUFFIX']) + for f in TEST_HARNESS_DLLS + ], + 'dest': 'bin', + }, + { + 'source': buildconfig.topobjdir, + 'base': 'dist/plugins', + 'patterns': [ + '%s%s%s' % (buildconfig.substs['DLL_PREFIX'], f, buildconfig.substs['DLL_SUFFIX']) + for f in TEST_PLUGIN_DLLS + ], + 'dest': 'bin/plugins', + }, + { + 'source': buildconfig.topobjdir, + 'base': 'dist/plugins', + 'patterns': TEST_PLUGIN_DIRS, + 'dest': 'bin/plugins', + }, + { + 'source': buildconfig.topobjdir, + 'base': 'dist/bin', + 'patterns': GMP_TEST_PLUGIN_DIRS, + 'dest': 'bin/plugins', + }, + { + 'source': buildconfig.topobjdir, + 'base': 'dist/bin', + 'patterns': [ + 'dmd.py', + 'fix_linux_stack.py', + 'fix_macosx_stack.py', + 'fix_stack_using_bpsyms.py', + ], + 'dest': 'bin', + }, + { + 'source': buildconfig.topobjdir, + 'base': 'dist/bin/components', + 'patterns': [ + 'httpd.js', + 'httpd.manifest', + 'test_necko.xpt', + ], + 'dest': 'bin/components', + }, + { + 'source': buildconfig.topsrcdir, + 'base': 'build/pgo/certs', + 'pattern': '**', + 'dest': 'certs', + } + ], + 'cppunittest': [ + { + 'source': STAGE, + 'base': '', + 'pattern': 'cppunittest/**', + }, + # We don't ship these files if startup cache is disabled, which is + # rare. But it shouldn't matter for test archives. + { + 'source': buildconfig.topsrcdir, + 'base': 'startupcache/test', + 'pattern': 'TestStartupCacheTelemetry.*', + 'dest': 'cppunittest', + }, + { + 'source': buildconfig.topsrcdir, + 'base': 'testing', + 'pattern': 'runcppunittests.py', + 'dest': 'cppunittest', + }, + { + 'source': buildconfig.topsrcdir, + 'base': 'testing', + 'pattern': 'remotecppunittests.py', + 'dest': 'cppunittest', + }, + { + 'source': buildconfig.topsrcdir, + 'base': 'testing', + 'pattern': 'cppunittest.ini', + 'dest': 'cppunittest', + }, + { + 'source': buildconfig.topobjdir, + 'base': '', + 'pattern': 'mozinfo.json', + 'dest': 'cppunittest', + }, + ], + 'gtest': [ + { + 'source': STAGE, + 'base': '', + 'pattern': 'gtest/**', + }, + ], + 'mochitest': [ + { + 'source': buildconfig.topobjdir, + 'base': '_tests/testing', + 'pattern': 'mochitest/**', + }, + { + 'source': STAGE, + 'base': '', + 'pattern': 'mochitest/**', + }, + { + 'source': buildconfig.topobjdir, + 'base': '', + 'pattern': 'mozinfo.json', + 'dest': 'mochitest' + } + ], + 'mozharness': [ + { + 'source': buildconfig.topsrcdir, + 'base': 'testing', + 'pattern': 'mozharness/**', + }, + ], + 'reftest': [ + { + 'source': buildconfig.topobjdir, + 'base': '_tests', + 'pattern': 'reftest/**', + }, + { + 'source': buildconfig.topobjdir, + 'base': '', + 'pattern': 'mozinfo.json', + 'dest': 'reftest', + }, + { + 'source': buildconfig.topsrcdir, + 'base': '', + 'manifests': [ + 'layout/reftests/reftest.list', + 'testing/crashtest/crashtests.list', + ], + 'dest': 'reftest/tests', + } + ], + 'talos': [ + { + 'source': buildconfig.topsrcdir, + 'base': 'testing', + 'pattern': 'talos/**', + }, + ], + 'web-platform': [ + { + 'source': buildconfig.topsrcdir, + 'base': 'testing', + 'pattern': 'web-platform/meta/**', + }, + { + 'source': buildconfig.topsrcdir, + 'base': 'testing', + 'pattern': 'web-platform/mozilla/**', + }, + { + 'source': buildconfig.topsrcdir, + 'base': 'testing', + 'pattern': 'web-platform/tests/**', + }, + { + 'source': buildconfig.topobjdir, + 'base': '_tests', + 'pattern': 'web-platform/**', + }, + { + 'source': buildconfig.topobjdir, + 'base': '', + 'pattern': 'mozinfo.json', + 'dest': 'web-platform', + }, + ], + 'xpcshell': [ + { + 'source': buildconfig.topobjdir, + 'base': '_tests/xpcshell', + 'pattern': '**', + 'dest': 'xpcshell/tests', + }, + { + 'source': buildconfig.topsrcdir, + 'base': 'testing/xpcshell', + 'patterns': [ + 'head.js', + 'mach_test_package_commands.py', + 'moz-http2/**', + 'moz-spdy/**', + 'node-http2/**', + 'node-spdy/**', + 'remotexpcshelltests.py', + 'runtestsb2g.py', + 'runxpcshelltests.py', + 'xpcshellcommandline.py', + ], + 'dest': 'xpcshell', + }, + { + 'source': STAGE, + 'base': '', + 'pattern': 'xpcshell/**', + }, + { + 'source': buildconfig.topobjdir, + 'base': '', + 'pattern': 'mozinfo.json', + 'dest': 'xpcshell', + }, + { + 'source': buildconfig.topobjdir, + 'base': 'build', + 'pattern': 'automation.py', + 'dest': 'xpcshell', + }, + ], +} + + +# "common" is our catch all archive and it ignores things from other archives. +# Verify nothing sneaks into ARCHIVE_FILES without a corresponding exclusion +# rule in the "common" archive. +for k, v in ARCHIVE_FILES.items(): + # Skip mozharness because it isn't staged. + if k in ('common', 'mozharness'): + continue + + ignores = set(itertools.chain(*(e.get('ignore', []) + for e in ARCHIVE_FILES['common']))) + + if not any(p.startswith('%s/' % k) for p in ignores): + raise Exception('"common" ignore list probably should contain %s' % k) + + +def find_files(archive): + for entry in ARCHIVE_FILES[archive]: + source = entry['source'] + dest = entry.get('dest') + base = entry.get('base', '') + + pattern = entry.get('pattern') + patterns = entry.get('patterns', []) + if pattern: + patterns.append(pattern) + + manifest = entry.get('manifest') + manifests = entry.get('manifests', []) + if manifest: + manifests.append(manifest) + if manifests: + dirs = find_manifest_dirs(buildconfig.topsrcdir, manifests) + patterns.extend({'{}/**'.format(d) for d in dirs}) + + ignore = list(entry.get('ignore', [])) + ignore.extend([ + '**/.flake8', + '**/.mkdir.done', + '**/*.pyc', + ]) + + common_kwargs = { + 'find_executables': False, + 'find_dotfiles': True, + 'ignore': ignore, + } + + finder = FileFinder(os.path.join(source, base), **common_kwargs) + + for pattern in patterns: + for p, f in finder.find(pattern): + if dest: + p = mozpath.join(dest, p) + yield p, f + + +def find_manifest_dirs(topsrcdir, manifests): + """Routine to retrieve directories specified in a manifest, relative to topsrcdir. + + It does not recurse into manifests, as we currently have no need for that. + """ + dirs = set() + + for p in manifests: + p = os.path.join(topsrcdir, p) + + if p.endswith('.ini'): + test_manifest = TestManifest() + test_manifest.read(p) + dirs |= set([os.path.dirname(m) for m in test_manifest.manifests()]) + + elif p.endswith('.list'): + m = ReftestManifest() + m.load(p) + dirs |= m.dirs + + else: + raise Exception('"{}" is not a supported manifest format.'.format( + os.path.splitext(p)[1])) + + dirs = {mozpath.normpath(d[len(topsrcdir):]).lstrip('/') for d in dirs} + + # Filter out children captured by parent directories because duplicates + # will confuse things later on. + def parents(p): + while True: + p = mozpath.dirname(p) + if not p: + break + yield p + + seen = set() + for d in sorted(dirs, key=len): + if not any(p in seen for p in parents(d)): + seen.add(d) + + return sorted(seen) + + +def main(argv): + parser = argparse.ArgumentParser( + description='Produce test archives') + parser.add_argument('archive', help='Which archive to generate') + parser.add_argument('outputfile', help='File to write output to') + + args = parser.parse_args(argv) + + if not args.outputfile.endswith('.zip'): + raise Exception('expected zip output file') + + file_count = 0 + t_start = time.time() + ensureParentDir(args.outputfile) + with open(args.outputfile, 'wb') as fh: + # Experimentation revealed that level 5 is significantly faster and has + # marginally larger sizes than higher values and is the sweet spot + # for optimal compression. Read the detailed commit message that + # introduced this for raw numbers. + with JarWriter(fileobj=fh, optimize=False, compress_level=5) as writer: + res = find_files(args.archive) + for p, f in res: + writer.add(p.encode('utf-8'), f.read(), mode=f.mode, skip_duplicates=True) + file_count += 1 + + duration = time.time() - t_start + zip_size = os.path.getsize(args.outputfile) + basename = os.path.basename(args.outputfile) + print('Wrote %d files in %d bytes to %s in %.2fs' % ( + file_count, zip_size, basename, duration)) + + +if __name__ == '__main__': + sys.exit(main(sys.argv[1:])) diff --git a/python/mozbuild/mozbuild/action/webidl.py b/python/mozbuild/mozbuild/action/webidl.py new file mode 100644 index 000000000..d595c728e --- /dev/null +++ b/python/mozbuild/mozbuild/action/webidl.py @@ -0,0 +1,19 @@ +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, You can obtain one at http://mozilla.org/MPL/2.0/. + +from __future__ import absolute_import + +import sys + +from mozwebidlcodegen import BuildSystemWebIDL + + +def main(argv): + """Perform WebIDL code generation required by the build system.""" + manager = BuildSystemWebIDL.from_environment().manager + manager.generate_build_files() + + +if __name__ == '__main__': + sys.exit(main(sys.argv[1:])) diff --git a/python/mozbuild/mozbuild/action/xpccheck.py b/python/mozbuild/mozbuild/action/xpccheck.py new file mode 100644 index 000000000..c3170a8da --- /dev/null +++ b/python/mozbuild/mozbuild/action/xpccheck.py @@ -0,0 +1,83 @@ +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, You can obtain one at http://mozilla.org/MPL/2.0/. + +'''A generic script to verify all test files are in the +corresponding .ini file. + +Usage: xpccheck.py [ ...] +''' + +from __future__ import absolute_import + +import sys +import os +from glob import glob +import manifestparser + +def getIniTests(testdir): + mp = manifestparser.ManifestParser(strict=False) + mp.read(os.path.join(testdir, 'xpcshell.ini')) + return mp.tests + +def verifyDirectory(initests, directory): + files = glob(os.path.join(os.path.abspath(directory), "test_*")) + for f in files: + if (not os.path.isfile(f)): + continue + + name = os.path.basename(f) + if name.endswith('.in'): + name = name[:-3] + + if not name.endswith('.js'): + continue + + found = False + for test in initests: + if os.path.join(os.path.abspath(directory), name) == test['path']: + found = True + break + + if not found: + print >>sys.stderr, "TEST-UNEXPECTED-FAIL | xpccheck | test %s is missing from test manifest %s!" % (name, os.path.join(directory, 'xpcshell.ini')) + sys.exit(1) + +def verifyIniFile(initests, directory): + files = glob(os.path.join(os.path.abspath(directory), "test_*")) + for test in initests: + name = test['path'].split('/')[-1] + + found = False + for f in files: + + fname = f.split('/')[-1] + if fname.endswith('.in'): + fname = '.in'.join(fname.split('.in')[:-1]) + + if os.path.join(os.path.abspath(directory), fname) == test['path']: + found = True + break + + if not found: + print >>sys.stderr, "TEST-UNEXPECTED-FAIL | xpccheck | found %s in xpcshell.ini and not in directory '%s'" % (name, directory) + sys.exit(1) + +def main(argv): + if len(argv) < 2: + print >>sys.stderr, "Usage: xpccheck.py [ ...]" + sys.exit(1) + + topsrcdir = argv[0] + for d in argv[1:]: + # xpcshell-unpack is a copy of xpcshell sibling directory and in the Makefile + # we copy all files (including xpcshell.ini from the sibling directory. + if d.endswith('toolkit/mozapps/extensions/test/xpcshell-unpack'): + continue + + initests = getIniTests(d) + verifyDirectory(initests, d) + verifyIniFile(initests, d) + +if __name__ == '__main__': + main(sys.argv[1:]) diff --git a/python/mozbuild/mozbuild/action/xpidl-process.py b/python/mozbuild/mozbuild/action/xpidl-process.py new file mode 100755 index 000000000..07ea3cf96 --- /dev/null +++ b/python/mozbuild/mozbuild/action/xpidl-process.py @@ -0,0 +1,94 @@ +#!/usr/bin/env python +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, You can obtain one at http://mozilla.org/MPL/2.0/. + +# This script is used to generate an output header and xpt file for +# input IDL file(s). It's purpose is to directly support the build +# system. The API will change to meet the needs of the build system. + +from __future__ import absolute_import + +import argparse +import os +import sys + +from io import BytesIO + +from buildconfig import topsrcdir +from xpidl.header import print_header +from xpidl.typelib import write_typelib +from xpidl.xpidl import IDLParser +from xpt import xpt_link + +from mozbuild.makeutil import Makefile +from mozbuild.pythonutil import iter_modules_in_path +from mozbuild.util import FileAvoidWrite + + +def process(input_dir, inc_paths, cache_dir, header_dir, xpt_dir, deps_dir, module, stems): + p = IDLParser(outputdir=cache_dir) + + xpts = {} + mk = Makefile() + rule = mk.create_rule() + + # Write out dependencies for Python modules we import. If this list isn't + # up to date, we will not re-process XPIDL files if the processor changes. + rule.add_dependencies(iter_modules_in_path(topsrcdir)) + + for stem in stems: + path = os.path.join(input_dir, '%s.idl' % stem) + idl_data = open(path).read() + + idl = p.parse(idl_data, filename=path) + idl.resolve([input_dir] + inc_paths, p) + + header_path = os.path.join(header_dir, '%s.h' % stem) + + xpt = BytesIO() + write_typelib(idl, xpt, path) + xpt.seek(0) + xpts[stem] = xpt + + rule.add_dependencies(idl.deps) + + with FileAvoidWrite(header_path) as fh: + print_header(idl, fh, path) + + # TODO use FileAvoidWrite once it supports binary mode. + xpt_path = os.path.join(xpt_dir, '%s.xpt' % module) + xpt_link(xpts.values()).write(xpt_path) + + rule.add_targets([xpt_path]) + if deps_dir: + deps_path = os.path.join(deps_dir, '%s.pp' % module) + with FileAvoidWrite(deps_path) as fh: + mk.dump(fh) + + +def main(argv): + parser = argparse.ArgumentParser() + parser.add_argument('--cache-dir', + help='Directory in which to find or write cached lexer data.') + parser.add_argument('--depsdir', + help='Directory in which to write dependency files.') + parser.add_argument('inputdir', + help='Directory in which to find source .idl files.') + parser.add_argument('headerdir', + help='Directory in which to write header files.') + parser.add_argument('xptdir', + help='Directory in which to write xpt file.') + parser.add_argument('module', + help='Final module name to use for linked output xpt file.') + parser.add_argument('idls', nargs='+', + help='Source .idl file(s). Specified as stems only.') + parser.add_argument('-I', dest='incpath', action='append', default=[], + help='Extra directories where to look for included .idl files.') + + args = parser.parse_args(argv) + process(args.inputdir, args.incpath, args.cache_dir, args.headerdir, + args.xptdir, args.depsdir, args.module, args.idls) + +if __name__ == '__main__': + main(sys.argv[1:]) diff --git a/python/mozbuild/mozbuild/action/zip.py b/python/mozbuild/mozbuild/action/zip.py new file mode 100644 index 000000000..143d7766e --- /dev/null +++ b/python/mozbuild/mozbuild/action/zip.py @@ -0,0 +1,39 @@ +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, You can obtain one at http://mozilla.org/MPL/2.0/. + +# This script creates a zip file, but will also strip any binaries +# it finds before adding them to the zip. + +from __future__ import absolute_import + +from mozpack.files import FileFinder +from mozpack.copier import Jarrer +from mozpack.errors import errors + +import argparse +import mozpack.path as mozpath +import sys + +def main(args): + parser = argparse.ArgumentParser() + parser.add_argument("-C", metavar='DIR', default=".", + help="Change to given directory before considering " + "other paths") + parser.add_argument("zip", help="Path to zip file to write") + parser.add_argument("input", nargs="+", + help="Path to files to add to zip") + args = parser.parse_args(args) + + jarrer = Jarrer(optimize=False) + + with errors.accumulate(): + finder = FileFinder(args.C) + for path in args.input: + for p, f in finder.find(path): + jarrer.add(p, f) + jarrer.copy(mozpath.join(args.C, args.zip)) + + +if __name__ == '__main__': + main(sys.argv[1:]) diff --git a/python/mozbuild/mozbuild/android_version_code.py b/python/mozbuild/mozbuild/android_version_code.py new file mode 100644 index 000000000..69ce22b8e --- /dev/null +++ b/python/mozbuild/mozbuild/android_version_code.py @@ -0,0 +1,167 @@ +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, You can obtain one at http://mozilla.org/MPL/2.0/. + +from __future__ import absolute_import, print_function + +import argparse +import math +import sys +import time + +# Builds before this build ID use the v0 version scheme. Builds after this +# build ID use the v1 version scheme. +V1_CUTOFF = 20150801000000 # YYYYmmddHHMMSS + +def android_version_code_v0(buildid, cpu_arch=None, min_sdk=0, max_sdk=0): + base = int(str(buildid)[:10]) + # None is interpreted as arm. + if not cpu_arch or cpu_arch in ['armeabi', 'armeabi-v7a']: + # Increment by MIN_SDK_VERSION -- this adds 9 to every build ID as a + # minimum. Our split APK starts at 15. + return base + min_sdk + 0 + elif cpu_arch in ['x86']: + # Increment the version code by 3 for x86 builds so they are offered to + # x86 phones that have ARM emulators, beating the 2-point advantage that + # the v15+ ARMv7 APK has. If we change our splits in the future, we'll + # need to do this further still. + return base + min_sdk + 3 + else: + raise ValueError("Don't know how to compute android:versionCode " + "for CPU arch %s" % cpu_arch) + +def android_version_code_v1(buildid, cpu_arch=None, min_sdk=0, max_sdk=0): + '''Generate a v1 android:versionCode. + + The important consideration is that version codes be monotonically + increasing (per Android package name) for all published builds. The input + build IDs are based on timestamps and hence are always monotonically + increasing. + + The generated v1 version codes look like (in binary): + + 0111 1000 0010 tttt tttt tttt tttt txpg + + The 17 bits labelled 't' represent the number of hours since midnight on + September 1, 2015. (2015090100 in YYYYMMMDDHH format.) This yields a + little under 15 years worth of hourly build identifiers, since 2**17 / (366 + * 24) =~ 14.92. + + The bits labelled 'x', 'p', and 'g' are feature flags. + + The bit labelled 'x' is 1 if the build is for an x86 architecture and 0 + otherwise, which means the build is for an ARM architecture. (Fennec no + longer supports ARMv6, so ARM is equivalent to ARMv7 and above.) + + The bit labelled 'p' is a placeholder that is always 0 (for now). + + Firefox no longer supports API 14 or earlier. + + This version code computation allows for a split on API levels that allowed + us to ship builds specifically for Gingerbread (API 9-10); we preserve + that functionality for sanity's sake, and to allow us to reintroduce a + split in the future. + + At present, the bit labelled 'g' is 1 if the build is an ARM build + targeting API 15+, which will always be the case. + + We throw an explanatory exception when we are within one calendar year of + running out of build events. This gives lots of time to update the version + scheme. The responsible individual should then bump the range (to allow + builds to continue) and use the time remaining to update the version scheme + via the reserved high order bits. + + N.B.: the reserved 0 bit to the left of the highest order 't' bit can, + sometimes, be used to bump the version scheme. In addition, by reducing the + granularity of the build identifiers (for example, moving to identifying + builds every 2 or 4 hours), the version scheme may be adjusted further still + without losing a (valuable) high order bit. + ''' + def hours_since_cutoff(buildid): + # The ID is formatted like YYYYMMDDHHMMSS (using + # datetime.now().strftime('%Y%m%d%H%M%S'); see build/variables.py). + # The inverse function is time.strptime. + # N.B.: the time module expresses time as decimal seconds since the + # epoch. + fmt = '%Y%m%d%H%M%S' + build = time.strptime(str(buildid), fmt) + cutoff = time.strptime(str(V1_CUTOFF), fmt) + return int(math.floor((time.mktime(build) - time.mktime(cutoff)) / (60.0 * 60.0))) + + # Of the 21 low order bits, we take 17 bits for builds. + base = hours_since_cutoff(buildid) + if base < 0: + raise ValueError("Something has gone horribly wrong: cannot calculate " + "android:versionCode from build ID %s: hours underflow " + "bits allotted!" % buildid) + if base > 2**17: + raise ValueError("Something has gone horribly wrong: cannot calculate " + "android:versionCode from build ID %s: hours overflow " + "bits allotted!" % buildid) + if base > 2**17 - 366 * 24: + raise ValueError("Running out of low order bits calculating " + "android:versionCode from build ID %s: " + "; YOU HAVE ONE YEAR TO UPDATE THE VERSION SCHEME." % buildid) + + version = 0b1111000001000000000000000000000 + # We reserve 1 "middle" high order bit for the future, and 3 low order bits + # for architecture and APK splits. + version |= base << 3 + + # None is interpreted as arm. + if not cpu_arch or cpu_arch in ['armeabi', 'armeabi-v7a']: + # 0 is interpreted as SDK 9. + if not min_sdk or min_sdk == 9: + pass + # This used to compare to 11. The 15+ APK directly supersedes 11+, so + # we reuse this check. + elif min_sdk == 15: + version |= 1 << 0 + else: + raise ValueError("Don't know how to compute android:versionCode " + "for CPU arch %s and min SDK %s" % (cpu_arch, min_sdk)) + elif cpu_arch in ['x86']: + version |= 1 << 2 + else: + raise ValueError("Don't know how to compute android:versionCode " + "for CPU arch %s" % cpu_arch) + + return version + +def android_version_code(buildid, *args, **kwargs): + base = int(str(buildid)) + if base < V1_CUTOFF: + return android_version_code_v0(buildid, *args, **kwargs) + else: + return android_version_code_v1(buildid, *args, **kwargs) + + +def main(argv): + parser = argparse.ArgumentParser('Generate an android:versionCode', + add_help=False) + parser.add_argument('--verbose', action='store_true', + default=False, + help='Be verbose') + parser.add_argument('--with-android-cpu-arch', dest='cpu_arch', + choices=['armeabi', 'armeabi-v7a', 'mips', 'x86'], + help='The target CPU architecture') + parser.add_argument('--with-android-min-sdk-version', dest='min_sdk', + type=int, default=0, + help='The minimum target SDK') + parser.add_argument('--with-android-max-sdk-version', dest='max_sdk', + type=int, default=0, + help='The maximum target SDK') + parser.add_argument('buildid', type=int, + help='The input build ID') + + args = parser.parse_args(argv) + code = android_version_code(args.buildid, + cpu_arch=args.cpu_arch, + min_sdk=args.min_sdk, + max_sdk=args.max_sdk) + print(code) + return 0 + + +if __name__ == '__main__': + sys.exit(main(sys.argv[1:])) diff --git a/python/mozbuild/mozbuild/artifacts.py b/python/mozbuild/mozbuild/artifacts.py new file mode 100644 index 000000000..02538938f --- /dev/null +++ b/python/mozbuild/mozbuild/artifacts.py @@ -0,0 +1,1089 @@ +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, You can obtain one at http://mozilla.org/MPL/2.0/. + +''' +Fetch build artifacts from a Firefox tree. + +This provides an (at-the-moment special purpose) interface to download Android +artifacts from Mozilla's Task Cluster. + +This module performs the following steps: + +* find a candidate hg parent revision. At one time we used the local pushlog, + which required the mozext hg extension. This isn't feasible with git, and it + is only mildly less efficient to not use the pushlog, so we don't use it even + when querying hg. + +* map the candidate parent to candidate Task Cluster tasks and artifact + locations. Pushlog entries might not correspond to tasks (yet), and those + tasks might not produce the desired class of artifacts. + +* fetch fresh Task Cluster artifacts and purge old artifacts, using a simple + Least Recently Used cache. + +* post-process fresh artifacts, to speed future installation. In particular, + extract relevant files from Mac OS X DMG files into a friendly archive format + so we don't have to mount DMG files frequently. + +The bulk of the complexity is in managing and persisting several caches. If +we found a Python LRU cache that pickled cleanly, we could remove a lot of +this code! Sadly, I found no such candidate implementations, so we pickle +pylru caches manually. + +None of the instances (or the underlying caches) are safe for concurrent use. +A future need, perhaps. + +This module requires certain modules be importable from the ambient Python +environment. |mach artifact| ensures these modules are available, but other +consumers will need to arrange this themselves. +''' + + +from __future__ import absolute_import, print_function, unicode_literals + +import collections +import functools +import glob +import hashlib +import logging +import operator +import os +import pickle +import re +import requests +import shutil +import stat +import subprocess +import tarfile +import tempfile +import urlparse +import zipfile + +import pylru +import taskcluster + +from mozbuild.util import ( + ensureParentDir, + FileAvoidWrite, +) +import mozinstall +from mozpack.files import ( + JarFinder, + TarFinder, +) +from mozpack.mozjar import ( + JarReader, + JarWriter, +) +from mozpack.packager.unpack import UnpackFinder +import mozpack.path as mozpath +from mozregression.download_manager import ( + DownloadManager, +) +from mozregression.persist_limit import ( + PersistLimit, +) + +NUM_PUSHHEADS_TO_QUERY_PER_PARENT = 50 # Number of candidate pushheads to cache per parent changeset. + +# Number of parent changesets to consider as possible pushheads. +# There isn't really such a thing as a reasonable default here, because we don't +# know how many pushheads we'll need to look at to find a build with our artifacts, +# and we don't know how many changesets will be in each push. For now we assume +# we'll find a build in the last 50 pushes, assuming each push contains 10 changesets. +NUM_REVISIONS_TO_QUERY = 500 + +MAX_CACHED_TASKS = 400 # Number of pushheads to cache Task Cluster task data for. + +# Number of downloaded artifacts to cache. Each artifact can be very large, +# so don't make this to large! TODO: make this a size (like 500 megs) rather than an artifact count. +MAX_CACHED_ARTIFACTS = 6 + +# Downloaded artifacts are cached, and a subset of their contents extracted for +# easy installation. This is most noticeable on Mac OS X: since mounting and +# copying from DMG files is very slow, we extract the desired binaries to a +# separate archive for fast re-installation. +PROCESSED_SUFFIX = '.processed.jar' + +CANDIDATE_TREES = ( + 'mozilla-central', + 'integration/mozilla-inbound', + 'releases/mozilla-aurora' +) + +class ArtifactJob(object): + # These are a subset of TEST_HARNESS_BINS in testing/mochitest/Makefile.in. + # Each item is a pair of (pattern, (src_prefix, dest_prefix), where src_prefix + # is the prefix of the pattern relevant to its location in the archive, and + # dest_prefix is the prefix to be added that will yield the final path relative + # to dist/. + test_artifact_patterns = { + ('bin/BadCertServer', ('bin', 'bin')), + ('bin/GenerateOCSPResponse', ('bin', 'bin')), + ('bin/OCSPStaplingServer', ('bin', 'bin')), + ('bin/certutil', ('bin', 'bin')), + ('bin/fileid', ('bin', 'bin')), + ('bin/pk12util', ('bin', 'bin')), + ('bin/ssltunnel', ('bin', 'bin')), + ('bin/xpcshell', ('bin', 'bin')), + ('bin/plugins/*', ('bin/plugins', 'plugins')) + } + + # We can tell our input is a test archive by this suffix, which happens to + # be the same across platforms. + _test_archive_suffix = '.common.tests.zip' + + def __init__(self, package_re, tests_re, log=None, download_symbols=False): + self._package_re = re.compile(package_re) + self._tests_re = None + if tests_re: + self._tests_re = re.compile(tests_re) + self._log = log + self._symbols_archive_suffix = None + if download_symbols: + self._symbols_archive_suffix = 'crashreporter-symbols.zip' + + def log(self, *args, **kwargs): + if self._log: + self._log(*args, **kwargs) + + def find_candidate_artifacts(self, artifacts): + # TODO: Handle multiple artifacts, taking the latest one. + tests_artifact = None + for artifact in artifacts: + name = artifact['name'] + if self._package_re and self._package_re.match(name): + yield name + elif self._tests_re and self._tests_re.match(name): + tests_artifact = name + yield name + elif self._symbols_archive_suffix and name.endswith(self._symbols_archive_suffix): + yield name + else: + self.log(logging.DEBUG, 'artifact', + {'name': name}, + 'Not yielding artifact named {name} as a candidate artifact') + if self._tests_re and not tests_artifact: + raise ValueError('Expected tests archive matching "{re}", but ' + 'found none!'.format(re=self._tests_re)) + + def process_artifact(self, filename, processed_filename): + if filename.endswith(ArtifactJob._test_archive_suffix) and self._tests_re: + return self.process_tests_artifact(filename, processed_filename) + if self._symbols_archive_suffix and filename.endswith(self._symbols_archive_suffix): + return self.process_symbols_archive(filename, processed_filename) + return self.process_package_artifact(filename, processed_filename) + + def process_package_artifact(self, filename, processed_filename): + raise NotImplementedError("Subclasses must specialize process_package_artifact!") + + def process_tests_artifact(self, filename, processed_filename): + added_entry = False + + with JarWriter(file=processed_filename, optimize=False, compress_level=5) as writer: + reader = JarReader(filename) + for filename, entry in reader.entries.iteritems(): + for pattern, (src_prefix, dest_prefix) in self.test_artifact_patterns: + if not mozpath.match(filename, pattern): + continue + destpath = mozpath.relpath(filename, src_prefix) + destpath = mozpath.join(dest_prefix, destpath) + self.log(logging.INFO, 'artifact', + {'destpath': destpath}, + 'Adding {destpath} to processed archive') + mode = entry['external_attr'] >> 16 + writer.add(destpath.encode('utf-8'), reader[filename], mode=mode) + added_entry = True + + if not added_entry: + raise ValueError('Archive format changed! No pattern from "{patterns}"' + 'matched an archive path.'.format( + patterns=LinuxArtifactJob.test_artifact_patterns)) + + def process_symbols_archive(self, filename, processed_filename): + with JarWriter(file=processed_filename, optimize=False, compress_level=5) as writer: + reader = JarReader(filename) + for filename in reader.entries: + destpath = mozpath.join('crashreporter-symbols', filename) + self.log(logging.INFO, 'artifact', + {'destpath': destpath}, + 'Adding {destpath} to processed archive') + writer.add(destpath.encode('utf-8'), reader[filename]) + +class AndroidArtifactJob(ArtifactJob): + + product = 'mobile' + + package_artifact_patterns = { + 'application.ini', + 'platform.ini', + '**/*.so', + '**/interfaces.xpt', + } + + def process_artifact(self, filename, processed_filename): + # Extract all .so files into the root, which will get copied into dist/bin. + with JarWriter(file=processed_filename, optimize=False, compress_level=5) as writer: + for p, f in UnpackFinder(JarFinder(filename, JarReader(filename))): + if not any(mozpath.match(p, pat) for pat in self.package_artifact_patterns): + continue + + dirname, basename = os.path.split(p) + self.log(logging.INFO, 'artifact', + {'basename': basename}, + 'Adding {basename} to processed archive') + + basedir = 'bin' + if not basename.endswith('.so'): + basedir = mozpath.join('bin', dirname.lstrip('assets/')) + basename = mozpath.join(basedir, basename) + writer.add(basename.encode('utf-8'), f.open()) + + +class LinuxArtifactJob(ArtifactJob): + + product = 'firefox' + + package_artifact_patterns = { + 'firefox/application.ini', + 'firefox/crashreporter', + 'firefox/dependentlibs.list', + 'firefox/firefox', + 'firefox/firefox-bin', + 'firefox/minidump-analyzer', + 'firefox/platform.ini', + 'firefox/plugin-container', + 'firefox/updater', + 'firefox/**/*.so', + 'firefox/**/interfaces.xpt', + } + + def process_package_artifact(self, filename, processed_filename): + added_entry = False + + with JarWriter(file=processed_filename, optimize=False, compress_level=5) as writer: + with tarfile.open(filename) as reader: + for p, f in UnpackFinder(TarFinder(filename, reader)): + if not any(mozpath.match(p, pat) for pat in self.package_artifact_patterns): + continue + + # We strip off the relative "firefox/" bit from the path, + # but otherwise preserve it. + destpath = mozpath.join('bin', + mozpath.relpath(p, "firefox")) + self.log(logging.INFO, 'artifact', + {'destpath': destpath}, + 'Adding {destpath} to processed archive') + writer.add(destpath.encode('utf-8'), f.open(), mode=f.mode) + added_entry = True + + if not added_entry: + raise ValueError('Archive format changed! No pattern from "{patterns}" ' + 'matched an archive path.'.format( + patterns=LinuxArtifactJob.package_artifact_patterns)) + + +class MacArtifactJob(ArtifactJob): + + product = 'firefox' + + def process_package_artifact(self, filename, processed_filename): + tempdir = tempfile.mkdtemp() + try: + self.log(logging.INFO, 'artifact', + {'tempdir': tempdir}, + 'Unpacking DMG into {tempdir}') + mozinstall.install(filename, tempdir) # Doesn't handle already mounted DMG files nicely: + + # InstallError: Failed to install "/Users/nalexander/.mozbuild/package-frontend/b38eeeb54cdcf744-firefox-44.0a1.en-US.mac.dmg (local variable 'appDir' referenced before assignment)" + + # File "/Users/nalexander/Mozilla/gecko/mobile/android/mach_commands.py", line 250, in artifact_install + # return artifacts.install_from(source, self.distdir) + # File "/Users/nalexander/Mozilla/gecko/python/mozbuild/mozbuild/artifacts.py", line 457, in install_from + # return self.install_from_hg(source, distdir) + # File "/Users/nalexander/Mozilla/gecko/python/mozbuild/mozbuild/artifacts.py", line 445, in install_from_hg + # return self.install_from_url(url, distdir) + # File "/Users/nalexander/Mozilla/gecko/python/mozbuild/mozbuild/artifacts.py", line 418, in install_from_url + # return self.install_from_file(filename, distdir) + # File "/Users/nalexander/Mozilla/gecko/python/mozbuild/mozbuild/artifacts.py", line 336, in install_from_file + # mozinstall.install(filename, tempdir) + # File "/Users/nalexander/Mozilla/gecko/objdir-dce/_virtualenv/lib/python2.7/site-packages/mozinstall/mozinstall.py", line 117, in install + # install_dir = _install_dmg(src, dest) + # File "/Users/nalexander/Mozilla/gecko/objdir-dce/_virtualenv/lib/python2.7/site-packages/mozinstall/mozinstall.py", line 261, in _install_dmg + # subprocess.call('hdiutil detach %s -quiet' % appDir, + + bundle_dirs = glob.glob(mozpath.join(tempdir, '*.app')) + if len(bundle_dirs) != 1: + raise ValueError('Expected one source bundle, found: {}'.format(bundle_dirs)) + [source] = bundle_dirs + + # These get copied into dist/bin without the path, so "root/a/b/c" -> "dist/bin/c". + paths_no_keep_path = ('Contents/MacOS', [ + 'crashreporter.app/Contents/MacOS/crashreporter', + 'crashreporter.app/Contents/MacOS/minidump-analyzer', + 'firefox', + 'firefox-bin', + 'libfreebl3.dylib', + 'liblgpllibs.dylib', + # 'liblogalloc.dylib', + 'libmozglue.dylib', + 'libnss3.dylib', + 'libnssckbi.dylib', + 'libnssdbm3.dylib', + 'libplugin_child_interpose.dylib', + # 'libreplace_jemalloc.dylib', + # 'libreplace_malloc.dylib', + 'libmozavutil.dylib', + 'libmozavcodec.dylib', + 'libsoftokn3.dylib', + 'plugin-container.app/Contents/MacOS/plugin-container', + 'updater.app/Contents/MacOS/org.mozilla.updater', + # 'xpcshell', + 'XUL', + ]) + + # These get copied into dist/bin with the path, so "root/a/b/c" -> "dist/bin/a/b/c". + paths_keep_path = ('Contents/Resources', [ + 'browser/components/libbrowsercomps.dylib', + 'dependentlibs.list', + # 'firefox', + 'gmp-clearkey/0.1/libclearkey.dylib', + # 'gmp-fake/1.0/libfake.dylib', + # 'gmp-fakeopenh264/1.0/libfakeopenh264.dylib', + '**/interfaces.xpt', + ]) + + with JarWriter(file=processed_filename, optimize=False, compress_level=5) as writer: + root, paths = paths_no_keep_path + finder = UnpackFinder(mozpath.join(source, root)) + for path in paths: + for p, f in finder.find(path): + self.log(logging.INFO, 'artifact', + {'path': p}, + 'Adding {path} to processed archive') + destpath = mozpath.join('bin', os.path.basename(p)) + writer.add(destpath.encode('utf-8'), f, mode=f.mode) + + root, paths = paths_keep_path + finder = UnpackFinder(mozpath.join(source, root)) + for path in paths: + for p, f in finder.find(path): + self.log(logging.INFO, 'artifact', + {'path': p}, + 'Adding {path} to processed archive') + destpath = mozpath.join('bin', p) + writer.add(destpath.encode('utf-8'), f.open(), mode=f.mode) + + finally: + try: + shutil.rmtree(tempdir) + except (OSError, IOError): + self.log(logging.WARN, 'artifact', + {'tempdir': tempdir}, + 'Unable to delete {tempdir}') + pass + + +class WinArtifactJob(ArtifactJob): + package_artifact_patterns = { + 'firefox/dependentlibs.list', + 'firefox/platform.ini', + 'firefox/application.ini', + 'firefox/**/*.dll', + 'firefox/*.exe', + 'firefox/**/interfaces.xpt', + } + + product = 'firefox' + + # These are a subset of TEST_HARNESS_BINS in testing/mochitest/Makefile.in. + test_artifact_patterns = { + ('bin/BadCertServer.exe', ('bin', 'bin')), + ('bin/GenerateOCSPResponse.exe', ('bin', 'bin')), + ('bin/OCSPStaplingServer.exe', ('bin', 'bin')), + ('bin/certutil.exe', ('bin', 'bin')), + ('bin/fileid.exe', ('bin', 'bin')), + ('bin/pk12util.exe', ('bin', 'bin')), + ('bin/ssltunnel.exe', ('bin', 'bin')), + ('bin/xpcshell.exe', ('bin', 'bin')), + ('bin/plugins/*', ('bin/plugins', 'plugins')) + } + + def process_package_artifact(self, filename, processed_filename): + added_entry = False + with JarWriter(file=processed_filename, optimize=False, compress_level=5) as writer: + for p, f in UnpackFinder(JarFinder(filename, JarReader(filename))): + if not any(mozpath.match(p, pat) for pat in self.package_artifact_patterns): + continue + + # strip off the relative "firefox/" bit from the path: + basename = mozpath.relpath(p, "firefox") + basename = mozpath.join('bin', basename) + self.log(logging.INFO, 'artifact', + {'basename': basename}, + 'Adding {basename} to processed archive') + writer.add(basename.encode('utf-8'), f.open(), mode=f.mode) + added_entry = True + + if not added_entry: + raise ValueError('Archive format changed! No pattern from "{patterns}"' + 'matched an archive path.'.format( + patterns=self.artifact_patterns)) + +# Keep the keys of this map in sync with the |mach artifact| --job +# options. The keys of this map correspond to entries at +# https://tools.taskcluster.net/index/artifacts/#gecko.v2.mozilla-central.latest/gecko.v2.mozilla-central.latest +# The values correpsond to a pair of (, ). +JOB_DETAILS = { + 'android-api-15-opt': (AndroidArtifactJob, ('public/build/target.apk', + None)), + 'android-api-15-debug': (AndroidArtifactJob, ('public/build/target.apk', + None)), + 'android-x86-opt': (AndroidArtifactJob, ('public/build/target.apk', + None)), + 'linux-opt': (LinuxArtifactJob, ('public/build/firefox-(.*)\.linux-i686\.tar\.bz2', + 'public/build/firefox-(.*)\.common\.tests\.zip')), + 'linux-debug': (LinuxArtifactJob, ('public/build/firefox-(.*)\.linux-i686\.tar\.bz2', + 'public/build/firefox-(.*)\.common\.tests\.zip')), + 'linux64-opt': (LinuxArtifactJob, ('public/build/firefox-(.*)\.linux-x86_64\.tar\.bz2', + 'public/build/firefox-(.*)\.common\.tests\.zip')), + 'linux64-debug': (LinuxArtifactJob, ('public/build/target\.tar\.bz2', + 'public/build/target\.common\.tests\.zip')), + 'macosx64-opt': (MacArtifactJob, ('public/build/firefox-(.*)\.mac\.dmg', + 'public/build/firefox-(.*)\.common\.tests\.zip')), + 'macosx64-debug': (MacArtifactJob, ('public/build/firefox-(.*)\.mac64\.dmg', + 'public/build/firefox-(.*)\.common\.tests\.zip')), + 'win32-opt': (WinArtifactJob, ('public/build/firefox-(.*)\.win32.zip', + 'public/build/firefox-(.*)\.common\.tests\.zip')), + 'win32-debug': (WinArtifactJob, ('public/build/firefox-(.*)\.win32.zip', + 'public/build/firefox-(.*)\.common\.tests\.zip')), + 'win64-opt': (WinArtifactJob, ('public/build/firefox-(.*)\.win64.zip', + 'public/build/firefox-(.*)\.common\.tests\.zip')), + 'win64-debug': (WinArtifactJob, ('public/build/firefox-(.*)\.win64.zip', + 'public/build/firefox-(.*)\.common\.tests\.zip')), +} + + + +def get_job_details(job, log=None, download_symbols=False): + cls, (package_re, tests_re) = JOB_DETAILS[job] + return cls(package_re, tests_re, log=log, download_symbols=download_symbols) + +def cachedmethod(cachefunc): + '''Decorator to wrap a class or instance method with a memoizing callable that + saves results in a (possibly shared) cache. + ''' + def decorator(method): + def wrapper(self, *args, **kwargs): + mapping = cachefunc(self) + if mapping is None: + return method(self, *args, **kwargs) + key = (method.__name__, args, tuple(sorted(kwargs.items()))) + try: + value = mapping[key] + return value + except KeyError: + pass + result = method(self, *args, **kwargs) + mapping[key] = result + return result + return functools.update_wrapper(wrapper, method) + return decorator + + +class CacheManager(object): + '''Maintain an LRU cache. Provide simple persistence, including support for + loading and saving the state using a "with" block. Allow clearing the cache + and printing the cache for debugging. + + Provide simple logging. + ''' + + def __init__(self, cache_dir, cache_name, cache_size, cache_callback=None, log=None, skip_cache=False): + self._skip_cache = skip_cache + self._cache = pylru.lrucache(cache_size, callback=cache_callback) + self._cache_filename = mozpath.join(cache_dir, cache_name + '-cache.pickle') + self._log = log + + def log(self, *args, **kwargs): + if self._log: + self._log(*args, **kwargs) + + def load_cache(self): + if self._skip_cache: + self.log(logging.DEBUG, 'artifact', + {}, + 'Skipping cache: ignoring load_cache!') + return + + try: + items = pickle.load(open(self._cache_filename, 'rb')) + for key, value in items: + self._cache[key] = value + except Exception as e: + # Corrupt cache, perhaps? Sadly, pickle raises many different + # exceptions, so it's not worth trying to be fine grained here. + # We ignore any exception, so the cache is effectively dropped. + self.log(logging.INFO, 'artifact', + {'filename': self._cache_filename, 'exception': repr(e)}, + 'Ignoring exception unpickling cache file {filename}: {exception}') + pass + + def dump_cache(self): + if self._skip_cache: + self.log(logging.DEBUG, 'artifact', + {}, + 'Skipping cache: ignoring dump_cache!') + return + + ensureParentDir(self._cache_filename) + pickle.dump(list(reversed(list(self._cache.items()))), open(self._cache_filename, 'wb'), -1) + + def clear_cache(self): + if self._skip_cache: + self.log(logging.DEBUG, 'artifact', + {}, + 'Skipping cache: ignoring clear_cache!') + return + + with self: + self._cache.clear() + + def print_cache(self): + with self: + for item in self._cache.items(): + self.log(logging.INFO, 'artifact', + {'item': item}, + '{item}') + + def print_last_item(self, args, sorted_kwargs, result): + # By default, show nothing. + pass + + def print_last(self): + # We use the persisted LRU caches to our advantage. The first item is + # most recent. + with self: + item = next(self._cache.items(), None) + if item is not None: + (name, args, sorted_kwargs), result = item + self.print_last_item(args, sorted_kwargs, result) + else: + self.log(logging.WARN, 'artifact', + {}, + 'No last cached item found.') + + def __enter__(self): + self.load_cache() + return self + + def __exit__(self, type, value, traceback): + self.dump_cache() + +class PushheadCache(CacheManager): + '''Helps map tree/revision pairs to parent pushheads according to the pushlog.''' + + def __init__(self, cache_dir, log=None, skip_cache=False): + CacheManager.__init__(self, cache_dir, 'pushhead_cache', MAX_CACHED_TASKS, log=log, skip_cache=skip_cache) + + @cachedmethod(operator.attrgetter('_cache')) + def parent_pushhead_id(self, tree, revision): + cset_url_tmpl = ('https://hg.mozilla.org/{tree}/json-pushes?' + 'changeset={changeset}&version=2&tipsonly=1') + req = requests.get(cset_url_tmpl.format(tree=tree, changeset=revision), + headers={'Accept': 'application/json'}) + if req.status_code not in range(200, 300): + raise ValueError + result = req.json() + [found_pushid] = result['pushes'].keys() + return int(found_pushid) + + @cachedmethod(operator.attrgetter('_cache')) + def pushid_range(self, tree, start, end): + pushid_url_tmpl = ('https://hg.mozilla.org/{tree}/json-pushes?' + 'startID={start}&endID={end}&version=2&tipsonly=1') + + req = requests.get(pushid_url_tmpl.format(tree=tree, start=start, + end=end), + headers={'Accept': 'application/json'}) + result = req.json() + return [ + p['changesets'][-1] for p in result['pushes'].values() + ] + +class TaskCache(CacheManager): + '''Map candidate pushheads to Task Cluster task IDs and artifact URLs.''' + + def __init__(self, cache_dir, log=None, skip_cache=False): + CacheManager.__init__(self, cache_dir, 'artifact_url', MAX_CACHED_TASKS, log=log, skip_cache=skip_cache) + self._index = taskcluster.Index() + self._queue = taskcluster.Queue() + + @cachedmethod(operator.attrgetter('_cache')) + def artifact_urls(self, tree, job, rev, download_symbols): + try: + artifact_job = get_job_details(job, log=self._log, download_symbols=download_symbols) + except KeyError: + self.log(logging.INFO, 'artifact', + {'job': job}, + 'Unknown job {job}') + raise KeyError("Unknown job") + + # Grab the second part of the repo name, which is generally how things + # are indexed. Eg: 'integration/mozilla-inbound' is indexed as + # 'mozilla-inbound' + tree = tree.split('/')[1] if '/' in tree else tree + + namespace = 'gecko.v2.{tree}.revision.{rev}.{product}.{job}'.format( + rev=rev, + tree=tree, + product=artifact_job.product, + job=job, + ) + self.log(logging.DEBUG, 'artifact', + {'namespace': namespace}, + 'Searching Taskcluster index with namespace: {namespace}') + try: + task = self._index.findTask(namespace) + except Exception: + # Not all revisions correspond to pushes that produce the job we + # care about; and even those that do may not have completed yet. + raise ValueError('Task for {namespace} does not exist (yet)!'.format(namespace=namespace)) + taskId = task['taskId'] + + artifacts = self._queue.listLatestArtifacts(taskId)['artifacts'] + + urls = [] + for artifact_name in artifact_job.find_candidate_artifacts(artifacts): + # We can easily extract the task ID from the URL. We can't easily + # extract the build ID; we use the .ini files embedded in the + # downloaded artifact for this. We could also use the uploaded + # public/build/buildprops.json for this purpose. + url = self._queue.buildUrl('getLatestArtifact', taskId, artifact_name) + urls.append(url) + if not urls: + raise ValueError('Task for {namespace} existed, but no artifacts found!'.format(namespace=namespace)) + return urls + + def print_last_item(self, args, sorted_kwargs, result): + tree, job, rev = args + self.log(logging.INFO, 'artifact', + {'rev': rev}, + 'Last installed binaries from hg parent revision {rev}') + + +class ArtifactCache(CacheManager): + '''Fetch Task Cluster artifact URLs and purge least recently used artifacts from disk.''' + + def __init__(self, cache_dir, log=None, skip_cache=False): + # TODO: instead of storing N artifact packages, store M megabytes. + CacheManager.__init__(self, cache_dir, 'fetch', MAX_CACHED_ARTIFACTS, cache_callback=self.delete_file, log=log, skip_cache=skip_cache) + self._cache_dir = cache_dir + size_limit = 1024 * 1024 * 1024 # 1Gb in bytes. + file_limit = 4 # But always keep at least 4 old artifacts around. + persist_limit = PersistLimit(size_limit, file_limit) + self._download_manager = DownloadManager(self._cache_dir, persist_limit=persist_limit) + self._last_dl_update = -1 + + def delete_file(self, key, value): + try: + os.remove(value) + self.log(logging.INFO, 'artifact', + {'filename': value}, + 'Purged artifact {filename}') + except (OSError, IOError): + pass + + try: + os.remove(value + PROCESSED_SUFFIX) + self.log(logging.INFO, 'artifact', + {'filename': value + PROCESSED_SUFFIX}, + 'Purged processed artifact {filename}') + except (OSError, IOError): + pass + + @cachedmethod(operator.attrgetter('_cache')) + def fetch(self, url, force=False): + # We download to a temporary name like HASH[:16]-basename to + # differentiate among URLs with the same basenames. We used to then + # extract the build ID from the downloaded artifact and use it to make a + # human readable unique name, but extracting build IDs is time consuming + # (especially on Mac OS X, where we must mount a large DMG file). + hash = hashlib.sha256(url).hexdigest()[:16] + fname = hash + '-' + os.path.basename(url) + + path = os.path.abspath(mozpath.join(self._cache_dir, fname)) + if self._skip_cache and os.path.exists(path): + self.log(logging.DEBUG, 'artifact', + {'path': path}, + 'Skipping cache: removing cached downloaded artifact {path}') + os.remove(path) + + self.log(logging.INFO, 'artifact', + {'path': path}, + 'Downloading to temporary location {path}') + try: + dl = self._download_manager.download(url, fname) + + def download_progress(dl, bytes_so_far, total_size): + percent = (float(bytes_so_far) / total_size) * 100 + now = int(percent / 5) + if now == self._last_dl_update: + return + self._last_dl_update = now + self.log(logging.INFO, 'artifact', + {'bytes_so_far': bytes_so_far, 'total_size': total_size, 'percent': percent}, + 'Downloading... {percent:02.1f} %') + + if dl: + dl.set_progress(download_progress) + dl.wait() + self.log(logging.INFO, 'artifact', + {'path': os.path.abspath(mozpath.join(self._cache_dir, fname))}, + 'Downloaded artifact to {path}') + return os.path.abspath(mozpath.join(self._cache_dir, fname)) + finally: + # Cancel any background downloads in progress. + self._download_manager.cancel() + + def print_last_item(self, args, sorted_kwargs, result): + url, = args + self.log(logging.INFO, 'artifact', + {'url': url}, + 'Last installed binaries from url {url}') + self.log(logging.INFO, 'artifact', + {'filename': result}, + 'Last installed binaries from local file {filename}') + self.log(logging.INFO, 'artifact', + {'filename': result + PROCESSED_SUFFIX}, + 'Last installed binaries from local processed file {filename}') + + +class Artifacts(object): + '''Maintain state to efficiently fetch build artifacts from a Firefox tree.''' + + def __init__(self, tree, substs, defines, job=None, log=None, + cache_dir='.', hg=None, git=None, skip_cache=False, + topsrcdir=None): + if (hg and git) or (not hg and not git): + raise ValueError("Must provide path to exactly one of hg and git") + + self._substs = substs + self._download_symbols = self._substs.get('MOZ_ARTIFACT_BUILD_SYMBOLS', False) + self._defines = defines + self._tree = tree + self._job = job or self._guess_artifact_job() + self._log = log + self._hg = hg + self._git = git + self._cache_dir = cache_dir + self._skip_cache = skip_cache + self._topsrcdir = topsrcdir + + try: + self._artifact_job = get_job_details(self._job, log=self._log, download_symbols=self._download_symbols) + except KeyError: + self.log(logging.INFO, 'artifact', + {'job': self._job}, + 'Unknown job {job}') + raise KeyError("Unknown job") + + self._task_cache = TaskCache(self._cache_dir, log=self._log, skip_cache=self._skip_cache) + self._artifact_cache = ArtifactCache(self._cache_dir, log=self._log, skip_cache=self._skip_cache) + self._pushhead_cache = PushheadCache(self._cache_dir, log=self._log, skip_cache=self._skip_cache) + + def log(self, *args, **kwargs): + if self._log: + self._log(*args, **kwargs) + + def _guess_artifact_job(self): + # Add the "-debug" suffix to the guessed artifact job name + # if MOZ_DEBUG is enabled. + if self._substs.get('MOZ_DEBUG'): + target_suffix = '-debug' + else: + target_suffix = '-opt' + + if self._substs.get('MOZ_BUILD_APP', '') == 'mobile/android': + if self._substs['ANDROID_CPU_ARCH'] == 'x86': + return 'android-x86-opt' + return 'android-api-15' + target_suffix + + target_64bit = False + if self._substs['target_cpu'] == 'x86_64': + target_64bit = True + + if self._defines.get('XP_LINUX', False): + return ('linux64' if target_64bit else 'linux') + target_suffix + if self._defines.get('XP_WIN', False): + return ('win64' if target_64bit else 'win32') + target_suffix + if self._defines.get('XP_MACOSX', False): + # We only produce unified builds in automation, so the target_cpu + # check is not relevant. + return 'macosx64' + target_suffix + raise Exception('Cannot determine default job for |mach artifact|!') + + def _pushheads_from_rev(self, rev, count): + """Queries hg.mozilla.org's json-pushlog for pushheads that are nearby + ancestors or `rev`. Multiple trees are queried, as the `rev` may + already have been pushed to multiple repositories. For each repository + containing `rev`, the pushhead introducing `rev` and the previous + `count` pushheads from that point are included in the output. + """ + + with self._pushhead_cache as pushhead_cache: + found_pushids = {} + for tree in CANDIDATE_TREES: + self.log(logging.INFO, 'artifact', + {'tree': tree, + 'rev': rev}, + 'Attempting to find a pushhead containing {rev} on {tree}.') + try: + pushid = pushhead_cache.parent_pushhead_id(tree, rev) + found_pushids[tree] = pushid + except ValueError: + continue + + candidate_pushheads = collections.defaultdict(list) + + for tree, pushid in found_pushids.iteritems(): + end = pushid + start = pushid - NUM_PUSHHEADS_TO_QUERY_PER_PARENT + + self.log(logging.INFO, 'artifact', + {'tree': tree, + 'pushid': pushid, + 'num': NUM_PUSHHEADS_TO_QUERY_PER_PARENT}, + 'Retrieving the last {num} pushheads starting with id {pushid} on {tree}') + for pushhead in pushhead_cache.pushid_range(tree, start, end): + candidate_pushheads[pushhead].append(tree) + + return candidate_pushheads + + def _get_hg_revisions_from_git(self): + rev_list = subprocess.check_output([ + self._git, 'rev-list', '--topo-order', + '--max-count={num}'.format(num=NUM_REVISIONS_TO_QUERY), + 'HEAD', + ], cwd=self._topsrcdir) + + hg_hash_list = subprocess.check_output([ + self._git, 'cinnabar', 'git2hg' + ] + rev_list.splitlines(), cwd=self._topsrcdir) + + zeroes = "0" * 40 + + hashes = [] + for hg_hash in hg_hash_list.splitlines(): + hg_hash = hg_hash.strip() + if not hg_hash or hg_hash == zeroes: + continue + hashes.append(hg_hash) + return hashes + + def _get_recent_public_revisions(self): + """Returns recent ancestors of the working parent that are likely to + to be known to Mozilla automation. + + If we're using git, retrieves hg revisions from git-cinnabar. + """ + if self._git: + return self._get_hg_revisions_from_git() + + return subprocess.check_output([ + self._hg, 'log', + '--template', '{node}\n', + '-r', 'last(public() and ::., {num})'.format( + num=NUM_REVISIONS_TO_QUERY) + ], cwd=self._topsrcdir).splitlines() + + def _find_pushheads(self): + """Returns an iterator of recent pushhead revisions, starting with the + working parent. + """ + + last_revs = self._get_recent_public_revisions() + candidate_pushheads = self._pushheads_from_rev(last_revs[0].rstrip(), + NUM_PUSHHEADS_TO_QUERY_PER_PARENT) + count = 0 + for rev in last_revs: + rev = rev.rstrip() + if not rev: + continue + if rev not in candidate_pushheads: + continue + count += 1 + yield candidate_pushheads[rev], rev + + if not count: + raise Exception('Could not find any candidate pushheads in the last {num} revisions.\n' + 'Search started with {rev}, which must be known to Mozilla automation.\n\n' + 'see https://developer.mozilla.org/en-US/docs/Artifact_builds'.format( + rev=last_revs[0], num=NUM_PUSHHEADS_TO_QUERY_PER_PARENT)) + + def find_pushhead_artifacts(self, task_cache, job, tree, pushhead): + try: + urls = task_cache.artifact_urls(tree, job, pushhead, self._download_symbols) + except ValueError: + return None + if urls: + self.log(logging.INFO, 'artifact', + {'pushhead': pushhead, + 'tree': tree}, + 'Installing from remote pushhead {pushhead} on {tree}') + return urls + return None + + def install_from_file(self, filename, distdir): + self.log(logging.INFO, 'artifact', + {'filename': filename}, + 'Installing from {filename}') + + # Do we need to post-process? + processed_filename = filename + PROCESSED_SUFFIX + + if self._skip_cache and os.path.exists(processed_filename): + self.log(logging.DEBUG, 'artifact', + {'path': processed_filename}, + 'Skipping cache: removing cached processed artifact {path}') + os.remove(processed_filename) + + if not os.path.exists(processed_filename): + self.log(logging.INFO, 'artifact', + {'filename': filename}, + 'Processing contents of {filename}') + self.log(logging.INFO, 'artifact', + {'processed_filename': processed_filename}, + 'Writing processed {processed_filename}') + self._artifact_job.process_artifact(filename, processed_filename) + + self.log(logging.INFO, 'artifact', + {'processed_filename': processed_filename}, + 'Installing from processed {processed_filename}') + + # Copy all .so files, avoiding modification where possible. + ensureParentDir(mozpath.join(distdir, '.dummy')) + + with zipfile.ZipFile(processed_filename) as zf: + for info in zf.infolist(): + if info.filename.endswith('.ini'): + continue + n = mozpath.join(distdir, info.filename) + fh = FileAvoidWrite(n, mode='rb') + shutil.copyfileobj(zf.open(info), fh) + file_existed, file_updated = fh.close() + self.log(logging.INFO, 'artifact', + {'updating': 'Updating' if file_updated else 'Not updating', 'filename': n}, + '{updating} {filename}') + if not file_existed or file_updated: + # Libraries and binaries may need to be marked executable, + # depending on platform. + perms = info.external_attr >> 16 # See http://stackoverflow.com/a/434689. + perms |= stat.S_IWUSR | stat.S_IRUSR | stat.S_IRGRP | stat.S_IROTH # u+w, a+r. + os.chmod(n, perms) + return 0 + + def install_from_url(self, url, distdir): + self.log(logging.INFO, 'artifact', + {'url': url}, + 'Installing from {url}') + with self._artifact_cache as artifact_cache: # The with block handles persistence. + filename = artifact_cache.fetch(url) + return self.install_from_file(filename, distdir) + + def _install_from_hg_pushheads(self, hg_pushheads, distdir): + """Iterate pairs (hg_hash, {tree-set}) associating hg revision hashes + and tree-sets they are known to be in, trying to download and + install from each. + """ + + urls = None + count = 0 + # with blocks handle handle persistence. + with self._task_cache as task_cache: + for trees, hg_hash in hg_pushheads: + for tree in trees: + count += 1 + self.log(logging.DEBUG, 'artifact', + {'hg_hash': hg_hash, + 'tree': tree}, + 'Trying to find artifacts for hg revision {hg_hash} on tree {tree}.') + urls = self.find_pushhead_artifacts(task_cache, self._job, tree, hg_hash) + if urls: + for url in urls: + if self.install_from_url(url, distdir): + return 1 + return 0 + + self.log(logging.ERROR, 'artifact', + {'count': count}, + 'Tried {count} pushheads, no built artifacts found.') + return 1 + + def install_from_recent(self, distdir): + hg_pushheads = self._find_pushheads() + return self._install_from_hg_pushheads(hg_pushheads, distdir) + + def install_from_revset(self, revset, distdir): + if self._hg: + revision = subprocess.check_output([self._hg, 'log', '--template', '{node}\n', + '-r', revset], cwd=self._topsrcdir).strip() + if len(revision.split('\n')) != 1: + raise ValueError('hg revision specification must resolve to exactly one commit') + else: + revision = subprocess.check_output([self._git, 'rev-parse', revset], cwd=self._topsrcdir).strip() + revision = subprocess.check_output([self._git, 'cinnabar', 'git2hg', revision], cwd=self._topsrcdir).strip() + if len(revision.split('\n')) != 1: + raise ValueError('hg revision specification must resolve to exactly one commit') + if revision == "0" * 40: + raise ValueError('git revision specification must resolve to a commit known to hg') + + self.log(logging.INFO, 'artifact', + {'revset': revset, + 'revision': revision}, + 'Will only accept artifacts from a pushhead at {revision} ' + '(matched revset "{revset}").') + pushheads = [(list(CANDIDATE_TREES), revision)] + return self._install_from_hg_pushheads(pushheads, distdir) + + def install_from(self, source, distdir): + """Install artifacts from a ``source`` into the given ``distdir``. + """ + if source and os.path.isfile(source): + return self.install_from_file(source, distdir) + elif source and urlparse.urlparse(source).scheme: + return self.install_from_url(source, distdir) + else: + if source is None and 'MOZ_ARTIFACT_REVISION' in os.environ: + source = os.environ['MOZ_ARTIFACT_REVISION'] + + if source: + return self.install_from_revset(source, distdir) + + return self.install_from_recent(distdir) + + + def print_last(self): + self.log(logging.INFO, 'artifact', + {}, + 'Printing last used artifact details.') + self._task_cache.print_last() + self._artifact_cache.print_last() + self._pushhead_cache.print_last() + + def clear_cache(self): + self.log(logging.INFO, 'artifact', + {}, + 'Deleting cached artifacts and caches.') + self._task_cache.clear_cache() + self._artifact_cache.clear_cache() + self._pushhead_cache.clear_cache() + + def print_cache(self): + self.log(logging.INFO, 'artifact', + {}, + 'Printing cached artifacts and caches.') + self._task_cache.print_cache() + self._artifact_cache.print_cache() + self._pushhead_cache.print_cache() diff --git a/python/mozbuild/mozbuild/backend/__init__.py b/python/mozbuild/mozbuild/backend/__init__.py new file mode 100644 index 000000000..64bcb87d9 --- /dev/null +++ b/python/mozbuild/mozbuild/backend/__init__.py @@ -0,0 +1,26 @@ +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, You can obtain one at http://mozilla.org/MPL/2.0/. + +backends = { + 'AndroidEclipse': 'mozbuild.backend.android_eclipse', + 'ChromeMap': 'mozbuild.codecoverage.chrome_map', + 'CompileDB': 'mozbuild.compilation.database', + 'CppEclipse': 'mozbuild.backend.cpp_eclipse', + 'FasterMake': 'mozbuild.backend.fastermake', + 'FasterMake+RecursiveMake': None, + 'RecursiveMake': 'mozbuild.backend.recursivemake', + 'Tup': 'mozbuild.backend.tup', + 'VisualStudio': 'mozbuild.backend.visualstudio', +} + + +def get_backend_class(name): + if '+' in name: + from mozbuild.backend.base import HybridBackend + return HybridBackend(*(get_backend_class(name) + for name in name.split('+'))) + + class_name = '%sBackend' % name + module = __import__(backends[name], globals(), locals(), [class_name]) + return getattr(module, class_name) diff --git a/python/mozbuild/mozbuild/backend/android_eclipse.py b/python/mozbuild/mozbuild/backend/android_eclipse.py new file mode 100644 index 000000000..f17eb8d34 --- /dev/null +++ b/python/mozbuild/mozbuild/backend/android_eclipse.py @@ -0,0 +1,267 @@ +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, You can obtain one at http://mozilla.org/MPL/2.0/. + +from __future__ import absolute_import, unicode_literals + +import itertools +import os +import time +import types +import xml.dom.minidom as minidom +import xml.etree.ElementTree as ET + +from mozpack.copier import FileCopier +from mozpack.files import (FileFinder, PreprocessedFile) +from mozpack.manifests import InstallManifest +import mozpack.path as mozpath + +from .common import CommonBackend +from ..frontend.data import ( + AndroidEclipseProjectData, + ContextDerived, + ContextWrapped, +) +from ..makeutil import Makefile +from ..util import ensureParentDir +from mozbuild.base import ( + ExecutionSummary, + MachCommandConditions, +) + + +def pretty_print(element): + """Return a pretty-printed XML string for an Element. + """ + s = ET.tostring(element, 'utf-8') + # minidom wraps element in a Document node; firstChild strips it. + return minidom.parseString(s).firstChild.toprettyxml(indent=' ') + + +class AndroidEclipseBackend(CommonBackend): + """Backend that generates Android Eclipse project files. + """ + def __init__(self, environment): + if not MachCommandConditions.is_android(environment): + raise Exception( + 'The Android Eclipse backend is not available with this ' + 'configuration.') + + super(AndroidEclipseBackend, self).__init__(environment) + + def summary(self): + return ExecutionSummary( + 'AndroidEclipse backend executed in {execution_time:.2f}s\n' + 'Wrote {projects:d} Android Eclipse projects to {path:s}; ' + '{created:d} created; {updated:d} updated', + execution_time=self._execution_time, + projects=self._created_count + self._updated_count, + path=mozpath.join(self.environment.topobjdir, 'android_eclipse'), + created=self._created_count, + updated=self._updated_count, + ) + + def consume_object(self, obj): + """Write out Android Eclipse project files.""" + + if not isinstance(obj, ContextDerived): + return False + + if CommonBackend.consume_object(self, obj): + # If CommonBackend acknowledged the object, we're done with it. + return True + + # Handle the one case we care about specially. + if isinstance(obj, ContextWrapped) and isinstance(obj.wrapped, AndroidEclipseProjectData): + self._process_android_eclipse_project_data(obj.wrapped, obj.srcdir, obj.objdir) + + # We don't want to handle most things, so we just acknowledge all objects + return True + + def consume_finished(self): + """The common backend handles WebIDL and test files. We don't handle + these, so we don't call our superclass. + """ + + def _Element_for_classpathentry(self, cpe): + """Turn a ClassPathEntry into an XML Element, like one of: + + + + + + + + """ + e = ET.Element('classpathentry') + e.set('kind', 'src') + e.set('including', '**/*.java') + e.set('path', cpe.path) + if cpe.exclude_patterns: + e.set('excluding', '|'.join(sorted(cpe.exclude_patterns))) + if cpe.ignore_warnings: + attrs = ET.SubElement(e, 'attributes') + attr = ET.SubElement(attrs, 'attribute') + attr.set('name', 'ignore_optional_problems') + attr.set('value', 'true') + return e + + def _Element_for_referenced_project(self, name): + """Turn a referenced project name into an XML Element, like: + + """ + e = ET.Element('classpathentry') + e.set('kind', 'src') + e.set('combineaccessrules', 'false') + # All project directories are in the same root; this + # reference is absolute in the Eclipse namespace. + e.set('path', '/' + name) + return e + + def _Element_for_extra_jar(self, name): + """Turn a referenced JAR name into an XML Element, like: + + """ + e = ET.Element('classpathentry') + e.set('kind', 'lib') + e.set('exported', 'true') + e.set('path', name) + return e + + def _Element_for_filtered_resources(self, filtered_resources): + """Turn a list of filtered resource arguments like + ['1.0-projectRelativePath-matches-false-false-*org/mozilla/gecko/resources/**'] + into an XML Element, like: + + + 1393009101322 + + 30 + + org.eclipse.ui.ide.multiFilter + 1.0-projectRelativePath-matches-false-false-*org/mozilla/gecko/resources/** + + + + + The id is random; the values are magic.""" + + id = int(1000 * time.time()) + filteredResources = ET.Element('filteredResources') + for arg in sorted(filtered_resources): + e = ET.SubElement(filteredResources, 'filter') + ET.SubElement(e, 'id').text = str(id) + id += 1 + ET.SubElement(e, 'name') + ET.SubElement(e, 'type').text = '30' # It's magic! + matcher = ET.SubElement(e, 'matcher') + ET.SubElement(matcher, 'id').text = 'org.eclipse.ui.ide.multiFilter' + ET.SubElement(matcher, 'arguments').text = str(arg) + return filteredResources + + def _manifest_for_project(self, srcdir, project): + manifest = InstallManifest() + + if project.manifest: + manifest.add_copy(mozpath.join(srcdir, project.manifest), 'AndroidManifest.xml') + + if project.res: + manifest.add_symlink(mozpath.join(srcdir, project.res), 'res') + else: + # Eclipse expects a res directory no matter what, so we + # make an empty directory if the project doesn't specify. + res = os.path.abspath(mozpath.join(os.path.dirname(__file__), + 'templates', 'android_eclipse_empty_resource_directory')) + manifest.add_pattern_copy(res, '.**', 'res') + + if project.assets: + manifest.add_symlink(mozpath.join(srcdir, project.assets), 'assets') + + for cpe in project._classpathentries: + manifest.add_symlink(mozpath.join(srcdir, cpe.srcdir), cpe.dstdir) + + # JARs and native libraries go in the same place. For now, we're adding + # class path entries with the full path to required JAR files (which + # makes sense for JARs in the source directory, but probably doesn't for + # JARs in the object directory). This could be a problem because we only + # know the contents of (a subdirectory of) libs/ after a successful + # build and package, which is after build-backend time. At the cost of + # some flexibility, we explicitly copy certain libraries here; if the + # libraries aren't present -- namely, when the tree hasn't been packaged + # -- this fails. That's by design, to avoid crashes on device caused by + # missing native libraries. + for src, dst in project.libs: + manifest.add_copy(mozpath.join(srcdir, src), dst) + + return manifest + + def _process_android_eclipse_project_data(self, data, srcdir, objdir): + # This can't be relative to the environment's topsrcdir, + # because during testing topsrcdir is faked. + template_directory = os.path.abspath(mozpath.join(os.path.dirname(__file__), + 'templates', 'android_eclipse')) + + project_directory = mozpath.join(self.environment.topobjdir, 'android_eclipse', data.name) + manifest_path = mozpath.join(self.environment.topobjdir, 'android_eclipse', '%s.manifest' % data.name) + + manifest = self._manifest_for_project(srcdir, data) + ensureParentDir(manifest_path) + manifest.write(path=manifest_path) + + classpathentries = [] + for cpe in sorted(data._classpathentries, key=lambda x: x.path): + e = self._Element_for_classpathentry(cpe) + classpathentries.append(ET.tostring(e)) + + for name in sorted(data.referenced_projects): + e = self._Element_for_referenced_project(name) + classpathentries.append(ET.tostring(e)) + + for name in sorted(data.extra_jars): + e = self._Element_for_extra_jar(mozpath.join(srcdir, name)) + classpathentries.append(ET.tostring(e)) + + defines = {} + defines['IDE_OBJDIR'] = objdir + defines['IDE_TOPOBJDIR'] = self.environment.topobjdir + defines['IDE_SRCDIR'] = srcdir + defines['IDE_TOPSRCDIR'] = self.environment.topsrcdir + defines['IDE_PROJECT_NAME'] = data.name + defines['IDE_PACKAGE_NAME'] = data.package_name + defines['IDE_PROJECT_DIRECTORY'] = project_directory + defines['IDE_RELSRCDIR'] = mozpath.relpath(srcdir, self.environment.topsrcdir) + defines['IDE_CLASSPATH_ENTRIES'] = '\n'.join('\t' + cpe for cpe in classpathentries) + defines['IDE_RECURSIVE_MAKE_TARGETS'] = ' '.join(sorted(data.recursive_make_targets)) + # Like android.library=true + defines['IDE_PROJECT_LIBRARY_SETTING'] = 'android.library=true' if data.is_library else '' + # Like android.library.reference.1=FennecBrandingResources + defines['IDE_PROJECT_LIBRARY_REFERENCES'] = '\n'.join( + 'android.library.reference.%s=%s' % (i + 1, ref) + for i, ref in enumerate(sorted(data.included_projects))) + if data.filtered_resources: + filteredResources = self._Element_for_filtered_resources(data.filtered_resources) + defines['IDE_PROJECT_FILTERED_RESOURCES'] = pretty_print(filteredResources).strip() + else: + defines['IDE_PROJECT_FILTERED_RESOURCES'] = '' + defines['ANDROID_TARGET_SDK'] = self.environment.substs['ANDROID_TARGET_SDK'] + defines['MOZ_ANDROID_MIN_SDK_VERSION'] = self.environment.defines['MOZ_ANDROID_MIN_SDK_VERSION'] + + copier = FileCopier() + finder = FileFinder(template_directory) + for input_filename, f in itertools.chain(finder.find('**'), finder.find('.**')): + if input_filename == 'AndroidManifest.xml' and not data.is_library: + # Main projects supply their own manifests. + continue + copier.add(input_filename, PreprocessedFile( + mozpath.join(finder.base, input_filename), + depfile_path=None, + marker='#', + defines=defines, + extra_depends={mozpath.join(finder.base, input_filename)})) + + # When we re-create the build backend, we kill everything that was there. + if os.path.isdir(project_directory): + self._updated_count += 1 + else: + self._created_count += 1 + copier.copy(project_directory, skip_if_older=False, remove_unaccounted=True) diff --git a/python/mozbuild/mozbuild/backend/base.py b/python/mozbuild/mozbuild/backend/base.py new file mode 100644 index 000000000..f5e0c2d3c --- /dev/null +++ b/python/mozbuild/mozbuild/backend/base.py @@ -0,0 +1,317 @@ +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, You can obtain one at http://mozilla.org/MPL/2.0/. + +from __future__ import absolute_import, unicode_literals + +from abc import ( + ABCMeta, + abstractmethod, +) + +import errno +import itertools +import os +import time + +from contextlib import contextmanager + +from mach.mixin.logging import LoggingMixin + +import mozpack.path as mozpath +from ..preprocessor import Preprocessor +from ..pythonutil import iter_modules_in_path +from ..util import ( + FileAvoidWrite, + simple_diff, +) +from ..frontend.data import ContextDerived +from .configenvironment import ConfigEnvironment +from mozbuild.base import ExecutionSummary + + +class BuildBackend(LoggingMixin): + """Abstract base class for build backends. + + A build backend is merely a consumer of the build configuration (the output + of the frontend processing). It does something with said data. What exactly + is the discretion of the specific implementation. + """ + + __metaclass__ = ABCMeta + + def __init__(self, environment): + assert isinstance(environment, ConfigEnvironment) + + self.populate_logger() + + self.environment = environment + + # Files whose modification should cause a new read and backend + # generation. + self.backend_input_files = set() + + # Files generated by the backend. + self._backend_output_files = set() + + self._environments = {} + self._environments[environment.topobjdir] = environment + + # The number of backend files created. + self._created_count = 0 + + # The number of backend files updated. + self._updated_count = 0 + + # The number of unchanged backend files. + self._unchanged_count = 0 + + # The number of deleted backend files. + self._deleted_count = 0 + + # The total wall time spent in the backend. This counts the time the + # backend writes out files, etc. + self._execution_time = 0.0 + + # Mapping of changed file paths to diffs of the changes. + self.file_diffs = {} + + self.dry_run = False + + self._init() + + def summary(self): + return ExecutionSummary( + self.__class__.__name__.replace('Backend', '') + + ' backend executed in {execution_time:.2f}s\n ' + '{total:d} total backend files; ' + '{created:d} created; ' + '{updated:d} updated; ' + '{unchanged:d} unchanged; ' + '{deleted:d} deleted', + execution_time=self._execution_time, + total=self._created_count + self._updated_count + + self._unchanged_count, + created=self._created_count, + updated=self._updated_count, + unchanged=self._unchanged_count, + deleted=self._deleted_count) + + def _init(self): + """Hook point for child classes to perform actions during __init__. + + This exists so child classes don't need to implement __init__. + """ + + def consume(self, objs): + """Consume a stream of TreeMetadata instances. + + This is the main method of the interface. This is what takes the + frontend output and does something with it. + + Child classes are not expected to implement this method. Instead, the + base class consumes objects and calls methods (possibly) implemented by + child classes. + """ + + # Previously generated files. + list_file = mozpath.join(self.environment.topobjdir, 'backend.%s' + % self.__class__.__name__) + backend_output_list = set() + if os.path.exists(list_file): + with open(list_file) as fh: + backend_output_list.update(mozpath.normsep(p) + for p in fh.read().splitlines()) + + for obj in objs: + obj_start = time.time() + if (not self.consume_object(obj) and + not isinstance(self, PartialBackend)): + raise Exception('Unhandled object of type %s' % type(obj)) + self._execution_time += time.time() - obj_start + + if (isinstance(obj, ContextDerived) and + not isinstance(self, PartialBackend)): + self.backend_input_files |= obj.context_all_paths + + # Pull in all loaded Python as dependencies so any Python changes that + # could influence our output result in a rescan. + self.backend_input_files |= set(iter_modules_in_path( + self.environment.topsrcdir, self.environment.topobjdir)) + + finished_start = time.time() + self.consume_finished() + self._execution_time += time.time() - finished_start + + # Purge backend files created in previous run, but not created anymore + delete_files = backend_output_list - self._backend_output_files + for path in delete_files: + full_path = mozpath.join(self.environment.topobjdir, path) + try: + with open(full_path, 'r') as existing: + old_content = existing.read() + if old_content: + self.file_diffs[full_path] = simple_diff( + full_path, old_content.splitlines(), None) + except IOError: + pass + try: + if not self.dry_run: + os.unlink(full_path) + self._deleted_count += 1 + except OSError: + pass + # Remove now empty directories + for dir in set(mozpath.dirname(d) for d in delete_files): + try: + os.removedirs(dir) + except OSError: + pass + + # Write out the list of backend files generated, if it changed. + if self._deleted_count or self._created_count or \ + not os.path.exists(list_file): + with self._write_file(list_file) as fh: + fh.write('\n'.join(sorted(self._backend_output_files))) + else: + # Always update its mtime. + with open(list_file, 'a'): + os.utime(list_file, None) + + # Write out the list of input files for the backend + with self._write_file('%s.in' % list_file) as fh: + fh.write('\n'.join(sorted( + mozpath.normsep(f) for f in self.backend_input_files))) + + @abstractmethod + def consume_object(self, obj): + """Consumes an individual TreeMetadata instance. + + This is the main method used by child classes to react to build + metadata. + """ + + def consume_finished(self): + """Called when consume() has completed handling all objects.""" + + def build(self, config, output, jobs, verbose): + """Called when 'mach build' is executed. + + This should return the status value of a subprocess, where 0 denotes + success and any other value is an error code. A return value of None + indicates that the default 'make -f client.mk' should run. + """ + return None + + @contextmanager + def _write_file(self, path=None, fh=None, mode='rU'): + """Context manager to write a file. + + This is a glorified wrapper around FileAvoidWrite with integration to + update the summary data on this instance. + + Example usage: + + with self._write_file('foo.txt') as fh: + fh.write('hello world') + """ + + if path is not None: + assert fh is None + fh = FileAvoidWrite(path, capture_diff=True, dry_run=self.dry_run, + mode=mode) + else: + assert fh is not None + + dirname = mozpath.dirname(fh.name) + try: + os.makedirs(dirname) + except OSError as error: + if error.errno != errno.EEXIST: + raise + + yield fh + + self._backend_output_files.add(mozpath.relpath(fh.name, self.environment.topobjdir)) + existed, updated = fh.close() + if fh.diff: + self.file_diffs[fh.name] = fh.diff + if not existed: + self._created_count += 1 + elif updated: + self._updated_count += 1 + else: + self._unchanged_count += 1 + + @contextmanager + def _get_preprocessor(self, obj): + '''Returns a preprocessor with a few predefined values depending on + the given BaseConfigSubstitution(-like) object, and all the substs + in the current environment.''' + pp = Preprocessor() + srcdir = mozpath.dirname(obj.input_path) + pp.context.update({ + k: ' '.join(v) if isinstance(v, list) else v + for k, v in obj.config.substs.iteritems() + }) + pp.context.update( + top_srcdir=obj.topsrcdir, + topobjdir=obj.topobjdir, + srcdir=srcdir, + relativesrcdir=mozpath.relpath(srcdir, obj.topsrcdir) or '.', + DEPTH=mozpath.relpath(obj.topobjdir, mozpath.dirname(obj.output_path)) or '.', + ) + pp.do_filter('attemptSubstitution') + pp.setMarker(None) + with self._write_file(obj.output_path) as fh: + pp.out = fh + yield pp + + +class PartialBackend(BuildBackend): + """A PartialBackend is a BuildBackend declaring that its consume_object + method may not handle all build configuration objects it's passed, and + that it's fine.""" + + +def HybridBackend(*backends): + """A HybridBackend is the combination of one or more PartialBackends + with a non-partial BuildBackend. + + Build configuration objects are passed to each backend, stopping at the + first of them that declares having handled them. + """ + assert len(backends) >= 2 + assert all(issubclass(b, PartialBackend) for b in backends[:-1]) + assert not(issubclass(backends[-1], PartialBackend)) + assert all(issubclass(b, BuildBackend) for b in backends) + + class TheHybridBackend(BuildBackend): + def __init__(self, environment): + self._backends = [b(environment) for b in backends] + super(TheHybridBackend, self).__init__(environment) + + def consume_object(self, obj): + return any(b.consume_object(obj) for b in self._backends) + + def consume_finished(self): + for backend in self._backends: + backend.consume_finished() + + for attr in ('_execution_time', '_created_count', '_updated_count', + '_unchanged_count', '_deleted_count'): + setattr(self, attr, + sum(getattr(b, attr) for b in self._backends)) + + for b in self._backends: + self.file_diffs.update(b.file_diffs) + for attr in ('backend_input_files', '_backend_output_files'): + files = getattr(self, attr) + files |= getattr(b, attr) + + name = '+'.join(itertools.chain( + (b.__name__.replace('Backend', '') for b in backends[:1]), + (b.__name__ for b in backends[-1:]) + )) + + return type(str(name), (TheHybridBackend,), {}) diff --git a/python/mozbuild/mozbuild/backend/common.py b/python/mozbuild/mozbuild/backend/common.py new file mode 100644 index 000000000..12b2a27c4 --- /dev/null +++ b/python/mozbuild/mozbuild/backend/common.py @@ -0,0 +1,567 @@ +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, You can obtain one at http://mozilla.org/MPL/2.0/. + +from __future__ import absolute_import, unicode_literals + +import cPickle as pickle +import itertools +import json +import os + +import mozpack.path as mozpath + +from mozbuild.backend.base import BuildBackend + +from mozbuild.frontend.context import ( + Context, + Path, + RenamedSourcePath, + VARIABLES, +) +from mozbuild.frontend.data import ( + BaseProgram, + ChromeManifestEntry, + ConfigFileSubstitution, + ExampleWebIDLInterface, + IPDLFile, + FinalTargetPreprocessedFiles, + FinalTargetFiles, + GeneratedEventWebIDLFile, + GeneratedWebIDLFile, + PreprocessedTestWebIDLFile, + PreprocessedWebIDLFile, + SharedLibrary, + TestManifest, + TestWebIDLFile, + UnifiedSources, + XPIDLFile, + WebIDLFile, +) +from mozbuild.jar import ( + DeprecatedJarManifest, + JarManifestParser, +) +from mozbuild.preprocessor import Preprocessor +from mozpack.chrome.manifest import parse_manifest_line + +from collections import defaultdict + +from mozbuild.util import group_unified_files + +class XPIDLManager(object): + """Helps manage XPCOM IDLs in the context of the build system.""" + def __init__(self, config): + self.config = config + self.topsrcdir = config.topsrcdir + self.topobjdir = config.topobjdir + + self.idls = {} + self.modules = {} + self.interface_manifests = {} + self.chrome_manifests = set() + + def register_idl(self, idl, allow_existing=False): + """Registers an IDL file with this instance. + + The IDL file will be built, installed, etc. + """ + basename = mozpath.basename(idl.source_path) + root = mozpath.splitext(basename)[0] + xpt = '%s.xpt' % idl.module + manifest = mozpath.join(idl.install_target, 'components', 'interfaces.manifest') + chrome_manifest = mozpath.join(idl.install_target, 'chrome.manifest') + + entry = { + 'source': idl.source_path, + 'module': idl.module, + 'basename': basename, + 'root': root, + 'manifest': manifest, + } + + if not allow_existing and entry['basename'] in self.idls: + raise Exception('IDL already registered: %s' % entry['basename']) + + self.idls[entry['basename']] = entry + t = self.modules.setdefault(entry['module'], (idl.install_target, set())) + t[1].add(entry['root']) + + if idl.add_to_manifest: + self.interface_manifests.setdefault(manifest, set()).add(xpt) + self.chrome_manifests.add(chrome_manifest) + + +class WebIDLCollection(object): + """Collects WebIDL info referenced during the build.""" + + def __init__(self): + self.sources = set() + self.generated_sources = set() + self.generated_events_sources = set() + self.preprocessed_sources = set() + self.test_sources = set() + self.preprocessed_test_sources = set() + self.example_interfaces = set() + + def all_regular_sources(self): + return self.sources | self.generated_sources | \ + self.generated_events_sources | self.preprocessed_sources + + def all_regular_basenames(self): + return [os.path.basename(source) for source in self.all_regular_sources()] + + def all_regular_stems(self): + return [os.path.splitext(b)[0] for b in self.all_regular_basenames()] + + def all_regular_bindinggen_stems(self): + for stem in self.all_regular_stems(): + yield '%sBinding' % stem + + for source in self.generated_events_sources: + yield os.path.splitext(os.path.basename(source))[0] + + def all_regular_cpp_basenames(self): + for stem in self.all_regular_bindinggen_stems(): + yield '%s.cpp' % stem + + def all_test_sources(self): + return self.test_sources | self.preprocessed_test_sources + + def all_test_basenames(self): + return [os.path.basename(source) for source in self.all_test_sources()] + + def all_test_stems(self): + return [os.path.splitext(b)[0] for b in self.all_test_basenames()] + + def all_test_cpp_basenames(self): + return ['%sBinding.cpp' % s for s in self.all_test_stems()] + + def all_static_sources(self): + return self.sources | self.generated_events_sources | \ + self.test_sources + + def all_non_static_sources(self): + return self.generated_sources | self.all_preprocessed_sources() + + def all_non_static_basenames(self): + return [os.path.basename(s) for s in self.all_non_static_sources()] + + def all_preprocessed_sources(self): + return self.preprocessed_sources | self.preprocessed_test_sources + + def all_sources(self): + return set(self.all_regular_sources()) | set(self.all_test_sources()) + + def all_basenames(self): + return [os.path.basename(source) for source in self.all_sources()] + + def all_stems(self): + return [os.path.splitext(b)[0] for b in self.all_basenames()] + + def generated_events_basenames(self): + return [os.path.basename(s) for s in self.generated_events_sources] + + def generated_events_stems(self): + return [os.path.splitext(b)[0] for b in self.generated_events_basenames()] + + +class TestManager(object): + """Helps hold state related to tests.""" + + def __init__(self, config): + self.config = config + self.topsrcdir = mozpath.normpath(config.topsrcdir) + + self.tests_by_path = defaultdict(list) + self.installs_by_path = defaultdict(list) + self.deferred_installs = set() + self.manifest_defaults = {} + + def add(self, t, flavor, topsrcdir): + t = dict(t) + t['flavor'] = flavor + + path = mozpath.normpath(t['path']) + assert mozpath.basedir(path, [topsrcdir]) + + key = path[len(topsrcdir)+1:] + t['file_relpath'] = key + t['dir_relpath'] = mozpath.dirname(key) + + self.tests_by_path[key].append(t) + + def add_defaults(self, manifest): + if not hasattr(manifest, 'manifest_defaults'): + return + for sub_manifest, defaults in manifest.manifest_defaults.items(): + self.manifest_defaults[sub_manifest] = defaults + + def add_installs(self, obj, topsrcdir): + for src, (dest, _) in obj.installs.iteritems(): + key = src[len(topsrcdir)+1:] + self.installs_by_path[key].append((src, dest)) + for src, pat, dest in obj.pattern_installs: + key = mozpath.join(src[len(topsrcdir)+1:], pat) + self.installs_by_path[key].append((src, pat, dest)) + for path in obj.deferred_installs: + self.deferred_installs.add(path[2:]) + + +class BinariesCollection(object): + """Tracks state of binaries produced by the build.""" + + def __init__(self): + self.shared_libraries = [] + self.programs = [] + + +class CommonBackend(BuildBackend): + """Holds logic common to all build backends.""" + + def _init(self): + self._idl_manager = XPIDLManager(self.environment) + self._test_manager = TestManager(self.environment) + self._webidls = WebIDLCollection() + self._binaries = BinariesCollection() + self._configs = set() + self._ipdl_sources = set() + + def consume_object(self, obj): + self._configs.add(obj.config) + + if isinstance(obj, TestManifest): + for test in obj.tests: + self._test_manager.add(test, obj.flavor, obj.topsrcdir) + self._test_manager.add_defaults(obj.manifest) + self._test_manager.add_installs(obj, obj.topsrcdir) + + elif isinstance(obj, XPIDLFile): + # TODO bug 1240134 tracks not processing XPIDL files during + # artifact builds. + self._idl_manager.register_idl(obj) + + elif isinstance(obj, ConfigFileSubstitution): + # Do not handle ConfigFileSubstitution for Makefiles. Leave that + # to other + if mozpath.basename(obj.output_path) == 'Makefile': + return False + with self._get_preprocessor(obj) as pp: + pp.do_include(obj.input_path) + self.backend_input_files.add(obj.input_path) + + # We should consider aggregating WebIDL types in emitter.py. + elif isinstance(obj, WebIDLFile): + # WebIDL isn't relevant to artifact builds. + if self.environment.is_artifact_build: + return True + + self._webidls.sources.add(mozpath.join(obj.srcdir, obj.basename)) + + elif isinstance(obj, GeneratedEventWebIDLFile): + # WebIDL isn't relevant to artifact builds. + if self.environment.is_artifact_build: + return True + + self._webidls.generated_events_sources.add(mozpath.join( + obj.srcdir, obj.basename)) + + elif isinstance(obj, TestWebIDLFile): + # WebIDL isn't relevant to artifact builds. + if self.environment.is_artifact_build: + return True + + self._webidls.test_sources.add(mozpath.join(obj.srcdir, + obj.basename)) + + elif isinstance(obj, PreprocessedTestWebIDLFile): + # WebIDL isn't relevant to artifact builds. + if self.environment.is_artifact_build: + return True + + self._webidls.preprocessed_test_sources.add(mozpath.join( + obj.srcdir, obj.basename)) + + elif isinstance(obj, GeneratedWebIDLFile): + # WebIDL isn't relevant to artifact builds. + if self.environment.is_artifact_build: + return True + + self._webidls.generated_sources.add(mozpath.join(obj.srcdir, + obj.basename)) + + elif isinstance(obj, PreprocessedWebIDLFile): + # WebIDL isn't relevant to artifact builds. + if self.environment.is_artifact_build: + return True + + self._webidls.preprocessed_sources.add(mozpath.join( + obj.srcdir, obj.basename)) + + elif isinstance(obj, ExampleWebIDLInterface): + # WebIDL isn't relevant to artifact builds. + if self.environment.is_artifact_build: + return True + + self._webidls.example_interfaces.add(obj.name) + + elif isinstance(obj, IPDLFile): + # IPDL isn't relevant to artifact builds. + if self.environment.is_artifact_build: + return True + + self._ipdl_sources.add(mozpath.join(obj.srcdir, obj.basename)) + + elif isinstance(obj, UnifiedSources): + # Unified sources aren't relevant to artifact builds. + if self.environment.is_artifact_build: + return True + + if obj.have_unified_mapping: + self._write_unified_files(obj.unified_source_mapping, obj.objdir) + if hasattr(self, '_process_unified_sources'): + self._process_unified_sources(obj) + + elif isinstance(obj, BaseProgram): + self._binaries.programs.append(obj) + return False + + elif isinstance(obj, SharedLibrary): + self._binaries.shared_libraries.append(obj) + return False + + else: + return False + + return True + + def consume_finished(self): + if len(self._idl_manager.idls): + self._handle_idl_manager(self._idl_manager) + + self._handle_webidl_collection(self._webidls) + + sorted_ipdl_sources = list(sorted(self._ipdl_sources)) + + def files_from(ipdl): + base = mozpath.basename(ipdl) + root, ext = mozpath.splitext(base) + + # Both .ipdl and .ipdlh become .cpp files + files = ['%s.cpp' % root] + if ext == '.ipdl': + # .ipdl also becomes Child/Parent.cpp files + files.extend(['%sChild.cpp' % root, + '%sParent.cpp' % root]) + return files + + ipdl_dir = mozpath.join(self.environment.topobjdir, 'ipc', 'ipdl') + + ipdl_cppsrcs = list(itertools.chain(*[files_from(p) for p in sorted_ipdl_sources])) + unified_source_mapping = list(group_unified_files(ipdl_cppsrcs, + unified_prefix='UnifiedProtocols', + unified_suffix='cpp', + files_per_unified_file=16)) + + self._write_unified_files(unified_source_mapping, ipdl_dir, poison_windows_h=False) + self._handle_ipdl_sources(ipdl_dir, sorted_ipdl_sources, unified_source_mapping) + + for config in self._configs: + self.backend_input_files.add(config.source) + + # Write out a machine-readable file describing every test. + topobjdir = self.environment.topobjdir + with self._write_file(mozpath.join(topobjdir, 'all-tests.pkl'), mode='rb') as fh: + pickle.dump(dict(self._test_manager.tests_by_path), fh, protocol=2) + + with self._write_file(mozpath.join(topobjdir, 'test-defaults.pkl'), mode='rb') as fh: + pickle.dump(self._test_manager.manifest_defaults, fh, protocol=2) + + path = mozpath.join(self.environment.topobjdir, 'test-installs.pkl') + with self._write_file(path, mode='rb') as fh: + pickle.dump({k: v for k, v in self._test_manager.installs_by_path.items() + if k in self._test_manager.deferred_installs}, + fh, + protocol=2) + + # Write out a machine-readable file describing binaries. + with self._write_file(mozpath.join(topobjdir, 'binaries.json')) as fh: + d = { + 'shared_libraries': [s.to_dict() for s in self._binaries.shared_libraries], + 'programs': [p.to_dict() for p in self._binaries.programs], + } + json.dump(d, fh, sort_keys=True, indent=4) + + def _handle_webidl_collection(self, webidls): + if not webidls.all_stems(): + return + + bindings_dir = mozpath.join(self.environment.topobjdir, 'dom', 'bindings') + + all_inputs = set(webidls.all_static_sources()) + for s in webidls.all_non_static_basenames(): + all_inputs.add(mozpath.join(bindings_dir, s)) + + generated_events_stems = webidls.generated_events_stems() + exported_stems = webidls.all_regular_stems() + + # The WebIDL manager reads configuration from a JSON file. So, we + # need to write this file early. + o = dict( + webidls=sorted(all_inputs), + generated_events_stems=sorted(generated_events_stems), + exported_stems=sorted(exported_stems), + example_interfaces=sorted(webidls.example_interfaces), + ) + + file_lists = mozpath.join(bindings_dir, 'file-lists.json') + with self._write_file(file_lists) as fh: + json.dump(o, fh, sort_keys=True, indent=2) + + import mozwebidlcodegen + + manager = mozwebidlcodegen.create_build_system_manager( + self.environment.topsrcdir, + self.environment.topobjdir, + mozpath.join(self.environment.topobjdir, 'dist') + ) + + # Bindings are compiled in unified mode to speed up compilation and + # to reduce linker memory size. Note that test bindings are separated + # from regular ones so tests bindings aren't shipped. + unified_source_mapping = list(group_unified_files(webidls.all_regular_cpp_basenames(), + unified_prefix='UnifiedBindings', + unified_suffix='cpp', + files_per_unified_file=32)) + self._write_unified_files(unified_source_mapping, bindings_dir, + poison_windows_h=True) + self._handle_webidl_build(bindings_dir, unified_source_mapping, + webidls, + manager.expected_build_output_files(), + manager.GLOBAL_DEFINE_FILES) + + def _write_unified_file(self, unified_file, source_filenames, + output_directory, poison_windows_h=False): + with self._write_file(mozpath.join(output_directory, unified_file)) as f: + f.write('#define MOZ_UNIFIED_BUILD\n') + includeTemplate = '#include "%(cppfile)s"' + if poison_windows_h: + includeTemplate += ( + '\n' + '#ifdef _WINDOWS_\n' + '#error "%(cppfile)s included windows.h"\n' + "#endif") + includeTemplate += ( + '\n' + '#ifdef PL_ARENA_CONST_ALIGN_MASK\n' + '#error "%(cppfile)s uses PL_ARENA_CONST_ALIGN_MASK, ' + 'so it cannot be built in unified mode."\n' + '#undef PL_ARENA_CONST_ALIGN_MASK\n' + '#endif\n' + '#ifdef INITGUID\n' + '#error "%(cppfile)s defines INITGUID, ' + 'so it cannot be built in unified mode."\n' + '#undef INITGUID\n' + '#endif') + f.write('\n'.join(includeTemplate % { "cppfile": s } for + s in source_filenames)) + + def _write_unified_files(self, unified_source_mapping, output_directory, + poison_windows_h=False): + for unified_file, source_filenames in unified_source_mapping: + self._write_unified_file(unified_file, source_filenames, + output_directory, poison_windows_h) + + def _consume_jar_manifest(self, obj): + # Ideally, this would all be handled somehow in the emitter, but + # this would require all the magic surrounding l10n and addons in + # the recursive make backend to die, which is not going to happen + # any time soon enough. + # Notably missing: + # - DEFINES from config/config.mk + # - L10n support + # - The equivalent of -e when USE_EXTENSION_MANIFEST is set in + # moz.build, but it doesn't matter in dist/bin. + pp = Preprocessor() + if obj.defines: + pp.context.update(obj.defines.defines) + pp.context.update(self.environment.defines) + pp.context.update( + AB_CD='en-US', + BUILD_FASTER=1, + ) + pp.out = JarManifestParser() + try: + pp.do_include(obj.path.full_path) + except DeprecatedJarManifest as e: + raise DeprecatedJarManifest('Parsing error while processing %s: %s' + % (obj.path.full_path, e.message)) + self.backend_input_files |= pp.includes + + for jarinfo in pp.out: + jar_context = Context( + allowed_variables=VARIABLES, config=obj._context.config) + jar_context.push_source(obj._context.main_path) + jar_context.push_source(obj.path.full_path) + + install_target = obj.install_target + if jarinfo.base: + install_target = mozpath.normpath( + mozpath.join(install_target, jarinfo.base)) + jar_context['FINAL_TARGET'] = install_target + if obj.defines: + jar_context['DEFINES'] = obj.defines.defines + files = jar_context['FINAL_TARGET_FILES'] + files_pp = jar_context['FINAL_TARGET_PP_FILES'] + + for e in jarinfo.entries: + if e.is_locale: + if jarinfo.relativesrcdir: + src = '/%s' % jarinfo.relativesrcdir + else: + src = '' + src = mozpath.join(src, 'en-US', e.source) + else: + src = e.source + + src = Path(jar_context, src) + + if '*' not in e.source and not os.path.exists(src.full_path): + if e.is_locale: + raise Exception( + '%s: Cannot find %s' % (obj.path, e.source)) + if e.source.startswith('/'): + src = Path(jar_context, '!' + e.source) + else: + # This actually gets awkward if the jar.mn is not + # in the same directory as the moz.build declaring + # it, but it's how it works in the recursive make, + # not that anything relies on that, but it's simpler. + src = Path(obj._context, '!' + e.source) + + output_basename = mozpath.basename(e.output) + if output_basename != src.target_basename: + src = RenamedSourcePath(jar_context, + (src, output_basename)) + path = mozpath.dirname(mozpath.join(jarinfo.name, e.output)) + + if e.preprocess: + if '*' in e.source: + raise Exception('%s: Wildcards are not supported with ' + 'preprocessing' % obj.path) + files_pp[path] += [src] + else: + files[path] += [src] + + if files: + self.consume_object(FinalTargetFiles(jar_context, files)) + if files_pp: + self.consume_object( + FinalTargetPreprocessedFiles(jar_context, files_pp)) + + for m in jarinfo.chrome_manifests: + entry = parse_manifest_line( + mozpath.dirname(jarinfo.name), + m.replace('%', mozpath.basename(jarinfo.name) + '/')) + self.consume_object(ChromeManifestEntry( + jar_context, '%s.manifest' % jarinfo.name, entry)) diff --git a/python/mozbuild/mozbuild/backend/configenvironment.py b/python/mozbuild/mozbuild/backend/configenvironment.py new file mode 100644 index 000000000..331309af6 --- /dev/null +++ b/python/mozbuild/mozbuild/backend/configenvironment.py @@ -0,0 +1,199 @@ +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, You can obtain one at http://mozilla.org/MPL/2.0/. + +from __future__ import absolute_import + +import os +import sys + +from collections import Iterable +from types import StringTypes, ModuleType + +import mozpack.path as mozpath + +from mozbuild.util import ReadOnlyDict +from mozbuild.shellutil import quote as shell_quote + + +if sys.version_info.major == 2: + text_type = unicode +else: + text_type = str + + +class BuildConfig(object): + """Represents the output of configure.""" + + _CODE_CACHE = {} + + def __init__(self): + self.topsrcdir = None + self.topobjdir = None + self.defines = {} + self.non_global_defines = [] + self.substs = {} + self.files = [] + self.mozconfig = None + + @classmethod + def from_config_status(cls, path): + """Create an instance from a config.status file.""" + code_cache = cls._CODE_CACHE + mtime = os.path.getmtime(path) + + # cache the compiled code as it can be reused + # we cache it the first time, or if the file changed + if not path in code_cache or code_cache[path][0] != mtime: + # Add config.status manually to sys.modules so it gets picked up by + # iter_modules_in_path() for automatic dependencies. + mod = ModuleType('config.status') + mod.__file__ = path + sys.modules['config.status'] = mod + + with open(path, 'rt') as fh: + source = fh.read() + code_cache[path] = ( + mtime, + compile(source, path, 'exec', dont_inherit=1) + ) + + g = { + '__builtins__': __builtins__, + '__file__': path, + } + l = {} + exec(code_cache[path][1], g, l) + + config = BuildConfig() + + for name in l['__all__']: + setattr(config, name, l[name]) + + return config + + +class ConfigEnvironment(object): + """Perform actions associated with a configured but bare objdir. + + The purpose of this class is to preprocess files from the source directory + and output results in the object directory. + + There are two types of files: config files and config headers, + each treated through a different member function. + + Creating a ConfigEnvironment requires a few arguments: + - topsrcdir and topobjdir are, respectively, the top source and + the top object directory. + - defines is a dict filled from AC_DEFINE and AC_DEFINE_UNQUOTED in + autoconf. + - non_global_defines are a list of names appearing in defines above + that are not meant to be exported in ACDEFINES (see below) + - substs is a dict filled from AC_SUBST in autoconf. + + ConfigEnvironment automatically defines one additional substs variable + from all the defines not appearing in non_global_defines: + - ACDEFINES contains the defines in the form -DNAME=VALUE, for use on + preprocessor command lines. The order in which defines were given + when creating the ConfigEnvironment is preserved. + and two other additional subst variables from all the other substs: + - ALLSUBSTS contains the substs in the form NAME = VALUE, in sorted + order, for use in autoconf.mk. It includes ACDEFINES + Only substs with a VALUE are included, such that the resulting file + doesn't change when new empty substs are added. + This results in less invalidation of build dependencies in the case + of autoconf.mk.. + - ALLEMPTYSUBSTS contains the substs with an empty value, in the form + NAME =. + + ConfigEnvironment expects a "top_srcdir" subst to be set with the top + source directory, in msys format on windows. It is used to derive a + "srcdir" subst when treating config files. It can either be an absolute + path or a path relative to the topobjdir. + """ + + def __init__(self, topsrcdir, topobjdir, defines=None, + non_global_defines=None, substs=None, source=None, mozconfig=None): + + if not source: + source = mozpath.join(topobjdir, 'config.status') + self.source = source + self.defines = ReadOnlyDict(defines or {}) + self.non_global_defines = non_global_defines or [] + self.substs = dict(substs or {}) + self.topsrcdir = mozpath.abspath(topsrcdir) + self.topobjdir = mozpath.abspath(topobjdir) + self.mozconfig = mozpath.abspath(mozconfig) if mozconfig else None + self.lib_prefix = self.substs.get('LIB_PREFIX', '') + if 'LIB_SUFFIX' in self.substs: + self.lib_suffix = '.%s' % self.substs['LIB_SUFFIX'] + self.dll_prefix = self.substs.get('DLL_PREFIX', '') + self.dll_suffix = self.substs.get('DLL_SUFFIX', '') + if self.substs.get('IMPORT_LIB_SUFFIX'): + self.import_prefix = self.lib_prefix + self.import_suffix = '.%s' % self.substs['IMPORT_LIB_SUFFIX'] + else: + self.import_prefix = self.dll_prefix + self.import_suffix = self.dll_suffix + + global_defines = [name for name in self.defines + if not name in self.non_global_defines] + self.substs['ACDEFINES'] = ' '.join(['-D%s=%s' % (name, + shell_quote(self.defines[name]).replace('$', '$$')) + for name in sorted(global_defines)]) + def serialize(obj): + if isinstance(obj, StringTypes): + return obj + if isinstance(obj, Iterable): + return ' '.join(obj) + raise Exception('Unhandled type %s', type(obj)) + self.substs['ALLSUBSTS'] = '\n'.join(sorted(['%s = %s' % (name, + serialize(self.substs[name])) for name in self.substs if self.substs[name]])) + self.substs['ALLEMPTYSUBSTS'] = '\n'.join(sorted(['%s =' % name + for name in self.substs if not self.substs[name]])) + + self.substs = ReadOnlyDict(self.substs) + + self.external_source_dir = None + external = self.substs.get('EXTERNAL_SOURCE_DIR', '') + if external: + external = mozpath.normpath(external) + if not os.path.isabs(external): + external = mozpath.join(self.topsrcdir, external) + self.external_source_dir = mozpath.normpath(external) + + # Populate a Unicode version of substs. This is an optimization to make + # moz.build reading faster, since each sandbox needs a Unicode version + # of these variables and doing it over a thousand times is a hotspot + # during sandbox execution! + # Bug 844509 tracks moving everything to Unicode. + self.substs_unicode = {} + + def decode(v): + if not isinstance(v, text_type): + try: + return v.decode('utf-8') + except UnicodeDecodeError: + return v.decode('utf-8', 'replace') + + for k, v in self.substs.items(): + if not isinstance(v, StringTypes): + if isinstance(v, Iterable): + type(v)(decode(i) for i in v) + elif not isinstance(v, text_type): + v = decode(v) + + self.substs_unicode[k] = v + + self.substs_unicode = ReadOnlyDict(self.substs_unicode) + + @property + def is_artifact_build(self): + return self.substs.get('MOZ_ARTIFACT_BUILDS', False) + + @staticmethod + def from_config_status(path): + config = BuildConfig.from_config_status(path) + + return ConfigEnvironment(config.topsrcdir, config.topobjdir, + config.defines, config.non_global_defines, config.substs, path) diff --git a/python/mozbuild/mozbuild/backend/cpp_eclipse.py b/python/mozbuild/mozbuild/backend/cpp_eclipse.py new file mode 100644 index 000000000..cbdbdde8c --- /dev/null +++ b/python/mozbuild/mozbuild/backend/cpp_eclipse.py @@ -0,0 +1,698 @@ +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, You can obtain one at http://mozilla.org/MPL/2.0/. + +from __future__ import absolute_import + +import errno +import random +import os +import subprocess +import types +import xml.etree.ElementTree as ET +from .common import CommonBackend + +from ..frontend.data import ( + Defines, +) +from mozbuild.base import ExecutionSummary + +# TODO Have ./mach eclipse generate the workspace and index it: +# /Users/bgirard/mozilla/eclipse/eclipse/eclipse/eclipse -application org.eclipse.cdt.managedbuilder.core.headlessbuild -data $PWD/workspace -importAll $PWD/eclipse +# Open eclipse: +# /Users/bgirard/mozilla/eclipse/eclipse/eclipse/eclipse -data $PWD/workspace + +class CppEclipseBackend(CommonBackend): + """Backend that generates Cpp Eclipse project files. + """ + + def __init__(self, environment): + if os.name == 'nt': + raise Exception('Eclipse is not supported on Windows. ' + 'Consider using Visual Studio instead.') + super(CppEclipseBackend, self).__init__(environment) + + def _init(self): + CommonBackend._init(self) + + self._paths_to_defines = {} + self._project_name = 'Gecko' + self._workspace_dir = self._get_workspace_path() + self._project_dir = os.path.join(self._workspace_dir, self._project_name) + self._overwriting_workspace = os.path.isdir(self._workspace_dir) + + self._macbundle = self.environment.substs['MOZ_MACBUNDLE_NAME'] + self._appname = self.environment.substs['MOZ_APP_NAME'] + self._bin_suffix = self.environment.substs['BIN_SUFFIX'] + self._cxx = self.environment.substs['CXX'] + # Note: We need the C Pre Processor (CPP) flags, not the CXX flags + self._cppflags = self.environment.substs.get('CPPFLAGS', '') + + def summary(self): + return ExecutionSummary( + 'CppEclipse backend executed in {execution_time:.2f}s\n' + 'Generated Cpp Eclipse workspace in "{workspace:s}".\n' + 'If missing, import the project using File > Import > General > Existing Project into workspace\n' + '\n' + 'Run with: eclipse -data {workspace:s}\n', + execution_time=self._execution_time, + workspace=self._workspace_dir) + + def _get_workspace_path(self): + return CppEclipseBackend.get_workspace_path(self.environment.topsrcdir, self.environment.topobjdir) + + @staticmethod + def get_workspace_path(topsrcdir, topobjdir): + # Eclipse doesn't support having the workspace inside the srcdir. + # Since most people have their objdir inside their srcdir it's easier + # and more consistent to just put the workspace along side the srcdir + srcdir_parent = os.path.dirname(topsrcdir) + workspace_dirname = "eclipse_" + os.path.basename(topobjdir) + return os.path.join(srcdir_parent, workspace_dirname) + + def consume_object(self, obj): + reldir = getattr(obj, 'relativedir', None) + + # Note that unlike VS, Eclipse' indexer seem to crawl the headers and + # isn't picky about the local includes. + if isinstance(obj, Defines): + self._paths_to_defines.setdefault(reldir, {}).update(obj.defines) + + return True + + def consume_finished(self): + settings_dir = os.path.join(self._project_dir, '.settings') + launch_dir = os.path.join(self._project_dir, 'RunConfigurations') + workspace_settings_dir = os.path.join(self._workspace_dir, '.metadata/.plugins/org.eclipse.core.runtime/.settings') + workspace_language_dir = os.path.join(self._workspace_dir, '.metadata/.plugins/org.eclipse.cdt.core') + + for dir_name in [self._project_dir, settings_dir, launch_dir, workspace_settings_dir, workspace_language_dir]: + try: + os.makedirs(dir_name) + except OSError as e: + if e.errno != errno.EEXIST: + raise + + project_path = os.path.join(self._project_dir, '.project') + with open(project_path, 'wb') as fh: + self._write_project(fh) + + cproject_path = os.path.join(self._project_dir, '.cproject') + with open(cproject_path, 'wb') as fh: + self._write_cproject(fh) + + language_path = os.path.join(settings_dir, 'language.settings.xml') + with open(language_path, 'wb') as fh: + self._write_language_settings(fh) + + workspace_language_path = os.path.join(workspace_language_dir, 'language.settings.xml') + with open(workspace_language_path, 'wb') as fh: + workspace_lang_settings = WORKSPACE_LANGUAGE_SETTINGS_TEMPLATE + workspace_lang_settings = workspace_lang_settings.replace("@COMPILER_FLAGS@", self._cxx + " " + self._cppflags); + fh.write(workspace_lang_settings) + + self._write_launch_files(launch_dir) + + # This will show up as an 'unmanged' formatter. This can be named by generating + # another file. + formatter_prefs_path = os.path.join(settings_dir, 'org.eclipse.cdt.core.prefs') + with open(formatter_prefs_path, 'wb') as fh: + fh.write(FORMATTER_SETTINGS); + + editor_prefs_path = os.path.join(workspace_settings_dir, "org.eclipse.ui.editors.prefs"); + with open(editor_prefs_path, 'wb') as fh: + fh.write(EDITOR_SETTINGS); + + # Now import the project into the workspace + self._import_project() + + def _import_project(self): + # If the workspace already exists then don't import the project again because + # eclipse doesn't handle this properly + if self._overwriting_workspace: + return + + # We disable the indexer otherwise we're forced to index + # the whole codebase when importing the project. Indexing the project can take 20 minutes. + self._write_noindex() + + try: + process = subprocess.check_call( + ["eclipse", "-application", "-nosplash", + "org.eclipse.cdt.managedbuilder.core.headlessbuild", + "-data", self._workspace_dir, "-importAll", self._project_dir]) + finally: + self._remove_noindex() + + def _write_noindex(self): + noindex_path = os.path.join(self._project_dir, '.settings/org.eclipse.cdt.core.prefs') + with open(noindex_path, 'wb') as fh: + fh.write(NOINDEX_TEMPLATE); + + def _remove_noindex(self): + noindex_path = os.path.join(self._project_dir, '.settings/org.eclipse.cdt.core.prefs') + os.remove(noindex_path) + + def _define_entry(self, name, value): + define = ET.Element('entry') + define.set('kind', 'macro') + define.set('name', name) + define.set('value', value) + return ET.tostring(define) + + def _write_language_settings(self, fh): + settings = LANGUAGE_SETTINGS_TEMPLATE + + settings = settings.replace('@GLOBAL_INCLUDE_PATH@', os.path.join(self.environment.topobjdir, 'dist/include')) + settings = settings.replace('@NSPR_INCLUDE_PATH@', os.path.join(self.environment.topobjdir, 'dist/include/nspr')) + settings = settings.replace('@IPDL_INCLUDE_PATH@', os.path.join(self.environment.topobjdir, 'ipc/ipdl/_ipdlheaders')) + settings = settings.replace('@PREINCLUDE_FILE_PATH@', os.path.join(self.environment.topobjdir, 'dist/include/mozilla-config.h')) + settings = settings.replace('@DEFINE_MOZILLA_INTERNAL_API@', self._define_entry('MOZILLA_INTERNAL_API', '1')) + settings = settings.replace("@COMPILER_FLAGS@", self._cxx + " " + self._cppflags); + + fh.write(settings) + + def _write_launch_files(self, launch_dir): + bin_dir = os.path.join(self.environment.topobjdir, 'dist') + + # TODO Improve binary detection + if self._macbundle: + exe_path = os.path.join(bin_dir, self._macbundle, 'Contents/MacOS') + else: + exe_path = os.path.join(bin_dir, 'bin') + + exe_path = os.path.join(exe_path, self._appname + self._bin_suffix) + + if self.environment.substs['MOZ_WIDGET_TOOLKIT'] != 'gonk': + main_gecko_launch = os.path.join(launch_dir, 'gecko.launch') + with open(main_gecko_launch, 'wb') as fh: + launch = GECKO_LAUNCH_CONFIG_TEMPLATE + launch = launch.replace('@LAUNCH_PROGRAM@', exe_path) + launch = launch.replace('@LAUNCH_ARGS@', '-P -no-remote') + fh.write(launch) + + if self.environment.substs['MOZ_WIDGET_TOOLKIT'] == 'gonk': + b2g_flash = os.path.join(launch_dir, 'b2g-flash.launch') + with open(b2g_flash, 'wb') as fh: + # We assume that the srcdir is inside the b2g tree. + # If that's not the case the user can always adjust the path + # from the eclipse IDE. + fastxul_path = os.path.join(self.environment.topsrcdir, '..', 'scripts', 'fastxul.sh') + launch = B2GFLASH_LAUNCH_CONFIG_TEMPLATE + launch = launch.replace('@LAUNCH_PROGRAM@', fastxul_path) + launch = launch.replace('@OBJDIR@', self.environment.topobjdir) + fh.write(launch) + + #TODO Add more launch configs (and delegate calls to mach) + + def _write_project(self, fh): + project = PROJECT_TEMPLATE; + + project = project.replace('@PROJECT_NAME@', self._project_name) + project = project.replace('@PROJECT_TOPSRCDIR@', self.environment.topsrcdir) + fh.write(project) + + def _write_cproject(self, fh): + cproject_header = CPROJECT_TEMPLATE_HEADER + cproject_header = cproject_header.replace('@PROJECT_TOPSRCDIR@', self.environment.topobjdir) + cproject_header = cproject_header.replace('@MACH_COMMAND@', os.path.join(self.environment.topsrcdir, 'mach')) + fh.write(cproject_header) + + for path, defines in self._paths_to_defines.items(): + folderinfo = CPROJECT_TEMPLATE_FOLDER_INFO_HEADER + folderinfo = folderinfo.replace('@FOLDER_ID@', str(random.randint(1000000, 99999999999))) + folderinfo = folderinfo.replace('@FOLDER_NAME@', 'tree/' + path) + fh.write(folderinfo) + for k, v in defines.items(): + define = ET.Element('listOptionValue') + define.set('builtIn', 'false') + define.set('value', str(k) + "=" + str(v)) + fh.write(ET.tostring(define)) + fh.write(CPROJECT_TEMPLATE_FOLDER_INFO_FOOTER) + + + fh.write(CPROJECT_TEMPLATE_FOOTER) + + +PROJECT_TEMPLATE = """ + + @PROJECT_NAME@ + + + + + + org.eclipse.cdt.managedbuilder.core.genmakebuilder + clean,full,incremental, + + + + + org.eclipse.cdt.managedbuilder.core.ScannerConfigBuilder + + + + + + + org.eclipse.cdt.core.cnature + org.eclipse.cdt.core.ccnature + org.eclipse.cdt.managedbuilder.core.managedBuildNature + org.eclipse.cdt.managedbuilder.core.ScannerConfigNature + + + + tree + 2 + @PROJECT_TOPSRCDIR@ + + + + + 17111971 + tree + 30 + + org.eclipse.ui.ide.multiFilter + 1.0-name-matches-false-false-obj-* + + + + 14081994 + tree + 22 + + org.eclipse.ui.ide.multiFilter + 1.0-name-matches-false-false-*.rej + + + + 25121970 + tree + 22 + + org.eclipse.ui.ide.multiFilter + 1.0-name-matches-false-false-*.orig + + + + 10102004 + tree + 10 + + org.eclipse.ui.ide.multiFilter + 1.0-name-matches-false-false-.hg + + + + 23122002 + tree + 22 + + org.eclipse.ui.ide.multiFilter + 1.0-name-matches-false-false-*.pyc + + + + +""" + +CPROJECT_TEMPLATE_HEADER = """ + + + + + + + + + + + + + + + + + + + + + + + + +""" +CPROJECT_TEMPLATE_FOLDER_INFO_HEADER = """ + + + + + + + + + +""" +CPROJECT_TEMPLATE_FILEINFO = """ + + + + + +""" +CPROJECT_TEMPLATE_FOOTER = """ + + + + + + + + + + + + + + + + + + +""" + +WORKSPACE_LANGUAGE_SETTINGS_TEMPLATE = """ + + + + + + + + +""" + +LANGUAGE_SETTINGS_TEMPLATE = """ + + + + + + + + + + + + + + + + + + + + @DEFINE_MOZILLA_INTERNAL_API@ + + + + + + + + + + + +""" + +GECKO_LAUNCH_CONFIG_TEMPLATE = """ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +""" + +B2GFLASH_LAUNCH_CONFIG_TEMPLATE = """ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +""" + + +EDITOR_SETTINGS = """eclipse.preferences.version=1 +lineNumberRuler=true +overviewRuler_migration=migrated_3.1 +printMargin=true +printMarginColumn=80 +showCarriageReturn=false +showEnclosedSpaces=false +showLeadingSpaces=false +showLineFeed=false +showWhitespaceCharacters=true +spacesForTabs=true +tabWidth=2 +undoHistorySize=200 +""" + +FORMATTER_SETTINGS = """eclipse.preferences.version=1 +org.eclipse.cdt.core.formatter.alignment_for_arguments_in_method_invocation=16 +org.eclipse.cdt.core.formatter.alignment_for_assignment=16 +org.eclipse.cdt.core.formatter.alignment_for_base_clause_in_type_declaration=80 +org.eclipse.cdt.core.formatter.alignment_for_binary_expression=16 +org.eclipse.cdt.core.formatter.alignment_for_compact_if=16 +org.eclipse.cdt.core.formatter.alignment_for_conditional_expression=34 +org.eclipse.cdt.core.formatter.alignment_for_conditional_expression_chain=18 +org.eclipse.cdt.core.formatter.alignment_for_constructor_initializer_list=48 +org.eclipse.cdt.core.formatter.alignment_for_declarator_list=16 +org.eclipse.cdt.core.formatter.alignment_for_enumerator_list=48 +org.eclipse.cdt.core.formatter.alignment_for_expression_list=0 +org.eclipse.cdt.core.formatter.alignment_for_expressions_in_array_initializer=16 +org.eclipse.cdt.core.formatter.alignment_for_member_access=0 +org.eclipse.cdt.core.formatter.alignment_for_overloaded_left_shift_chain=16 +org.eclipse.cdt.core.formatter.alignment_for_parameters_in_method_declaration=16 +org.eclipse.cdt.core.formatter.alignment_for_throws_clause_in_method_declaration=16 +org.eclipse.cdt.core.formatter.brace_position_for_array_initializer=end_of_line +org.eclipse.cdt.core.formatter.brace_position_for_block=end_of_line +org.eclipse.cdt.core.formatter.brace_position_for_block_in_case=next_line_shifted +org.eclipse.cdt.core.formatter.brace_position_for_method_declaration=next_line +org.eclipse.cdt.core.formatter.brace_position_for_namespace_declaration=end_of_line +org.eclipse.cdt.core.formatter.brace_position_for_switch=end_of_line +org.eclipse.cdt.core.formatter.brace_position_for_type_declaration=next_line +org.eclipse.cdt.core.formatter.comment.min_distance_between_code_and_line_comment=1 +org.eclipse.cdt.core.formatter.comment.never_indent_line_comments_on_first_column=true +org.eclipse.cdt.core.formatter.comment.preserve_white_space_between_code_and_line_comments=true +org.eclipse.cdt.core.formatter.compact_else_if=true +org.eclipse.cdt.core.formatter.continuation_indentation=2 +org.eclipse.cdt.core.formatter.continuation_indentation_for_array_initializer=2 +org.eclipse.cdt.core.formatter.format_guardian_clause_on_one_line=false +org.eclipse.cdt.core.formatter.indent_access_specifier_compare_to_type_header=false +org.eclipse.cdt.core.formatter.indent_access_specifier_extra_spaces=0 +org.eclipse.cdt.core.formatter.indent_body_declarations_compare_to_access_specifier=true +org.eclipse.cdt.core.formatter.indent_body_declarations_compare_to_namespace_header=false +org.eclipse.cdt.core.formatter.indent_breaks_compare_to_cases=true +org.eclipse.cdt.core.formatter.indent_declaration_compare_to_template_header=true +org.eclipse.cdt.core.formatter.indent_empty_lines=false +org.eclipse.cdt.core.formatter.indent_statements_compare_to_block=true +org.eclipse.cdt.core.formatter.indent_statements_compare_to_body=true +org.eclipse.cdt.core.formatter.indent_switchstatements_compare_to_cases=true +org.eclipse.cdt.core.formatter.indent_switchstatements_compare_to_switch=false +org.eclipse.cdt.core.formatter.indentation.size=2 +org.eclipse.cdt.core.formatter.insert_new_line_after_opening_brace_in_array_initializer=do not insert +org.eclipse.cdt.core.formatter.insert_new_line_after_template_declaration=insert +org.eclipse.cdt.core.formatter.insert_new_line_at_end_of_file_if_missing=do not insert +org.eclipse.cdt.core.formatter.insert_new_line_before_catch_in_try_statement=do not insert +org.eclipse.cdt.core.formatter.insert_new_line_before_closing_brace_in_array_initializer=do not insert +org.eclipse.cdt.core.formatter.insert_new_line_before_colon_in_constructor_initializer_list=do not insert +org.eclipse.cdt.core.formatter.insert_new_line_before_else_in_if_statement=do not insert +org.eclipse.cdt.core.formatter.insert_new_line_before_identifier_in_function_declaration=insert +org.eclipse.cdt.core.formatter.insert_new_line_before_while_in_do_statement=do not insert +org.eclipse.cdt.core.formatter.insert_new_line_in_empty_block=insert +org.eclipse.cdt.core.formatter.insert_space_after_assignment_operator=insert +org.eclipse.cdt.core.formatter.insert_space_after_binary_operator=insert +org.eclipse.cdt.core.formatter.insert_space_after_closing_angle_bracket_in_template_arguments=insert +org.eclipse.cdt.core.formatter.insert_space_after_closing_angle_bracket_in_template_parameters=insert +org.eclipse.cdt.core.formatter.insert_space_after_closing_brace_in_block=insert +org.eclipse.cdt.core.formatter.insert_space_after_closing_paren_in_cast=insert +org.eclipse.cdt.core.formatter.insert_space_after_colon_in_base_clause=insert +org.eclipse.cdt.core.formatter.insert_space_after_colon_in_case=insert +org.eclipse.cdt.core.formatter.insert_space_after_colon_in_conditional=insert +org.eclipse.cdt.core.formatter.insert_space_after_colon_in_labeled_statement=insert +org.eclipse.cdt.core.formatter.insert_space_after_comma_in_array_initializer=insert +org.eclipse.cdt.core.formatter.insert_space_after_comma_in_base_types=insert +org.eclipse.cdt.core.formatter.insert_space_after_comma_in_declarator_list=insert +org.eclipse.cdt.core.formatter.insert_space_after_comma_in_enum_declarations=insert +org.eclipse.cdt.core.formatter.insert_space_after_comma_in_expression_list=insert +org.eclipse.cdt.core.formatter.insert_space_after_comma_in_method_declaration_parameters=insert +org.eclipse.cdt.core.formatter.insert_space_after_comma_in_method_declaration_throws=insert +org.eclipse.cdt.core.formatter.insert_space_after_comma_in_method_invocation_arguments=insert +org.eclipse.cdt.core.formatter.insert_space_after_comma_in_template_arguments=insert +org.eclipse.cdt.core.formatter.insert_space_after_comma_in_template_parameters=insert +org.eclipse.cdt.core.formatter.insert_space_after_opening_angle_bracket_in_template_arguments=do not insert +org.eclipse.cdt.core.formatter.insert_space_after_opening_angle_bracket_in_template_parameters=do not insert +org.eclipse.cdt.core.formatter.insert_space_after_opening_brace_in_array_initializer=insert +org.eclipse.cdt.core.formatter.insert_space_after_opening_bracket=do not insert +org.eclipse.cdt.core.formatter.insert_space_after_opening_paren_in_cast=do not insert +org.eclipse.cdt.core.formatter.insert_space_after_opening_paren_in_catch=do not insert +org.eclipse.cdt.core.formatter.insert_space_after_opening_paren_in_exception_specification=do not insert +org.eclipse.cdt.core.formatter.insert_space_after_opening_paren_in_for=do not insert +org.eclipse.cdt.core.formatter.insert_space_after_opening_paren_in_if=do not insert +org.eclipse.cdt.core.formatter.insert_space_after_opening_paren_in_method_declaration=do not insert +org.eclipse.cdt.core.formatter.insert_space_after_opening_paren_in_method_invocation=do not insert +org.eclipse.cdt.core.formatter.insert_space_after_opening_paren_in_parenthesized_expression=do not insert +org.eclipse.cdt.core.formatter.insert_space_after_opening_paren_in_switch=do not insert +org.eclipse.cdt.core.formatter.insert_space_after_opening_paren_in_while=do not insert +org.eclipse.cdt.core.formatter.insert_space_after_postfix_operator=do not insert +org.eclipse.cdt.core.formatter.insert_space_after_prefix_operator=do not insert +org.eclipse.cdt.core.formatter.insert_space_after_question_in_conditional=insert +org.eclipse.cdt.core.formatter.insert_space_after_semicolon_in_for=insert +org.eclipse.cdt.core.formatter.insert_space_after_unary_operator=do not insert +org.eclipse.cdt.core.formatter.insert_space_before_assignment_operator=insert +org.eclipse.cdt.core.formatter.insert_space_before_binary_operator=insert +org.eclipse.cdt.core.formatter.insert_space_before_closing_angle_bracket_in_template_arguments=do not insert +org.eclipse.cdt.core.formatter.insert_space_before_closing_angle_bracket_in_template_parameters=do not insert +org.eclipse.cdt.core.formatter.insert_space_before_closing_brace_in_array_initializer=insert +org.eclipse.cdt.core.formatter.insert_space_before_closing_bracket=do not insert +org.eclipse.cdt.core.formatter.insert_space_before_closing_paren_in_cast=do not insert +org.eclipse.cdt.core.formatter.insert_space_before_closing_paren_in_catch=do not insert +org.eclipse.cdt.core.formatter.insert_space_before_closing_paren_in_exception_specification=do not insert +org.eclipse.cdt.core.formatter.insert_space_before_closing_paren_in_for=do not insert +org.eclipse.cdt.core.formatter.insert_space_before_closing_paren_in_if=do not insert +org.eclipse.cdt.core.formatter.insert_space_before_closing_paren_in_method_declaration=do not insert +org.eclipse.cdt.core.formatter.insert_space_before_closing_paren_in_method_invocation=do not insert +org.eclipse.cdt.core.formatter.insert_space_before_closing_paren_in_parenthesized_expression=do not insert +org.eclipse.cdt.core.formatter.insert_space_before_closing_paren_in_switch=do not insert +org.eclipse.cdt.core.formatter.insert_space_before_closing_paren_in_while=do not insert +org.eclipse.cdt.core.formatter.insert_space_before_colon_in_base_clause=insert +org.eclipse.cdt.core.formatter.insert_space_before_colon_in_case=do not insert +org.eclipse.cdt.core.formatter.insert_space_before_colon_in_conditional=insert +org.eclipse.cdt.core.formatter.insert_space_before_colon_in_default=do not insert +org.eclipse.cdt.core.formatter.insert_space_before_colon_in_labeled_statement=do not insert +org.eclipse.cdt.core.formatter.insert_space_before_comma_in_array_initializer=do not insert +org.eclipse.cdt.core.formatter.insert_space_before_comma_in_base_types=do not insert +org.eclipse.cdt.core.formatter.insert_space_before_comma_in_declarator_list=do not insert +org.eclipse.cdt.core.formatter.insert_space_before_comma_in_enum_declarations=do not insert +org.eclipse.cdt.core.formatter.insert_space_before_comma_in_expression_list=do not insert +org.eclipse.cdt.core.formatter.insert_space_before_comma_in_method_declaration_parameters=do not insert +org.eclipse.cdt.core.formatter.insert_space_before_comma_in_method_declaration_throws=do not insert +org.eclipse.cdt.core.formatter.insert_space_before_comma_in_method_invocation_arguments=do not insert +org.eclipse.cdt.core.formatter.insert_space_before_comma_in_template_arguments=do not insert +org.eclipse.cdt.core.formatter.insert_space_before_comma_in_template_parameters=do not insert +org.eclipse.cdt.core.formatter.insert_space_before_opening_angle_bracket_in_template_arguments=do not insert +org.eclipse.cdt.core.formatter.insert_space_before_opening_angle_bracket_in_template_parameters=do not insert +org.eclipse.cdt.core.formatter.insert_space_before_opening_brace_in_array_initializer=insert +org.eclipse.cdt.core.formatter.insert_space_before_opening_brace_in_block=insert +org.eclipse.cdt.core.formatter.insert_space_before_opening_brace_in_method_declaration=insert +org.eclipse.cdt.core.formatter.insert_space_before_opening_brace_in_namespace_declaration=insert +org.eclipse.cdt.core.formatter.insert_space_before_opening_brace_in_switch=insert +org.eclipse.cdt.core.formatter.insert_space_before_opening_brace_in_type_declaration=insert +org.eclipse.cdt.core.formatter.insert_space_before_opening_bracket=do not insert +org.eclipse.cdt.core.formatter.insert_space_before_opening_paren_in_catch=insert +org.eclipse.cdt.core.formatter.insert_space_before_opening_paren_in_exception_specification=insert +org.eclipse.cdt.core.formatter.insert_space_before_opening_paren_in_for=insert +org.eclipse.cdt.core.formatter.insert_space_before_opening_paren_in_if=insert +org.eclipse.cdt.core.formatter.insert_space_before_opening_paren_in_method_declaration=do not insert +org.eclipse.cdt.core.formatter.insert_space_before_opening_paren_in_method_invocation=do not insert +org.eclipse.cdt.core.formatter.insert_space_before_opening_paren_in_parenthesized_expression=do not insert +org.eclipse.cdt.core.formatter.insert_space_before_opening_paren_in_switch=insert +org.eclipse.cdt.core.formatter.insert_space_before_opening_paren_in_while=insert +org.eclipse.cdt.core.formatter.insert_space_before_postfix_operator=do not insert +org.eclipse.cdt.core.formatter.insert_space_before_prefix_operator=do not insert +org.eclipse.cdt.core.formatter.insert_space_before_question_in_conditional=insert +org.eclipse.cdt.core.formatter.insert_space_before_semicolon=do not insert +org.eclipse.cdt.core.formatter.insert_space_before_semicolon_in_for=do not insert +org.eclipse.cdt.core.formatter.insert_space_before_unary_operator=do not insert +org.eclipse.cdt.core.formatter.insert_space_between_empty_braces_in_array_initializer=do not insert +org.eclipse.cdt.core.formatter.insert_space_between_empty_brackets=do not insert +org.eclipse.cdt.core.formatter.insert_space_between_empty_parens_in_exception_specification=do not insert +org.eclipse.cdt.core.formatter.insert_space_between_empty_parens_in_method_declaration=do not insert +org.eclipse.cdt.core.formatter.insert_space_between_empty_parens_in_method_invocation=do not insert +org.eclipse.cdt.core.formatter.join_wrapped_lines=false +org.eclipse.cdt.core.formatter.keep_else_statement_on_same_line=false +org.eclipse.cdt.core.formatter.keep_empty_array_initializer_on_one_line=false +org.eclipse.cdt.core.formatter.keep_imple_if_on_one_line=false +org.eclipse.cdt.core.formatter.keep_then_statement_on_same_line=false +org.eclipse.cdt.core.formatter.lineSplit=80 +org.eclipse.cdt.core.formatter.number_of_empty_lines_to_preserve=1 +org.eclipse.cdt.core.formatter.put_empty_statement_on_new_line=true +org.eclipse.cdt.core.formatter.tabulation.char=space +org.eclipse.cdt.core.formatter.tabulation.size=2 +org.eclipse.cdt.core.formatter.use_tabs_only_for_leading_indentations=false +""" + +NOINDEX_TEMPLATE = """eclipse.preferences.version=1 +indexer/indexerId=org.eclipse.cdt.core.nullIndexer +""" diff --git a/python/mozbuild/mozbuild/backend/fastermake.py b/python/mozbuild/mozbuild/backend/fastermake.py new file mode 100644 index 000000000..d55928e8c --- /dev/null +++ b/python/mozbuild/mozbuild/backend/fastermake.py @@ -0,0 +1,165 @@ +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, You can obtain one at http://mozilla.org/MPL/2.0/. + +from __future__ import absolute_import, unicode_literals, print_function + +from mozbuild.backend.base import PartialBackend +from mozbuild.backend.common import CommonBackend +from mozbuild.frontend.context import ( + ObjDirPath, +) +from mozbuild.frontend.data import ( + ChromeManifestEntry, + FinalTargetPreprocessedFiles, + FinalTargetFiles, + JARManifest, + XPIDLFile, +) +from mozbuild.makeutil import Makefile +from mozbuild.util import OrderedDefaultDict +from mozpack.manifests import InstallManifest +import mozpack.path as mozpath + + +class FasterMakeBackend(CommonBackend, PartialBackend): + def _init(self): + super(FasterMakeBackend, self)._init() + + self._manifest_entries = OrderedDefaultDict(set) + + self._install_manifests = OrderedDefaultDict(InstallManifest) + + self._dependencies = OrderedDefaultDict(list) + + self._has_xpidl = False + + def _add_preprocess(self, obj, path, dest, target=None, **kwargs): + if target is None: + target = mozpath.basename(path) + # This matches what PP_TARGETS do in config/rules. + if target.endswith('.in'): + target = target[:-3] + if target.endswith('.css'): + kwargs['marker'] = '%' + depfile = mozpath.join( + self.environment.topobjdir, 'faster', '.deps', + mozpath.join(obj.install_target, dest, target).replace('/', '_')) + self._install_manifests[obj.install_target].add_preprocess( + mozpath.join(obj.srcdir, path), + mozpath.join(dest, target), + depfile, + **kwargs) + + def consume_object(self, obj): + if isinstance(obj, JARManifest) and \ + obj.install_target.startswith('dist/bin'): + self._consume_jar_manifest(obj) + + elif isinstance(obj, (FinalTargetFiles, + FinalTargetPreprocessedFiles)) and \ + obj.install_target.startswith('dist/bin'): + defines = obj.defines or {} + if defines: + defines = defines.defines + for path, files in obj.files.walk(): + for f in files: + if isinstance(obj, FinalTargetPreprocessedFiles): + self._add_preprocess(obj, f.full_path, path, + target=f.target_basename, + defines=defines) + elif '*' in f: + def _prefix(s): + for p in mozpath.split(s): + if '*' not in p: + yield p + '/' + prefix = ''.join(_prefix(f.full_path)) + + self._install_manifests[obj.install_target] \ + .add_pattern_symlink( + prefix, + f.full_path[len(prefix):], + mozpath.join(path, f.target_basename)) + else: + self._install_manifests[obj.install_target].add_symlink( + f.full_path, + mozpath.join(path, f.target_basename) + ) + if isinstance(f, ObjDirPath): + dep_target = 'install-%s' % obj.install_target + self._dependencies[dep_target].append( + mozpath.relpath(f.full_path, + self.environment.topobjdir)) + + elif isinstance(obj, ChromeManifestEntry) and \ + obj.install_target.startswith('dist/bin'): + top_level = mozpath.join(obj.install_target, 'chrome.manifest') + if obj.path != top_level: + entry = 'manifest %s' % mozpath.relpath(obj.path, + obj.install_target) + self._manifest_entries[top_level].add(entry) + self._manifest_entries[obj.path].add(str(obj.entry)) + + elif isinstance(obj, XPIDLFile): + self._has_xpidl = True + # We're not actually handling XPIDL files. + return False + + else: + return False + + return True + + def consume_finished(self): + mk = Makefile() + # Add the default rule at the very beginning. + mk.create_rule(['default']) + mk.add_statement('TOPSRCDIR = %s' % self.environment.topsrcdir) + mk.add_statement('TOPOBJDIR = %s' % self.environment.topobjdir) + if not self._has_xpidl: + mk.add_statement('NO_XPIDL = 1') + + # Add a few necessary variables inherited from configure + for var in ( + 'PYTHON', + 'ACDEFINES', + 'MOZ_BUILD_APP', + 'MOZ_WIDGET_TOOLKIT', + ): + value = self.environment.substs.get(var) + if value is not None: + mk.add_statement('%s = %s' % (var, value)) + + install_manifests_bases = self._install_manifests.keys() + + # Add information for chrome manifest generation + manifest_targets = [] + + for target, entries in self._manifest_entries.iteritems(): + manifest_targets.append(target) + install_target = mozpath.basedir(target, install_manifests_bases) + self._install_manifests[install_target].add_content( + ''.join('%s\n' % e for e in sorted(entries)), + mozpath.relpath(target, install_target)) + + # Add information for install manifests. + mk.add_statement('INSTALL_MANIFESTS = %s' + % ' '.join(self._install_manifests.keys())) + + # Add dependencies we infered: + for target, deps in self._dependencies.iteritems(): + mk.create_rule([target]).add_dependencies( + '$(TOPOBJDIR)/%s' % d for d in deps) + + mk.add_statement('include $(TOPSRCDIR)/config/faster/rules.mk') + + for base, install_manifest in self._install_manifests.iteritems(): + with self._write_file( + mozpath.join(self.environment.topobjdir, 'faster', + 'install_%s' % base.replace('/', '_'))) as fh: + install_manifest.write(fileobj=fh) + + with self._write_file( + mozpath.join(self.environment.topobjdir, 'faster', + 'Makefile')) as fh: + mk.dump(fh, removal_guard=False) diff --git a/python/mozbuild/mozbuild/backend/mach_commands.py b/python/mozbuild/mozbuild/backend/mach_commands.py new file mode 100644 index 000000000..5608d40b1 --- /dev/null +++ b/python/mozbuild/mozbuild/backend/mach_commands.py @@ -0,0 +1,132 @@ +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, You can obtain one at http://mozilla.org/MPL/2.0/. + +from __future__ import absolute_import, print_function, unicode_literals + +import argparse +import os +import sys +import subprocess +import which + +from mozbuild.base import ( + MachCommandBase, +) + +from mach.decorators import ( + CommandArgument, + CommandProvider, + Command, +) + +@CommandProvider +class MachCommands(MachCommandBase): + @Command('ide', category='devenv', + description='Generate a project and launch an IDE.') + @CommandArgument('ide', choices=['eclipse', 'visualstudio', 'androidstudio', 'intellij']) + @CommandArgument('args', nargs=argparse.REMAINDER) + def eclipse(self, ide, args): + if ide == 'eclipse': + backend = 'CppEclipse' + elif ide == 'visualstudio': + backend = 'VisualStudio' + elif ide == 'androidstudio' or ide == 'intellij': + # The build backend for Android Studio and IntelliJ is just the regular one. + backend = 'RecursiveMake' + + if ide == 'eclipse': + try: + which.which('eclipse') + except which.WhichError: + print('Eclipse CDT 8.4 or later must be installed in your PATH.') + print('Download: http://www.eclipse.org/cdt/downloads.php') + return 1 + elif ide == 'androidstudio' or ide =='intellij': + studio = ['studio'] if ide == 'androidstudio' else ['idea'] + if sys.platform != 'darwin': + try: + which.which(studio[0]) + except: + self.print_ide_error(ide) + return 1 + else: + # In order of preference! + for d in self.get_mac_ide_preferences(ide): + if os.path.isdir(d): + studio = ['open', '-a', d] + break + else: + print('Android Studio or IntelliJ IDEA 14 is not installed in /Applications.') + return 1 + + # Here we refresh the whole build. 'build export' is sufficient here and is probably more + # correct but it's also nice having a single target to get a fully built and indexed + # project (gives a easy target to use before go out to lunch). + res = self._mach_context.commands.dispatch('build', self._mach_context) + if res != 0: + return 1 + + if ide in ('androidstudio', 'intellij'): + res = self._mach_context.commands.dispatch('package', self._mach_context) + if res != 0: + return 1 + else: + # Generate or refresh the IDE backend. + python = self.virtualenv_manager.python_path + config_status = os.path.join(self.topobjdir, 'config.status') + args = [python, config_status, '--backend=%s' % backend] + res = self._run_command_in_objdir(args=args, pass_thru=True, ensure_exit_code=False) + if res != 0: + return 1 + + + if ide == 'eclipse': + eclipse_workspace_dir = self.get_eclipse_workspace_path() + process = subprocess.check_call(['eclipse', '-data', eclipse_workspace_dir]) + elif ide == 'visualstudio': + visual_studio_workspace_dir = self.get_visualstudio_workspace_path() + process = subprocess.check_call(['explorer.exe', visual_studio_workspace_dir]) + elif ide == 'androidstudio' or ide == 'intellij': + gradle_dir = None + if self.is_gradle_project_already_imported(): + gradle_dir = self.get_gradle_project_path() + else: + gradle_dir = self.get_gradle_import_path() + process = subprocess.check_call(studio + [gradle_dir]) + + def get_eclipse_workspace_path(self): + from mozbuild.backend.cpp_eclipse import CppEclipseBackend + return CppEclipseBackend.get_workspace_path(self.topsrcdir, self.topobjdir) + + def get_visualstudio_workspace_path(self): + return os.path.join(self.topobjdir, 'msvc', 'mozilla.sln') + + def get_gradle_project_path(self): + return os.path.join(self.topobjdir, 'mobile', 'android', 'gradle') + + def get_gradle_import_path(self): + return os.path.join(self.get_gradle_project_path(), 'build.gradle') + + def is_gradle_project_already_imported(self): + gradle_project_path = os.path.join(self.get_gradle_project_path(), '.idea') + return os.path.exists(gradle_project_path) + + def get_mac_ide_preferences(self, ide): + if sys.platform == 'darwin': + if ide == 'androidstudio': + return ['/Applications/Android Studio.app'] + else: + return [ + '/Applications/IntelliJ IDEA 14 EAP.app', + '/Applications/IntelliJ IDEA 14.app', + '/Applications/IntelliJ IDEA 14 CE EAP.app', + '/Applications/IntelliJ IDEA 14 CE.app'] + + def print_ide_error(self, ide): + if ide == 'androidstudio': + print('Android Studio is not installed in your PATH.') + print('You can generate a command-line launcher from Android Studio->Tools->Create Command-line launcher with script name \'studio\'') + elif ide == 'intellij': + print('IntelliJ is not installed in your PATH.') + print('You can generate a command-line launcher from IntelliJ IDEA->Tools->Create Command-line launcher with script name \'idea\'') diff --git a/python/mozbuild/mozbuild/backend/recursivemake.py b/python/mozbuild/mozbuild/backend/recursivemake.py new file mode 100644 index 000000000..132dcf944 --- /dev/null +++ b/python/mozbuild/mozbuild/backend/recursivemake.py @@ -0,0 +1,1513 @@ +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, You can obtain one at http://mozilla.org/MPL/2.0/. + +from __future__ import absolute_import, unicode_literals + +import logging +import os +import re + +from collections import ( + defaultdict, + namedtuple, +) +from StringIO import StringIO +from itertools import chain + +from mozpack.manifests import ( + InstallManifest, +) +import mozpack.path as mozpath + +from mozbuild.frontend.context import ( + AbsolutePath, + Path, + RenamedSourcePath, + SourcePath, + ObjDirPath, +) +from .common import CommonBackend +from ..frontend.data import ( + AndroidAssetsDirs, + AndroidResDirs, + AndroidExtraResDirs, + AndroidExtraPackages, + AndroidEclipseProjectData, + ChromeManifestEntry, + ConfigFileSubstitution, + ContextDerived, + ContextWrapped, + Defines, + DirectoryTraversal, + ExternalLibrary, + FinalTargetFiles, + FinalTargetPreprocessedFiles, + GeneratedFile, + GeneratedSources, + HostDefines, + HostLibrary, + HostProgram, + HostSimpleProgram, + HostSources, + InstallationTarget, + JARManifest, + JavaJarData, + Library, + LocalInclude, + ObjdirFiles, + ObjdirPreprocessedFiles, + PerSourceFlag, + Program, + RustLibrary, + SharedLibrary, + SimpleProgram, + Sources, + StaticLibrary, + TestManifest, + VariablePassthru, + XPIDLFile, +) +from ..util import ( + ensureParentDir, + FileAvoidWrite, +) +from ..makeutil import Makefile +from mozbuild.shellutil import quote as shell_quote + +MOZBUILD_VARIABLES = [ + b'ANDROID_APK_NAME', + b'ANDROID_APK_PACKAGE', + b'ANDROID_ASSETS_DIRS', + b'ANDROID_EXTRA_PACKAGES', + b'ANDROID_EXTRA_RES_DIRS', + b'ANDROID_GENERATED_RESFILES', + b'ANDROID_RES_DIRS', + b'ASFLAGS', + b'CMSRCS', + b'CMMSRCS', + b'CPP_UNIT_TESTS', + b'DIRS', + b'DIST_INSTALL', + b'EXTRA_DSO_LDOPTS', + b'EXTRA_JS_MODULES', + b'EXTRA_PP_COMPONENTS', + b'EXTRA_PP_JS_MODULES', + b'FORCE_SHARED_LIB', + b'FORCE_STATIC_LIB', + b'FINAL_LIBRARY', + b'HOST_CFLAGS', + b'HOST_CSRCS', + b'HOST_CMMSRCS', + b'HOST_CXXFLAGS', + b'HOST_EXTRA_LIBS', + b'HOST_LIBRARY_NAME', + b'HOST_PROGRAM', + b'HOST_SIMPLE_PROGRAMS', + b'IS_COMPONENT', + b'JAR_MANIFEST', + b'JAVA_JAR_TARGETS', + b'LD_VERSION_SCRIPT', + b'LIBRARY_NAME', + b'LIBS', + b'MAKE_FRAMEWORK', + b'MODULE', + b'NO_DIST_INSTALL', + b'NO_EXPAND_LIBS', + b'NO_INTERFACES_MANIFEST', + b'NO_JS_MANIFEST', + b'OS_LIBS', + b'PARALLEL_DIRS', + b'PREF_JS_EXPORTS', + b'PROGRAM', + b'PYTHON_UNIT_TESTS', + b'RESOURCE_FILES', + b'SDK_HEADERS', + b'SDK_LIBRARY', + b'SHARED_LIBRARY_LIBS', + b'SHARED_LIBRARY_NAME', + b'SIMPLE_PROGRAMS', + b'SONAME', + b'STATIC_LIBRARY_NAME', + b'TEST_DIRS', + b'TOOL_DIRS', + # XXX config/Makefile.in specifies this in a make invocation + #'USE_EXTENSION_MANIFEST', + b'XPCSHELL_TESTS', + b'XPIDL_MODULE', +] + +DEPRECATED_VARIABLES = [ + b'ANDROID_RESFILES', + b'EXPORT_LIBRARY', + b'EXTRA_LIBS', + b'HOST_LIBS', + b'LIBXUL_LIBRARY', + b'MOCHITEST_A11Y_FILES', + b'MOCHITEST_BROWSER_FILES', + b'MOCHITEST_BROWSER_FILES_PARTS', + b'MOCHITEST_CHROME_FILES', + b'MOCHITEST_FILES', + b'MOCHITEST_FILES_PARTS', + b'MOCHITEST_METRO_FILES', + b'MOCHITEST_ROBOCOP_FILES', + b'MODULE_OPTIMIZE_FLAGS', + b'MOZ_CHROME_FILE_FORMAT', + b'SHORT_LIBNAME', + b'TESTING_JS_MODULES', + b'TESTING_JS_MODULE_DIR', +] + +MOZBUILD_VARIABLES_MESSAGE = 'It should only be defined in moz.build files.' + +DEPRECATED_VARIABLES_MESSAGE = ( + 'This variable has been deprecated. It does nothing. It must be removed ' + 'in order to build.' +) + + +def make_quote(s): + return s.replace('#', '\#').replace('$', '$$') + + +class BackendMakeFile(object): + """Represents a generated backend.mk file. + + This is both a wrapper around a file handle as well as a container that + holds accumulated state. + + It's worth taking a moment to explain the make dependencies. The + generated backend.mk as well as the Makefile.in (if it exists) are in the + GLOBAL_DEPS list. This means that if one of them changes, all targets + in that Makefile are invalidated. backend.mk also depends on all of its + input files. + + It's worth considering the effect of file mtimes on build behavior. + + Since we perform an "all or none" traversal of moz.build files (the whole + tree is scanned as opposed to individual files), if we were to blindly + write backend.mk files, the net effect of updating a single mozbuild file + in the tree is all backend.mk files have new mtimes. This would in turn + invalidate all make targets across the whole tree! This would effectively + undermine incremental builds as any mozbuild change would cause the entire + tree to rebuild! + + The solution is to not update the mtimes of backend.mk files unless they + actually change. We use FileAvoidWrite to accomplish this. + """ + + def __init__(self, srcdir, objdir, environment, topsrcdir, topobjdir): + self.topsrcdir = topsrcdir + self.srcdir = srcdir + self.objdir = objdir + self.relobjdir = mozpath.relpath(objdir, topobjdir) + self.environment = environment + self.name = mozpath.join(objdir, 'backend.mk') + + self.xpt_name = None + + self.fh = FileAvoidWrite(self.name, capture_diff=True) + self.fh.write('# THIS FILE WAS AUTOMATICALLY GENERATED. DO NOT EDIT.\n') + self.fh.write('\n') + + def write(self, buf): + self.fh.write(buf) + + def write_once(self, buf): + if isinstance(buf, unicode): + buf = buf.encode('utf-8') + if b'\n' + buf not in self.fh.getvalue(): + self.write(buf) + + # For compatibility with makeutil.Makefile + def add_statement(self, stmt): + self.write('%s\n' % stmt) + + def close(self): + if self.xpt_name: + # We just recompile all xpidls because it's easier and less error + # prone. + self.fh.write('NONRECURSIVE_TARGETS += export\n') + self.fh.write('NONRECURSIVE_TARGETS_export += xpidl\n') + self.fh.write('NONRECURSIVE_TARGETS_export_xpidl_DIRECTORY = ' + '$(DEPTH)/xpcom/xpidl\n') + self.fh.write('NONRECURSIVE_TARGETS_export_xpidl_TARGETS += ' + 'export\n') + + return self.fh.close() + + @property + def diff(self): + return self.fh.diff + + +class RecursiveMakeTraversal(object): + """ + Helper class to keep track of how the "traditional" recursive make backend + recurses subdirectories. This is useful until all adhoc rules are removed + from Makefiles. + + Each directory may have one or more types of subdirectories: + - (normal) dirs + - tests + """ + SubDirectoryCategories = ['dirs', 'tests'] + SubDirectoriesTuple = namedtuple('SubDirectories', SubDirectoryCategories) + class SubDirectories(SubDirectoriesTuple): + def __new__(self): + return RecursiveMakeTraversal.SubDirectoriesTuple.__new__(self, [], []) + + def __init__(self): + self._traversal = {} + + def add(self, dir, dirs=[], tests=[]): + """ + Adds a directory to traversal, registering its subdirectories, + sorted by categories. If the directory was already added to + traversal, adds the new subdirectories to the already known lists. + """ + subdirs = self._traversal.setdefault(dir, self.SubDirectories()) + for key, value in (('dirs', dirs), ('tests', tests)): + assert(key in self.SubDirectoryCategories) + getattr(subdirs, key).extend(value) + + @staticmethod + def default_filter(current, subdirs): + """ + Default filter for use with compute_dependencies and traverse. + """ + return current, [], subdirs.dirs + subdirs.tests + + def call_filter(self, current, filter): + """ + Helper function to call a filter from compute_dependencies and + traverse. + """ + return filter(current, self._traversal.get(current, + self.SubDirectories())) + + def compute_dependencies(self, filter=None): + """ + Compute make dependencies corresponding to the registered directory + traversal. + + filter is a function with the following signature: + def filter(current, subdirs) + where current is the directory being traversed, and subdirs the + SubDirectories instance corresponding to it. + The filter function returns a tuple (filtered_current, filtered_parallel, + filtered_dirs) where filtered_current is either current or None if + the current directory is to be skipped, and filtered_parallel and + filtered_dirs are lists of parallel directories and sequential + directories, which can be rearranged from whatever is given in the + SubDirectories members. + + The default filter corresponds to a default recursive traversal. + """ + filter = filter or self.default_filter + + deps = {} + + def recurse(start_node, prev_nodes=None): + current, parallel, sequential = self.call_filter(start_node, filter) + if current is not None: + if start_node != '': + deps[start_node] = prev_nodes + prev_nodes = (start_node,) + if not start_node in self._traversal: + return prev_nodes + parallel_nodes = [] + for node in parallel: + nodes = recurse(node, prev_nodes) + if nodes and nodes != ('',): + parallel_nodes.extend(nodes) + if parallel_nodes: + prev_nodes = tuple(parallel_nodes) + for dir in sequential: + prev_nodes = recurse(dir, prev_nodes) + return prev_nodes + + return recurse(''), deps + + def traverse(self, start, filter=None): + """ + Iterate over the filtered subdirectories, following the traditional + make traversal order. + """ + if filter is None: + filter = self.default_filter + + current, parallel, sequential = self.call_filter(start, filter) + if current is not None: + yield start + if not start in self._traversal: + return + for node in parallel: + for n in self.traverse(node, filter): + yield n + for dir in sequential: + for d in self.traverse(dir, filter): + yield d + + def get_subdirs(self, dir): + """ + Returns all direct subdirectories under the given directory. + """ + return self._traversal.get(dir, self.SubDirectories()) + + +class RecursiveMakeBackend(CommonBackend): + """Backend that integrates with the existing recursive make build system. + + This backend facilitates the transition from Makefile.in to moz.build + files. + + This backend performs Makefile.in -> Makefile conversion. It also writes + out .mk files containing content derived from moz.build files. Both are + consumed by the recursive make builder. + + This backend may eventually evolve to write out non-recursive make files. + However, as long as there are Makefile.in files in the tree, we are tied to + recursive make and thus will need this backend. + """ + + def _init(self): + CommonBackend._init(self) + + self._backend_files = {} + self._idl_dirs = set() + + self._makefile_in_count = 0 + self._makefile_out_count = 0 + + self._test_manifests = {} + + self.backend_input_files.add(mozpath.join(self.environment.topobjdir, + 'config', 'autoconf.mk')) + + self._install_manifests = defaultdict(InstallManifest) + # The build system relies on some install manifests always existing + # even if they are empty, because the directories are still filled + # by the build system itself, and the install manifests are only + # used for a "magic" rm -rf. + self._install_manifests['dist_public'] + self._install_manifests['dist_private'] + self._install_manifests['dist_sdk'] + + self._traversal = RecursiveMakeTraversal() + self._compile_graph = defaultdict(set) + + self._no_skip = { + 'export': set(), + 'libs': set(), + 'misc': set(), + 'tools': set(), + } + + def summary(self): + summary = super(RecursiveMakeBackend, self).summary() + summary.extend('; {makefile_in:d} -> {makefile_out:d} Makefile', + makefile_in=self._makefile_in_count, + makefile_out=self._makefile_out_count) + return summary + + def _get_backend_file_for(self, obj): + if obj.objdir not in self._backend_files: + self._backend_files[obj.objdir] = \ + BackendMakeFile(obj.srcdir, obj.objdir, obj.config, + obj.topsrcdir, self.environment.topobjdir) + return self._backend_files[obj.objdir] + + def consume_object(self, obj): + """Write out build files necessary to build with recursive make.""" + + if not isinstance(obj, ContextDerived): + return False + + backend_file = self._get_backend_file_for(obj) + + consumed = CommonBackend.consume_object(self, obj) + + # CommonBackend handles XPIDLFile and TestManifest, but we want to do + # some extra things for them. + if isinstance(obj, XPIDLFile): + backend_file.xpt_name = '%s.xpt' % obj.module + self._idl_dirs.add(obj.relobjdir) + + elif isinstance(obj, TestManifest): + self._process_test_manifest(obj, backend_file) + + # If CommonBackend acknowledged the object, we're done with it. + if consumed: + return True + + if not isinstance(obj, Defines): + self.consume_object(obj.defines) + + if isinstance(obj, DirectoryTraversal): + self._process_directory_traversal(obj, backend_file) + elif isinstance(obj, ConfigFileSubstitution): + # Other ConfigFileSubstitution should have been acked by + # CommonBackend. + assert os.path.basename(obj.output_path) == 'Makefile' + self._create_makefile(obj) + elif isinstance(obj, (Sources, GeneratedSources)): + suffix_map = { + '.s': 'ASFILES', + '.c': 'CSRCS', + '.m': 'CMSRCS', + '.mm': 'CMMSRCS', + '.cpp': 'CPPSRCS', + '.rs': 'RSSRCS', + '.S': 'SSRCS', + } + variables = [suffix_map[obj.canonical_suffix]] + if isinstance(obj, GeneratedSources): + variables.append('GARBAGE') + base = backend_file.objdir + else: + base = backend_file.srcdir + for f in sorted(obj.files): + f = mozpath.relpath(f, base) + for var in variables: + backend_file.write('%s += %s\n' % (var, f)) + elif isinstance(obj, HostSources): + suffix_map = { + '.c': 'HOST_CSRCS', + '.mm': 'HOST_CMMSRCS', + '.cpp': 'HOST_CPPSRCS', + } + var = suffix_map[obj.canonical_suffix] + for f in sorted(obj.files): + backend_file.write('%s += %s\n' % ( + var, mozpath.relpath(f, backend_file.srcdir))) + elif isinstance(obj, VariablePassthru): + # Sorted so output is consistent and we don't bump mtimes. + for k, v in sorted(obj.variables.items()): + if k == 'HAS_MISC_RULE': + self._no_skip['misc'].add(backend_file.relobjdir) + continue + if isinstance(v, list): + for item in v: + backend_file.write( + '%s += %s\n' % (k, make_quote(shell_quote(item)))) + elif isinstance(v, bool): + if v: + backend_file.write('%s := 1\n' % k) + else: + backend_file.write('%s := %s\n' % (k, v)) + elif isinstance(obj, HostDefines): + self._process_defines(obj, backend_file, which='HOST_DEFINES') + elif isinstance(obj, Defines): + self._process_defines(obj, backend_file) + + elif isinstance(obj, GeneratedFile): + export_suffixes = ( + '.c', + '.cpp', + '.h', + '.inc', + '.py', + ) + tier = 'export' if any(f.endswith(export_suffixes) for f in obj.outputs) else 'misc' + self._no_skip[tier].add(backend_file.relobjdir) + first_output = obj.outputs[0] + dep_file = "%s.pp" % first_output + backend_file.write('%s:: %s\n' % (tier, first_output)) + for output in obj.outputs: + if output != first_output: + backend_file.write('%s: %s ;\n' % (output, first_output)) + backend_file.write('GARBAGE += %s\n' % output) + backend_file.write('EXTRA_MDDEPEND_FILES += %s\n' % dep_file) + if obj.script: + backend_file.write("""{output}: {script}{inputs}{backend} +\t$(REPORT_BUILD) +\t$(call py_action,file_generate,{script} {method} {output} $(MDDEPDIR)/{dep_file}{inputs}{flags}) + +""".format(output=first_output, + dep_file=dep_file, + inputs=' ' + ' '.join([self._pretty_path(f, backend_file) for f in obj.inputs]) if obj.inputs else '', + flags=' ' + ' '.join(obj.flags) if obj.flags else '', + backend=' backend.mk' if obj.flags else '', + script=obj.script, + method=obj.method)) + + elif isinstance(obj, JARManifest): + self._no_skip['libs'].add(backend_file.relobjdir) + backend_file.write('JAR_MANIFEST := %s\n' % obj.path.full_path) + + elif isinstance(obj, Program): + self._process_program(obj.program, backend_file) + self._process_linked_libraries(obj, backend_file) + + elif isinstance(obj, HostProgram): + self._process_host_program(obj.program, backend_file) + self._process_linked_libraries(obj, backend_file) + + elif isinstance(obj, SimpleProgram): + self._process_simple_program(obj, backend_file) + self._process_linked_libraries(obj, backend_file) + + elif isinstance(obj, HostSimpleProgram): + self._process_host_simple_program(obj.program, backend_file) + self._process_linked_libraries(obj, backend_file) + + elif isinstance(obj, LocalInclude): + self._process_local_include(obj.path, backend_file) + + elif isinstance(obj, PerSourceFlag): + self._process_per_source_flag(obj, backend_file) + + elif isinstance(obj, InstallationTarget): + self._process_installation_target(obj, backend_file) + + elif isinstance(obj, ContextWrapped): + # Process a rich build system object from the front-end + # as-is. Please follow precedent and handle CamelCaseData + # in a function named _process_camel_case_data. At some + # point in the future, this unwrapping process may be + # automated. + if isinstance(obj.wrapped, JavaJarData): + self._process_java_jar_data(obj.wrapped, backend_file) + elif isinstance(obj.wrapped, AndroidEclipseProjectData): + self._process_android_eclipse_project_data(obj.wrapped, backend_file) + else: + return False + + elif isinstance(obj, RustLibrary): + self.backend_input_files.add(obj.cargo_file) + self._process_rust_library(obj, backend_file) + # No need to call _process_linked_libraries, because Rust + # libraries are self-contained objects at this point. + + elif isinstance(obj, SharedLibrary): + self._process_shared_library(obj, backend_file) + self._process_linked_libraries(obj, backend_file) + + elif isinstance(obj, StaticLibrary): + self._process_static_library(obj, backend_file) + self._process_linked_libraries(obj, backend_file) + + elif isinstance(obj, HostLibrary): + self._process_host_library(obj, backend_file) + self._process_linked_libraries(obj, backend_file) + + elif isinstance(obj, FinalTargetFiles): + self._process_final_target_files(obj, obj.files, backend_file) + + elif isinstance(obj, FinalTargetPreprocessedFiles): + self._process_final_target_pp_files(obj, obj.files, backend_file, 'DIST_FILES') + + elif isinstance(obj, ObjdirFiles): + self._process_objdir_files(obj, obj.files, backend_file) + + elif isinstance(obj, ObjdirPreprocessedFiles): + self._process_final_target_pp_files(obj, obj.files, backend_file, 'OBJDIR_PP_FILES') + + elif isinstance(obj, AndroidResDirs): + # Order matters. + for p in obj.paths: + backend_file.write('ANDROID_RES_DIRS += %s\n' % p.full_path) + + elif isinstance(obj, AndroidAssetsDirs): + # Order matters. + for p in obj.paths: + backend_file.write('ANDROID_ASSETS_DIRS += %s\n' % p.full_path) + + elif isinstance(obj, AndroidExtraResDirs): + # Order does not matter. + for p in sorted(set(p.full_path for p in obj.paths)): + backend_file.write('ANDROID_EXTRA_RES_DIRS += %s\n' % p) + + elif isinstance(obj, AndroidExtraPackages): + # Order does not matter. + for p in sorted(set(obj.packages)): + backend_file.write('ANDROID_EXTRA_PACKAGES += %s\n' % p) + + elif isinstance(obj, ChromeManifestEntry): + self._process_chrome_manifest_entry(obj, backend_file) + + else: + return False + + return True + + def _fill_root_mk(self): + """ + Create two files, root.mk and root-deps.mk, the first containing + convenience variables, and the other dependency definitions for a + hopefully proper directory traversal. + """ + for tier, no_skip in self._no_skip.items(): + self.log(logging.DEBUG, 'fill_root_mk', { + 'number': len(no_skip), 'tier': tier + }, 'Using {number} directories during {tier}') + + def should_skip(tier, dir): + if tier in self._no_skip: + return dir not in self._no_skip[tier] + return False + + # Traverse directories in parallel, and skip static dirs + def parallel_filter(current, subdirs): + all_subdirs = subdirs.dirs + subdirs.tests + if should_skip(tier, current) or current.startswith('subtiers/'): + current = None + return current, all_subdirs, [] + + # build everything in parallel, including static dirs + # Because of bug 925236 and possible other unknown race conditions, + # don't parallelize the libs tier. + def libs_filter(current, subdirs): + if should_skip('libs', current) or current.startswith('subtiers/'): + current = None + return current, [], subdirs.dirs + subdirs.tests + + # Because of bug 925236 and possible other unknown race conditions, + # don't parallelize the tools tier. There aren't many directories for + # this tier anyways. + def tools_filter(current, subdirs): + if should_skip('tools', current) or current.startswith('subtiers/'): + current = None + return current, [], subdirs.dirs + subdirs.tests + + filters = [ + ('export', parallel_filter), + ('libs', libs_filter), + ('misc', parallel_filter), + ('tools', tools_filter), + ] + + root_deps_mk = Makefile() + + # Fill the dependencies for traversal of each tier. + for tier, filter in filters: + main, all_deps = \ + self._traversal.compute_dependencies(filter) + for dir, deps in all_deps.items(): + if deps is not None or (dir in self._idl_dirs \ + and tier == 'export'): + rule = root_deps_mk.create_rule(['%s/%s' % (dir, tier)]) + if deps: + rule.add_dependencies('%s/%s' % (d, tier) for d in deps if d) + if dir in self._idl_dirs and tier == 'export': + rule.add_dependencies(['xpcom/xpidl/%s' % tier]) + rule = root_deps_mk.create_rule(['recurse_%s' % tier]) + if main: + rule.add_dependencies('%s/%s' % (d, tier) for d in main) + + all_compile_deps = reduce(lambda x,y: x|y, + self._compile_graph.values()) if self._compile_graph else set() + compile_roots = set(self._compile_graph.keys()) - all_compile_deps + + rule = root_deps_mk.create_rule(['recurse_compile']) + rule.add_dependencies(compile_roots) + for target, deps in sorted(self._compile_graph.items()): + if deps: + rule = root_deps_mk.create_rule([target]) + rule.add_dependencies(deps) + + root_mk = Makefile() + + # Fill root.mk with the convenience variables. + for tier, filter in filters: + all_dirs = self._traversal.traverse('', filter) + root_mk.add_statement('%s_dirs := %s' % (tier, ' '.join(all_dirs))) + + # Need a list of compile targets because we can't use pattern rules: + # https://savannah.gnu.org/bugs/index.php?42833 + root_mk.add_statement('compile_targets := %s' % ' '.join(sorted( + set(self._compile_graph.keys()) | all_compile_deps))) + + root_mk.add_statement('include root-deps.mk') + + with self._write_file( + mozpath.join(self.environment.topobjdir, 'root.mk')) as root: + root_mk.dump(root, removal_guard=False) + + with self._write_file( + mozpath.join(self.environment.topobjdir, 'root-deps.mk')) as root_deps: + root_deps_mk.dump(root_deps, removal_guard=False) + + def _add_unified_build_rules(self, makefile, unified_source_mapping, + unified_files_makefile_variable='unified_files', + include_curdir_build_rules=True): + + # In case it's a generator. + unified_source_mapping = sorted(unified_source_mapping) + + explanation = "\n" \ + "# We build files in 'unified' mode by including several files\n" \ + "# together into a single source file. This cuts down on\n" \ + "# compilation times and debug information size." + makefile.add_statement(explanation) + + all_sources = ' '.join(source for source, _ in unified_source_mapping) + makefile.add_statement('%s := %s' % (unified_files_makefile_variable, + all_sources)) + + if include_curdir_build_rules: + makefile.add_statement('\n' + '# Make sometimes gets confused between "foo" and "$(CURDIR)/foo".\n' + '# Help it out by explicitly specifiying dependencies.') + makefile.add_statement('all_absolute_unified_files := \\\n' + ' $(addprefix $(CURDIR)/,$(%s))' + % unified_files_makefile_variable) + rule = makefile.create_rule(['$(all_absolute_unified_files)']) + rule.add_dependencies(['$(CURDIR)/%: %']) + + def _check_blacklisted_variables(self, makefile_in, makefile_content): + if b'EXTERNALLY_MANAGED_MAKE_FILE' in makefile_content: + # Bypass the variable restrictions for externally managed makefiles. + return + + for l in makefile_content.splitlines(): + l = l.strip() + # Don't check comments + if l.startswith(b'#'): + continue + for x in chain(MOZBUILD_VARIABLES, DEPRECATED_VARIABLES): + if x not in l: + continue + + # Finding the variable name in the Makefile is not enough: it + # may just appear as part of something else, like DIRS appears + # in GENERATED_DIRS. + if re.search(r'\b%s\s*[:?+]?=' % x, l): + if x in MOZBUILD_VARIABLES: + message = MOZBUILD_VARIABLES_MESSAGE + else: + message = DEPRECATED_VARIABLES_MESSAGE + raise Exception('Variable %s is defined in %s. %s' + % (x, makefile_in, message)) + + def consume_finished(self): + CommonBackend.consume_finished(self) + + for objdir, backend_file in sorted(self._backend_files.items()): + srcdir = backend_file.srcdir + with self._write_file(fh=backend_file) as bf: + makefile_in = mozpath.join(srcdir, 'Makefile.in') + makefile = mozpath.join(objdir, 'Makefile') + + # If Makefile.in exists, use it as a template. Otherwise, + # create a stub. + stub = not os.path.exists(makefile_in) + if not stub: + self.log(logging.DEBUG, 'substitute_makefile', + {'path': makefile}, 'Substituting makefile: {path}') + self._makefile_in_count += 1 + + # In the export and libs tiers, we don't skip directories + # containing a Makefile.in. + # topobjdir is handled separatedly, don't do anything for + # it. + if bf.relobjdir: + for tier in ('export', 'libs',): + self._no_skip[tier].add(bf.relobjdir) + else: + self.log(logging.DEBUG, 'stub_makefile', + {'path': makefile}, 'Creating stub Makefile: {path}') + + obj = self.Substitution() + obj.output_path = makefile + obj.input_path = makefile_in + obj.topsrcdir = backend_file.topsrcdir + obj.topobjdir = bf.environment.topobjdir + obj.config = bf.environment + self._create_makefile(obj, stub=stub) + with open(obj.output_path) as fh: + content = fh.read() + # Skip every directory but those with a Makefile + # containing a tools target, or XPI_PKGNAME or + # INSTALL_EXTENSION_ID. + for t in (b'XPI_PKGNAME', b'INSTALL_EXTENSION_ID', + b'tools'): + if t not in content: + continue + if t == b'tools' and not re.search('(?:^|\s)tools.*::', content, re.M): + continue + if objdir == self.environment.topobjdir: + continue + self._no_skip['tools'].add(mozpath.relpath(objdir, + self.environment.topobjdir)) + + # Detect any Makefile.ins that contain variables on the + # moz.build-only list + self._check_blacklisted_variables(makefile_in, content) + + self._fill_root_mk() + + # Make the master test manifest files. + for flavor, t in self._test_manifests.items(): + install_prefix, manifests = t + manifest_stem = mozpath.join(install_prefix, '%s.ini' % flavor) + self._write_master_test_manifest(mozpath.join( + self.environment.topobjdir, '_tests', manifest_stem), + manifests) + + # Catch duplicate inserts. + try: + self._install_manifests['_tests'].add_optional_exists(manifest_stem) + except ValueError: + pass + + self._write_manifests('install', self._install_manifests) + + ensureParentDir(mozpath.join(self.environment.topobjdir, 'dist', 'foo')) + + def _pretty_path_parts(self, path, backend_file): + assert isinstance(path, Path) + if isinstance(path, SourcePath): + if path.full_path.startswith(backend_file.srcdir): + return '$(srcdir)', path.full_path[len(backend_file.srcdir):] + if path.full_path.startswith(backend_file.topsrcdir): + return '$(topsrcdir)', path.full_path[len(backend_file.topsrcdir):] + elif isinstance(path, ObjDirPath): + if path.full_path.startswith(backend_file.objdir): + return '', path.full_path[len(backend_file.objdir) + 1:] + if path.full_path.startswith(self.environment.topobjdir): + return '$(DEPTH)', path.full_path[len(self.environment.topobjdir):] + + return '', path.full_path + + def _pretty_path(self, path, backend_file): + return ''.join(self._pretty_path_parts(path, backend_file)) + + def _process_unified_sources(self, obj): + backend_file = self._get_backend_file_for(obj) + + suffix_map = { + '.c': 'UNIFIED_CSRCS', + '.mm': 'UNIFIED_CMMSRCS', + '.cpp': 'UNIFIED_CPPSRCS', + } + + var = suffix_map[obj.canonical_suffix] + non_unified_var = var[len('UNIFIED_'):] + + if obj.have_unified_mapping: + self._add_unified_build_rules(backend_file, + obj.unified_source_mapping, + unified_files_makefile_variable=var, + include_curdir_build_rules=False) + backend_file.write('%s += $(%s)\n' % (non_unified_var, var)) + else: + # Sorted so output is consistent and we don't bump mtimes. + source_files = list(sorted(obj.files)) + + backend_file.write('%s += %s\n' % ( + non_unified_var, ' '.join(source_files))) + + def _process_directory_traversal(self, obj, backend_file): + """Process a data.DirectoryTraversal instance.""" + fh = backend_file.fh + + def relativize(base, dirs): + return (mozpath.relpath(d.translated, base) for d in dirs) + + if obj.dirs: + fh.write('DIRS := %s\n' % ' '.join( + relativize(backend_file.objdir, obj.dirs))) + self._traversal.add(backend_file.relobjdir, + dirs=relativize(self.environment.topobjdir, obj.dirs)) + + # The directory needs to be registered whether subdirectories have been + # registered or not. + self._traversal.add(backend_file.relobjdir) + + def _process_defines(self, obj, backend_file, which='DEFINES'): + """Output the DEFINES rules to the given backend file.""" + defines = list(obj.get_defines()) + if defines: + defines = ' '.join(shell_quote(d) for d in defines) + backend_file.write_once('%s += %s\n' % (which, defines)) + + def _process_installation_target(self, obj, backend_file): + # A few makefiles need to be able to override the following rules via + # make XPI_NAME=blah commands, so we default to the lazy evaluation as + # much as possible here to avoid breaking things. + if obj.xpiname: + backend_file.write('XPI_NAME = %s\n' % (obj.xpiname)) + if obj.subdir: + backend_file.write('DIST_SUBDIR = %s\n' % (obj.subdir)) + if obj.target and not obj.is_custom(): + backend_file.write('FINAL_TARGET = $(DEPTH)/%s\n' % (obj.target)) + else: + backend_file.write('FINAL_TARGET = $(if $(XPI_NAME),$(DIST)/xpi-stage/$(XPI_NAME),$(DIST)/bin)$(DIST_SUBDIR:%=/%)\n') + + if not obj.enabled: + backend_file.write('NO_DIST_INSTALL := 1\n') + + def _handle_idl_manager(self, manager): + build_files = self._install_manifests['xpidl'] + + for p in ('Makefile', 'backend.mk', '.deps/.mkdir.done'): + build_files.add_optional_exists(p) + + for idl in manager.idls.values(): + self._install_manifests['dist_idl'].add_symlink(idl['source'], + idl['basename']) + self._install_manifests['dist_include'].add_optional_exists('%s.h' + % idl['root']) + + for module in manager.modules: + build_files.add_optional_exists(mozpath.join('.deps', + '%s.pp' % module)) + + modules = manager.modules + xpt_modules = sorted(modules.keys()) + xpt_files = set() + registered_xpt_files = set() + + mk = Makefile() + + for module in xpt_modules: + install_target, sources = modules[module] + deps = sorted(sources) + + # It may seem strange to have the .idl files listed as + # prerequisites both here and in the auto-generated .pp files. + # It is necessary to list them here to handle the case where a + # new .idl is added to an xpt. If we add a new .idl and nothing + # else has changed, the new .idl won't be referenced anywhere + # except in the command invocation. Therefore, the .xpt won't + # be rebuilt because the dependencies say it is up to date. By + # listing the .idls here, we ensure the make file has a + # reference to the new .idl. Since the new .idl presumably has + # an mtime newer than the .xpt, it will trigger xpt generation. + xpt_path = '$(DEPTH)/%s/components/%s.xpt' % (install_target, module) + xpt_files.add(xpt_path) + mk.add_statement('%s_deps = %s' % (module, ' '.join(deps))) + + if install_target.startswith('dist/'): + path = mozpath.relpath(xpt_path, '$(DEPTH)/dist') + prefix, subpath = path.split('/', 1) + key = 'dist_%s' % prefix + + self._install_manifests[key].add_optional_exists(subpath) + + rules = StringIO() + mk.dump(rules, removal_guard=False) + + interfaces_manifests = [] + dist_dir = mozpath.join(self.environment.topobjdir, 'dist') + for manifest, entries in manager.interface_manifests.items(): + interfaces_manifests.append(mozpath.join('$(DEPTH)', manifest)) + for xpt in sorted(entries): + registered_xpt_files.add(mozpath.join( + '$(DEPTH)', mozpath.dirname(manifest), xpt)) + + if install_target.startswith('dist/'): + path = mozpath.join(self.environment.topobjdir, manifest) + path = mozpath.relpath(path, dist_dir) + prefix, subpath = path.split('/', 1) + key = 'dist_%s' % prefix + self._install_manifests[key].add_optional_exists(subpath) + + chrome_manifests = [mozpath.join('$(DEPTH)', m) for m in sorted(manager.chrome_manifests)] + + # Create dependency for output header so we force regeneration if the + # header was deleted. This ideally should not be necessary. However, + # some processes (such as PGO at the time this was implemented) wipe + # out dist/include without regard to our install manifests. + + obj = self.Substitution() + obj.output_path = mozpath.join(self.environment.topobjdir, 'config', + 'makefiles', 'xpidl', 'Makefile') + obj.input_path = mozpath.join(self.environment.topsrcdir, 'config', + 'makefiles', 'xpidl', 'Makefile.in') + obj.topsrcdir = self.environment.topsrcdir + obj.topobjdir = self.environment.topobjdir + obj.config = self.environment + self._create_makefile(obj, extra=dict( + chrome_manifests = ' '.join(chrome_manifests), + interfaces_manifests = ' '.join(interfaces_manifests), + xpidl_rules=rules.getvalue(), + xpidl_modules=' '.join(xpt_modules), + xpt_files=' '.join(sorted(xpt_files - registered_xpt_files)), + registered_xpt_files=' '.join(sorted(registered_xpt_files)), + )) + + def _process_program(self, program, backend_file): + backend_file.write('PROGRAM = %s\n' % program) + + def _process_host_program(self, program, backend_file): + backend_file.write('HOST_PROGRAM = %s\n' % program) + + def _process_simple_program(self, obj, backend_file): + if obj.is_unit_test: + backend_file.write('CPP_UNIT_TESTS += %s\n' % obj.program) + else: + backend_file.write('SIMPLE_PROGRAMS += %s\n' % obj.program) + + def _process_host_simple_program(self, program, backend_file): + backend_file.write('HOST_SIMPLE_PROGRAMS += %s\n' % program) + + def _process_test_manifest(self, obj, backend_file): + # Much of the logic in this function could be moved to CommonBackend. + self.backend_input_files.add(mozpath.join(obj.topsrcdir, + obj.manifest_relpath)) + + # Don't allow files to be defined multiple times unless it is allowed. + # We currently allow duplicates for non-test files or test files if + # the manifest is listed as a duplicate. + for source, (dest, is_test) in obj.installs.items(): + try: + self._install_manifests['_test_files'].add_symlink(source, dest) + except ValueError: + if not obj.dupe_manifest and is_test: + raise + + for base, pattern, dest in obj.pattern_installs: + try: + self._install_manifests['_test_files'].add_pattern_symlink(base, + pattern, dest) + except ValueError: + if not obj.dupe_manifest: + raise + + for dest in obj.external_installs: + try: + self._install_manifests['_test_files'].add_optional_exists(dest) + except ValueError: + if not obj.dupe_manifest: + raise + + m = self._test_manifests.setdefault(obj.flavor, + (obj.install_prefix, set())) + m[1].add(obj.manifest_obj_relpath) + + try: + from reftest import ReftestManifest + + if isinstance(obj.manifest, ReftestManifest): + # Mark included files as part of the build backend so changes + # result in re-config. + self.backend_input_files |= obj.manifest.manifests + except ImportError: + # Ignore errors caused by the reftest module not being present. + # This can happen when building SpiderMonkey standalone, for example. + pass + + def _process_local_include(self, local_include, backend_file): + d, path = self._pretty_path_parts(local_include, backend_file) + if isinstance(local_include, ObjDirPath) and not d: + # path doesn't start with a slash in this case + d = '$(CURDIR)/' + elif d == '$(DEPTH)': + d = '$(topobjdir)' + quoted_path = shell_quote(path) if path else path + if quoted_path != path: + path = quoted_path[0] + d + quoted_path[1:] + else: + path = d + path + backend_file.write('LOCAL_INCLUDES += -I%s\n' % path) + + def _process_per_source_flag(self, per_source_flag, backend_file): + for flag in per_source_flag.flags: + backend_file.write('%s_FLAGS += %s\n' % (mozpath.basename(per_source_flag.file_name), flag)) + + def _process_java_jar_data(self, jar, backend_file): + target = jar.name + backend_file.write('JAVA_JAR_TARGETS += %s\n' % target) + backend_file.write('%s_DEST := %s.jar\n' % (target, jar.name)) + if jar.sources: + backend_file.write('%s_JAVAFILES := %s\n' % + (target, ' '.join(jar.sources))) + if jar.generated_sources: + backend_file.write('%s_PP_JAVAFILES := %s\n' % + (target, ' '.join(mozpath.join('generated', f) for f in jar.generated_sources))) + if jar.extra_jars: + backend_file.write('%s_EXTRA_JARS := %s\n' % + (target, ' '.join(sorted(set(jar.extra_jars))))) + if jar.javac_flags: + backend_file.write('%s_JAVAC_FLAGS := %s\n' % + (target, ' '.join(jar.javac_flags))) + + def _process_android_eclipse_project_data(self, project, backend_file): + # We add a single target to the backend.mk corresponding to + # the moz.build defining the Android Eclipse project. This + # target depends on some targets to be fresh, and installs a + # manifest generated by the Android Eclipse build backend. The + # manifests for all projects live in $TOPOBJDIR/android_eclipse + # and are installed into subdirectories thereof. + + project_directory = mozpath.join(self.environment.topobjdir, 'android_eclipse', project.name) + manifest_path = mozpath.join(self.environment.topobjdir, 'android_eclipse', '%s.manifest' % project.name) + + fragment = Makefile() + rule = fragment.create_rule(targets=['ANDROID_ECLIPSE_PROJECT_%s' % project.name]) + rule.add_dependencies(project.recursive_make_targets) + args = ['--no-remove', + '--no-remove-all-directory-symlinks', + '--no-remove-empty-directories', + project_directory, + manifest_path] + rule.add_commands(['$(call py_action,process_install_manifest,%s)' % ' '.join(args)]) + fragment.dump(backend_file.fh, removal_guard=False) + + def _process_shared_library(self, libdef, backend_file): + backend_file.write_once('LIBRARY_NAME := %s\n' % libdef.basename) + backend_file.write('FORCE_SHARED_LIB := 1\n') + backend_file.write('IMPORT_LIBRARY := %s\n' % libdef.import_name) + backend_file.write('SHARED_LIBRARY := %s\n' % libdef.lib_name) + if libdef.variant == libdef.COMPONENT: + backend_file.write('IS_COMPONENT := 1\n') + if libdef.soname: + backend_file.write('DSO_SONAME := %s\n' % libdef.soname) + if libdef.is_sdk: + backend_file.write('SDK_LIBRARY := %s\n' % libdef.import_name) + if libdef.symbols_file: + backend_file.write('SYMBOLS_FILE := %s\n' % libdef.symbols_file) + if not libdef.cxx_link: + backend_file.write('LIB_IS_C_ONLY := 1\n') + + def _process_static_library(self, libdef, backend_file): + backend_file.write_once('LIBRARY_NAME := %s\n' % libdef.basename) + backend_file.write('FORCE_STATIC_LIB := 1\n') + backend_file.write('REAL_LIBRARY := %s\n' % libdef.lib_name) + if libdef.is_sdk: + backend_file.write('SDK_LIBRARY := %s\n' % libdef.import_name) + if libdef.no_expand_lib: + backend_file.write('NO_EXPAND_LIBS := 1\n') + + def _process_rust_library(self, libdef, backend_file): + backend_file.write_once('RUST_LIBRARY_FILE := %s\n' % libdef.import_name) + backend_file.write('CARGO_FILE := $(srcdir)/Cargo.toml') + + def _process_host_library(self, libdef, backend_file): + backend_file.write('HOST_LIBRARY_NAME = %s\n' % libdef.basename) + + def _build_target_for_obj(self, obj): + return '%s/%s' % (mozpath.relpath(obj.objdir, + self.environment.topobjdir), obj.KIND) + + def _process_linked_libraries(self, obj, backend_file): + def write_shared_and_system_libs(lib): + for l in lib.linked_libraries: + if isinstance(l, (StaticLibrary, RustLibrary)): + write_shared_and_system_libs(l) + else: + backend_file.write_once('SHARED_LIBS += %s/%s\n' + % (pretty_relpath(l), l.import_name)) + for l in lib.linked_system_libs: + backend_file.write_once('OS_LIBS += %s\n' % l) + + def pretty_relpath(lib): + return '$(DEPTH)/%s' % mozpath.relpath(lib.objdir, topobjdir) + + topobjdir = mozpath.normsep(obj.topobjdir) + # This will create the node even if there aren't any linked libraries. + build_target = self._build_target_for_obj(obj) + self._compile_graph[build_target] + + for lib in obj.linked_libraries: + if not isinstance(lib, ExternalLibrary): + self._compile_graph[build_target].add( + self._build_target_for_obj(lib)) + relpath = pretty_relpath(lib) + if isinstance(obj, Library): + if isinstance(lib, RustLibrary): + # We don't need to do anything here; we will handle + # linkage for any RustLibrary elsewhere. + continue + elif isinstance(lib, StaticLibrary): + backend_file.write_once('STATIC_LIBS += %s/%s\n' + % (relpath, lib.import_name)) + if isinstance(obj, SharedLibrary): + write_shared_and_system_libs(lib) + elif isinstance(obj, SharedLibrary): + assert lib.variant != lib.COMPONENT + backend_file.write_once('SHARED_LIBS += %s/%s\n' + % (relpath, lib.import_name)) + elif isinstance(obj, (Program, SimpleProgram)): + if isinstance(lib, StaticLibrary): + backend_file.write_once('STATIC_LIBS += %s/%s\n' + % (relpath, lib.import_name)) + write_shared_and_system_libs(lib) + else: + assert lib.variant != lib.COMPONENT + backend_file.write_once('SHARED_LIBS += %s/%s\n' + % (relpath, lib.import_name)) + elif isinstance(obj, (HostLibrary, HostProgram, HostSimpleProgram)): + assert isinstance(lib, HostLibrary) + backend_file.write_once('HOST_LIBS += %s/%s\n' + % (relpath, lib.import_name)) + + # We have to link any Rust libraries after all intermediate static + # libraries have been listed to ensure that the Rust libraries are + # searched after the C/C++ objects that might reference Rust symbols. + if isinstance(obj, SharedLibrary): + self._process_rust_libraries(obj, backend_file, pretty_relpath) + + for lib in obj.linked_system_libs: + if obj.KIND == 'target': + backend_file.write_once('OS_LIBS += %s\n' % lib) + else: + backend_file.write_once('HOST_EXTRA_LIBS += %s\n' % lib) + + # Process library-based defines + self._process_defines(obj.lib_defines, backend_file) + + def _process_rust_libraries(self, obj, backend_file, pretty_relpath): + assert isinstance(obj, SharedLibrary) + + # If this library does not depend on any Rust libraries, then we are done. + direct_linked = [l for l in obj.linked_libraries if isinstance(l, RustLibrary)] + if not direct_linked: + return + + # We should have already checked this in Linkable.link_library. + assert len(direct_linked) == 1 + + # TODO: see bug 1310063 for checking dependencies are set up correctly. + + direct_linked = direct_linked[0] + backend_file.write('RUST_STATIC_LIB_FOR_SHARED_LIB := %s/%s\n' % + (pretty_relpath(direct_linked), direct_linked.import_name)) + + def _process_final_target_files(self, obj, files, backend_file): + target = obj.install_target + path = mozpath.basedir(target, ( + 'dist/bin', + 'dist/xpi-stage', + '_tests', + 'dist/include', + 'dist/branding', + 'dist/sdk', + )) + if not path: + raise Exception("Cannot install to " + target) + + manifest = path.replace('/', '_') + install_manifest = self._install_manifests[manifest] + reltarget = mozpath.relpath(target, path) + + # Also emit the necessary rules to create $(DIST)/branding during + # partial tree builds. The locale makefiles rely on this working. + if path == 'dist/branding': + backend_file.write('NONRECURSIVE_TARGETS += export\n') + backend_file.write('NONRECURSIVE_TARGETS_export += branding\n') + backend_file.write('NONRECURSIVE_TARGETS_export_branding_DIRECTORY = $(DEPTH)\n') + backend_file.write('NONRECURSIVE_TARGETS_export_branding_TARGETS += install-dist/branding\n') + + for path, files in files.walk(): + target_var = (mozpath.join(target, path) + if path else target).replace('/', '_') + have_objdir_files = False + for f in files: + assert not isinstance(f, RenamedSourcePath) + dest = mozpath.join(reltarget, path, f.target_basename) + if not isinstance(f, ObjDirPath): + if '*' in f: + if f.startswith('/') or isinstance(f, AbsolutePath): + basepath, wild = os.path.split(f.full_path) + if '*' in basepath: + raise Exception("Wildcards are only supported in the filename part of " + "srcdir-relative or absolute paths.") + + install_manifest.add_pattern_symlink(basepath, wild, path) + else: + install_manifest.add_pattern_symlink(f.srcdir, f, path) + else: + install_manifest.add_symlink(f.full_path, dest) + else: + install_manifest.add_optional_exists(dest) + backend_file.write('%s_FILES += %s\n' % ( + target_var, self._pretty_path(f, backend_file))) + have_objdir_files = True + if have_objdir_files: + tier = 'export' if obj.install_target == 'dist/include' else 'misc' + self._no_skip[tier].add(backend_file.relobjdir) + backend_file.write('%s_DEST := $(DEPTH)/%s\n' + % (target_var, + mozpath.join(target, path))) + backend_file.write('%s_TARGET := %s\n' % (target_var, tier)) + backend_file.write('INSTALL_TARGETS += %s\n' % target_var) + + def _process_final_target_pp_files(self, obj, files, backend_file, name): + # Bug 1177710 - We'd like to install these via manifests as + # preprocessed files. But they currently depend on non-standard flags + # being added via some Makefiles, so for now we just pass them through + # to the underlying Makefile.in. + # + # Note that if this becomes a manifest, OBJDIR_PP_FILES will likely + # still need to use PP_TARGETS internally because we can't have an + # install manifest for the root of the objdir. + for i, (path, files) in enumerate(files.walk()): + self._no_skip['misc'].add(backend_file.relobjdir) + var = '%s_%d' % (name, i) + for f in files: + backend_file.write('%s += %s\n' % ( + var, self._pretty_path(f, backend_file))) + backend_file.write('%s_PATH := $(DEPTH)/%s\n' + % (var, mozpath.join(obj.install_target, path))) + backend_file.write('%s_TARGET := misc\n' % var) + backend_file.write('PP_TARGETS += %s\n' % var) + + def _process_objdir_files(self, obj, files, backend_file): + # We can't use an install manifest for the root of the objdir, since it + # would delete all the other files that get put there by the build + # system. + for i, (path, files) in enumerate(files.walk()): + self._no_skip['misc'].add(backend_file.relobjdir) + for f in files: + backend_file.write('OBJDIR_%d_FILES += %s\n' % ( + i, self._pretty_path(f, backend_file))) + backend_file.write('OBJDIR_%d_DEST := $(topobjdir)/%s\n' % (i, path)) + backend_file.write('OBJDIR_%d_TARGET := misc\n' % i) + backend_file.write('INSTALL_TARGETS += OBJDIR_%d\n' % i) + + def _process_chrome_manifest_entry(self, obj, backend_file): + fragment = Makefile() + rule = fragment.create_rule(targets=['misc:']) + + top_level = mozpath.join(obj.install_target, 'chrome.manifest') + if obj.path != top_level: + args = [ + mozpath.join('$(DEPTH)', top_level), + make_quote(shell_quote('manifest %s' % + mozpath.relpath(obj.path, + obj.install_target))), + ] + rule.add_commands(['$(call py_action,buildlist,%s)' % + ' '.join(args)]) + args = [ + mozpath.join('$(DEPTH)', obj.path), + make_quote(shell_quote(str(obj.entry))), + ] + rule.add_commands(['$(call py_action,buildlist,%s)' % ' '.join(args)]) + fragment.dump(backend_file.fh, removal_guard=False) + + self._no_skip['misc'].add(obj.relativedir) + + def _write_manifests(self, dest, manifests): + man_dir = mozpath.join(self.environment.topobjdir, '_build_manifests', + dest) + + for k, manifest in manifests.items(): + with self._write_file(mozpath.join(man_dir, k)) as fh: + manifest.write(fileobj=fh) + + def _write_master_test_manifest(self, path, manifests): + with self._write_file(path) as master: + master.write( + '; THIS FILE WAS AUTOMATICALLY GENERATED. DO NOT MODIFY BY HAND.\n\n') + + for manifest in sorted(manifests): + master.write('[include:%s]\n' % manifest) + + class Substitution(object): + """BaseConfigSubstitution-like class for use with _create_makefile.""" + __slots__ = ( + 'input_path', + 'output_path', + 'topsrcdir', + 'topobjdir', + 'config', + ) + + def _create_makefile(self, obj, stub=False, extra=None): + '''Creates the given makefile. Makefiles are treated the same as + config files, but some additional header and footer is added to the + output. + + When the stub argument is True, no source file is used, and a stub + makefile with the default header and footer only is created. + ''' + with self._get_preprocessor(obj) as pp: + if extra: + pp.context.update(extra) + if not pp.context.get('autoconfmk', ''): + pp.context['autoconfmk'] = 'autoconf.mk' + pp.handleLine(b'# THIS FILE WAS AUTOMATICALLY GENERATED. DO NOT MODIFY BY HAND.\n'); + pp.handleLine(b'DEPTH := @DEPTH@\n') + pp.handleLine(b'topobjdir := @topobjdir@\n') + pp.handleLine(b'topsrcdir := @top_srcdir@\n') + pp.handleLine(b'srcdir := @srcdir@\n') + pp.handleLine(b'VPATH := @srcdir@\n') + pp.handleLine(b'relativesrcdir := @relativesrcdir@\n') + pp.handleLine(b'include $(DEPTH)/config/@autoconfmk@\n') + if not stub: + pp.do_include(obj.input_path) + # Empty line to avoid failures when last line in Makefile.in ends + # with a backslash. + pp.handleLine(b'\n') + pp.handleLine(b'include $(topsrcdir)/config/recurse.mk\n') + if not stub: + # Adding the Makefile.in here has the desired side-effect + # that if the Makefile.in disappears, this will force + # moz.build traversal. This means that when we remove empty + # Makefile.in files, the old file will get replaced with + # the autogenerated one automatically. + self.backend_input_files.add(obj.input_path) + + self._makefile_out_count += 1 + + def _handle_linked_rust_crates(self, obj, extern_crate_file): + backend_file = self._get_backend_file_for(obj) + + backend_file.write('RS_STATICLIB_CRATE_SRC := %s\n' % extern_crate_file) + + def _handle_ipdl_sources(self, ipdl_dir, sorted_ipdl_sources, + unified_ipdl_cppsrcs_mapping): + # Write out a master list of all IPDL source files. + mk = Makefile() + + mk.add_statement('ALL_IPDLSRCS := %s' % ' '.join(sorted_ipdl_sources)) + + self._add_unified_build_rules(mk, unified_ipdl_cppsrcs_mapping, + unified_files_makefile_variable='CPPSRCS') + + mk.add_statement('IPDLDIRS := %s' % ' '.join(sorted(set(mozpath.dirname(p) + for p in self._ipdl_sources)))) + + with self._write_file(mozpath.join(ipdl_dir, 'ipdlsrcs.mk')) as ipdls: + mk.dump(ipdls, removal_guard=False) + + def _handle_webidl_build(self, bindings_dir, unified_source_mapping, + webidls, expected_build_output_files, + global_define_files): + include_dir = mozpath.join(self.environment.topobjdir, 'dist', + 'include') + for f in expected_build_output_files: + if f.startswith(include_dir): + self._install_manifests['dist_include'].add_optional_exists( + mozpath.relpath(f, include_dir)) + + # We pass WebIDL info to make via a completely generated make file. + mk = Makefile() + mk.add_statement('nonstatic_webidl_files := %s' % ' '.join( + sorted(webidls.all_non_static_basenames()))) + mk.add_statement('globalgen_sources := %s' % ' '.join( + sorted(global_define_files))) + mk.add_statement('test_sources := %s' % ' '.join( + sorted('%sBinding.cpp' % s for s in webidls.all_test_stems()))) + + # Add rules to preprocess bindings. + # This should ideally be using PP_TARGETS. However, since the input + # filenames match the output filenames, the existing PP_TARGETS rules + # result in circular dependencies and other make weirdness. One + # solution is to rename the input or output files repsectively. See + # bug 928195 comment 129. + for source in sorted(webidls.all_preprocessed_sources()): + basename = os.path.basename(source) + rule = mk.create_rule([basename]) + rule.add_dependencies([source, '$(GLOBAL_DEPS)']) + rule.add_commands([ + # Remove the file before writing so bindings that go from + # static to preprocessed don't end up writing to a symlink, + # which would modify content in the source directory. + '$(RM) $@', + '$(call py_action,preprocessor,$(DEFINES) $(ACDEFINES) ' + '$< -o $@)' + ]) + + self._add_unified_build_rules(mk, + unified_source_mapping, + unified_files_makefile_variable='unified_binding_cpp_files') + + webidls_mk = mozpath.join(bindings_dir, 'webidlsrcs.mk') + with self._write_file(webidls_mk) as fh: + mk.dump(fh, removal_guard=False) diff --git a/python/mozbuild/mozbuild/backend/templates/android_eclipse/.classpath b/python/mozbuild/mozbuild/backend/templates/android_eclipse/.classpath new file mode 100644 index 000000000..7c51c539c --- /dev/null +++ b/python/mozbuild/mozbuild/backend/templates/android_eclipse/.classpath @@ -0,0 +1,10 @@ +#filter substitution + + + + + + + +@IDE_CLASSPATH_ENTRIES@ + diff --git a/python/mozbuild/mozbuild/backend/templates/android_eclipse/.externalToolBuilders/com.android.ide.eclipse.adt.ApkBuilder.launch b/python/mozbuild/mozbuild/backend/templates/android_eclipse/.externalToolBuilders/com.android.ide.eclipse.adt.ApkBuilder.launch new file mode 100644 index 000000000..3005dee45 --- /dev/null +++ b/python/mozbuild/mozbuild/backend/templates/android_eclipse/.externalToolBuilders/com.android.ide.eclipse.adt.ApkBuilder.launch @@ -0,0 +1,8 @@ +#filter substitution + + + + + + + diff --git a/python/mozbuild/mozbuild/backend/templates/android_eclipse/.externalToolBuilders/com.android.ide.eclipse.adt.PreCompilerBuilder.launch b/python/mozbuild/mozbuild/backend/templates/android_eclipse/.externalToolBuilders/com.android.ide.eclipse.adt.PreCompilerBuilder.launch new file mode 100644 index 000000000..9fa599f5f --- /dev/null +++ b/python/mozbuild/mozbuild/backend/templates/android_eclipse/.externalToolBuilders/com.android.ide.eclipse.adt.PreCompilerBuilder.launch @@ -0,0 +1,8 @@ +#filter substitution + + + + + + + diff --git a/python/mozbuild/mozbuild/backend/templates/android_eclipse/.externalToolBuilders/com.android.ide.eclipse.adt.ResourceManagerBuilder.launch b/python/mozbuild/mozbuild/backend/templates/android_eclipse/.externalToolBuilders/com.android.ide.eclipse.adt.ResourceManagerBuilder.launch new file mode 100644 index 000000000..20d1c3f4e --- /dev/null +++ b/python/mozbuild/mozbuild/backend/templates/android_eclipse/.externalToolBuilders/com.android.ide.eclipse.adt.ResourceManagerBuilder.launch @@ -0,0 +1,8 @@ +#filter substitution + + + + + + + diff --git a/python/mozbuild/mozbuild/backend/templates/android_eclipse/.externalToolBuilders/org.eclipse.jdt.core.javabuilder.launch b/python/mozbuild/mozbuild/backend/templates/android_eclipse/.externalToolBuilders/org.eclipse.jdt.core.javabuilder.launch new file mode 100644 index 000000000..ed5bf6885 --- /dev/null +++ b/python/mozbuild/mozbuild/backend/templates/android_eclipse/.externalToolBuilders/org.eclipse.jdt.core.javabuilder.launch @@ -0,0 +1,8 @@ +#filter substitution + + + + + + + diff --git a/python/mozbuild/mozbuild/backend/templates/android_eclipse/AndroidManifest.xml b/python/mozbuild/mozbuild/backend/templates/android_eclipse/AndroidManifest.xml new file mode 100644 index 000000000..57d8aca8c --- /dev/null +++ b/python/mozbuild/mozbuild/backend/templates/android_eclipse/AndroidManifest.xml @@ -0,0 +1,11 @@ +#filter substitution + + + + + diff --git a/python/mozbuild/mozbuild/backend/templates/android_eclipse/gen/tmp b/python/mozbuild/mozbuild/backend/templates/android_eclipse/gen/tmp new file mode 100644 index 000000000..c1c78936f --- /dev/null +++ b/python/mozbuild/mozbuild/backend/templates/android_eclipse/gen/tmp @@ -0,0 +1 @@ +#filter substitution diff --git a/python/mozbuild/mozbuild/backend/templates/android_eclipse/lint.xml b/python/mozbuild/mozbuild/backend/templates/android_eclipse/lint.xml new file mode 100644 index 000000000..43ad15dc9 --- /dev/null +++ b/python/mozbuild/mozbuild/backend/templates/android_eclipse/lint.xml @@ -0,0 +1,5 @@ +#filter substitution + + + + diff --git a/python/mozbuild/mozbuild/backend/templates/android_eclipse/project.properties b/python/mozbuild/mozbuild/backend/templates/android_eclipse/project.properties new file mode 100644 index 000000000..2106d9646 --- /dev/null +++ b/python/mozbuild/mozbuild/backend/templates/android_eclipse/project.properties @@ -0,0 +1,14 @@ +#filter substitution +# This file is automatically generated by Android Tools. +# Do not modify this file -- YOUR CHANGES WILL BE ERASED! +# +# This file must be checked in Version Control Systems. +# +# To customize properties used by the Ant build system edit +# "ant.properties", and override values to adapt the script to your +# project structure. + +# Project target. +target=android-L +@IDE_PROJECT_LIBRARY_SETTING@ +@IDE_PROJECT_LIBRARY_REFERENCES@ diff --git a/python/mozbuild/mozbuild/backend/templates/android_eclipse_empty_resource_directory/.not_an_android_resource b/python/mozbuild/mozbuild/backend/templates/android_eclipse_empty_resource_directory/.not_an_android_resource new file mode 100644 index 000000000..8ffce0692 --- /dev/null +++ b/python/mozbuild/mozbuild/backend/templates/android_eclipse_empty_resource_directory/.not_an_android_resource @@ -0,0 +1,5 @@ +This file is named such that it is ignored by Android aapt. The file +itself ensures that the AndroidEclipse build backend can create an +empty res/ directory for projects explicitly specifying that it has no +resource directory. This is necessary because the Android Eclipse +plugin requires that each project have a res/ directory. diff --git a/python/mozbuild/mozbuild/backend/tup.py b/python/mozbuild/mozbuild/backend/tup.py new file mode 100644 index 000000000..0f7250eb0 --- /dev/null +++ b/python/mozbuild/mozbuild/backend/tup.py @@ -0,0 +1,344 @@ +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, You can obtain one at http://mozilla.org/MPL/2.0/. + +from __future__ import absolute_import, unicode_literals + +import os + +import mozpack.path as mozpath +from mozbuild.base import MozbuildObject +from mozbuild.backend.base import PartialBackend, HybridBackend +from mozbuild.backend.recursivemake import RecursiveMakeBackend +from mozbuild.shellutil import quote as shell_quote + +from .common import CommonBackend +from ..frontend.data import ( + ContextDerived, + Defines, + FinalTargetPreprocessedFiles, + GeneratedFile, + HostDefines, + ObjdirPreprocessedFiles, +) +from ..util import ( + FileAvoidWrite, +) + + +class BackendTupfile(object): + """Represents a generated Tupfile. + """ + + def __init__(self, srcdir, objdir, environment, topsrcdir, topobjdir): + self.topsrcdir = topsrcdir + self.srcdir = srcdir + self.objdir = objdir + self.relobjdir = mozpath.relpath(objdir, topobjdir) + self.environment = environment + self.name = mozpath.join(objdir, 'Tupfile') + self.rules_included = False + self.shell_exported = False + self.defines = [] + self.host_defines = [] + self.delayed_generated_files = [] + + self.fh = FileAvoidWrite(self.name, capture_diff=True) + self.fh.write('# THIS FILE WAS AUTOMATICALLY GENERATED. DO NOT EDIT.\n') + self.fh.write('\n') + + def write(self, buf): + self.fh.write(buf) + + def include_rules(self): + if not self.rules_included: + self.write('include_rules\n') + self.rules_included = True + + def rule(self, cmd, inputs=None, outputs=None, display=None, extra_outputs=None, check_unchanged=False): + inputs = inputs or [] + outputs = outputs or [] + display = display or "" + self.include_rules() + flags = "" + if check_unchanged: + # This flag causes tup to compare the outputs with the previous run + # of the command, and skip the rest of the DAG for any that are the + # same. + flags += "o" + + if display: + caret_text = flags + ' ' + display + else: + caret_text = flags + + self.write(': %(inputs)s |> %(display)s%(cmd)s |> %(outputs)s%(extra_outputs)s\n' % { + 'inputs': ' '.join(inputs), + 'display': '^%s^ ' % caret_text if caret_text else '', + 'cmd': ' '.join(cmd), + 'outputs': ' '.join(outputs), + 'extra_outputs': ' | ' + ' '.join(extra_outputs) if extra_outputs else '', + }) + + def export_shell(self): + if not self.shell_exported: + # These are used by mach/mixin/process.py to determine the current + # shell. + for var in ('SHELL', 'MOZILLABUILD', 'COMSPEC'): + self.write('export %s\n' % var) + self.shell_exported = True + + def close(self): + return self.fh.close() + + @property + def diff(self): + return self.fh.diff + + +class TupOnly(CommonBackend, PartialBackend): + """Backend that generates Tupfiles for the tup build system. + """ + + def _init(self): + CommonBackend._init(self) + + self._backend_files = {} + self._cmd = MozbuildObject.from_environment() + + def _get_backend_file(self, relativedir): + objdir = mozpath.join(self.environment.topobjdir, relativedir) + srcdir = mozpath.join(self.environment.topsrcdir, relativedir) + if objdir not in self._backend_files: + self._backend_files[objdir] = \ + BackendTupfile(srcdir, objdir, self.environment, + self.environment.topsrcdir, self.environment.topobjdir) + return self._backend_files[objdir] + + def _get_backend_file_for(self, obj): + return self._get_backend_file(obj.relativedir) + + def _py_action(self, action): + cmd = [ + '$(PYTHON)', + '-m', + 'mozbuild.action.%s' % action, + ] + return cmd + + def consume_object(self, obj): + """Write out build files necessary to build with tup.""" + + if not isinstance(obj, ContextDerived): + return False + + consumed = CommonBackend.consume_object(self, obj) + + # Even if CommonBackend acknowledged the object, we still need to let + # the RecursiveMake backend also handle these objects. + if consumed: + return False + + backend_file = self._get_backend_file_for(obj) + + if isinstance(obj, GeneratedFile): + # These files are already generated by make before tup runs. + skip_files = ( + 'buildid.h', + 'source-repo.h', + ) + if any(f in skip_files for f in obj.outputs): + # Let the RecursiveMake backend handle these. + return False + + if 'application.ini.h' in obj.outputs: + # application.ini.h is a special case since we need to process + # the FINAL_TARGET_PP_FILES for application.ini before running + # the GENERATED_FILES script, and tup doesn't handle the rules + # out of order. + backend_file.delayed_generated_files.append(obj) + else: + self._process_generated_file(backend_file, obj) + elif isinstance(obj, Defines): + self._process_defines(backend_file, obj) + elif isinstance(obj, HostDefines): + self._process_defines(backend_file, obj, host=True) + elif isinstance(obj, FinalTargetPreprocessedFiles): + self._process_final_target_pp_files(obj, backend_file) + elif isinstance(obj, ObjdirPreprocessedFiles): + self._process_final_target_pp_files(obj, backend_file) + + return True + + def consume_finished(self): + CommonBackend.consume_finished(self) + + for objdir, backend_file in sorted(self._backend_files.items()): + for obj in backend_file.delayed_generated_files: + self._process_generated_file(backend_file, obj) + with self._write_file(fh=backend_file): + pass + + with self._write_file(mozpath.join(self.environment.topobjdir, 'Tuprules.tup')) as fh: + acdefines = [name for name in self.environment.defines + if not name in self.environment.non_global_defines] + acdefines_flags = ' '.join(['-D%s=%s' % (name, + shell_quote(self.environment.defines[name])) + for name in sorted(acdefines)]) + # TODO: AB_CD only exists in Makefiles at the moment. + acdefines_flags += ' -DAB_CD=en-US' + + fh.write('MOZ_OBJ_ROOT = $(TUP_CWD)\n') + fh.write('DIST = $(MOZ_OBJ_ROOT)/dist\n') + fh.write('ACDEFINES = %s\n' % acdefines_flags) + fh.write('topsrcdir = $(MOZ_OBJ_ROOT)/%s\n' % ( + os.path.relpath(self.environment.topsrcdir, self.environment.topobjdir) + )) + fh.write('PYTHON = $(MOZ_OBJ_ROOT)/_virtualenv/bin/python -B\n') + fh.write('PYTHON_PATH = $(PYTHON) $(topsrcdir)/config/pythonpath.py\n') + fh.write('PLY_INCLUDE = -I$(topsrcdir)/other-licenses/ply\n') + fh.write('IDL_PARSER_DIR = $(topsrcdir)/xpcom/idl-parser\n') + fh.write('IDL_PARSER_CACHE_DIR = $(MOZ_OBJ_ROOT)/xpcom/idl-parser/xpidl\n') + + # Run 'tup init' if necessary. + if not os.path.exists(mozpath.join(self.environment.topsrcdir, ".tup")): + tup = self.environment.substs.get('TUP', 'tup') + self._cmd.run_process(cwd=self.environment.topsrcdir, log_name='tup', args=[tup, 'init']) + + def _process_generated_file(self, backend_file, obj): + # TODO: These are directories that don't work in the tup backend + # yet, because things they depend on aren't built yet. + skip_directories = ( + 'layout/style/test', # HostSimplePrograms + 'toolkit/library', # libxul.so + ) + if obj.script and obj.method and obj.relobjdir not in skip_directories: + backend_file.export_shell() + cmd = self._py_action('file_generate') + cmd.extend([ + obj.script, + obj.method, + obj.outputs[0], + '%s.pp' % obj.outputs[0], # deps file required + ]) + full_inputs = [f.full_path for f in obj.inputs] + cmd.extend(full_inputs) + + outputs = [] + outputs.extend(obj.outputs) + outputs.append('%s.pp' % obj.outputs[0]) + + backend_file.rule( + display='python {script}:{method} -> [%o]'.format(script=obj.script, method=obj.method), + cmd=cmd, + inputs=full_inputs, + outputs=outputs, + ) + + def _process_defines(self, backend_file, obj, host=False): + defines = list(obj.get_defines()) + if defines: + if host: + backend_file.host_defines = defines + else: + backend_file.defines = defines + + def _process_final_target_pp_files(self, obj, backend_file): + for i, (path, files) in enumerate(obj.files.walk()): + for f in files: + self._preprocess(backend_file, f.full_path, + destdir=mozpath.join(self.environment.topobjdir, obj.install_target, path)) + + def _handle_idl_manager(self, manager): + backend_file = self._get_backend_file('xpcom/xpidl') + backend_file.export_shell() + + for module, data in sorted(manager.modules.iteritems()): + dest, idls = data + cmd = [ + '$(PYTHON_PATH)', + '$(PLY_INCLUDE)', + '-I$(IDL_PARSER_DIR)', + '-I$(IDL_PARSER_CACHE_DIR)', + '$(topsrcdir)/python/mozbuild/mozbuild/action/xpidl-process.py', + '--cache-dir', '$(IDL_PARSER_CACHE_DIR)', + '$(DIST)/idl', + '$(DIST)/include', + '$(MOZ_OBJ_ROOT)/%s/components' % dest, + module, + ] + cmd.extend(sorted(idls)) + + outputs = ['$(MOZ_OBJ_ROOT)/%s/components/%s.xpt' % (dest, module)] + outputs.extend(['$(MOZ_OBJ_ROOT)/dist/include/%s.h' % f for f in sorted(idls)]) + backend_file.rule( + inputs=[ + '$(MOZ_OBJ_ROOT)/xpcom/idl-parser/xpidl/xpidllex.py', + '$(MOZ_OBJ_ROOT)/xpcom/idl-parser/xpidl/xpidlyacc.py', + ], + display='XPIDL %s' % module, + cmd=cmd, + outputs=outputs, + ) + + def _preprocess(self, backend_file, input_file, destdir=None): + cmd = self._py_action('preprocessor') + cmd.extend(backend_file.defines) + cmd.extend(['$(ACDEFINES)', '%f', '-o', '%o']) + + base_input = mozpath.basename(input_file) + if base_input.endswith('.in'): + base_input = mozpath.splitext(base_input)[0] + output = mozpath.join(destdir, base_input) if destdir else base_input + + backend_file.rule( + inputs=[input_file], + display='Preprocess %o', + cmd=cmd, + outputs=[output], + ) + + def _handle_ipdl_sources(self, ipdl_dir, sorted_ipdl_sources, + unified_ipdl_cppsrcs_mapping): + # TODO: This isn't implemented yet in the tup backend, but it is called + # by the CommonBackend. + pass + + def _handle_webidl_build(self, bindings_dir, unified_source_mapping, + webidls, expected_build_output_files, + global_define_files): + backend_file = self._get_backend_file('dom/bindings') + backend_file.export_shell() + + for source in sorted(webidls.all_preprocessed_sources()): + self._preprocess(backend_file, source) + + cmd = self._py_action('webidl') + cmd.append(mozpath.join(self.environment.topsrcdir, 'dom', 'bindings')) + + # The WebIDLCodegenManager knows all of the .cpp and .h files that will + # be created (expected_build_output_files), but there are a few + # additional files that are also created by the webidl py_action. + outputs = [ + '_cache/webidlyacc.py', + 'codegen.json', + 'codegen.pp', + 'parser.out', + ] + outputs.extend(expected_build_output_files) + + backend_file.rule( + display='WebIDL code generation', + cmd=cmd, + inputs=webidls.all_non_static_basenames(), + outputs=outputs, + check_unchanged=True, + ) + + +class TupBackend(HybridBackend(TupOnly, RecursiveMakeBackend)): + def build(self, config, output, jobs, verbose): + status = config._run_make(directory=self.environment.topobjdir, target='tup', + line_handler=output.on_line, log=False, print_directory=False, + ensure_exit_code=False, num_jobs=jobs, silent=not verbose) + return status diff --git a/python/mozbuild/mozbuild/backend/visualstudio.py b/python/mozbuild/mozbuild/backend/visualstudio.py new file mode 100644 index 000000000..86e97d13d --- /dev/null +++ b/python/mozbuild/mozbuild/backend/visualstudio.py @@ -0,0 +1,582 @@ +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, You can obtain one at http://mozilla.org/MPL/2.0/. + +# This file contains a build backend for generating Visual Studio project +# files. + +from __future__ import absolute_import, unicode_literals + +import errno +import os +import re +import types +import uuid + +from xml.dom import getDOMImplementation + +from mozpack.files import FileFinder + +from .common import CommonBackend +from ..frontend.data import ( + Defines, + GeneratedSources, + HostProgram, + HostSources, + Library, + LocalInclude, + Program, + Sources, + UnifiedSources, +) +from mozbuild.base import ExecutionSummary + + +MSBUILD_NAMESPACE = 'http://schemas.microsoft.com/developer/msbuild/2003' + +def get_id(name): + return str(uuid.uuid5(uuid.NAMESPACE_URL, name)).upper() + +def visual_studio_product_to_solution_version(version): + if version == '2015': + return '12.00', '14' + else: + raise Exception('Unknown version seen: %s' % version) + +def visual_studio_product_to_platform_toolset_version(version): + if version == '2015': + return 'v140' + else: + raise Exception('Unknown version seen: %s' % version) + +class VisualStudioBackend(CommonBackend): + """Generate Visual Studio project files. + + This backend is used to produce Visual Studio projects and a solution + to foster developing Firefox with Visual Studio. + + This backend is currently considered experimental. There are many things + not optimal about how it works. + """ + + def _init(self): + CommonBackend._init(self) + + # These should eventually evolve into parameters. + self._out_dir = os.path.join(self.environment.topobjdir, 'msvc') + self._projsubdir = 'projects' + + self._version = self.environment.substs.get('MSVS_VERSION', '2015') + + self._paths_to_sources = {} + self._paths_to_includes = {} + self._paths_to_defines = {} + self._paths_to_configs = {} + self._libs_to_paths = {} + self._progs_to_paths = {} + + def summary(self): + return ExecutionSummary( + 'VisualStudio backend executed in {execution_time:.2f}s\n' + 'Generated Visual Studio solution at {path:s}', + execution_time=self._execution_time, + path=os.path.join(self._out_dir, 'mozilla.sln')) + + def consume_object(self, obj): + reldir = getattr(obj, 'relativedir', None) + + if hasattr(obj, 'config') and reldir not in self._paths_to_configs: + self._paths_to_configs[reldir] = obj.config + + if isinstance(obj, Sources): + self._add_sources(reldir, obj) + + elif isinstance(obj, HostSources): + self._add_sources(reldir, obj) + + elif isinstance(obj, GeneratedSources): + self._add_sources(reldir, obj) + + elif isinstance(obj, UnifiedSources): + # XXX we should be letting CommonBackend.consume_object call this + # for us instead. + self._process_unified_sources(obj); + + elif isinstance(obj, Library): + self._libs_to_paths[obj.basename] = reldir + + elif isinstance(obj, Program) or isinstance(obj, HostProgram): + self._progs_to_paths[obj.program] = reldir + + elif isinstance(obj, Defines): + self._paths_to_defines.setdefault(reldir, {}).update(obj.defines) + + elif isinstance(obj, LocalInclude): + includes = self._paths_to_includes.setdefault(reldir, []) + includes.append(obj.path.full_path) + + # Just acknowledge everything. + return True + + def _add_sources(self, reldir, obj): + s = self._paths_to_sources.setdefault(reldir, set()) + s.update(obj.files) + + def _process_unified_sources(self, obj): + reldir = getattr(obj, 'relativedir', None) + + s = self._paths_to_sources.setdefault(reldir, set()) + s.update(obj.files) + + def consume_finished(self): + out_dir = self._out_dir + out_proj_dir = os.path.join(self._out_dir, self._projsubdir) + + projects = self._write_projects_for_sources(self._libs_to_paths, + "library", out_proj_dir) + projects.update(self._write_projects_for_sources(self._progs_to_paths, + "binary", out_proj_dir)) + + # Generate projects that can be used to build common targets. + for target in ('export', 'binaries', 'tools', 'full'): + basename = 'target_%s' % target + command = '$(SolutionDir)\\mach.bat build' + if target != 'full': + command += ' %s' % target + + project_id = self._write_vs_project(out_proj_dir, basename, target, + build_command=command, + clean_command='$(SolutionDir)\\mach.bat build clean') + + projects[basename] = (project_id, basename, target) + + # A project that can be used to regenerate the visual studio projects. + basename = 'target_vs' + project_id = self._write_vs_project(out_proj_dir, basename, 'visual-studio', + build_command='$(SolutionDir)\\mach.bat build-backend -b VisualStudio') + projects[basename] = (project_id, basename, 'visual-studio') + + # Write out a shared property file with common variables. + props_path = os.path.join(out_proj_dir, 'mozilla.props') + with self._write_file(props_path, mode='rb') as fh: + self._write_props(fh) + + # Generate some wrapper scripts that allow us to invoke mach inside + # a MozillaBuild-like environment. We currently only use the batch + # script. We'd like to use the PowerShell script. However, it seems + # to buffer output from within Visual Studio (surely this is + # configurable) and the default execution policy of PowerShell doesn't + # allow custom scripts to be executed. + with self._write_file(os.path.join(out_dir, 'mach.bat'), mode='rb') as fh: + self._write_mach_batch(fh) + + with self._write_file(os.path.join(out_dir, 'mach.ps1'), mode='rb') as fh: + self._write_mach_powershell(fh) + + # Write out a solution file to tie it all together. + solution_path = os.path.join(out_dir, 'mozilla.sln') + with self._write_file(solution_path, mode='rb') as fh: + self._write_solution(fh, projects) + + def _write_projects_for_sources(self, sources, prefix, out_dir): + projects = {} + for item, path in sorted(sources.items()): + config = self._paths_to_configs.get(path, None) + sources = self._paths_to_sources.get(path, set()) + sources = set(os.path.join('$(TopSrcDir)', path, s) for s in sources) + sources = set(os.path.normpath(s) for s in sources) + + finder = FileFinder(os.path.join(self.environment.topsrcdir, path), + find_executables=False) + + headers = [t[0] for t in finder.find('*.h')] + headers = [os.path.normpath(os.path.join('$(TopSrcDir)', + path, f)) for f in headers] + + includes = [ + os.path.join('$(TopSrcDir)', path), + os.path.join('$(TopObjDir)', path), + ] + includes.extend(self._paths_to_includes.get(path, [])) + includes.append('$(TopObjDir)\\dist\\include\\nss') + includes.append('$(TopObjDir)\\dist\\include') + + for v in ('NSPR_CFLAGS', 'NSS_CFLAGS', 'MOZ_JPEG_CFLAGS', + 'MOZ_PNG_CFLAGS', 'MOZ_ZLIB_CFLAGS', 'MOZ_PIXMAN_CFLAGS'): + if not config: + break + + args = config.substs.get(v, []) + + for i, arg in enumerate(args): + if arg.startswith('-I'): + includes.append(os.path.normpath(arg[2:])) + + # Pull in system defaults. + includes.append('$(DefaultIncludes)') + + includes = [os.path.normpath(i) for i in includes] + + defines = [] + for k, v in self._paths_to_defines.get(path, {}).items(): + if v is True: + defines.append(k) + else: + defines.append('%s=%s' % (k, v)) + + debugger=None + if prefix == 'binary': + if item.startswith(self.environment.substs['MOZ_APP_NAME']): + debugger = ('$(TopObjDir)\\dist\\bin\\%s' % item, '-no-remote') + else: + debugger = ('$(TopObjDir)\\dist\\bin\\%s' % item, '') + + basename = '%s_%s' % (prefix, item) + + project_id = self._write_vs_project(out_dir, basename, item, + includes=includes, + forced_includes=['$(TopObjDir)\\dist\\include\\mozilla-config.h'], + defines=defines, + headers=headers, + sources=sources, + debugger=debugger) + + projects[basename] = (project_id, basename, item) + + return projects + + def _write_solution(self, fh, projects): + # Visual Studio appears to write out its current version in the + # solution file. Instead of trying to figure out what version it will + # write, try to parse the version out of the existing file and use it + # verbatim. + vs_version = None + try: + with open(fh.name, 'rb') as sfh: + for line in sfh: + if line.startswith(b'VisualStudioVersion = '): + vs_version = line.split(b' = ', 1)[1].strip() + except IOError as e: + if e.errno != errno.ENOENT: + raise + + format_version, comment_version = visual_studio_product_to_solution_version(self._version) + # This is a Visual C++ Project type. + project_type = '8BC9CEB8-8B4A-11D0-8D11-00A0C91BC942' + + # Visual Studio seems to require this header. + fh.write('Microsoft Visual Studio Solution File, Format Version %s\r\n' % + format_version) + fh.write('# Visual Studio %s\r\n' % comment_version) + + if vs_version: + fh.write('VisualStudioVersion = %s\r\n' % vs_version) + + # Corresponds to VS2013. + fh.write('MinimumVisualStudioVersion = 12.0.31101.0\r\n') + + binaries_id = projects['target_binaries'][0] + + # Write out entries for each project. + for key in sorted(projects): + project_id, basename, name = projects[key] + path = os.path.join(self._projsubdir, '%s.vcxproj' % basename) + + fh.write('Project("{%s}") = "%s", "%s", "{%s}"\r\n' % ( + project_type, name, path, project_id)) + + # Make all libraries depend on the binaries target. + if key.startswith('library_'): + fh.write('\tProjectSection(ProjectDependencies) = postProject\r\n') + fh.write('\t\t{%s} = {%s}\r\n' % (binaries_id, binaries_id)) + fh.write('\tEndProjectSection\r\n') + + fh.write('EndProject\r\n') + + # Write out solution folders for organizing things. + + # This is the UUID you use for solution folders. + container_id = '2150E333-8FDC-42A3-9474-1A3956D46DE8' + + def write_container(desc): + cid = get_id(desc.encode('utf-8')) + fh.write('Project("{%s}") = "%s", "%s", "{%s}"\r\n' % ( + container_id, desc, desc, cid)) + fh.write('EndProject\r\n') + + return cid + + library_id = write_container('Libraries') + target_id = write_container('Build Targets') + binary_id = write_container('Binaries') + + fh.write('Global\r\n') + + # Make every project a member of our one configuration. + fh.write('\tGlobalSection(SolutionConfigurationPlatforms) = preSolution\r\n') + fh.write('\t\tBuild|Win32 = Build|Win32\r\n') + fh.write('\tEndGlobalSection\r\n') + + # Set every project's active configuration to the one configuration and + # set up the default build project. + fh.write('\tGlobalSection(ProjectConfigurationPlatforms) = postSolution\r\n') + for name, project in sorted(projects.items()): + fh.write('\t\t{%s}.Build|Win32.ActiveCfg = Build|Win32\r\n' % project[0]) + + # Only build the full build target by default. + # It's important we don't write multiple entries here because they + # conflict! + if name == 'target_full': + fh.write('\t\t{%s}.Build|Win32.Build.0 = Build|Win32\r\n' % project[0]) + + fh.write('\tEndGlobalSection\r\n') + + fh.write('\tGlobalSection(SolutionProperties) = preSolution\r\n') + fh.write('\t\tHideSolutionNode = FALSE\r\n') + fh.write('\tEndGlobalSection\r\n') + + # Associate projects with containers. + fh.write('\tGlobalSection(NestedProjects) = preSolution\r\n') + for key in sorted(projects): + project_id = projects[key][0] + + if key.startswith('library_'): + container_id = library_id + elif key.startswith('target_'): + container_id = target_id + elif key.startswith('binary_'): + container_id = binary_id + else: + raise Exception('Unknown project type: %s' % key) + + fh.write('\t\t{%s} = {%s}\r\n' % (project_id, container_id)) + fh.write('\tEndGlobalSection\r\n') + + fh.write('EndGlobal\r\n') + + def _write_props(self, fh): + impl = getDOMImplementation() + doc = impl.createDocument(MSBUILD_NAMESPACE, 'Project', None) + + project = doc.documentElement + project.setAttribute('xmlns', MSBUILD_NAMESPACE) + project.setAttribute('ToolsVersion', '4.0') + + ig = project.appendChild(doc.createElement('ImportGroup')) + ig.setAttribute('Label', 'PropertySheets') + + pg = project.appendChild(doc.createElement('PropertyGroup')) + pg.setAttribute('Label', 'UserMacros') + + ig = project.appendChild(doc.createElement('ItemGroup')) + + def add_var(k, v): + e = pg.appendChild(doc.createElement(k)) + e.appendChild(doc.createTextNode(v)) + + e = ig.appendChild(doc.createElement('BuildMacro')) + e.setAttribute('Include', k) + + e = e.appendChild(doc.createElement('Value')) + e.appendChild(doc.createTextNode('$(%s)' % k)) + + add_var('TopObjDir', os.path.normpath(self.environment.topobjdir)) + add_var('TopSrcDir', os.path.normpath(self.environment.topsrcdir)) + add_var('PYTHON', '$(TopObjDir)\\_virtualenv\\Scripts\\python.exe') + add_var('MACH', '$(TopSrcDir)\\mach') + + # From MozillaBuild. + add_var('DefaultIncludes', os.environ.get('INCLUDE', '')) + + fh.write(b'\xef\xbb\xbf') + doc.writexml(fh, addindent=' ', newl='\r\n') + + def _relevant_environment_variables(self): + # Write out the environment variables, presumably coming from + # MozillaBuild. + for k, v in sorted(os.environ.items()): + if not re.match('^[a-zA-Z0-9_]+$', k): + continue + + if k in ('OLDPWD', 'PS1'): + continue + + if k.startswith('_'): + continue + + yield k, v + + yield 'TOPSRCDIR', self.environment.topsrcdir + yield 'TOPOBJDIR', self.environment.topobjdir + + def _write_mach_powershell(self, fh): + for k, v in self._relevant_environment_variables(): + fh.write(b'$env:%s = "%s"\r\n' % (k, v)) + + relpath = os.path.relpath(self.environment.topsrcdir, + self.environment.topobjdir).replace('\\', '/') + + fh.write(b'$bashargs = "%s/mach", "--log-no-times"\r\n' % relpath) + fh.write(b'$bashargs = $bashargs + $args\r\n') + + fh.write(b"$expanded = $bashargs -join ' '\r\n") + fh.write(b'$procargs = "-c", $expanded\r\n') + + fh.write(b'Start-Process -WorkingDirectory $env:TOPOBJDIR ' + b'-FilePath $env:MOZILLABUILD\\msys\\bin\\bash ' + b'-ArgumentList $procargs ' + b'-Wait -NoNewWindow\r\n') + + def _write_mach_batch(self, fh): + """Write out a batch script that builds the tree. + + The script "bootstraps" into the MozillaBuild environment by setting + the environment variables that are active in the current MozillaBuild + environment. Then, it builds the tree. + """ + for k, v in self._relevant_environment_variables(): + fh.write(b'SET "%s=%s"\r\n' % (k, v)) + + fh.write(b'cd %TOPOBJDIR%\r\n') + + # We need to convert Windows-native paths to msys paths. Easiest way is + # relative paths, since munging c:\ to /c/ is slightly more + # complicated. + relpath = os.path.relpath(self.environment.topsrcdir, + self.environment.topobjdir).replace('\\', '/') + + # We go through mach because it has the logic for choosing the most + # appropriate build tool. + fh.write(b'"%%MOZILLABUILD%%\\msys\\bin\\bash" ' + b'-c "%s/mach --log-no-times %%1 %%2 %%3 %%4 %%5 %%6 %%7"' % relpath) + + def _write_vs_project(self, out_dir, basename, name, **kwargs): + root = '%s.vcxproj' % basename + project_id = get_id(basename.encode('utf-8')) + + with self._write_file(os.path.join(out_dir, root), mode='rb') as fh: + project_id, name = VisualStudioBackend.write_vs_project(fh, + self._version, project_id, name, **kwargs) + + with self._write_file(os.path.join(out_dir, '%s.user' % root), mode='rb') as fh: + fh.write('\r\n') + fh.write('\r\n' % + MSBUILD_NAMESPACE) + fh.write('\r\n') + + return project_id + + @staticmethod + def write_vs_project(fh, version, project_id, name, includes=[], + forced_includes=[], defines=[], + build_command=None, clean_command=None, + debugger=None, headers=[], sources=[]): + + impl = getDOMImplementation() + doc = impl.createDocument(MSBUILD_NAMESPACE, 'Project', None) + + project = doc.documentElement + project.setAttribute('DefaultTargets', 'Build') + project.setAttribute('ToolsVersion', '4.0') + project.setAttribute('xmlns', MSBUILD_NAMESPACE) + + ig = project.appendChild(doc.createElement('ItemGroup')) + ig.setAttribute('Label', 'ProjectConfigurations') + + pc = ig.appendChild(doc.createElement('ProjectConfiguration')) + pc.setAttribute('Include', 'Build|Win32') + + c = pc.appendChild(doc.createElement('Configuration')) + c.appendChild(doc.createTextNode('Build')) + + p = pc.appendChild(doc.createElement('Platform')) + p.appendChild(doc.createTextNode('Win32')) + + pg = project.appendChild(doc.createElement('PropertyGroup')) + pg.setAttribute('Label', 'Globals') + + n = pg.appendChild(doc.createElement('ProjectName')) + n.appendChild(doc.createTextNode(name)) + + k = pg.appendChild(doc.createElement('Keyword')) + k.appendChild(doc.createTextNode('MakeFileProj')) + + g = pg.appendChild(doc.createElement('ProjectGuid')) + g.appendChild(doc.createTextNode('{%s}' % project_id)) + + rn = pg.appendChild(doc.createElement('RootNamespace')) + rn.appendChild(doc.createTextNode('mozilla')) + + pts = pg.appendChild(doc.createElement('PlatformToolset')) + pts.appendChild(doc.createTextNode(visual_studio_product_to_platform_toolset_version(version))) + + i = project.appendChild(doc.createElement('Import')) + i.setAttribute('Project', '$(VCTargetsPath)\\Microsoft.Cpp.Default.props') + + ig = project.appendChild(doc.createElement('ImportGroup')) + ig.setAttribute('Label', 'ExtensionTargets') + + ig = project.appendChild(doc.createElement('ImportGroup')) + ig.setAttribute('Label', 'ExtensionSettings') + + ig = project.appendChild(doc.createElement('ImportGroup')) + ig.setAttribute('Label', 'PropertySheets') + i = ig.appendChild(doc.createElement('Import')) + i.setAttribute('Project', 'mozilla.props') + + pg = project.appendChild(doc.createElement('PropertyGroup')) + pg.setAttribute('Label', 'Configuration') + ct = pg.appendChild(doc.createElement('ConfigurationType')) + ct.appendChild(doc.createTextNode('Makefile')) + + pg = project.appendChild(doc.createElement('PropertyGroup')) + pg.setAttribute('Condition', "'$(Configuration)|$(Platform)'=='Build|Win32'") + + if build_command: + n = pg.appendChild(doc.createElement('NMakeBuildCommandLine')) + n.appendChild(doc.createTextNode(build_command)) + + if clean_command: + n = pg.appendChild(doc.createElement('NMakeCleanCommandLine')) + n.appendChild(doc.createTextNode(clean_command)) + + if includes: + n = pg.appendChild(doc.createElement('NMakeIncludeSearchPath')) + n.appendChild(doc.createTextNode(';'.join(includes))) + + if forced_includes: + n = pg.appendChild(doc.createElement('NMakeForcedIncludes')) + n.appendChild(doc.createTextNode(';'.join(forced_includes))) + + if defines: + n = pg.appendChild(doc.createElement('NMakePreprocessorDefinitions')) + n.appendChild(doc.createTextNode(';'.join(defines))) + + if debugger: + n = pg.appendChild(doc.createElement('LocalDebuggerCommand')) + n.appendChild(doc.createTextNode(debugger[0])) + + n = pg.appendChild(doc.createElement('LocalDebuggerCommandArguments')) + n.appendChild(doc.createTextNode(debugger[1])) + + i = project.appendChild(doc.createElement('Import')) + i.setAttribute('Project', '$(VCTargetsPath)\\Microsoft.Cpp.props') + + i = project.appendChild(doc.createElement('Import')) + i.setAttribute('Project', '$(VCTargetsPath)\\Microsoft.Cpp.targets') + + # Now add files to the project. + ig = project.appendChild(doc.createElement('ItemGroup')) + for header in sorted(headers or []): + n = ig.appendChild(doc.createElement('ClInclude')) + n.setAttribute('Include', header) + + ig = project.appendChild(doc.createElement('ItemGroup')) + for source in sorted(sources or []): + n = ig.appendChild(doc.createElement('ClCompile')) + n.setAttribute('Include', source) + + fh.write(b'\xef\xbb\xbf') + doc.writexml(fh, addindent=' ', newl='\r\n') + + return project_id, name diff --git a/python/mozbuild/mozbuild/base.py b/python/mozbuild/mozbuild/base.py new file mode 100644 index 000000000..a50b8ff89 --- /dev/null +++ b/python/mozbuild/mozbuild/base.py @@ -0,0 +1,850 @@ +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, You can obtain one at http://mozilla.org/MPL/2.0/. + +from __future__ import absolute_import, print_function, unicode_literals + +import json +import logging +import mozpack.path as mozpath +import multiprocessing +import os +import subprocess +import sys +import which + +from mach.mixin.logging import LoggingMixin +from mach.mixin.process import ProcessExecutionMixin +from mozversioncontrol import get_repository_object + +from .backend.configenvironment import ConfigEnvironment +from .controller.clobber import Clobberer +from .mozconfig import ( + MozconfigFindException, + MozconfigLoadException, + MozconfigLoader, +) +from .util import memoized_property +from .virtualenv import VirtualenvManager + + +_config_guess_output = [] + + +def ancestors(path): + """Emit the parent directories of a path.""" + while path: + yield path + newpath = os.path.dirname(path) + if newpath == path: + break + path = newpath + +def samepath(path1, path2): + if hasattr(os.path, 'samefile'): + return os.path.samefile(path1, path2) + return os.path.normcase(os.path.realpath(path1)) == \ + os.path.normcase(os.path.realpath(path2)) + +class BadEnvironmentException(Exception): + """Base class for errors raised when the build environment is not sane.""" + + +class BuildEnvironmentNotFoundException(BadEnvironmentException): + """Raised when we could not find a build environment.""" + + +class ObjdirMismatchException(BadEnvironmentException): + """Raised when the current dir is an objdir and doesn't match the mozconfig.""" + def __init__(self, objdir1, objdir2): + self.objdir1 = objdir1 + self.objdir2 = objdir2 + + def __str__(self): + return "Objdir mismatch: %s != %s" % (self.objdir1, self.objdir2) + + +class MozbuildObject(ProcessExecutionMixin): + """Base class providing basic functionality useful to many modules. + + Modules in this package typically require common functionality such as + accessing the current config, getting the location of the source directory, + running processes, etc. This classes provides that functionality. Other + modules can inherit from this class to obtain this functionality easily. + """ + def __init__(self, topsrcdir, settings, log_manager, topobjdir=None, + mozconfig=MozconfigLoader.AUTODETECT): + """Create a new Mozbuild object instance. + + Instances are bound to a source directory, a ConfigSettings instance, + and a LogManager instance. The topobjdir may be passed in as well. If + it isn't, it will be calculated from the active mozconfig. + """ + self.topsrcdir = mozpath.normsep(topsrcdir) + self.settings = settings + + self.populate_logger() + self.log_manager = log_manager + + self._make = None + self._topobjdir = mozpath.normsep(topobjdir) if topobjdir else topobjdir + self._mozconfig = mozconfig + self._config_environment = None + self._virtualenv_manager = None + + @classmethod + def from_environment(cls, cwd=None, detect_virtualenv_mozinfo=True): + """Create a MozbuildObject by detecting the proper one from the env. + + This examines environment state like the current working directory and + creates a MozbuildObject from the found source directory, mozconfig, etc. + + The role of this function is to identify a topsrcdir, topobjdir, and + mozconfig file. + + If the current working directory is inside a known objdir, we always + use the topsrcdir and mozconfig associated with that objdir. + + If the current working directory is inside a known srcdir, we use that + topsrcdir and look for mozconfigs using the default mechanism, which + looks inside environment variables. + + If the current Python interpreter is running from a virtualenv inside + an objdir, we use that as our objdir. + + If we're not inside a srcdir or objdir, an exception is raised. + + detect_virtualenv_mozinfo determines whether we should look for a + mozinfo.json file relative to the virtualenv directory. This was + added to facilitate testing. Callers likely shouldn't change the + default. + """ + + cwd = cwd or os.getcwd() + topsrcdir = None + topobjdir = None + mozconfig = MozconfigLoader.AUTODETECT + + def load_mozinfo(path): + info = json.load(open(path, 'rt')) + topsrcdir = info.get('topsrcdir') + topobjdir = os.path.dirname(path) + mozconfig = info.get('mozconfig') + return topsrcdir, topobjdir, mozconfig + + for dir_path in ancestors(cwd): + # If we find a mozinfo.json, we are in the objdir. + mozinfo_path = os.path.join(dir_path, 'mozinfo.json') + if os.path.isfile(mozinfo_path): + topsrcdir, topobjdir, mozconfig = load_mozinfo(mozinfo_path) + break + + # We choose an arbitrary file as an indicator that this is a + # srcdir. We go with ourself because why not! + our_path = os.path.join(dir_path, 'python', 'mozbuild', 'mozbuild', 'base.py') + if os.path.isfile(our_path): + topsrcdir = dir_path + break + + # See if we're running from a Python virtualenv that's inside an objdir. + mozinfo_path = os.path.join(os.path.dirname(sys.prefix), "mozinfo.json") + if detect_virtualenv_mozinfo and os.path.isfile(mozinfo_path): + topsrcdir, topobjdir, mozconfig = load_mozinfo(mozinfo_path) + + # If we were successful, we're only guaranteed to find a topsrcdir. If + # we couldn't find that, there's nothing we can do. + if not topsrcdir: + raise BuildEnvironmentNotFoundException( + 'Could not find Mozilla source tree or build environment.') + + topsrcdir = mozpath.normsep(topsrcdir) + if topobjdir: + topobjdir = mozpath.normsep(os.path.normpath(topobjdir)) + + if topsrcdir == topobjdir: + raise BadEnvironmentException('The object directory appears ' + 'to be the same as your source directory (%s). This build ' + 'configuration is not supported.' % topsrcdir) + + # If we can't resolve topobjdir, oh well. We'll figure out when we need + # one. + return cls(topsrcdir, None, None, topobjdir=topobjdir, + mozconfig=mozconfig) + + def resolve_mozconfig_topobjdir(self, default=None): + topobjdir = self.mozconfig['topobjdir'] or default + if not topobjdir: + return None + + if '@CONFIG_GUESS@' in topobjdir: + topobjdir = topobjdir.replace('@CONFIG_GUESS@', + self.resolve_config_guess()) + + if not os.path.isabs(topobjdir): + topobjdir = os.path.abspath(os.path.join(self.topsrcdir, topobjdir)) + + return mozpath.normsep(os.path.normpath(topobjdir)) + + @property + def topobjdir(self): + if self._topobjdir is None: + self._topobjdir = self.resolve_mozconfig_topobjdir( + default='obj-@CONFIG_GUESS@') + + return self._topobjdir + + @property + def virtualenv_manager(self): + if self._virtualenv_manager is None: + self._virtualenv_manager = VirtualenvManager(self.topsrcdir, + self.topobjdir, os.path.join(self.topobjdir, '_virtualenv'), + sys.stdout, os.path.join(self.topsrcdir, 'build', + 'virtualenv_packages.txt')) + + return self._virtualenv_manager + + @property + def mozconfig(self): + """Returns information about the current mozconfig file. + + This a dict as returned by MozconfigLoader.read_mozconfig() + """ + if not isinstance(self._mozconfig, dict): + loader = MozconfigLoader(self.topsrcdir) + self._mozconfig = loader.read_mozconfig(path=self._mozconfig, + moz_build_app=os.environ.get('MOZ_CURRENT_PROJECT')) + + return self._mozconfig + + @property + def config_environment(self): + """Returns the ConfigEnvironment for the current build configuration. + + This property is only available once configure has executed. + + If configure's output is not available, this will raise. + """ + if self._config_environment: + return self._config_environment + + config_status = os.path.join(self.topobjdir, 'config.status') + + if not os.path.exists(config_status): + raise BuildEnvironmentNotFoundException('config.status not available. Run configure.') + + self._config_environment = \ + ConfigEnvironment.from_config_status(config_status) + + return self._config_environment + + @property + def defines(self): + return self.config_environment.defines + + @property + def non_global_defines(self): + return self.config_environment.non_global_defines + + @property + def substs(self): + return self.config_environment.substs + + @property + def distdir(self): + return os.path.join(self.topobjdir, 'dist') + + @property + def bindir(self): + return os.path.join(self.topobjdir, 'dist', 'bin') + + @property + def includedir(self): + return os.path.join(self.topobjdir, 'dist', 'include') + + @property + def statedir(self): + return os.path.join(self.topobjdir, '.mozbuild') + + @memoized_property + def extra_environment_variables(self): + '''Some extra environment variables are stored in .mozconfig.mk. + This functions extracts and returns them.''' + from mozbuild import shellutil + mozconfig_mk = os.path.join(self.topobjdir, '.mozconfig.mk') + env = {} + with open(mozconfig_mk) as fh: + for line in fh: + if line.startswith('export '): + exports = shellutil.split(line)[1:] + for e in exports: + if '=' in e: + key, value = e.split('=') + env[key] = value + return env + + @memoized_property + def repository(self): + '''Get a `mozversioncontrol.Repository` object for the + top source directory.''' + return get_repository_object(self.topsrcdir) + + def is_clobber_needed(self): + if not os.path.exists(self.topobjdir): + return False + return Clobberer(self.topsrcdir, self.topobjdir).clobber_needed() + + def get_binary_path(self, what='app', validate_exists=True, where='default'): + """Obtain the path to a compiled binary for this build configuration. + + The what argument is the program or tool being sought after. See the + code implementation for supported values. + + If validate_exists is True (the default), we will ensure the found path + exists before returning, raising an exception if it doesn't. + + If where is 'staged-package', we will return the path to the binary in + the package staging directory. + + If no arguments are specified, we will return the main binary for the + configured XUL application. + """ + + if where not in ('default', 'staged-package'): + raise Exception("Don't know location %s" % where) + + substs = self.substs + + stem = self.distdir + if where == 'staged-package': + stem = os.path.join(stem, substs['MOZ_APP_NAME']) + + if substs['OS_ARCH'] == 'Darwin': + if substs['MOZ_BUILD_APP'] == 'xulrunner': + stem = os.path.join(stem, 'XUL.framework'); + else: + stem = os.path.join(stem, substs['MOZ_MACBUNDLE_NAME'], 'Contents', + 'MacOS') + elif where == 'default': + stem = os.path.join(stem, 'bin') + + leaf = None + + leaf = (substs['MOZ_APP_NAME'] if what == 'app' else what) + substs['BIN_SUFFIX'] + path = os.path.join(stem, leaf) + + if validate_exists and not os.path.exists(path): + raise Exception('Binary expected at %s does not exist.' % path) + + return path + + def resolve_config_guess(self): + make_extra = self.mozconfig['make_extra'] or [] + make_extra = dict(m.split('=', 1) for m in make_extra) + + config_guess = make_extra.get('CONFIG_GUESS', None) + + if config_guess: + return config_guess + + # config.guess results should be constant for process lifetime. Cache + # it. + if _config_guess_output: + return _config_guess_output[0] + + p = os.path.join(self.topsrcdir, 'build', 'autoconf', 'config.guess') + + # This is a little kludgy. We need access to the normalize_command + # function. However, that's a method of a mach mixin, so we need a + # class instance. Ideally the function should be accessible as a + # standalone function. + o = MozbuildObject(self.topsrcdir, None, None, None) + args = o._normalize_command([p], True) + + _config_guess_output.append( + subprocess.check_output(args, cwd=self.topsrcdir).strip()) + return _config_guess_output[0] + + def notify(self, msg): + """Show a desktop notification with the supplied message + + On Linux and Mac, this will show a desktop notification with the message, + but on Windows we can only flash the screen. + """ + moz_nospam = os.environ.get('MOZ_NOSPAM') + if moz_nospam: + return + + try: + if sys.platform.startswith('darwin'): + try: + notifier = which.which('terminal-notifier') + except which.WhichError: + raise Exception('Install terminal-notifier to get ' + 'a notification when the build finishes.') + self.run_process([notifier, '-title', + 'Mozilla Build System', '-group', 'mozbuild', + '-message', msg], ensure_exit_code=False) + elif sys.platform.startswith('linux'): + try: + import dbus + except ImportError: + raise Exception('Install the python dbus module to ' + 'get a notification when the build finishes.') + bus = dbus.SessionBus() + notify = bus.get_object('org.freedesktop.Notifications', + '/org/freedesktop/Notifications') + method = notify.get_dbus_method('Notify', + 'org.freedesktop.Notifications') + method('Mozilla Build System', 0, '', msg, '', [], [], -1) + elif sys.platform.startswith('win'): + from ctypes import Structure, windll, POINTER, sizeof + from ctypes.wintypes import DWORD, HANDLE, WINFUNCTYPE, BOOL, UINT + class FLASHWINDOW(Structure): + _fields_ = [("cbSize", UINT), + ("hwnd", HANDLE), + ("dwFlags", DWORD), + ("uCount", UINT), + ("dwTimeout", DWORD)] + FlashWindowExProto = WINFUNCTYPE(BOOL, POINTER(FLASHWINDOW)) + FlashWindowEx = FlashWindowExProto(("FlashWindowEx", windll.user32)) + FLASHW_CAPTION = 0x01 + FLASHW_TRAY = 0x02 + FLASHW_TIMERNOFG = 0x0C + + # GetConsoleWindows returns NULL if no console is attached. We + # can't flash nothing. + console = windll.kernel32.GetConsoleWindow() + if not console: + return + + params = FLASHWINDOW(sizeof(FLASHWINDOW), + console, + FLASHW_CAPTION | FLASHW_TRAY | FLASHW_TIMERNOFG, 3, 0) + FlashWindowEx(params) + except Exception as e: + self.log(logging.WARNING, 'notifier-failed', {'error': + e.message}, 'Notification center failed: {error}') + + def _ensure_objdir_exists(self): + if os.path.isdir(self.statedir): + return + + os.makedirs(self.statedir) + + def _ensure_state_subdir_exists(self, subdir): + path = os.path.join(self.statedir, subdir) + + if os.path.isdir(path): + return + + os.makedirs(path) + + def _get_state_filename(self, filename, subdir=None): + path = self.statedir + + if subdir: + path = os.path.join(path, subdir) + + return os.path.join(path, filename) + + def _wrap_path_argument(self, arg): + return PathArgument(arg, self.topsrcdir, self.topobjdir) + + def _run_make(self, directory=None, filename=None, target=None, log=True, + srcdir=False, allow_parallel=True, line_handler=None, + append_env=None, explicit_env=None, ignore_errors=False, + ensure_exit_code=0, silent=True, print_directory=True, + pass_thru=False, num_jobs=0): + """Invoke make. + + directory -- Relative directory to look for Makefile in. + filename -- Explicit makefile to run. + target -- Makefile target(s) to make. Can be a string or iterable of + strings. + srcdir -- If True, invoke make from the source directory tree. + Otherwise, make will be invoked from the object directory. + silent -- If True (the default), run make in silent mode. + print_directory -- If True (the default), have make print directories + while doing traversal. + """ + self._ensure_objdir_exists() + + args = self._make_path() + + if directory: + args.extend(['-C', directory.replace(os.sep, '/')]) + + if filename: + args.extend(['-f', filename]) + + if num_jobs == 0 and self.mozconfig['make_flags']: + flags = iter(self.mozconfig['make_flags']) + for flag in flags: + if flag == '-j': + try: + flag = flags.next() + except StopIteration: + break + try: + num_jobs = int(flag) + except ValueError: + args.append(flag) + elif flag.startswith('-j'): + try: + num_jobs = int(flag[2:]) + except (ValueError, IndexError): + break + else: + args.append(flag) + + if allow_parallel: + if num_jobs > 0: + args.append('-j%d' % num_jobs) + else: + args.append('-j%d' % multiprocessing.cpu_count()) + elif num_jobs > 0: + args.append('MOZ_PARALLEL_BUILD=%d' % num_jobs) + + if ignore_errors: + args.append('-k') + + if silent: + args.append('-s') + + # Print entering/leaving directory messages. Some consumers look at + # these to measure progress. + if print_directory: + args.append('-w') + + if isinstance(target, list): + args.extend(target) + elif target: + args.append(target) + + fn = self._run_command_in_objdir + + if srcdir: + fn = self._run_command_in_srcdir + + append_env = dict(append_env or ()) + append_env[b'MACH'] = '1' + + params = { + 'args': args, + 'line_handler': line_handler, + 'append_env': append_env, + 'explicit_env': explicit_env, + 'log_level': logging.INFO, + 'require_unix_environment': False, + 'ensure_exit_code': ensure_exit_code, + 'pass_thru': pass_thru, + + # Make manages its children, so mozprocess doesn't need to bother. + # Having mozprocess manage children can also have side-effects when + # building on Windows. See bug 796840. + 'ignore_children': True, + } + + if log: + params['log_name'] = 'make' + + return fn(**params) + + def _make_path(self): + baseconfig = os.path.join(self.topsrcdir, 'config', 'baseconfig.mk') + + def is_xcode_lisense_error(output): + return self._is_osx() and 'Agreeing to the Xcode' in output + + def validate_make(make): + if os.path.exists(baseconfig) and os.path.exists(make): + cmd = [make, '-f', baseconfig] + if self._is_windows(): + cmd.append('HOST_OS_ARCH=WINNT') + try: + subprocess.check_output(cmd, stderr=subprocess.STDOUT) + except subprocess.CalledProcessError as e: + return False, is_xcode_lisense_error(e.output) + return True, False + return False, False + + xcode_lisense_error = False + possible_makes = ['gmake', 'make', 'mozmake', 'gnumake', 'mingw32-make'] + + if 'MAKE' in os.environ: + make = os.environ['MAKE'] + possible_makes.insert(0, make) + + for test in possible_makes: + if os.path.isabs(test): + make = test + else: + try: + make = which.which(test) + except which.WhichError: + continue + result, xcode_lisense_error_tmp = validate_make(make) + if result: + return [make] + if xcode_lisense_error_tmp: + xcode_lisense_error = True + + if xcode_lisense_error: + raise Exception('Xcode requires accepting to the license agreement.\n' + 'Please run Xcode and accept the license agreement.') + + if self._is_windows(): + raise Exception('Could not find a suitable make implementation.\n' + 'Please use MozillaBuild 1.9 or newer') + else: + raise Exception('Could not find a suitable make implementation.') + + def _run_command_in_srcdir(self, **args): + return self.run_process(cwd=self.topsrcdir, **args) + + def _run_command_in_objdir(self, **args): + return self.run_process(cwd=self.topobjdir, **args) + + def _is_windows(self): + return os.name in ('nt', 'ce') + + def _is_osx(self): + return 'darwin' in str(sys.platform).lower() + + def _spawn(self, cls): + """Create a new MozbuildObject-derived class instance from ourselves. + + This is used as a convenience method to create other + MozbuildObject-derived class instances. It can only be used on + classes that have the same constructor arguments as us. + """ + + return cls(self.topsrcdir, self.settings, self.log_manager, + topobjdir=self.topobjdir) + + def _activate_virtualenv(self): + self.virtualenv_manager.ensure() + self.virtualenv_manager.activate() + + +class MachCommandBase(MozbuildObject): + """Base class for mach command providers that wish to be MozbuildObjects. + + This provides a level of indirection so MozbuildObject can be refactored + without having to change everything that inherits from it. + """ + + def __init__(self, context): + # Attempt to discover topobjdir through environment detection, as it is + # more reliable than mozconfig when cwd is inside an objdir. + topsrcdir = context.topdir + topobjdir = None + detect_virtualenv_mozinfo = True + if hasattr(context, 'detect_virtualenv_mozinfo'): + detect_virtualenv_mozinfo = getattr(context, + 'detect_virtualenv_mozinfo') + try: + dummy = MozbuildObject.from_environment(cwd=context.cwd, + detect_virtualenv_mozinfo=detect_virtualenv_mozinfo) + topsrcdir = dummy.topsrcdir + topobjdir = dummy._topobjdir + if topobjdir: + # If we're inside a objdir and the found mozconfig resolves to + # another objdir, we abort. The reasoning here is that if you + # are inside an objdir you probably want to perform actions on + # that objdir, not another one. This prevents accidental usage + # of the wrong objdir when the current objdir is ambiguous. + config_topobjdir = dummy.resolve_mozconfig_topobjdir() + + try: + universal_bin = dummy.substs.get('UNIVERSAL_BINARY') + except: + universal_bin = False + + if config_topobjdir and not (samepath(topobjdir, config_topobjdir) or + universal_bin and topobjdir.startswith(config_topobjdir)): + raise ObjdirMismatchException(topobjdir, config_topobjdir) + except BuildEnvironmentNotFoundException: + pass + except ObjdirMismatchException as e: + print('Ambiguous object directory detected. We detected that ' + 'both %s and %s could be object directories. This is ' + 'typically caused by having a mozconfig pointing to a ' + 'different object directory from the current working ' + 'directory. To solve this problem, ensure you do not have a ' + 'default mozconfig in searched paths.' % (e.objdir1, + e.objdir2)) + sys.exit(1) + + except MozconfigLoadException as e: + print('Error loading mozconfig: ' + e.path) + print('') + print(e.message) + if e.output: + print('') + print('mozconfig output:') + print('') + for line in e.output: + print(line) + + sys.exit(1) + + MozbuildObject.__init__(self, topsrcdir, context.settings, + context.log_manager, topobjdir=topobjdir) + + self._mach_context = context + + # Incur mozconfig processing so we have unified error handling for + # errors. Otherwise, the exceptions could bubble back to mach's error + # handler. + try: + self.mozconfig + + except MozconfigFindException as e: + print(e.message) + sys.exit(1) + + except MozconfigLoadException as e: + print('Error loading mozconfig: ' + e.path) + print('') + print(e.message) + if e.output: + print('') + print('mozconfig output:') + print('') + for line in e.output: + print(line) + + sys.exit(1) + + # Always keep a log of the last command, but don't do that for mach + # invokations from scripts (especially not the ones done by the build + # system itself). + if (os.isatty(sys.stdout.fileno()) and + not getattr(self, 'NO_AUTO_LOG', False)): + self._ensure_state_subdir_exists('.') + logfile = self._get_state_filename('last_log.json') + try: + fd = open(logfile, "wb") + self.log_manager.add_json_handler(fd) + except Exception as e: + self.log(logging.WARNING, 'mach', {'error': e}, + 'Log will not be kept for this command: {error}.') + + +class MachCommandConditions(object): + """A series of commonly used condition functions which can be applied to + mach commands with providers deriving from MachCommandBase. + """ + @staticmethod + def is_firefox(cls): + """Must have a Firefox build.""" + if hasattr(cls, 'substs'): + return cls.substs.get('MOZ_BUILD_APP') == 'browser' + return False + + @staticmethod + def is_mulet(cls): + """Must have a Mulet build.""" + if hasattr(cls, 'substs'): + return cls.substs.get('MOZ_BUILD_APP') == 'b2g/dev' + return False + + @staticmethod + def is_b2g(cls): + """Must have a B2G build.""" + if hasattr(cls, 'substs'): + return cls.substs.get('MOZ_WIDGET_TOOLKIT') == 'gonk' + return False + + @staticmethod + def is_b2g_desktop(cls): + """Must have a B2G desktop build.""" + if hasattr(cls, 'substs'): + return cls.substs.get('MOZ_BUILD_APP') == 'b2g' and \ + cls.substs.get('MOZ_WIDGET_TOOLKIT') != 'gonk' + return False + + @staticmethod + def is_emulator(cls): + """Must have a B2G build with an emulator configured.""" + try: + return MachCommandConditions.is_b2g(cls) and \ + cls.device_name.startswith('emulator') + except AttributeError: + return False + + @staticmethod + def is_android(cls): + """Must have an Android build.""" + if hasattr(cls, 'substs'): + return cls.substs.get('MOZ_WIDGET_TOOLKIT') == 'android' + return False + + @staticmethod + def is_hg(cls): + """Must have a mercurial source checkout.""" + if hasattr(cls, 'substs'): + top_srcdir = cls.substs.get('top_srcdir') + return top_srcdir and os.path.isdir(os.path.join(top_srcdir, '.hg')) + return False + + @staticmethod + def is_git(cls): + """Must have a git source checkout.""" + if hasattr(cls, 'substs'): + top_srcdir = cls.substs.get('top_srcdir') + return top_srcdir and os.path.isdir(os.path.join(top_srcdir, '.git')) + return False + + +class PathArgument(object): + """Parse a filesystem path argument and transform it in various ways.""" + + def __init__(self, arg, topsrcdir, topobjdir, cwd=None): + self.arg = arg + self.topsrcdir = topsrcdir + self.topobjdir = topobjdir + self.cwd = os.getcwd() if cwd is None else cwd + + def relpath(self): + """Return a path relative to the topsrcdir or topobjdir. + + If the argument is a path to a location in one of the base directories + (topsrcdir or topobjdir), then strip off the base directory part and + just return the path within the base directory.""" + + abspath = os.path.abspath(os.path.join(self.cwd, self.arg)) + + # If that path is within topsrcdir or topobjdir, return an equivalent + # path relative to that base directory. + for base_dir in [self.topobjdir, self.topsrcdir]: + if abspath.startswith(os.path.abspath(base_dir)): + return mozpath.relpath(abspath, base_dir) + + return mozpath.normsep(self.arg) + + def srcdir_path(self): + return mozpath.join(self.topsrcdir, self.relpath()) + + def objdir_path(self): + return mozpath.join(self.topobjdir, self.relpath()) + + +class ExecutionSummary(dict): + """Helper for execution summaries.""" + + def __init__(self, summary_format, **data): + self._summary_format = '' + assert 'execution_time' in data + self.extend(summary_format, **data) + + def extend(self, summary_format, **data): + self._summary_format += summary_format + self.update(data) + + def __str__(self): + return self._summary_format.format(**self) + + def __getattr__(self, key): + return self[key] diff --git a/python/mozbuild/mozbuild/codecoverage/__init__.py b/python/mozbuild/mozbuild/codecoverage/__init__.py new file mode 100644 index 000000000..e69de29bb diff --git a/python/mozbuild/mozbuild/codecoverage/chrome_map.py b/python/mozbuild/mozbuild/codecoverage/chrome_map.py new file mode 100644 index 000000000..81c3c9a07 --- /dev/null +++ b/python/mozbuild/mozbuild/codecoverage/chrome_map.py @@ -0,0 +1,105 @@ +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, You can obtain one at http://mozilla.org/MPL/2.0/. + +from collections import defaultdict +import json +import os +import urlparse + +from mach.config import ConfigSettings +from mach.logging import LoggingManager +from mozbuild.backend.common import CommonBackend +from mozbuild.base import MozbuildObject +from mozbuild.frontend.data import ( + FinalTargetFiles, + FinalTargetPreprocessedFiles, +) +from mozbuild.frontend.data import JARManifest, ChromeManifestEntry +from mozpack.chrome.manifest import ( + Manifest, + ManifestChrome, + ManifestOverride, + ManifestResource, + parse_manifest, +) +import mozpack.path as mozpath + + +class ChromeManifestHandler(object): + def __init__(self): + self.overrides = {} + self.chrome_mapping = defaultdict(set) + + def handle_manifest_entry(self, entry): + format_strings = { + "content": "chrome://%s/content/", + "resource": "resource://%s/", + "locale": "chrome://%s/locale/", + "skin": "chrome://%s/skin/", + } + + if isinstance(entry, (ManifestChrome, ManifestResource)): + if isinstance(entry, ManifestResource): + dest = entry.target + url = urlparse.urlparse(dest) + if not url.scheme: + dest = mozpath.normpath(mozpath.join(entry.base, dest)) + if url.scheme == 'file': + dest = mozpath.normpath(url.path) + else: + dest = mozpath.normpath(entry.path) + + base_uri = format_strings[entry.type] % entry.name + self.chrome_mapping[base_uri].add(dest) + if isinstance(entry, ManifestOverride): + self.overrides[entry.overloaded] = entry.overload + if isinstance(entry, Manifest): + for e in parse_manifest(None, entry.path): + self.handle_manifest_entry(e) + +class ChromeMapBackend(CommonBackend): + def _init(self): + CommonBackend._init(self) + + log_manager = LoggingManager() + self._cmd = MozbuildObject(self.environment.topsrcdir, ConfigSettings(), + log_manager, self.environment.topobjdir) + self._install_mapping = {} + self.manifest_handler = ChromeManifestHandler() + + def consume_object(self, obj): + if isinstance(obj, JARManifest): + self._consume_jar_manifest(obj) + if isinstance(obj, ChromeManifestEntry): + self.manifest_handler.handle_manifest_entry(obj.entry) + if isinstance(obj, (FinalTargetFiles, + FinalTargetPreprocessedFiles)): + self._handle_final_target_files(obj) + return True + + def _handle_final_target_files(self, obj): + for path, files in obj.files.walk(): + for f in files: + dest = mozpath.join(obj.install_target, path, f.target_basename) + is_pp = isinstance(obj, + FinalTargetPreprocessedFiles) + self._install_mapping[dest] = f.full_path, is_pp + + def consume_finished(self): + # Our result has three parts: + # A map from url prefixes to objdir directories: + # { "chrome://mozapps/content/": [ "dist/bin/chrome/toolkit/content/mozapps" ], ... } + # A map of overrides. + # A map from objdir paths to sourcedir paths, and a flag for whether the source was preprocessed: + # { "dist/bin/browser/chrome/browser/content/browser/aboutSessionRestore.js": + # [ "$topsrcdir/browser/components/sessionstore/content/aboutSessionRestore.js", false ], ... } + outputfile = os.path.join(self.environment.topobjdir, 'chrome-map.json') + with self._write_file(outputfile) as fh: + chrome_mapping = self.manifest_handler.chrome_mapping + overrides = self.manifest_handler.overrides + json.dump([ + {k: list(v) for k, v in chrome_mapping.iteritems()}, + overrides, + self._install_mapping, + ], fh, sort_keys=True, indent=2) diff --git a/python/mozbuild/mozbuild/codecoverage/packager.py b/python/mozbuild/mozbuild/codecoverage/packager.py new file mode 100644 index 000000000..3a4f359f6 --- /dev/null +++ b/python/mozbuild/mozbuild/codecoverage/packager.py @@ -0,0 +1,43 @@ +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, You can obtain one at http://mozilla.org/MPL/2.0/. + +from __future__ import absolute_import, print_function + +import argparse +import sys + +from mozpack.files import FileFinder +from mozpack.copier import Jarrer + +def package_gcno_tree(root, output_file): + # XXX JarWriter doesn't support unicode strings, see bug 1056859 + if isinstance(root, unicode): + root = root.encode('utf-8') + + finder = FileFinder(root) + jarrer = Jarrer(optimize=False) + for p, f in finder.find("**/*.gcno"): + jarrer.add(p, f) + jarrer.copy(output_file) + + +def cli(args=sys.argv[1:]): + parser = argparse.ArgumentParser() + parser.add_argument('-o', '--output-file', + dest='output_file', + help='Path to save packaged data to.') + parser.add_argument('--root', + dest='root', + default=None, + help='Root directory to search from.') + args = parser.parse_args(args) + + if not args.root: + from buildconfig import topobjdir + args.root = topobjdir + + return package_gcno_tree(args.root, args.output_file) + +if __name__ == '__main__': + sys.exit(cli()) diff --git a/python/mozbuild/mozbuild/compilation/__init__.py b/python/mozbuild/mozbuild/compilation/__init__.py new file mode 100644 index 000000000..e69de29bb diff --git a/python/mozbuild/mozbuild/compilation/codecomplete.py b/python/mozbuild/mozbuild/compilation/codecomplete.py new file mode 100644 index 000000000..05583961a --- /dev/null +++ b/python/mozbuild/mozbuild/compilation/codecomplete.py @@ -0,0 +1,63 @@ +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this file, +# You can obtain one at http://mozilla.org/MPL/2.0/. + +# This modules provides functionality for dealing with code completion. + +from __future__ import absolute_import + +import os + +from mach.decorators import ( + CommandArgument, + CommandProvider, + Command, +) + +from mozbuild.base import MachCommandBase +from mozbuild.shellutil import ( + split as shell_split, + quote as shell_quote, +) + + +@CommandProvider +class Introspection(MachCommandBase): + """Instropection commands.""" + + @Command('compileflags', category='devenv', + description='Display the compilation flags for a given source file') + @CommandArgument('what', default=None, + help='Source file to display compilation flags for') + def compileflags(self, what): + from mozbuild.util import resolve_target_to_make + from mozbuild.compilation import util + + if not util.check_top_objdir(self.topobjdir): + return 1 + + path_arg = self._wrap_path_argument(what) + + make_dir, make_target = resolve_target_to_make(self.topobjdir, + path_arg.relpath()) + + if make_dir is None and make_target is None: + return 1 + + build_vars = util.get_build_vars(make_dir, self) + + if what.endswith('.c'): + cc = 'CC' + name = 'COMPILE_CFLAGS' + else: + cc = 'CXX' + name = 'COMPILE_CXXFLAGS' + + if name not in build_vars: + return + + # Drop the first flag since that is the pathname of the compiler. + flags = (shell_split(build_vars[cc]) + shell_split(build_vars[name]))[1:] + + print(' '.join(shell_quote(arg) + for arg in util.sanitize_cflags(flags))) diff --git a/python/mozbuild/mozbuild/compilation/database.py b/python/mozbuild/mozbuild/compilation/database.py new file mode 100644 index 000000000..4193e1bcf --- /dev/null +++ b/python/mozbuild/mozbuild/compilation/database.py @@ -0,0 +1,252 @@ +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this file, +# You can obtain one at http://mozilla.org/MPL/2.0/. + +# This modules provides functionality for dealing with code completion. + +import os +import types + +from mozbuild.compilation import util +from mozbuild.backend.common import CommonBackend +from mozbuild.frontend.data import ( + Sources, + GeneratedSources, + DirectoryTraversal, + Defines, + Linkable, + LocalInclude, + VariablePassthru, + SimpleProgram, +) +from mozbuild.shellutil import ( + quote as shell_quote, +) +from mozbuild.util import expand_variables +import mozpack.path as mozpath +from collections import ( + defaultdict, + OrderedDict, +) + + +class CompileDBBackend(CommonBackend): + def _init(self): + CommonBackend._init(self) + if not util.check_top_objdir(self.environment.topobjdir): + raise Exception() + + # The database we're going to dump out to. + self._db = OrderedDict() + + # The cache for per-directory flags + self._flags = {} + + self._envs = {} + self._includes = defaultdict(list) + self._defines = defaultdict(list) + self._local_flags = defaultdict(dict) + self._extra_includes = defaultdict(list) + self._gyp_dirs = set() + self._dist_include_testing = '-I%s' % mozpath.join( + self.environment.topobjdir, 'dist', 'include', 'testing') + + def consume_object(self, obj): + # Those are difficult directories, that will be handled later. + if obj.relativedir in ( + 'build/unix/elfhack', + 'build/unix/elfhack/inject', + 'build/clang-plugin', + 'build/clang-plugin/tests', + 'security/sandbox/win/wow_helper', + 'toolkit/crashreporter/google-breakpad/src/common'): + return True + + consumed = CommonBackend.consume_object(self, obj) + + if consumed: + return True + + if isinstance(obj, DirectoryTraversal): + self._envs[obj.objdir] = obj.config + for var in ('STL_FLAGS', 'VISIBILITY_FLAGS', 'WARNINGS_AS_ERRORS'): + value = obj.config.substs.get(var) + if value: + self._local_flags[obj.objdir][var] = value + + elif isinstance(obj, (Sources, GeneratedSources)): + # For other sources, include each source file. + for f in obj.files: + self._build_db_line(obj.objdir, obj.relativedir, obj.config, f, + obj.canonical_suffix) + + elif isinstance(obj, LocalInclude): + self._includes[obj.objdir].append('-I%s' % mozpath.normpath( + obj.path.full_path)) + + elif isinstance(obj, Linkable): + if isinstance(obj.defines, Defines): # As opposed to HostDefines + for d in obj.defines.get_defines(): + if d not in self._defines[obj.objdir]: + self._defines[obj.objdir].append(d) + self._defines[obj.objdir].extend(obj.lib_defines.get_defines()) + if isinstance(obj, SimpleProgram) and obj.is_unit_test: + if (self._dist_include_testing not in + self._extra_includes[obj.objdir]): + self._extra_includes[obj.objdir].append( + self._dist_include_testing) + + elif isinstance(obj, VariablePassthru): + if obj.variables.get('IS_GYP_DIR'): + self._gyp_dirs.add(obj.objdir) + for var in ('MOZBUILD_CFLAGS', 'MOZBUILD_CXXFLAGS', + 'MOZBUILD_CMFLAGS', 'MOZBUILD_CMMFLAGS', + 'RTL_FLAGS', 'VISIBILITY_FLAGS'): + if var in obj.variables: + self._local_flags[obj.objdir][var] = obj.variables[var] + if (obj.variables.get('DISABLE_STL_WRAPPING') and + 'STL_FLAGS' in self._local_flags[obj.objdir]): + del self._local_flags[obj.objdir]['STL_FLAGS'] + if (obj.variables.get('ALLOW_COMPILER_WARNINGS') and + 'WARNINGS_AS_ERRORS' in self._local_flags[obj.objdir]): + del self._local_flags[obj.objdir]['WARNINGS_AS_ERRORS'] + + return True + + def consume_finished(self): + CommonBackend.consume_finished(self) + + db = [] + + for (directory, filename), cmd in self._db.iteritems(): + env = self._envs[directory] + cmd = list(cmd) + cmd.append(filename) + local_extra = list(self._extra_includes[directory]) + if directory not in self._gyp_dirs: + for var in ( + 'NSPR_CFLAGS', + 'NSS_CFLAGS', + 'MOZ_JPEG_CFLAGS', + 'MOZ_PNG_CFLAGS', + 'MOZ_ZLIB_CFLAGS', + 'MOZ_PIXMAN_CFLAGS', + ): + f = env.substs.get(var) + if f: + local_extra.extend(f) + variables = { + 'LOCAL_INCLUDES': self._includes[directory], + 'DEFINES': self._defines[directory], + 'EXTRA_INCLUDES': local_extra, + 'DIST': mozpath.join(env.topobjdir, 'dist'), + 'DEPTH': env.topobjdir, + 'MOZILLA_DIR': env.topsrcdir, + 'topsrcdir': env.topsrcdir, + 'topobjdir': env.topobjdir, + } + variables.update(self._local_flags[directory]) + c = [] + for a in cmd: + a = expand_variables(a, variables).split() + if not a: + continue + if isinstance(a, types.StringTypes): + c.append(a) + else: + c.extend(a) + db.append({ + 'directory': directory, + 'command': ' '.join(shell_quote(a) for a in c), + 'file': filename, + }) + + import json + # Output the database (a JSON file) to objdir/compile_commands.json + outputfile = os.path.join(self.environment.topobjdir, 'compile_commands.json') + with self._write_file(outputfile) as jsonout: + json.dump(db, jsonout, indent=0) + + def _process_unified_sources(self, obj): + # For unified sources, only include the unified source file. + # Note that unified sources are never used for host sources. + for f in obj.unified_source_mapping: + self._build_db_line(obj.objdir, obj.relativedir, obj.config, f[0], + obj.canonical_suffix) + + def _handle_idl_manager(self, idl_manager): + pass + + def _handle_ipdl_sources(self, ipdl_dir, sorted_ipdl_sources, + unified_ipdl_cppsrcs_mapping): + for f in unified_ipdl_cppsrcs_mapping: + self._build_db_line(ipdl_dir, None, self.environment, f[0], + '.cpp') + + def _handle_webidl_build(self, bindings_dir, unified_source_mapping, + webidls, expected_build_output_files, + global_define_files): + for f in unified_source_mapping: + self._build_db_line(bindings_dir, None, self.environment, f[0], + '.cpp') + + COMPILERS = { + '.c': 'CC', + '.cpp': 'CXX', + '.m': 'CC', + '.mm': 'CXX', + } + + CFLAGS = { + '.c': 'CFLAGS', + '.cpp': 'CXXFLAGS', + '.m': 'CFLAGS', + '.mm': 'CXXFLAGS', + } + + def _build_db_line(self, objdir, reldir, cenv, filename, canonical_suffix): + if canonical_suffix not in self.COMPILERS: + return + db = self._db.setdefault((objdir, filename), + cenv.substs[self.COMPILERS[canonical_suffix]].split() + + ['-o', '/dev/null', '-c']) + reldir = reldir or mozpath.relpath(objdir, cenv.topobjdir) + + def append_var(name): + value = cenv.substs.get(name) + if not value: + return + if isinstance(value, types.StringTypes): + value = value.split() + db.extend(value) + + if canonical_suffix in ('.mm', '.cpp'): + db.append('$(STL_FLAGS)') + + db.extend(( + '$(VISIBILITY_FLAGS)', + '$(DEFINES)', + '-I%s' % mozpath.join(cenv.topsrcdir, reldir), + '-I%s' % objdir, + '$(LOCAL_INCLUDES)', + '-I%s/dist/include' % cenv.topobjdir, + '$(EXTRA_INCLUDES)', + )) + append_var('DSO_CFLAGS') + append_var('DSO_PIC_CFLAGS') + if canonical_suffix in ('.c', '.cpp'): + db.append('$(RTL_FLAGS)') + append_var('OS_COMPILE_%s' % self.CFLAGS[canonical_suffix]) + append_var('OS_CPPFLAGS') + append_var('OS_%s' % self.CFLAGS[canonical_suffix]) + append_var('MOZ_DEBUG_FLAGS') + append_var('MOZ_OPTIMIZE_FLAGS') + append_var('MOZ_FRAMEPTR_FLAGS') + db.append('$(WARNINGS_AS_ERRORS)') + db.append('$(MOZBUILD_%s)' % self.CFLAGS[canonical_suffix]) + if canonical_suffix == '.m': + append_var('OS_COMPILE_CMFLAGS') + db.append('$(MOZBUILD_CMFLAGS)') + elif canonical_suffix == '.mm': + append_var('OS_COMPILE_CMMFLAGS') + db.append('$(MOZBUILD_CMMFLAGS)') diff --git a/python/mozbuild/mozbuild/compilation/util.py b/python/mozbuild/mozbuild/compilation/util.py new file mode 100644 index 000000000..32ff2f876 --- /dev/null +++ b/python/mozbuild/mozbuild/compilation/util.py @@ -0,0 +1,54 @@ +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this file, +# You can obtain one at http://mozilla.org/MPL/2.0/. + +import os +from mozbuild import shellutil + +def check_top_objdir(topobjdir): + top_make = os.path.join(topobjdir, 'Makefile') + if not os.path.exists(top_make): + print('Your tree has not been built yet. Please run ' + '|mach build| with no arguments.') + return False + return True + +def get_build_vars(directory, cmd): + build_vars = {} + + def on_line(line): + elements = [s.strip() for s in line.split('=', 1)] + + if len(elements) != 2: + return + + build_vars[elements[0]] = elements[1] + + try: + old_logger = cmd.log_manager.replace_terminal_handler(None) + cmd._run_make(directory=directory, target='showbuild', log=False, + print_directory=False, allow_parallel=False, silent=True, + line_handler=on_line) + finally: + cmd.log_manager.replace_terminal_handler(old_logger) + + return build_vars + +def sanitize_cflags(flags): + # We filter out -Xclang arguments as clang based tools typically choke on + # passing these flags down to the clang driver. -Xclang tells the clang + # driver driver to pass whatever comes after it down to clang cc1, which is + # why we skip -Xclang and the argument immediately after it. Here is an + # example: the following two invocations pass |-foo -bar -baz| to cc1: + # clang -cc1 -foo -bar -baz + # clang -Xclang -foo -Xclang -bar -Xclang -baz + sanitized = [] + saw_xclang = False + for flag in flags: + if flag == '-Xclang': + saw_xclang = True + elif saw_xclang: + saw_xclang = False + else: + sanitized.append(flag) + return sanitized diff --git a/python/mozbuild/mozbuild/compilation/warnings.py b/python/mozbuild/mozbuild/compilation/warnings.py new file mode 100644 index 000000000..8fb20ccbf --- /dev/null +++ b/python/mozbuild/mozbuild/compilation/warnings.py @@ -0,0 +1,376 @@ +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this file, +# You can obtain one at http://mozilla.org/MPL/2.0/. + +# This modules provides functionality for dealing with compiler warnings. + +from __future__ import absolute_import, unicode_literals + +import errno +import json +import os +import re + +from mozbuild.util import hash_file +import mozpack.path as mozpath + + +# Regular expression to strip ANSI color sequences from a string. This is +# needed to properly analyze Clang compiler output, which may be colorized. +# It assumes ANSI escape sequences. +RE_STRIP_COLORS = re.compile(r'\x1b\[[\d;]+m') + +# This captures Clang diagnostics with the standard formatting. +RE_CLANG_WARNING = re.compile(r""" + (?P[^:]+) + : + (?P\d+) + : + (?P\d+) + : + \swarning:\s + (?P.+) + \[(?P[^\]]+) + """, re.X) + +# This captures Visual Studio's warning format. +RE_MSVC_WARNING = re.compile(r""" + (?P.*) + \((?P\d+)\) + \s?:\swarning\s + (?P[^:]+) + :\s + (?P.*) + """, re.X) + +IN_FILE_INCLUDED_FROM = 'In file included from ' + + +class CompilerWarning(dict): + """Represents an individual compiler warning.""" + + def __init__(self): + dict.__init__(self) + + self['filename'] = None + self['line'] = None + self['column'] = None + self['message'] = None + self['flag'] = None + + # Since we inherit from dict, functools.total_ordering gets confused. + # Thus, we define a key function, a generic comparison, and then + # implement all the rich operators with those; approach is from: + # http://regebro.wordpress.com/2010/12/13/python-implementing-rich-comparison-the-correct-way/ + def _cmpkey(self): + return (self['filename'], self['line'], self['column']) + + def _compare(self, other, func): + if not isinstance(other, CompilerWarning): + return NotImplemented + + return func(self._cmpkey(), other._cmpkey()) + + def __eq__(self, other): + return self._compare(other, lambda s,o: s == o) + + def __neq__(self, other): + return self._compare(other, lambda s,o: s != o) + + def __lt__(self, other): + return self._compare(other, lambda s,o: s < o) + + def __le__(self, other): + return self._compare(other, lambda s,o: s <= o) + + def __gt__(self, other): + return self._compare(other, lambda s,o: s > o) + + def __ge__(self, other): + return self._compare(other, lambda s,o: s >= o) + + def __hash__(self): + """Define so this can exist inside a set, etc.""" + return hash(tuple(sorted(self.items()))) + + +class WarningsDatabase(object): + """Holds a collection of warnings. + + The warnings database is a semi-intelligent container that holds warnings + encountered during builds. + + The warnings database is backed by a JSON file. But, that is transparent + to consumers. + + Under most circumstances, the warnings database is insert only. When a + warning is encountered, the caller simply blindly inserts it into the + database. The database figures out whether it is a dupe, etc. + + During the course of development, it is common for warnings to change + slightly as source code changes. For example, line numbers will disagree. + The WarningsDatabase handles this by storing the hash of a file a warning + occurred in. At warning insert time, if the hash of the file does not match + what is stored in the database, the existing warnings for that file are + purged from the database. + + Callers should periodically prune old, invalid warnings from the database + by calling prune(). A good time to do this is at the end of a build. + """ + def __init__(self): + """Create an empty database.""" + self._files = {} + + def __len__(self): + i = 0 + for value in self._files.values(): + i += len(value['warnings']) + + return i + + def __iter__(self): + for value in self._files.values(): + for warning in value['warnings']: + yield warning + + def __contains__(self, item): + for value in self._files.values(): + for warning in value['warnings']: + if warning == item: + return True + + return False + + @property + def warnings(self): + """All the CompilerWarning instances in this database.""" + for value in self._files.values(): + for w in value['warnings']: + yield w + + def type_counts(self, dirpath=None): + """Returns a mapping of warning types to their counts.""" + + types = {} + for value in self._files.values(): + for warning in value['warnings']: + if dirpath and not mozpath.normsep(warning['filename']).startswith(dirpath): + continue + flag = warning['flag'] + count = types.get(flag, 0) + count += 1 + + types[flag] = count + + return types + + def has_file(self, filename): + """Whether we have any warnings for the specified file.""" + return filename in self._files + + def warnings_for_file(self, filename): + """Obtain the warnings for the specified file.""" + f = self._files.get(filename, {'warnings': []}) + + for warning in f['warnings']: + yield warning + + def insert(self, warning, compute_hash=True): + assert isinstance(warning, CompilerWarning) + + filename = warning['filename'] + + new_hash = None + + if compute_hash: + new_hash = hash_file(filename) + + if filename in self._files: + if new_hash != self._files[filename]['hash']: + del self._files[filename] + + value = self._files.get(filename, { + 'hash': new_hash, + 'warnings': set(), + }) + + value['warnings'].add(warning) + + self._files[filename] = value + + def prune(self): + """Prune the contents of the database. + + This removes warnings that are no longer valid. A warning is no longer + valid if the file it was in no longer exists or if the content has + changed. + + The check for changed content catches the case where a file previously + contained warnings but no longer does. + """ + + # Need to calculate up front since we are mutating original object. + filenames = self._files.keys() + for filename in filenames: + if not os.path.exists(filename): + del self._files[filename] + continue + + if self._files[filename]['hash'] is None: + continue + + current_hash = hash_file(filename) + if current_hash != self._files[filename]['hash']: + del self._files[filename] + continue + + def serialize(self, fh): + """Serialize the database to an open file handle.""" + obj = {'files': {}} + + # All this hackery because JSON can't handle sets. + for k, v in self._files.iteritems(): + obj['files'][k] = {} + + for k2, v2 in v.iteritems(): + normalized = v2 + + if k2 == 'warnings': + normalized = [w for w in v2] + + obj['files'][k][k2] = normalized + + json.dump(obj, fh, indent=2) + + def deserialize(self, fh): + """Load serialized content from a handle into the current instance.""" + obj = json.load(fh) + + self._files = obj['files'] + + # Normalize data types. + for filename, value in self._files.iteritems(): + for k, v in value.iteritems(): + if k != 'warnings': + continue + + normalized = set() + for d in v: + w = CompilerWarning() + w.update(d) + normalized.add(w) + + self._files[filename]['warnings'] = normalized + + def load_from_file(self, filename): + """Load the database from a file.""" + with open(filename, 'rb') as fh: + self.deserialize(fh) + + def save_to_file(self, filename): + """Save the database to a file.""" + try: + # Ensure the directory exists + os.makedirs(os.path.dirname(filename)) + except OSError as e: + if e.errno != errno.EEXIST: + raise + with open(filename, 'wb') as fh: + self.serialize(fh) + + +class WarningsCollector(object): + """Collects warnings from text data. + + Instances of this class receive data (usually the output of compiler + invocations) and parse it into warnings and add these warnings to a + database. + + The collector works by incrementally receiving data, usually line-by-line + output from the compiler. Therefore, it can maintain state to parse + multi-line warning messages. + """ + def __init__(self, database=None, objdir=None, resolve_files=True): + self.database = database + self.objdir = objdir + self.resolve_files = resolve_files + self.included_from = [] + + if database is None: + self.database = WarningsDatabase() + + def process_line(self, line): + """Take a line of text and process it for a warning.""" + + filtered = RE_STRIP_COLORS.sub('', line) + + # Clang warnings in files included from the one(s) being compiled will + # start with "In file included from /path/to/file:line:". Here, we + # record those. + if filtered.startswith(IN_FILE_INCLUDED_FROM): + included_from = filtered[len(IN_FILE_INCLUDED_FROM):] + + parts = included_from.split(':') + + self.included_from.append(parts[0]) + + return + + warning = CompilerWarning() + filename = None + + # TODO make more efficient so we run minimal regexp matches. + match_clang = RE_CLANG_WARNING.match(filtered) + match_msvc = RE_MSVC_WARNING.match(filtered) + if match_clang: + d = match_clang.groupdict() + + filename = d['file'] + warning['line'] = int(d['line']) + warning['column'] = int(d['column']) + warning['flag'] = d['flag'] + warning['message'] = d['message'].rstrip() + + elif match_msvc: + d = match_msvc.groupdict() + + filename = d['file'] + warning['line'] = int(d['line']) + warning['flag'] = d['flag'] + warning['message'] = d['message'].rstrip() + else: + self.included_from = [] + return None + + filename = os.path.normpath(filename) + + # Sometimes we get relative includes. These typically point to files in + # the object directory. We try to resolve the relative path. + if not os.path.isabs(filename): + filename = self._normalize_relative_path(filename) + + if not os.path.exists(filename) and self.resolve_files: + raise Exception('Could not find file containing warning: %s' % + filename) + + warning['filename'] = filename + + self.database.insert(warning, compute_hash=self.resolve_files) + + return warning + + def _normalize_relative_path(self, filename): + # Special case files in dist/include. + idx = filename.find('/dist/include') + if idx != -1: + return self.objdir + filename[idx:] + + for included_from in self.included_from: + source_dir = os.path.dirname(included_from) + + candidate = os.path.normpath(os.path.join(source_dir, filename)) + + if os.path.exists(candidate): + return candidate + + return filename diff --git a/python/mozbuild/mozbuild/config_status.py b/python/mozbuild/mozbuild/config_status.py new file mode 100644 index 000000000..343dcc3a2 --- /dev/null +++ b/python/mozbuild/mozbuild/config_status.py @@ -0,0 +1,182 @@ +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, You can obtain one at http://mozilla.org/MPL/2.0/. + +# Combined with build/autoconf/config.status.m4, ConfigStatus is an almost +# drop-in replacement for autoconf 2.13's config.status, with features +# borrowed from autoconf > 2.5, and additional features. + +from __future__ import absolute_import, print_function + +import logging +import os +import subprocess +import sys +import time + +from argparse import ArgumentParser + +from mach.logging import LoggingManager +from mozbuild.backend.configenvironment import ConfigEnvironment +from mozbuild.base import MachCommandConditions +from mozbuild.frontend.emitter import TreeMetadataEmitter +from mozbuild.frontend.reader import BuildReader +from mozbuild.mozinfo import write_mozinfo +from itertools import chain + +from mozbuild.backend import ( + backends, + get_backend_class, +) + + +log_manager = LoggingManager() + + +ANDROID_IDE_ADVERTISEMENT = ''' +============= +ADVERTISEMENT + +You are building Firefox for Android. After your build completes, you can open +the top source directory in IntelliJ or Android Studio directly and build using +Gradle. See the documentation at + +https://developer.mozilla.org/en-US/docs/Simple_Firefox_for_Android_build + +PLEASE BE AWARE THAT GRADLE AND INTELLIJ/ANDROID STUDIO SUPPORT IS EXPERIMENTAL. +You should verify any changes using |mach build|. +============= +'''.strip() + +VISUAL_STUDIO_ADVERTISEMENT = ''' +=============================== +Visual Studio Support Available + +You are building Firefox on Windows. You can generate Visual Studio +files by running: + + mach build-backend --backend=VisualStudio + +=============================== +'''.strip() + + +def config_status(topobjdir='.', topsrcdir='.', defines=None, + non_global_defines=None, substs=None, source=None, + mozconfig=None, args=sys.argv[1:]): + '''Main function, providing config.status functionality. + + Contrary to config.status, it doesn't use CONFIG_FILES or CONFIG_HEADERS + variables. + + Without the -n option, this program acts as config.status and considers + the current directory as the top object directory, even when config.status + is in a different directory. It will, however, treat the directory + containing config.status as the top object directory with the -n option. + + The options to this function are passed when creating the + ConfigEnvironment. These lists, as well as the actual wrapper script + around this function, are meant to be generated by configure. + See build/autoconf/config.status.m4. + ''' + + if 'CONFIG_FILES' in os.environ: + raise Exception('Using the CONFIG_FILES environment variable is not ' + 'supported.') + if 'CONFIG_HEADERS' in os.environ: + raise Exception('Using the CONFIG_HEADERS environment variable is not ' + 'supported.') + + if not os.path.isabs(topsrcdir): + raise Exception('topsrcdir must be defined as an absolute directory: ' + '%s' % topsrcdir) + + default_backends = ['RecursiveMake'] + default_backends = (substs or {}).get('BUILD_BACKENDS', ['RecursiveMake']) + + parser = ArgumentParser() + parser.add_argument('-v', '--verbose', dest='verbose', action='store_true', + help='display verbose output') + parser.add_argument('-n', dest='not_topobjdir', action='store_true', + help='do not consider current directory as top object directory') + parser.add_argument('-d', '--diff', action='store_true', + help='print diffs of changed files.') + parser.add_argument('-b', '--backend', nargs='+', choices=sorted(backends), + default=default_backends, + help='what backend to build (default: %s).' % + ' '.join(default_backends)) + parser.add_argument('--dry-run', action='store_true', + help='do everything except writing files out.') + options = parser.parse_args(args) + + # Without -n, the current directory is meant to be the top object directory + if not options.not_topobjdir: + topobjdir = os.path.abspath('.') + + env = ConfigEnvironment(topsrcdir, topobjdir, defines=defines, + non_global_defines=non_global_defines, substs=substs, + source=source, mozconfig=mozconfig) + + # mozinfo.json only needs written if configure changes and configure always + # passes this environment variable. + if 'WRITE_MOZINFO' in os.environ: + write_mozinfo(os.path.join(topobjdir, 'mozinfo.json'), env, os.environ) + + cpu_start = time.clock() + time_start = time.time() + + # Make appropriate backend instances, defaulting to RecursiveMakeBackend, + # or what is in BUILD_BACKENDS. + selected_backends = [get_backend_class(b)(env) for b in options.backend] + + if options.dry_run: + for b in selected_backends: + b.dry_run = True + + reader = BuildReader(env) + emitter = TreeMetadataEmitter(env) + # This won't actually do anything because of the magic of generators. + definitions = emitter.emit(reader.read_topsrcdir()) + + log_level = logging.DEBUG if options.verbose else logging.INFO + log_manager.add_terminal_logging(level=log_level) + log_manager.enable_unstructured() + + print('Reticulating splines...', file=sys.stderr) + if len(selected_backends) > 1: + definitions = list(definitions) + + for the_backend in selected_backends: + the_backend.consume(definitions) + + execution_time = 0.0 + for obj in chain((reader, emitter), selected_backends): + summary = obj.summary() + print(summary, file=sys.stderr) + execution_time += summary.execution_time + + cpu_time = time.clock() - cpu_start + wall_time = time.time() - time_start + efficiency = cpu_time / wall_time if wall_time else 100 + untracked = wall_time - execution_time + + print( + 'Total wall time: {:.2f}s; CPU time: {:.2f}s; Efficiency: ' + '{:.0%}; Untracked: {:.2f}s'.format( + wall_time, cpu_time, efficiency, untracked), + file=sys.stderr + ) + + if options.diff: + for the_backend in selected_backends: + for path, diff in sorted(the_backend.file_diffs.items()): + print('\n'.join(diff)) + + # Advertise Visual Studio if appropriate. + if os.name == 'nt' and 'VisualStudio' not in options.backend: + print(VISUAL_STUDIO_ADVERTISEMENT) + + # Advertise Eclipse if it is appropriate. + if MachCommandConditions.is_android(env): + if 'AndroidEclipse' not in options.backend: + print(ANDROID_IDE_ADVERTISEMENT) diff --git a/python/mozbuild/mozbuild/configure/__init__.py b/python/mozbuild/mozbuild/configure/__init__.py new file mode 100644 index 000000000..0fe640cae --- /dev/null +++ b/python/mozbuild/mozbuild/configure/__init__.py @@ -0,0 +1,935 @@ +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, You can obtain one at http://mozilla.org/MPL/2.0/. + +from __future__ import absolute_import, print_function, unicode_literals + +import inspect +import logging +import os +import re +import sys +import types +from collections import OrderedDict +from contextlib import contextmanager +from functools import wraps +from mozbuild.configure.options import ( + CommandLineHelper, + ConflictingOptionError, + InvalidOptionError, + NegativeOptionValue, + Option, + OptionValue, + PositiveOptionValue, +) +from mozbuild.configure.help import HelpFormatter +from mozbuild.configure.util import ( + ConfigureOutputHandler, + getpreferredencoding, + LineIO, +) +from mozbuild.util import ( + exec_, + memoize, + memoized_property, + ReadOnlyDict, + ReadOnlyNamespace, +) + +import mozpack.path as mozpath + + +class ConfigureError(Exception): + pass + + +class SandboxDependsFunction(object): + '''Sandbox-visible representation of @depends functions.''' + def __call__(self, *arg, **kwargs): + raise ConfigureError('The `%s` function may not be called' + % self.__name__) + + +class DependsFunction(object): + __slots__ = ( + 'func', 'dependencies', 'when', 'sandboxed', 'sandbox', '_result') + + def __init__(self, sandbox, func, dependencies, when=None): + assert isinstance(sandbox, ConfigureSandbox) + self.func = func + self.dependencies = dependencies + self.sandboxed = wraps(func)(SandboxDependsFunction()) + self.sandbox = sandbox + self.when = when + sandbox._depends[self.sandboxed] = self + + # Only @depends functions with a dependency on '--help' are executed + # immediately. Everything else is queued for later execution. + if sandbox._help_option in dependencies: + sandbox._value_for(self) + elif not sandbox._help: + sandbox._execution_queue.append((sandbox._value_for, (self,))) + + @property + def name(self): + return self.func.__name__ + + @property + def sandboxed_dependencies(self): + return [ + d.sandboxed if isinstance(d, DependsFunction) else d + for d in self.dependencies + ] + + @memoized_property + def result(self): + if self.when and not self.sandbox._value_for(self.when): + return None + + resolved_args = [self.sandbox._value_for(d) for d in self.dependencies] + return self.func(*resolved_args) + + def __repr__(self): + return '<%s.%s %s(%s)>' % ( + self.__class__.__module__, + self.__class__.__name__, + self.name, + ', '.join(repr(d) for d in self.dependencies), + ) + + +class CombinedDependsFunction(DependsFunction): + def __init__(self, sandbox, func, dependencies): + @memoize + @wraps(func) + def wrapper(*args): + return func(args) + + flatten_deps = [] + for d in dependencies: + if isinstance(d, CombinedDependsFunction) and d.func == wrapper: + for d2 in d.dependencies: + if d2 not in flatten_deps: + flatten_deps.append(d2) + elif d not in flatten_deps: + flatten_deps.append(d) + + # Automatically add a --help dependency if one of the dependencies + # depends on it. + for d in flatten_deps: + if (isinstance(d, DependsFunction) and + sandbox._help_option in d.dependencies): + flatten_deps.insert(0, sandbox._help_option) + break + + super(CombinedDependsFunction, self).__init__( + sandbox, wrapper, flatten_deps) + + @memoized_property + def result(self): + # Ignore --help for the combined result + deps = self.dependencies + if deps[0] == self.sandbox._help_option: + deps = deps[1:] + resolved_args = [self.sandbox._value_for(d) for d in deps] + return self.func(*resolved_args) + + def __eq__(self, other): + return (isinstance(other, self.__class__) and + self.func == other.func and + set(self.dependencies) == set(other.dependencies)) + + def __ne__(self, other): + return not self == other + +class SandboxedGlobal(dict): + '''Identifiable dict type for use as function global''' + + +def forbidden_import(*args, **kwargs): + raise ImportError('Importing modules is forbidden') + + +class ConfigureSandbox(dict): + """Represents a sandbox for executing Python code for build configuration. + This is a different kind of sandboxing than the one used for moz.build + processing. + + The sandbox has 9 primitives: + - option + - depends + - template + - imports + - include + - set_config + - set_define + - imply_option + - only_when + + `option`, `include`, `set_config`, `set_define` and `imply_option` are + functions. `depends`, `template`, and `imports` are decorators. `only_when` + is a context_manager. + + These primitives are declared as name_impl methods to this class and + the mapping name -> name_impl is done automatically in __getitem__. + + Additional primitives should be frowned upon to keep the sandbox itself as + simple as possible. Instead, helpers should be created within the sandbox + with the existing primitives. + + The sandbox is given, at creation, a dict where the yielded configuration + will be stored. + + config = {} + sandbox = ConfigureSandbox(config) + sandbox.run(path) + do_stuff(config) + """ + + # The default set of builtins. We expose unicode as str to make sandboxed + # files more python3-ready. + BUILTINS = ReadOnlyDict({ + b: __builtins__[b] + for b in ('None', 'False', 'True', 'int', 'bool', 'any', 'all', 'len', + 'list', 'tuple', 'set', 'dict', 'isinstance', 'getattr', + 'hasattr', 'enumerate', 'range', 'zip') + }, __import__=forbidden_import, str=unicode) + + # Expose a limited set of functions from os.path + OS = ReadOnlyNamespace(path=ReadOnlyNamespace(**{ + k: getattr(mozpath, k, getattr(os.path, k)) + for k in ('abspath', 'basename', 'dirname', 'isabs', 'join', + 'normcase', 'normpath', 'realpath', 'relpath') + })) + + def __init__(self, config, environ=os.environ, argv=sys.argv, + stdout=sys.stdout, stderr=sys.stderr, logger=None): + dict.__setitem__(self, '__builtins__', self.BUILTINS) + + self._paths = [] + self._all_paths = set() + self._templates = set() + # Associate SandboxDependsFunctions to DependsFunctions. + self._depends = {} + self._seen = set() + # Store the @imports added to a given function. + self._imports = {} + + self._options = OrderedDict() + # Store raw option (as per command line or environment) for each Option + self._raw_options = OrderedDict() + + # Store options added with `imply_option`, and the reason they were + # added (which can either have been given to `imply_option`, or + # inferred. Their order matters, so use a list. + self._implied_options = [] + + # Store all results from _prepare_function + self._prepared_functions = set() + + # Queue of functions to execute, with their arguments + self._execution_queue = [] + + # Store the `when`s associated to some options. + self._conditions = {} + + # A list of conditions to apply as a default `when` for every *_impl() + self._default_conditions = [] + + self._helper = CommandLineHelper(environ, argv) + + assert isinstance(config, dict) + self._config = config + + if logger is None: + logger = moz_logger = logging.getLogger('moz.configure') + logger.setLevel(logging.DEBUG) + formatter = logging.Formatter('%(levelname)s: %(message)s') + handler = ConfigureOutputHandler(stdout, stderr) + handler.setFormatter(formatter) + queue_debug = handler.queue_debug + logger.addHandler(handler) + + else: + assert isinstance(logger, logging.Logger) + moz_logger = None + @contextmanager + def queue_debug(): + yield + + # Some callers will manage to log a bytestring with characters in it + # that can't be converted to ascii. Make our log methods robust to this + # by detecting the encoding that a producer is likely to have used. + encoding = getpreferredencoding() + def wrapped_log_method(logger, key): + method = getattr(logger, key) + if not encoding: + return method + def wrapped(*args, **kwargs): + out_args = [ + arg.decode(encoding) if isinstance(arg, str) else arg + for arg in args + ] + return method(*out_args, **kwargs) + return wrapped + + log_namespace = { + k: wrapped_log_method(logger, k) + for k in ('debug', 'info', 'warning', 'error') + } + log_namespace['queue_debug'] = queue_debug + self.log_impl = ReadOnlyNamespace(**log_namespace) + + self._help = None + self._help_option = self.option_impl('--help', + help='print this message') + self._seen.add(self._help_option) + + self._always = DependsFunction(self, lambda: True, []) + self._never = DependsFunction(self, lambda: False, []) + + if self._value_for(self._help_option): + self._help = HelpFormatter(argv[0]) + self._help.add(self._help_option) + elif moz_logger: + handler = logging.FileHandler('config.log', mode='w', delay=True) + handler.setFormatter(formatter) + logger.addHandler(handler) + + def include_file(self, path): + '''Include one file in the sandbox. Users of this class probably want + + Note: this will execute all template invocations, as well as @depends + functions that depend on '--help', but nothing else. + ''' + + if self._paths: + path = mozpath.join(mozpath.dirname(self._paths[-1]), path) + path = mozpath.normpath(path) + if not mozpath.basedir(path, (mozpath.dirname(self._paths[0]),)): + raise ConfigureError( + 'Cannot include `%s` because it is not in a subdirectory ' + 'of `%s`' % (path, mozpath.dirname(self._paths[0]))) + else: + path = mozpath.realpath(mozpath.abspath(path)) + if path in self._all_paths: + raise ConfigureError( + 'Cannot include `%s` because it was included already.' % path) + self._paths.append(path) + self._all_paths.add(path) + + source = open(path, 'rb').read() + + code = compile(source, path, 'exec') + + exec_(code, self) + + self._paths.pop(-1) + + def run(self, path=None): + '''Executes the given file within the sandbox, as well as everything + pending from any other included file, and ensure the overall + consistency of the executed script(s).''' + if path: + self.include_file(path) + + for option in self._options.itervalues(): + # All options must be referenced by some @depends function + if option not in self._seen: + raise ConfigureError( + 'Option `%s` is not handled ; reference it with a @depends' + % option.option + ) + + self._value_for(option) + + # All implied options should exist. + for implied_option in self._implied_options: + value = self._resolve(implied_option.value, + need_help_dependency=False) + if value is not None: + raise ConfigureError( + '`%s`, emitted from `%s` line %d, is unknown.' + % (implied_option.option, implied_option.caller[1], + implied_option.caller[2])) + + # All options should have been removed (handled) by now. + for arg in self._helper: + without_value = arg.split('=', 1)[0] + raise InvalidOptionError('Unknown option: %s' % without_value) + + # Run the execution queue + for func, args in self._execution_queue: + func(*args) + + if self._help: + with LineIO(self.log_impl.info) as out: + self._help.usage(out) + + def __getitem__(self, key): + impl = '%s_impl' % key + func = getattr(self, impl, None) + if func: + return func + + return super(ConfigureSandbox, self).__getitem__(key) + + def __setitem__(self, key, value): + if (key in self.BUILTINS or key == '__builtins__' or + hasattr(self, '%s_impl' % key)): + raise KeyError('Cannot reassign builtins') + + if inspect.isfunction(value) and value not in self._templates: + value, _ = self._prepare_function(value) + + elif (not isinstance(value, SandboxDependsFunction) and + value not in self._templates and + not (inspect.isclass(value) and issubclass(value, Exception))): + raise KeyError('Cannot assign `%s` because it is neither a ' + '@depends nor a @template' % key) + + return super(ConfigureSandbox, self).__setitem__(key, value) + + def _resolve(self, arg, need_help_dependency=True): + if isinstance(arg, SandboxDependsFunction): + return self._value_for_depends(self._depends[arg], + need_help_dependency) + return arg + + def _value_for(self, obj, need_help_dependency=False): + if isinstance(obj, SandboxDependsFunction): + assert obj in self._depends + return self._value_for_depends(self._depends[obj], + need_help_dependency) + + elif isinstance(obj, DependsFunction): + return self._value_for_depends(obj, need_help_dependency) + + elif isinstance(obj, Option): + return self._value_for_option(obj) + + assert False + + @memoize + def _value_for_depends(self, obj, need_help_dependency=False): + assert not inspect.isgeneratorfunction(obj.func) + return obj.result + + @memoize + def _value_for_option(self, option): + implied = {} + for implied_option in self._implied_options[:]: + if implied_option.name not in (option.name, option.env): + continue + self._implied_options.remove(implied_option) + + if (implied_option.when and + not self._value_for(implied_option.when)): + continue + + value = self._resolve(implied_option.value, + need_help_dependency=False) + + if value is not None: + if isinstance(value, OptionValue): + pass + elif value is True: + value = PositiveOptionValue() + elif value is False or value == (): + value = NegativeOptionValue() + elif isinstance(value, types.StringTypes): + value = PositiveOptionValue((value,)) + elif isinstance(value, tuple): + value = PositiveOptionValue(value) + else: + raise TypeError("Unexpected type: '%s'" + % type(value).__name__) + + opt = value.format(implied_option.option) + self._helper.add(opt, 'implied') + implied[opt] = implied_option + + try: + value, option_string = self._helper.handle(option) + except ConflictingOptionError as e: + reason = implied[e.arg].reason + if isinstance(reason, Option): + reason = self._raw_options.get(reason) or reason.option + reason = reason.split('=', 1)[0] + raise InvalidOptionError( + "'%s' implied by '%s' conflicts with '%s' from the %s" + % (e.arg, reason, e.old_arg, e.old_origin)) + + if option_string: + self._raw_options[option] = option_string + + when = self._conditions.get(option) + if (when and not self._value_for(when, need_help_dependency=True) and + value is not None and value.origin != 'default'): + if value.origin == 'environment': + # The value we return doesn't really matter, because of the + # requirement for @depends to have the same when. + return None + raise InvalidOptionError( + '%s is not available in this configuration' + % option_string.split('=', 1)[0]) + + return value + + def _dependency(self, arg, callee_name, arg_name=None): + if isinstance(arg, types.StringTypes): + prefix, name, values = Option.split_option(arg) + if values != (): + raise ConfigureError("Option must not contain an '='") + if name not in self._options: + raise ConfigureError("'%s' is not a known option. " + "Maybe it's declared too late?" + % arg) + arg = self._options[name] + self._seen.add(arg) + elif isinstance(arg, SandboxDependsFunction): + assert arg in self._depends + arg = self._depends[arg] + else: + raise TypeError( + "Cannot use object of type '%s' as %sargument to %s" + % (type(arg).__name__, '`%s` ' % arg_name if arg_name else '', + callee_name)) + return arg + + def _normalize_when(self, when, callee_name): + if when is True: + when = self._always + elif when is False: + when = self._never + elif when is not None: + when = self._dependency(when, callee_name, 'when') + + if self._default_conditions: + # Create a pseudo @depends function for the combination of all + # default conditions and `when`. + dependencies = [when] if when else [] + dependencies.extend(self._default_conditions) + if len(dependencies) == 1: + return dependencies[0] + return CombinedDependsFunction(self, all, dependencies) + return when + + @contextmanager + def only_when_impl(self, when): + '''Implementation of only_when() + + `only_when` is a context manager that essentially makes calls to + other sandbox functions within the context block ignored. + ''' + when = self._normalize_when(when, 'only_when') + if when and self._default_conditions[-1:] != [when]: + self._default_conditions.append(when) + yield + self._default_conditions.pop() + else: + yield + + def option_impl(self, *args, **kwargs): + '''Implementation of option() + This function creates and returns an Option() object, passing it the + resolved arguments (uses the result of functions when functions are + passed). In most cases, the result of this function is not expected to + be used. + Command line argument/environment variable parsing for this Option is + handled here. + ''' + when = self._normalize_when(kwargs.get('when'), 'option') + args = [self._resolve(arg) for arg in args] + kwargs = {k: self._resolve(v) for k, v in kwargs.iteritems() + if k != 'when'} + option = Option(*args, **kwargs) + if when: + self._conditions[option] = when + if option.name in self._options: + raise ConfigureError('Option `%s` already defined' % option.option) + if option.env in self._options: + raise ConfigureError('Option `%s` already defined' % option.env) + if option.name: + self._options[option.name] = option + if option.env: + self._options[option.env] = option + + if self._help and (when is None or + self._value_for(when, need_help_dependency=True)): + self._help.add(option) + + return option + + def depends_impl(self, *args, **kwargs): + '''Implementation of @depends() + This function is a decorator. It returns a function that subsequently + takes a function and returns a dummy function. The dummy function + identifies the actual function for the sandbox, while preventing + further function calls from within the sandbox. + + @depends() takes a variable number of option strings or dummy function + references. The decorated function is called as soon as the decorator + is called, and the arguments it receives are the OptionValue or + function results corresponding to each of the arguments to @depends. + As an exception, when a HelpFormatter is attached, only functions that + have '--help' in their @depends argument list are called. + + The decorated function is altered to use a different global namespace + for its execution. This different global namespace exposes a limited + set of functions from os.path. + ''' + for k in kwargs: + if k != 'when': + raise TypeError( + "depends_impl() got an unexpected keyword argument '%s'" + % k) + + when = self._normalize_when(kwargs.get('when'), '@depends') + + if not when and not args: + raise ConfigureError('@depends needs at least one argument') + + dependencies = tuple(self._dependency(arg, '@depends') for arg in args) + + conditions = [ + self._conditions[d] + for d in dependencies + if d in self._conditions and isinstance(d, Option) + ] + for c in conditions: + if c != when: + raise ConfigureError('@depends function needs the same `when` ' + 'as options it depends on') + + def decorator(func): + if inspect.isgeneratorfunction(func): + raise ConfigureError( + 'Cannot decorate generator functions with @depends') + func, glob = self._prepare_function(func) + depends = DependsFunction(self, func, dependencies, when=when) + return depends.sandboxed + + return decorator + + def include_impl(self, what, when=None): + '''Implementation of include(). + Allows to include external files for execution in the sandbox. + It is possible to use a @depends function as argument, in which case + the result of the function is the file name to include. This latter + feature is only really meant for --enable-application/--enable-project. + ''' + with self.only_when_impl(when): + what = self._resolve(what) + if what: + if not isinstance(what, types.StringTypes): + raise TypeError("Unexpected type: '%s'" % type(what).__name__) + self.include_file(what) + + def template_impl(self, func): + '''Implementation of @template. + This function is a decorator. Template functions are called + immediately. They are altered so that their global namespace exposes + a limited set of functions from os.path, as well as `depends` and + `option`. + Templates allow to simplify repetitive constructs, or to implement + helper decorators and somesuch. + ''' + template, glob = self._prepare_function(func) + glob.update( + (k[:-len('_impl')], getattr(self, k)) + for k in dir(self) if k.endswith('_impl') and k != 'template_impl' + ) + glob.update((k, v) for k, v in self.iteritems() if k not in glob) + + # Any function argument to the template must be prepared to be sandboxed. + # If the template itself returns a function (in which case, it's very + # likely a decorator), that function must be prepared to be sandboxed as + # well. + def wrap_template(template): + isfunction = inspect.isfunction + + def maybe_prepare_function(obj): + if isfunction(obj): + func, _ = self._prepare_function(obj) + return func + return obj + + # The following function may end up being prepared to be sandboxed, + # so it mustn't depend on anything from the global scope in this + # file. It can however depend on variables from the closure, thus + # maybe_prepare_function and isfunction are declared above to be + # available there. + @wraps(template) + def wrapper(*args, **kwargs): + args = [maybe_prepare_function(arg) for arg in args] + kwargs = {k: maybe_prepare_function(v) + for k, v in kwargs.iteritems()} + ret = template(*args, **kwargs) + if isfunction(ret): + # We can't expect the sandboxed code to think about all the + # details of implementing decorators, so do some of the + # work for them. If the function takes exactly one function + # as argument and returns a function, it must be a + # decorator, so mark the returned function as wrapping the + # function passed in. + if len(args) == 1 and not kwargs and isfunction(args[0]): + ret = wraps(args[0])(ret) + return wrap_template(ret) + return ret + return wrapper + + wrapper = wrap_template(template) + self._templates.add(wrapper) + return wrapper + + RE_MODULE = re.compile('^[a-zA-Z0-9_\.]+$') + + def imports_impl(self, _import, _from=None, _as=None): + '''Implementation of @imports. + This decorator imports the given _import from the given _from module + optionally under a different _as name. + The options correspond to the various forms for the import builtin. + @imports('sys') + @imports(_from='mozpack', _import='path', _as='mozpath') + ''' + for value, required in ( + (_import, True), (_from, False), (_as, False)): + + if not isinstance(value, types.StringTypes) and ( + required or value is not None): + raise TypeError("Unexpected type: '%s'" % type(value).__name__) + if value is not None and not self.RE_MODULE.match(value): + raise ValueError("Invalid argument to @imports: '%s'" % value) + if _as and '.' in _as: + raise ValueError("Invalid argument to @imports: '%s'" % _as) + + def decorator(func): + if func in self._templates: + raise ConfigureError( + '@imports must appear after @template') + if func in self._depends: + raise ConfigureError( + '@imports must appear after @depends') + # For the imports to apply in the order they appear in the + # .configure file, we accumulate them in reverse order and apply + # them later. + imports = self._imports.setdefault(func, []) + imports.insert(0, (_from, _import, _as)) + return func + + return decorator + + def _apply_imports(self, func, glob): + for _from, _import, _as in self._imports.get(func, ()): + _from = '%s.' % _from if _from else '' + if _as: + glob[_as] = self._get_one_import('%s%s' % (_from, _import)) + else: + what = _import.split('.')[0] + glob[what] = self._get_one_import('%s%s' % (_from, what)) + + def _get_one_import(self, what): + # The special `__sandbox__` module gives access to the sandbox + # instance. + if what == '__sandbox__': + return self + # Special case for the open() builtin, because otherwise, using it + # fails with "IOError: file() constructor not accessible in + # restricted mode" + if what == '__builtin__.open': + return lambda *args, **kwargs: open(*args, **kwargs) + # Until this proves to be a performance problem, just construct an + # import statement and execute it. + import_line = '' + if '.' in what: + _from, what = what.rsplit('.', 1) + import_line += 'from %s ' % _from + import_line += 'import %s as imported' % what + glob = {} + exec_(import_line, {}, glob) + return glob['imported'] + + def _resolve_and_set(self, data, name, value, when=None): + # Don't set anything when --help was on the command line + if self._help: + return + if when and not self._value_for(when): + return + name = self._resolve(name, need_help_dependency=False) + if name is None: + return + if not isinstance(name, types.StringTypes): + raise TypeError("Unexpected type: '%s'" % type(name).__name__) + if name in data: + raise ConfigureError( + "Cannot add '%s' to configuration: Key already " + "exists" % name) + value = self._resolve(value, need_help_dependency=False) + if value is not None: + data[name] = value + + def set_config_impl(self, name, value, when=None): + '''Implementation of set_config(). + Set the configuration items with the given name to the given value. + Both `name` and `value` can be references to @depends functions, + in which case the result from these functions is used. If the result + of either function is None, the configuration item is not set. + ''' + when = self._normalize_when(when, 'set_config') + + self._execution_queue.append(( + self._resolve_and_set, (self._config, name, value, when))) + + def set_define_impl(self, name, value, when=None): + '''Implementation of set_define(). + Set the define with the given name to the given value. Both `name` and + `value` can be references to @depends functions, in which case the + result from these functions is used. If the result of either function + is None, the define is not set. If the result is False, the define is + explicitly undefined (-U). + ''' + when = self._normalize_when(when, 'set_define') + + defines = self._config.setdefault('DEFINES', {}) + self._execution_queue.append(( + self._resolve_and_set, (defines, name, value, when))) + + def imply_option_impl(self, option, value, reason=None, when=None): + '''Implementation of imply_option(). + Injects additional options as if they had been passed on the command + line. The `option` argument is a string as in option()'s `name` or + `env`. The option must be declared after `imply_option` references it. + The `value` argument indicates the value to pass to the option. + It can be: + - True. In this case `imply_option` injects the positive option + (--enable-foo/--with-foo). + imply_option('--enable-foo', True) + imply_option('--disable-foo', True) + are both equivalent to `--enable-foo` on the command line. + + - False. In this case `imply_option` injects the negative option + (--disable-foo/--without-foo). + imply_option('--enable-foo', False) + imply_option('--disable-foo', False) + are both equivalent to `--disable-foo` on the command line. + + - None. In this case `imply_option` does nothing. + imply_option('--enable-foo', None) + imply_option('--disable-foo', None) + are both equivalent to not passing any flag on the command line. + + - a string or a tuple. In this case `imply_option` injects the positive + option with the given value(s). + imply_option('--enable-foo', 'a') + imply_option('--disable-foo', 'a') + are both equivalent to `--enable-foo=a` on the command line. + imply_option('--enable-foo', ('a', 'b')) + imply_option('--disable-foo', ('a', 'b')) + are both equivalent to `--enable-foo=a,b` on the command line. + + Because imply_option('--disable-foo', ...) can be misleading, it is + recommended to use the positive form ('--enable' or '--with') for + `option`. + + The `value` argument can also be (and usually is) a reference to a + @depends function, in which case the result of that function will be + used as per the descripted mapping above. + + The `reason` argument indicates what caused the option to be implied. + It is necessary when it cannot be inferred from the `value`. + ''' + # Don't do anything when --help was on the command line + if self._help: + return + if not reason and isinstance(value, SandboxDependsFunction): + deps = self._depends[value].dependencies + possible_reasons = [d for d in deps if d != self._help_option] + if len(possible_reasons) == 1: + if isinstance(possible_reasons[0], Option): + reason = possible_reasons[0] + if not reason and (isinstance(value, (bool, tuple)) or + isinstance(value, types.StringTypes)): + # A reason can be provided automatically when imply_option + # is called with an immediate value. + _, filename, line, _, _, _ = inspect.stack()[1] + reason = "imply_option at %s:%s" % (filename, line) + + if not reason: + raise ConfigureError( + "Cannot infer what implies '%s'. Please add a `reason` to " + "the `imply_option` call." + % option) + + when = self._normalize_when(when, 'imply_option') + + prefix, name, values = Option.split_option(option) + if values != (): + raise ConfigureError("Implied option must not contain an '='") + + self._implied_options.append(ReadOnlyNamespace( + option=option, + prefix=prefix, + name=name, + value=value, + caller=inspect.stack()[1], + reason=reason, + when=when, + )) + + def _prepare_function(self, func): + '''Alter the given function global namespace with the common ground + for @depends, and @template. + ''' + if not inspect.isfunction(func): + raise TypeError("Unexpected type: '%s'" % type(func).__name__) + if func in self._prepared_functions: + return func, func.func_globals + + glob = SandboxedGlobal( + (k, v) for k, v in func.func_globals.iteritems() + if (inspect.isfunction(v) and v not in self._templates) or ( + inspect.isclass(v) and issubclass(v, Exception)) + ) + glob.update( + __builtins__=self.BUILTINS, + __file__=self._paths[-1] if self._paths else '', + __name__=self._paths[-1] if self._paths else '', + os=self.OS, + log=self.log_impl, + ) + + # The execution model in the sandbox doesn't guarantee the execution + # order will always be the same for a given function, and if it uses + # variables from a closure that are changed after the function is + # declared, depending when the function is executed, the value of the + # variable can differ. For consistency, we force the function to use + # the value from the earliest it can be run, which is at declaration. + # Note this is not entirely bullet proof (if the value is e.g. a list, + # the list contents could have changed), but covers the bases. + closure = None + if func.func_closure: + def makecell(content): + def f(): + content + return f.func_closure[0] + + closure = tuple(makecell(cell.cell_contents) + for cell in func.func_closure) + + new_func = wraps(func)(types.FunctionType( + func.func_code, + glob, + func.__name__, + func.func_defaults, + closure + )) + @wraps(new_func) + def wrapped(*args, **kwargs): + if func in self._imports: + self._apply_imports(func, glob) + del self._imports[func] + return new_func(*args, **kwargs) + + self._prepared_functions.add(wrapped) + return wrapped, glob diff --git a/python/mozbuild/mozbuild/configure/check_debug_ranges.py b/python/mozbuild/mozbuild/configure/check_debug_ranges.py new file mode 100644 index 000000000..ca312dff4 --- /dev/null +++ b/python/mozbuild/mozbuild/configure/check_debug_ranges.py @@ -0,0 +1,62 @@ +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, You can obtain one at http://mozilla.org/MPL/2.0/. + +# This script returns the number of items for the DW_AT_ranges corresponding +# to a given compilation unit. This is used as a helper to find a bug in some +# versions of GNU ld. + +from __future__ import absolute_import + +import subprocess +import sys +import re + +def get_range_for(compilation_unit, debug_info): + '''Returns the range offset for a given compilation unit + in a given debug_info.''' + name = ranges = '' + search_cu = False + for nfo in debug_info.splitlines(): + if 'DW_TAG_compile_unit' in nfo: + search_cu = True + elif 'DW_TAG_' in nfo or not nfo.strip(): + if name == compilation_unit and ranges != '': + return int(ranges, 16) + name = ranges = '' + search_cu = False + if search_cu: + if 'DW_AT_name' in nfo: + name = nfo.rsplit(None, 1)[1] + elif 'DW_AT_ranges' in nfo: + ranges = nfo.rsplit(None, 1)[1] + return None + +def get_range_length(range, debug_ranges): + '''Returns the number of items in the range starting at the + given offset.''' + length = 0 + for line in debug_ranges.splitlines(): + m = re.match('\s*([0-9a-fA-F]+)\s+([0-9a-fA-F]+)\s+([0-9a-fA-F]+)', line) + if m and int(m.group(1), 16) == range: + length += 1 + return length + +def main(bin, compilation_unit): + p = subprocess.Popen(['objdump', '-W', bin], stdout = subprocess.PIPE, stderr = subprocess.PIPE) + (out, err) = p.communicate() + sections = re.split('\n(Contents of the|The section) ', out) + debug_info = [s for s in sections if s.startswith('.debug_info')] + debug_ranges = [s for s in sections if s.startswith('.debug_ranges')] + if not debug_ranges or not debug_info: + return 0 + + range = get_range_for(compilation_unit, debug_info[0]) + if range is not None: + return get_range_length(range, debug_ranges[0]) + + return -1 + + +if __name__ == '__main__': + print main(*sys.argv[1:]) diff --git a/python/mozbuild/mozbuild/configure/constants.py b/python/mozbuild/mozbuild/configure/constants.py new file mode 100644 index 000000000..dfc7cf8ad --- /dev/null +++ b/python/mozbuild/mozbuild/configure/constants.py @@ -0,0 +1,103 @@ +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, You can obtain one at http://mozilla.org/MPL/2.0/. + +from __future__ import absolute_import, print_function, unicode_literals + +from mozbuild.util import EnumString +from collections import OrderedDict + + +CompilerType = EnumString.subclass( + 'clang', + 'clang-cl', + 'gcc', + 'msvc', +) + +OS = EnumString.subclass( + 'Android', + 'DragonFly', + 'FreeBSD', + 'GNU', + 'iOS', + 'NetBSD', + 'OpenBSD', + 'OSX', + 'WINNT', +) + +Kernel = EnumString.subclass( + 'Darwin', + 'DragonFly', + 'FreeBSD', + 'kFreeBSD', + 'Linux', + 'NetBSD', + 'OpenBSD', + 'WINNT', +) + +CPU_bitness = { + 'aarch64': 64, + 'Alpha': 32, + 'arm': 32, + 'hppa': 32, + 'ia64': 64, + 'mips32': 32, + 'mips64': 64, + 'ppc': 32, + 'ppc64': 64, + 's390': 32, + 's390x': 64, + 'sparc': 32, + 'sparc64': 64, + 'x86': 32, + 'x86_64': 64, +} + +CPU = EnumString.subclass(*CPU_bitness.keys()) + +Endianness = EnumString.subclass( + 'big', + 'little', +) + +WindowsBinaryType = EnumString.subclass( + 'win32', + 'win64', +) + +# The order of those checks matter +CPU_preprocessor_checks = OrderedDict(( + ('x86', '__i386__ || _M_IX86'), + ('x86_64', '__x86_64__ || _M_X64'), + ('arm', '__arm__ || _M_ARM'), + ('aarch64', '__aarch64__'), + ('ia64', '__ia64__'), + ('s390x', '__s390x__'), + ('s390', '__s390__'), + ('ppc64', '__powerpc64__'), + ('ppc', '__powerpc__'), + ('Alpha', '__alpha__'), + ('hppa', '__hppa__'), + ('sparc64', '__sparc__ && __arch64__'), + ('sparc', '__sparc__'), + ('mips64', '__mips64'), + ('mips32', '__mips__'), +)) + +assert sorted(CPU_preprocessor_checks.keys()) == sorted(CPU.POSSIBLE_VALUES) + +kernel_preprocessor_checks = { + 'Darwin': '__APPLE__', + 'DragonFly': '__DragonFly__', + 'FreeBSD': '__FreeBSD__', + 'kFreeBSD': '__FreeBSD_kernel__', + 'Linux': '__linux__', + 'NetBSD': '__NetBSD__', + 'OpenBSD': '__OpenBSD__', + 'WINNT': '_WIN32 || __CYGWIN__', +} + +assert sorted(kernel_preprocessor_checks.keys()) == sorted(Kernel.POSSIBLE_VALUES) diff --git a/python/mozbuild/mozbuild/configure/help.py b/python/mozbuild/mozbuild/configure/help.py new file mode 100644 index 000000000..cd7876fbd --- /dev/null +++ b/python/mozbuild/mozbuild/configure/help.py @@ -0,0 +1,45 @@ +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, You can obtain one at http://mozilla.org/MPL/2.0/. + +from __future__ import absolute_import, print_function, unicode_literals + +import os +from mozbuild.configure.options import Option + + +class HelpFormatter(object): + def __init__(self, argv0): + self.intro = ['Usage: %s [options]' % os.path.basename(argv0)] + self.options = ['Options: [defaults in brackets after descriptions]'] + self.env = ['Environment variables:'] + + def add(self, option): + assert isinstance(option, Option) + + if option.possible_origins == ('implied',): + # Don't display help if our option can only be implied. + return + + # TODO: improve formatting + target = self.options if option.name else self.env + opt = option.option + if option.choices: + opt += '={%s}' % ','.join(option.choices) + help = option.help or '' + if len(option.default): + if help: + help += ' ' + help += '[%s]' % ','.join(option.default) + + if len(opt) > 24 or not help: + target.append(' %s' % opt) + if help: + target.append('%s%s' % (' ' * 28, help)) + else: + target.append(' %-24s %s' % (opt, help)) + + def usage(self, out): + print('\n\n'.join('\n'.join(t) + for t in (self.intro, self.options, self.env)), + file=out) diff --git a/python/mozbuild/mozbuild/configure/libstdcxx.py b/python/mozbuild/mozbuild/configure/libstdcxx.py new file mode 100644 index 000000000..cab0ccb11 --- /dev/null +++ b/python/mozbuild/mozbuild/configure/libstdcxx.py @@ -0,0 +1,81 @@ +#!/usr/bin/python +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, You can obtain one at http://mozilla.org/MPL/2.0/. + + +# This script find the version of libstdc++ and prints it as single number +# with 8 bits per element. For example, GLIBCXX_3.4.10 becomes +# 3 << 16 | 4 << 8 | 10 = 197642. This format is easy to use +# in the C preprocessor. + +# We find out both the host and target versions. Since the output +# will be used from shell, we just print the two assignments and evaluate +# them from shell. + +from __future__ import absolute_import + +import os +import subprocess +import re + +re_for_ld = re.compile('.*\((.*)\).*') + +def parse_readelf_line(x): + """Return the version from a readelf line that looks like: + 0x00ec: Rev: 1 Flags: none Index: 8 Cnt: 2 Name: GLIBCXX_3.4.6 + """ + return x.split(':')[-1].split('_')[-1].strip() + +def parse_ld_line(x): + """Parse a line from the output of ld -t. The output of gold is just + the full path, gnu ld prints "-lstdc++ (path)". + """ + t = re_for_ld.match(x) + if t: + return t.groups()[0].strip() + return x.strip() + +def split_ver(v): + """Covert the string '1.2.3' into the list [1,2,3] + """ + return [int(x) for x in v.split('.')] + +def cmp_ver(a, b): + """Compare versions in the form 'a.b.c' + """ + for (i, j) in zip(split_ver(a), split_ver(b)): + if i != j: + return i - j + return 0 + +def encode_ver(v): + """Encode the version as a single number. + """ + t = split_ver(v) + return t[0] << 16 | t[1] << 8 | t[2] + +def find_version(e): + """Given the value of environment variable CXX or HOST_CXX, find the + version of the libstdc++ it uses. + """ + args = e.split() + args += ['-shared', '-Wl,-t'] + p = subprocess.Popen(args, stderr=subprocess.STDOUT, stdout=subprocess.PIPE) + candidates = [x for x in p.stdout if 'libstdc++.so' in x] + if not candidates: + return '' + assert len(candidates) == 1 + libstdcxx = parse_ld_line(candidates[-1]) + + p = subprocess.Popen(['readelf', '-V', libstdcxx], stdout=subprocess.PIPE) + versions = [parse_readelf_line(x) + for x in p.stdout.readlines() if 'Name: GLIBCXX' in x] + last_version = sorted(versions, cmp = cmp_ver)[-1] + return encode_ver(last_version) + +if __name__ == '__main__': + cxx_env = os.environ['CXX'] + print 'MOZ_LIBSTDCXX_TARGET_VERSION=%s' % find_version(cxx_env) + host_cxx_env = os.environ.get('HOST_CXX', cxx_env) + print 'MOZ_LIBSTDCXX_HOST_VERSION=%s' % find_version(host_cxx_env) diff --git a/python/mozbuild/mozbuild/configure/lint.py b/python/mozbuild/mozbuild/configure/lint.py new file mode 100644 index 000000000..e0a5c8328 --- /dev/null +++ b/python/mozbuild/mozbuild/configure/lint.py @@ -0,0 +1,78 @@ +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, You can obtain one at http://mozilla.org/MPL/2.0/. + +from __future__ import absolute_import, print_function, unicode_literals + +from StringIO import StringIO +from . import ( + CombinedDependsFunction, + ConfigureError, + ConfigureSandbox, + DependsFunction, +) +from .lint_util import disassemble_as_iter +from mozbuild.util import memoize + + +class LintSandbox(ConfigureSandbox): + def __init__(self, environ=None, argv=None, stdout=None, stderr=None): + out = StringIO() + stdout = stdout or out + stderr = stderr or out + environ = environ or {} + argv = argv or [] + self._wrapped = {} + super(LintSandbox, self).__init__({}, environ=environ, argv=argv, + stdout=stdout, stderr=stderr) + + def run(self, path=None): + if path: + self.include_file(path) + + def _missing_help_dependency(self, obj): + if isinstance(obj, CombinedDependsFunction): + return False + if isinstance(obj, DependsFunction): + if (self._help_option in obj.dependencies or + obj in (self._always, self._never)): + return False + func, glob = self._wrapped[obj.func] + # We allow missing --help dependencies for functions that: + # - don't use @imports + # - don't have a closure + # - don't use global variables + if func in self._imports or func.func_closure: + return True + for op, arg in disassemble_as_iter(func): + if op in ('LOAD_GLOBAL', 'STORE_GLOBAL'): + # There is a fake os module when one is not imported, + # and it's allowed for functions without a --help + # dependency. + if arg == 'os' and glob.get('os') is self.OS: + continue + return True + return False + + @memoize + def _value_for_depends(self, obj, need_help_dependency=False): + with_help = self._help_option in obj.dependencies + if with_help: + for arg in obj.dependencies: + if self._missing_help_dependency(arg): + raise ConfigureError( + "`%s` depends on '--help' and `%s`. " + "`%s` must depend on '--help'" + % (obj.name, arg.name, arg.name)) + elif ((self._help or need_help_dependency) and + self._missing_help_dependency(obj)): + raise ConfigureError("Missing @depends for `%s`: '--help'" % + obj.name) + return super(LintSandbox, self)._value_for_depends( + obj, need_help_dependency) + + def _prepare_function(self, func): + wrapped, glob = super(LintSandbox, self)._prepare_function(func) + if wrapped not in self._wrapped: + self._wrapped[wrapped] = func, glob + return wrapped, glob diff --git a/python/mozbuild/mozbuild/configure/lint_util.py b/python/mozbuild/mozbuild/configure/lint_util.py new file mode 100644 index 000000000..f1c2f8731 --- /dev/null +++ b/python/mozbuild/mozbuild/configure/lint_util.py @@ -0,0 +1,52 @@ +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, You can obtain one at http://mozilla.org/MPL/2.0/. + +from __future__ import absolute_import, print_function, unicode_literals + +import dis +import inspect + + +# dis.dis only outputs to stdout. This is a modified version that +# returns an iterator. +def disassemble_as_iter(co): + if inspect.ismethod(co): + co = co.im_func + if inspect.isfunction(co): + co = co.func_code + code = co.co_code + n = len(code) + i = 0 + extended_arg = 0 + free = None + while i < n: + c = code[i] + op = ord(c) + opname = dis.opname[op] + i += 1; + if op >= dis.HAVE_ARGUMENT: + arg = ord(code[i]) + ord(code[i + 1]) * 256 + extended_arg + extended_arg = 0 + i += 2 + if op == dis.EXTENDED_ARG: + extended_arg = arg * 65536L + continue + if op in dis.hasconst: + yield opname, co.co_consts[arg] + elif op in dis.hasname: + yield opname, co.co_names[arg] + elif op in dis.hasjrel: + yield opname, i + arg + elif op in dis.haslocal: + yield opname, co.co_varnames[arg] + elif op in dis.hascompare: + yield opname, dis.cmp_op[arg] + elif op in dis.hasfree: + if free is None: + free = co.co_cellvars + co.co_freevars + yield opname, free[arg] + else: + yield opname, None + else: + yield opname, None diff --git a/python/mozbuild/mozbuild/configure/options.py b/python/mozbuild/mozbuild/configure/options.py new file mode 100644 index 000000000..4310c8627 --- /dev/null +++ b/python/mozbuild/mozbuild/configure/options.py @@ -0,0 +1,485 @@ +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, You can obtain one at http://mozilla.org/MPL/2.0/. + +from __future__ import absolute_import, print_function, unicode_literals + +import os +import sys +import types +from collections import OrderedDict + + +def istupleofstrings(obj): + return isinstance(obj, tuple) and len(obj) and all( + isinstance(o, types.StringTypes) for o in obj) + + +class OptionValue(tuple): + '''Represents the value of a configure option. + + This class is not meant to be used directly. Use its subclasses instead. + + The `origin` attribute holds where the option comes from (e.g. environment, + command line, or default) + ''' + def __new__(cls, values=(), origin='unknown'): + return super(OptionValue, cls).__new__(cls, values) + + def __init__(self, values=(), origin='unknown'): + self.origin = origin + + def format(self, option): + if option.startswith('--'): + prefix, name, values = Option.split_option(option) + assert values == () + for prefix_set in ( + ('disable', 'enable'), + ('without', 'with'), + ): + if prefix in prefix_set: + prefix = prefix_set[int(bool(self))] + break + if prefix: + option = '--%s-%s' % (prefix, name) + elif self: + option = '--%s' % name + else: + return '' + if len(self): + return '%s=%s' % (option, ','.join(self)) + return option + elif self and not len(self): + return '%s=1' % option + return '%s=%s' % (option, ','.join(self)) + + def __eq__(self, other): + if type(other) != type(self): + return False + return super(OptionValue, self).__eq__(other) + + def __ne__(self, other): + return not self.__eq__(other) + + def __repr__(self): + return '%s%s' % (self.__class__.__name__, + super(OptionValue, self).__repr__()) + + +class PositiveOptionValue(OptionValue): + '''Represents the value for a positive option (--enable/--with/--foo) + in the form of a tuple for when values are given to the option (in the form + --option=value[,value2...]. + ''' + def __nonzero__(self): + return True + + +class NegativeOptionValue(OptionValue): + '''Represents the value for a negative option (--disable/--without) + + This is effectively an empty tuple with a `origin` attribute. + ''' + def __new__(cls, origin='unknown'): + return super(NegativeOptionValue, cls).__new__(cls, origin=origin) + + def __init__(self, origin='unknown'): + return super(NegativeOptionValue, self).__init__(origin=origin) + + +class InvalidOptionError(Exception): + pass + + +class ConflictingOptionError(InvalidOptionError): + def __init__(self, message, **format_data): + if format_data: + message = message.format(**format_data) + super(ConflictingOptionError, self).__init__(message) + for k, v in format_data.iteritems(): + setattr(self, k, v) + + +class Option(object): + '''Represents a configure option + + A configure option can be a command line flag or an environment variable + or both. + + - `name` is the full command line flag (e.g. --enable-foo). + - `env` is the environment variable name (e.g. ENV) + - `nargs` is the number of arguments the option may take. It can be a + number or the special values '?' (0 or 1), '*' (0 or more), or '+' (1 or + more). + - `default` can be used to give a default value to the option. When the + `name` of the option starts with '--enable-' or '--with-', the implied + default is an empty PositiveOptionValue. When it starts with '--disable-' + or '--without-', the implied default is a NegativeOptionValue. + - `choices` restricts the set of values that can be given to the option. + - `help` is the option description for use in the --help output. + - `possible_origins` is a tuple of strings that are origins accepted for + this option. Example origins are 'mozconfig', 'implied', and 'environment'. + ''' + __slots__ = ( + 'id', 'prefix', 'name', 'env', 'nargs', 'default', 'choices', 'help', + 'possible_origins', + ) + + def __init__(self, name=None, env=None, nargs=None, default=None, + possible_origins=None, choices=None, help=None): + if not name and not env: + raise InvalidOptionError( + 'At least an option name or an environment variable name must ' + 'be given') + if name: + if not isinstance(name, types.StringTypes): + raise InvalidOptionError('Option must be a string') + if not name.startswith('--'): + raise InvalidOptionError('Option must start with `--`') + if '=' in name: + raise InvalidOptionError('Option must not contain an `=`') + if not name.islower(): + raise InvalidOptionError('Option must be all lowercase') + if env: + if not isinstance(env, types.StringTypes): + raise InvalidOptionError( + 'Environment variable name must be a string') + if not env.isupper(): + raise InvalidOptionError( + 'Environment variable name must be all uppercase') + if nargs not in (None, '?', '*', '+') and not ( + isinstance(nargs, int) and nargs >= 0): + raise InvalidOptionError( + "nargs must be a positive integer, '?', '*' or '+'") + if (not isinstance(default, types.StringTypes) and + not isinstance(default, (bool, types.NoneType)) and + not istupleofstrings(default)): + raise InvalidOptionError( + 'default must be a bool, a string or a tuple of strings') + if choices and not istupleofstrings(choices): + raise InvalidOptionError( + 'choices must be a tuple of strings') + if not help: + raise InvalidOptionError('A help string must be provided') + if possible_origins and not istupleofstrings(possible_origins): + raise InvalidOptionError( + 'possible_origins must be a tuple of strings') + self.possible_origins = possible_origins + + if name: + prefix, name, values = self.split_option(name) + assert values == () + + # --disable and --without options mean the default is enabled. + # --enable and --with options mean the default is disabled. + # However, we allow a default to be given so that the default + # can be affected by other factors. + if prefix: + if default is None: + default = prefix in ('disable', 'without') + elif default is False: + prefix = { + 'disable': 'enable', + 'without': 'with', + }.get(prefix, prefix) + elif default is True: + prefix = { + 'enable': 'disable', + 'with': 'without', + }.get(prefix, prefix) + else: + prefix = '' + + self.prefix = prefix + self.name = name + self.env = env + if default in (None, False): + self.default = NegativeOptionValue(origin='default') + elif isinstance(default, tuple): + self.default = PositiveOptionValue(default, origin='default') + elif default is True: + self.default = PositiveOptionValue(origin='default') + else: + self.default = PositiveOptionValue((default,), origin='default') + if nargs is None: + nargs = 0 + if len(self.default) == 1: + nargs = '?' + elif len(self.default) > 1: + nargs = '*' + elif choices: + nargs = 1 + self.nargs = nargs + has_choices = choices is not None + if isinstance(self.default, PositiveOptionValue): + if has_choices and len(self.default) == 0: + raise InvalidOptionError( + 'A `default` must be given along with `choices`') + if not self._validate_nargs(len(self.default)): + raise InvalidOptionError( + "The given `default` doesn't satisfy `nargs`") + if has_choices and not all(d in choices for d in self.default): + raise InvalidOptionError( + 'The `default` value must be one of %s' % + ', '.join("'%s'" % c for c in choices)) + elif has_choices: + maxargs = self.maxargs + if len(choices) < maxargs and maxargs != sys.maxint: + raise InvalidOptionError('Not enough `choices` for `nargs`') + self.choices = choices + self.help = help + + @staticmethod + def split_option(option): + '''Split a flag or variable into a prefix, a name and values + + Variables come in the form NAME=values (no prefix). + Flags come in the form --name=values or --prefix-name=values + where prefix is one of 'with', 'without', 'enable' or 'disable'. + The '=values' part is optional. Values are separated with commas. + ''' + if not isinstance(option, types.StringTypes): + raise InvalidOptionError('Option must be a string') + + elements = option.split('=', 1) + name = elements[0] + values = tuple(elements[1].split(',')) if len(elements) == 2 else () + if name.startswith('--'): + name = name[2:] + if not name.islower(): + raise InvalidOptionError('Option must be all lowercase') + elements = name.split('-', 1) + prefix = elements[0] + if len(elements) == 2 and prefix in ('enable', 'disable', + 'with', 'without'): + return prefix, elements[1], values + else: + if name.startswith('-'): + raise InvalidOptionError( + 'Option must start with two dashes instead of one') + if name.islower(): + raise InvalidOptionError( + 'Environment variable name must be all uppercase') + return '', name, values + + @staticmethod + def _join_option(prefix, name): + # The constraints around name and env in __init__ make it so that + # we can distinguish between flags and environment variables with + # islower/isupper. + if name.isupper(): + assert not prefix + return name + elif prefix: + return '--%s-%s' % (prefix, name) + return '--%s' % name + + @property + def option(self): + if self.prefix or self.name: + return self._join_option(self.prefix, self.name) + else: + return self.env + + @property + def minargs(self): + if isinstance(self.nargs, int): + return self.nargs + return 1 if self.nargs == '+' else 0 + + @property + def maxargs(self): + if isinstance(self.nargs, int): + return self.nargs + return 1 if self.nargs == '?' else sys.maxint + + def _validate_nargs(self, num): + minargs, maxargs = self.minargs, self.maxargs + return num >= minargs and num <= maxargs + + def get_value(self, option=None, origin='unknown'): + '''Given a full command line option (e.g. --enable-foo=bar) or a + variable assignment (FOO=bar), returns the corresponding OptionValue. + + Note: variable assignments can come from either the environment or + from the command line (e.g. `../configure CFLAGS=-O2`) + ''' + if not option: + return self.default + + if self.possible_origins and origin not in self.possible_origins: + raise InvalidOptionError( + '%s can not be set by %s. Values are accepted from: %s' % + (option, origin, ', '.join(self.possible_origins))) + + prefix, name, values = self.split_option(option) + option = self._join_option(prefix, name) + + assert name in (self.name, self.env) + + if prefix in ('disable', 'without'): + if values != (): + raise InvalidOptionError('Cannot pass a value to %s' % option) + return NegativeOptionValue(origin=origin) + + if name == self.env: + if values == ('',): + return NegativeOptionValue(origin=origin) + if self.nargs in (0, '?', '*') and values == ('1',): + return PositiveOptionValue(origin=origin) + + values = PositiveOptionValue(values, origin=origin) + + if not self._validate_nargs(len(values)): + raise InvalidOptionError('%s takes %s value%s' % ( + option, + { + '?': '0 or 1', + '*': '0 or more', + '+': '1 or more', + }.get(self.nargs, str(self.nargs)), + 's' if (not isinstance(self.nargs, int) or + self.nargs != 1) else '' + )) + + if len(values) and self.choices: + relative_result = None + for val in values: + if self.nargs in ('+', '*'): + if val.startswith(('+', '-')): + if relative_result is None: + relative_result = list(self.default) + sign = val[0] + val = val[1:] + if sign == '+': + if val not in relative_result: + relative_result.append(val) + else: + try: + relative_result.remove(val) + except ValueError: + pass + + if val not in self.choices: + raise InvalidOptionError( + "'%s' is not one of %s" + % (val, ', '.join("'%s'" % c for c in self.choices))) + + if relative_result is not None: + values = PositiveOptionValue(relative_result, origin=origin) + + return values + + def __repr__(self): + return '<%s.%s [%s]>' % (self.__class__.__module__, + self.__class__.__name__, self.option) + + +class CommandLineHelper(object): + '''Helper class to handle the various ways options can be given either + on the command line of through the environment. + + For instance, an Option('--foo', env='FOO') can be passed as --foo on the + command line, or as FOO=1 in the environment *or* on the command line. + + If multiple variants are given, command line is prefered over the + environment, and if different values are given on the command line, the + last one wins. (This mimicks the behavior of autoconf, avoiding to break + existing mozconfigs using valid options in weird ways) + + Extra options can be added afterwards through API calls. For those, + conflicting values will raise an exception. + ''' + def __init__(self, environ=os.environ, argv=sys.argv): + self._environ = dict(environ) + self._args = OrderedDict() + self._extra_args = OrderedDict() + self._origins = {} + self._last = 0 + + for arg in argv[1:]: + self.add(arg, 'command-line', self._args) + + def add(self, arg, origin='command-line', args=None): + assert origin != 'default' + prefix, name, values = Option.split_option(arg) + if args is None: + args = self._extra_args + if args is self._extra_args and name in self._extra_args: + old_arg = self._extra_args[name][0] + old_prefix, _, old_values = Option.split_option(old_arg) + if prefix != old_prefix or values != old_values: + raise ConflictingOptionError( + "Cannot add '{arg}' to the {origin} set because it " + "conflicts with '{old_arg}' that was added earlier", + arg=arg, origin=origin, old_arg=old_arg, + old_origin=self._origins[old_arg]) + self._last += 1 + args[name] = arg, self._last + self._origins[arg] = origin + + def _prepare(self, option, args): + arg = None + origin = 'command-line' + from_name = args.get(option.name) + from_env = args.get(option.env) + if from_name and from_env: + arg1, pos1 = from_name + arg2, pos2 = from_env + arg, pos = (arg1, pos1) if abs(pos1) > abs(pos2) else (arg2, pos2) + if args is self._extra_args and (option.get_value(arg1) != + option.get_value(arg2)): + origin = self._origins[arg] + old_arg = arg2 if abs(pos1) > abs(pos2) else arg1 + raise ConflictingOptionError( + "Cannot add '{arg}' to the {origin} set because it " + "conflicts with '{old_arg}' that was added earlier", + arg=arg, origin=origin, old_arg=old_arg, + old_origin=self._origins[old_arg]) + elif from_name or from_env: + arg, pos = from_name if from_name else from_env + elif option.env and args is self._args: + env = self._environ.get(option.env) + if env is not None: + arg = '%s=%s' % (option.env, env) + origin = 'environment' + + origin = self._origins.get(arg, origin) + + for k in (option.name, option.env): + try: + del args[k] + except KeyError: + pass + + return arg, origin + + def handle(self, option): + '''Return the OptionValue corresponding to the given Option instance, + depending on the command line, environment, and extra arguments, and + the actual option or variable that set it. + Only works once for a given Option. + ''' + assert isinstance(option, Option) + + arg, origin = self._prepare(option, self._args) + ret = option.get_value(arg, origin) + + extra_arg, extra_origin = self._prepare(option, self._extra_args) + extra_ret = option.get_value(extra_arg, extra_origin) + + if extra_ret.origin == 'default': + return ret, arg + + if ret.origin != 'default' and extra_ret != ret: + raise ConflictingOptionError( + "Cannot add '{arg}' to the {origin} set because it conflicts " + "with {old_arg} from the {old_origin} set", arg=extra_arg, + origin=extra_ret.origin, old_arg=arg, old_origin=ret.origin) + + return extra_ret, extra_arg + + def __iter__(self): + for d in (self._args, self._extra_args): + for arg, pos in d.itervalues(): + yield arg diff --git a/python/mozbuild/mozbuild/configure/util.py b/python/mozbuild/mozbuild/configure/util.py new file mode 100644 index 000000000..c7a305282 --- /dev/null +++ b/python/mozbuild/mozbuild/configure/util.py @@ -0,0 +1,226 @@ +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, You can obtain one at http://mozilla.org/MPL/2.0/. + +from __future__ import absolute_import, print_function, unicode_literals + +import codecs +import itertools +import locale +import logging +import os +import sys +from collections import deque +from contextlib import contextmanager +from distutils.version import LooseVersion + +def getpreferredencoding(): + # locale._parse_localename makes locale.getpreferredencoding + # return None when LC_ALL is C, instead of e.g. 'US-ASCII' or + # 'ANSI_X3.4-1968' when it uses nl_langinfo. + encoding = None + try: + encoding = locale.getpreferredencoding() + except ValueError: + # On english OSX, LC_ALL is UTF-8 (not en-US.UTF-8), and + # that throws off locale._parse_localename, which ends up + # being used on e.g. homebrew python. + if os.environ.get('LC_ALL', '').upper() == 'UTF-8': + encoding = 'utf-8' + return encoding + +class Version(LooseVersion): + '''A simple subclass of distutils.version.LooseVersion. + Adds attributes for `major`, `minor`, `patch` for the first three + version components so users can easily pull out major/minor + versions, like: + + v = Version('1.2b') + v.major == 1 + v.minor == 2 + v.patch == 0 + ''' + def __init__(self, version): + # Can't use super, LooseVersion's base class is not a new-style class. + LooseVersion.__init__(self, version) + # Take the first three integer components, stopping at the first + # non-integer and padding the rest with zeroes. + (self.major, self.minor, self.patch) = list(itertools.chain( + itertools.takewhile(lambda x:isinstance(x, int), self.version), + (0, 0, 0)))[:3] + + + def __cmp__(self, other): + # LooseVersion checks isinstance(StringType), so work around it. + if isinstance(other, unicode): + other = other.encode('ascii') + return LooseVersion.__cmp__(self, other) + + +class ConfigureOutputHandler(logging.Handler): + '''A logging handler class that sends info messages to stdout and other + messages to stderr. + + Messages sent to stdout are not formatted with the attached Formatter. + Additionally, if they end with '... ', no newline character is printed, + making the next message printed follow the '... '. + + Only messages above log level INFO (included) are logged. + + Messages below that level can be kept until an ERROR message is received, + at which point the last `maxlen` accumulated messages below INFO are + printed out. This feature is only enabled under the `queue_debug` context + manager. + ''' + def __init__(self, stdout=sys.stdout, stderr=sys.stderr, maxlen=20): + super(ConfigureOutputHandler, self).__init__() + + # Python has this feature where it sets the encoding of pipes to + # ascii, which blatantly fails when trying to print out non-ascii. + def fix_encoding(fh): + try: + isatty = fh.isatty() + except AttributeError: + isatty = True + + if not isatty: + encoding = getpreferredencoding() + if encoding: + return codecs.getwriter(encoding)(fh) + return fh + + self._stdout = fix_encoding(stdout) + self._stderr = fix_encoding(stderr) if stdout != stderr else self._stdout + try: + fd1 = self._stdout.fileno() + fd2 = self._stderr.fileno() + self._same_output = self._is_same_output(fd1, fd2) + except AttributeError: + self._same_output = self._stdout == self._stderr + self._stdout_waiting = None + self._debug = deque(maxlen=maxlen + 1) + self._keep_if_debug = self.THROW + self._queue_is_active = False + + @staticmethod + def _is_same_output(fd1, fd2): + if fd1 == fd2: + return True + stat1 = os.fstat(fd1) + stat2 = os.fstat(fd2) + return stat1.st_ino == stat2.st_ino and stat1.st_dev == stat2.st_dev + + # possible values for _stdout_waiting + WAITING = 1 + INTERRUPTED = 2 + + # possible values for _keep_if_debug + THROW = 0 + KEEP = 1 + PRINT = 2 + + def emit(self, record): + try: + if record.levelno == logging.INFO: + stream = self._stdout + msg = record.getMessage() + if (self._stdout_waiting == self.INTERRUPTED and + self._same_output): + msg = ' ... %s' % msg + self._stdout_waiting = msg.endswith('... ') + if msg.endswith('... '): + self._stdout_waiting = self.WAITING + else: + self._stdout_waiting = None + msg = '%s\n' % msg + elif (record.levelno < logging.INFO and + self._keep_if_debug != self.PRINT): + if self._keep_if_debug == self.KEEP: + self._debug.append(record) + return + else: + if record.levelno >= logging.ERROR and len(self._debug): + self._emit_queue() + + if self._stdout_waiting == self.WAITING and self._same_output: + self._stdout_waiting = self.INTERRUPTED + self._stdout.write('\n') + self._stdout.flush() + stream = self._stderr + msg = '%s\n' % self.format(record) + stream.write(msg) + stream.flush() + except (KeyboardInterrupt, SystemExit): + raise + except: + self.handleError(record) + + @contextmanager + def queue_debug(self): + if self._queue_is_active: + yield + return + self._queue_is_active = True + self._keep_if_debug = self.KEEP + try: + yield + except Exception: + self._emit_queue() + # The exception will be handled and very probably printed out by + # something upper in the stack. + raise + finally: + self._queue_is_active = False + self._keep_if_debug = self.THROW + self._debug.clear() + + def _emit_queue(self): + self._keep_if_debug = self.PRINT + if len(self._debug) == self._debug.maxlen: + r = self._debug.popleft() + self.emit(logging.LogRecord( + r.name, r.levelno, r.pathname, r.lineno, + '', + (), None)) + while True: + try: + self.emit(self._debug.popleft()) + except IndexError: + break + self._keep_if_debug = self.KEEP + + +class LineIO(object): + '''File-like class that sends each line of the written data to a callback + (without carriage returns). + ''' + def __init__(self, callback): + self._callback = callback + self._buf = '' + self._encoding = getpreferredencoding() + + def write(self, buf): + if self._encoding and isinstance(buf, str): + buf = buf.decode(self._encoding) + lines = buf.splitlines() + if not lines: + return + if self._buf: + lines[0] = self._buf + lines[0] + self._buf = '' + if not buf.endswith('\n'): + self._buf = lines.pop() + + for line in lines: + self._callback(line) + + def close(self): + if self._buf: + self._callback(self._buf) + self._buf = '' + + def __enter__(self): + return self + + def __exit__(self, *args): + self.close() diff --git a/python/mozbuild/mozbuild/controller/__init__.py b/python/mozbuild/mozbuild/controller/__init__.py new file mode 100644 index 000000000..e69de29bb diff --git a/python/mozbuild/mozbuild/controller/building.py b/python/mozbuild/mozbuild/controller/building.py new file mode 100644 index 000000000..663f789b8 --- /dev/null +++ b/python/mozbuild/mozbuild/controller/building.py @@ -0,0 +1,680 @@ +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, You can obtain one at http://mozilla.org/MPL/2.0/. + +from __future__ import absolute_import, unicode_literals + +import getpass +import json +import logging +import os +import platform +import subprocess +import sys +import time +import which + +from collections import ( + namedtuple, + OrderedDict, +) + +try: + import psutil +except Exception: + psutil = None + +from mozsystemmonitor.resourcemonitor import SystemResourceMonitor + +import mozpack.path as mozpath + +from ..base import MozbuildObject + +from ..testing import install_test_files + +from ..compilation.warnings import ( + WarningsCollector, + WarningsDatabase, +) + +from textwrap import TextWrapper + +INSTALL_TESTS_CLOBBER = ''.join([TextWrapper().fill(line) + '\n' for line in +''' +The build system was unable to install tests because the CLOBBER file has \ +been updated. This means if you edited any test files, your changes may not \ +be picked up until a full/clobber build is performed. + +The easiest and fastest way to perform a clobber build is to run: + + $ mach clobber + $ mach build + +If you did not modify any test files, it is safe to ignore this message \ +and proceed with running tests. To do this run: + + $ touch {clobber_file} +'''.splitlines()]) + + + +BuildOutputResult = namedtuple('BuildOutputResult', + ('warning', 'state_changed', 'for_display')) + + +class TierStatus(object): + """Represents the state and progress of tier traversal. + + The build system is organized into linear phases called tiers. Each tier + executes in the order it was defined, 1 at a time. + """ + + def __init__(self, resources): + """Accepts a SystemResourceMonitor to record results against.""" + self.tiers = OrderedDict() + self.tier_status = OrderedDict() + self.resources = resources + + def set_tiers(self, tiers): + """Record the set of known tiers.""" + for tier in tiers: + self.tiers[tier] = dict( + begin_time=None, + finish_time=None, + duration=None, + ) + self.tier_status[tier] = None + + def begin_tier(self, tier): + """Record that execution of a tier has begun.""" + self.tier_status[tier] = 'active' + t = self.tiers[tier] + # We should ideally use a monotonic clock here. Unfortunately, we won't + # have one until Python 3. + t['begin_time'] = time.time() + self.resources.begin_phase(tier) + + def finish_tier(self, tier): + """Record that execution of a tier has finished.""" + self.tier_status[tier] = 'finished' + t = self.tiers[tier] + t['finish_time'] = time.time() + t['duration'] = self.resources.finish_phase(tier) + + def tiered_resource_usage(self): + """Obtains an object containing resource usage for tiers. + + The returned object is suitable for serialization. + """ + o = [] + + for tier, state in self.tiers.items(): + t_entry = dict( + name=tier, + start=state['begin_time'], + end=state['finish_time'], + duration=state['duration'], + ) + + self.add_resources_to_dict(t_entry, phase=tier) + + o.append(t_entry) + + return o + + def add_resources_to_dict(self, entry, start=None, end=None, phase=None): + """Helper function to append resource information to a dict.""" + cpu_percent = self.resources.aggregate_cpu_percent(start=start, + end=end, phase=phase, per_cpu=False) + cpu_times = self.resources.aggregate_cpu_times(start=start, end=end, + phase=phase, per_cpu=False) + io = self.resources.aggregate_io(start=start, end=end, phase=phase) + + if cpu_percent is None: + return entry + + entry['cpu_percent'] = cpu_percent + entry['cpu_times'] = list(cpu_times) + entry['io'] = list(io) + + return entry + + def add_resource_fields_to_dict(self, d): + for usage in self.resources.range_usage(): + cpu_times = self.resources.aggregate_cpu_times(per_cpu=False) + + d['cpu_times_fields'] = list(cpu_times._fields) + d['io_fields'] = list(usage.io._fields) + d['virt_fields'] = list(usage.virt._fields) + d['swap_fields'] = list(usage.swap._fields) + + return d + + +class BuildMonitor(MozbuildObject): + """Monitors the output of the build.""" + + def init(self, warnings_path): + """Create a new monitor. + + warnings_path is a path of a warnings database to use. + """ + self._warnings_path = warnings_path + self.resources = SystemResourceMonitor(poll_interval=1.0) + self._resources_started = False + + self.tiers = TierStatus(self.resources) + + self.warnings_database = WarningsDatabase() + if os.path.exists(warnings_path): + try: + self.warnings_database.load_from_file(warnings_path) + except ValueError: + os.remove(warnings_path) + + self._warnings_collector = WarningsCollector( + database=self.warnings_database, objdir=self.topobjdir) + + self.build_objects = [] + + def start(self): + """Record the start of the build.""" + self.start_time = time.time() + self._finder_start_cpu = self._get_finder_cpu_usage() + + def start_resource_recording(self): + # This should be merged into start() once bug 892342 lands. + self.resources.start() + self._resources_started = True + + def on_line(self, line): + """Consume a line of output from the build system. + + This will parse the line for state and determine whether more action is + needed. + + Returns a BuildOutputResult instance. + + In this named tuple, warning will be an object describing a new parsed + warning. Otherwise it will be None. + + state_changed indicates whether the build system changed state with + this line. If the build system changed state, the caller may want to + query this instance for the current state in order to update UI, etc. + + for_display is a boolean indicating whether the line is relevant to the + user. This is typically used to filter whether the line should be + presented to the user. + """ + if line.startswith('BUILDSTATUS'): + args = line.split()[1:] + + action = args.pop(0) + update_needed = True + + if action == 'TIERS': + self.tiers.set_tiers(args) + update_needed = False + elif action == 'TIER_START': + tier = args[0] + self.tiers.begin_tier(tier) + elif action == 'TIER_FINISH': + tier, = args + self.tiers.finish_tier(tier) + elif action == 'OBJECT_FILE': + self.build_objects.append(args[0]) + update_needed = False + else: + raise Exception('Unknown build status: %s' % action) + + return BuildOutputResult(None, update_needed, False) + + warning = None + + try: + warning = self._warnings_collector.process_line(line) + except: + pass + + return BuildOutputResult(warning, False, True) + + def stop_resource_recording(self): + if self._resources_started: + self.resources.stop() + + self._resources_started = False + + def finish(self, record_usage=True): + """Record the end of the build.""" + self.stop_resource_recording() + self.end_time = time.time() + self._finder_end_cpu = self._get_finder_cpu_usage() + self.elapsed = self.end_time - self.start_time + + self.warnings_database.prune() + self.warnings_database.save_to_file(self._warnings_path) + + if not record_usage: + return + + try: + usage = self.get_resource_usage() + if not usage: + return + + self.log_resource_usage(usage) + with open(self._get_state_filename('build_resources.json'), 'w') as fh: + json.dump(self.resources.as_dict(), fh, indent=2) + except Exception as e: + self.log(logging.WARNING, 'build_resources_error', + {'msg': str(e)}, + 'Exception when writing resource usage file: {msg}') + + def _get_finder_cpu_usage(self): + """Obtain the CPU usage of the Finder app on OS X. + + This is used to detect high CPU usage. + """ + if not sys.platform.startswith('darwin'): + return None + + if not psutil: + return None + + for proc in psutil.process_iter(): + if proc.name != 'Finder': + continue + + if proc.username != getpass.getuser(): + continue + + # Try to isolate system finder as opposed to other "Finder" + # processes. + if not proc.exe.endswith('CoreServices/Finder.app/Contents/MacOS/Finder'): + continue + + return proc.get_cpu_times() + + return None + + def have_high_finder_usage(self): + """Determine whether there was high Finder CPU usage during the build. + + Returns True if there was high Finder CPU usage, False if there wasn't, + or None if there is nothing to report. + """ + if not self._finder_start_cpu: + return None, None + + # We only measure if the measured range is sufficiently long. + if self.elapsed < 15: + return None, None + + if not self._finder_end_cpu: + return None, None + + start = self._finder_start_cpu + end = self._finder_end_cpu + + start_total = start.user + start.system + end_total = end.user + end.system + + cpu_seconds = end_total - start_total + + # If Finder used more than 25% of 1 core during the build, report an + # error. + finder_percent = cpu_seconds / self.elapsed * 100 + + return finder_percent > 25, finder_percent + + def have_excessive_swapping(self): + """Determine whether there was excessive swapping during the build. + + Returns a tuple of (excessive, swap_in, swap_out). All values are None + if no swap information is available. + """ + if not self.have_resource_usage: + return None, None, None + + swap_in = sum(m.swap.sin for m in self.resources.measurements) + swap_out = sum(m.swap.sout for m in self.resources.measurements) + + # The threshold of 1024 MB has been arbitrarily chosen. + # + # Choosing a proper value that is ideal for everyone is hard. We will + # likely iterate on the logic until people are generally satisfied. + # If a value is too low, the eventual warning produced does not carry + # much meaning. If the threshold is too high, people may not see the + # warning and the warning will thus be ineffective. + excessive = swap_in > 512 * 1048576 or swap_out > 512 * 1048576 + return excessive, swap_in, swap_out + + @property + def have_resource_usage(self): + """Whether resource usage is available.""" + return self.resources.start_time is not None + + def get_resource_usage(self): + """ Produce a data structure containing the low-level resource usage information. + + This data structure can e.g. be serialized into JSON and saved for + subsequent analysis. + + If no resource usage is available, None is returned. + """ + if not self.have_resource_usage: + return None + + cpu_percent = self.resources.aggregate_cpu_percent(phase=None, + per_cpu=False) + cpu_times = self.resources.aggregate_cpu_times(phase=None, + per_cpu=False) + io = self.resources.aggregate_io(phase=None) + + o = dict( + version=3, + argv=sys.argv, + start=self.start_time, + end=self.end_time, + duration=self.end_time - self.start_time, + resources=[], + cpu_percent=cpu_percent, + cpu_times=cpu_times, + io=io, + objects=self.build_objects + ) + + o['tiers'] = self.tiers.tiered_resource_usage() + + self.tiers.add_resource_fields_to_dict(o) + + for usage in self.resources.range_usage(): + cpu_percent = self.resources.aggregate_cpu_percent(usage.start, + usage.end, per_cpu=False) + cpu_times = self.resources.aggregate_cpu_times(usage.start, + usage.end, per_cpu=False) + + entry = dict( + start=usage.start, + end=usage.end, + virt=list(usage.virt), + swap=list(usage.swap), + ) + + self.tiers.add_resources_to_dict(entry, start=usage.start, + end=usage.end) + + o['resources'].append(entry) + + + # If the imports for this file ran before the in-tree virtualenv + # was bootstrapped (for instance, for a clobber build in automation), + # psutil might not be available. + # + # Treat psutil as optional to avoid an outright failure to log resources + # TODO: it would be nice to collect data on the storage device as well + # in this case. + o['system'] = {} + if psutil: + o['system'].update(dict( + logical_cpu_count=psutil.cpu_count(), + physical_cpu_count=psutil.cpu_count(logical=False), + swap_total=psutil.swap_memory()[0], + vmem_total=psutil.virtual_memory()[0], + )) + + return o + + def log_resource_usage(self, usage): + """Summarize the resource usage of this build in a log message.""" + + if not usage: + return + + params = dict( + duration=self.end_time - self.start_time, + cpu_percent=usage['cpu_percent'], + io_read_bytes=usage['io'].read_bytes, + io_write_bytes=usage['io'].write_bytes, + io_read_time=usage['io'].read_time, + io_write_time=usage['io'].write_time, + ) + + message = 'Overall system resources - Wall time: {duration:.0f}s; ' \ + 'CPU: {cpu_percent:.0f}%; ' \ + 'Read bytes: {io_read_bytes}; Write bytes: {io_write_bytes}; ' \ + 'Read time: {io_read_time}; Write time: {io_write_time}' + + self.log(logging.WARNING, 'resource_usage', params, message) + + excessive, sin, sout = self.have_excessive_swapping() + if excessive is not None and (sin or sout): + sin /= 1048576 + sout /= 1048576 + self.log(logging.WARNING, 'swap_activity', + {'sin': sin, 'sout': sout}, + 'Swap in/out (MB): {sin}/{sout}') + + def ccache_stats(self): + ccache_stats = None + + try: + ccache = which.which('ccache') + output = subprocess.check_output([ccache, '-s']) + ccache_stats = CCacheStats(output) + except which.WhichError: + pass + except ValueError as e: + self.log(logging.WARNING, 'ccache', {'msg': str(e)}, '{msg}') + + return ccache_stats + + +class CCacheStats(object): + """Holds statistics from ccache. + + Instances can be subtracted from each other to obtain differences. + print() or str() the object to show a ``ccache -s`` like output + of the captured stats. + + """ + STATS_KEYS = [ + # (key, description) + # Refer to stats.c in ccache project for all the descriptions. + ('cache_hit_direct', 'cache hit (direct)'), + ('cache_hit_preprocessed', 'cache hit (preprocessed)'), + ('cache_hit_rate', 'cache hit rate'), + ('cache_miss', 'cache miss'), + ('link', 'called for link'), + ('preprocessing', 'called for preprocessing'), + ('multiple', 'multiple source files'), + ('stdout', 'compiler produced stdout'), + ('no_output', 'compiler produced no output'), + ('empty_output', 'compiler produced empty output'), + ('failed', 'compile failed'), + ('error', 'ccache internal error'), + ('preprocessor_error', 'preprocessor error'), + ('cant_use_pch', "can't use precompiled header"), + ('compiler_missing', "couldn't find the compiler"), + ('cache_file_missing', 'cache file missing'), + ('bad_args', 'bad compiler arguments'), + ('unsupported_lang', 'unsupported source language'), + ('compiler_check_failed', 'compiler check failed'), + ('autoconf', 'autoconf compile/link'), + ('unsupported_compiler_option', 'unsupported compiler option'), + ('out_stdout', 'output to stdout'), + ('out_device', 'output to a non-regular file'), + ('no_input', 'no input file'), + ('bad_extra_file', 'error hashing extra file'), + ('num_cleanups', 'cleanups performed'), + ('cache_files', 'files in cache'), + ('cache_size', 'cache size'), + ('cache_max_size', 'max cache size'), + ] + + DIRECTORY_DESCRIPTION = "cache directory" + PRIMARY_CONFIG_DESCRIPTION = "primary config" + SECONDARY_CONFIG_DESCRIPTION = "secondary config (readonly)" + ABSOLUTE_KEYS = {'cache_files', 'cache_size', 'cache_max_size'} + FORMAT_KEYS = {'cache_size', 'cache_max_size'} + + GiB = 1024 ** 3 + MiB = 1024 ** 2 + KiB = 1024 + + def __init__(self, output=None): + """Construct an instance from the output of ccache -s.""" + self._values = {} + self.cache_dir = "" + self.primary_config = "" + self.secondary_config = "" + + if not output: + return + + for line in output.splitlines(): + line = line.strip() + if line: + self._parse_line(line) + + def _parse_line(self, line): + if line.startswith(self.DIRECTORY_DESCRIPTION): + self.cache_dir = self._strip_prefix(line, self.DIRECTORY_DESCRIPTION) + elif line.startswith(self.PRIMARY_CONFIG_DESCRIPTION): + self.primary_config = self._strip_prefix( + line, self.PRIMARY_CONFIG_DESCRIPTION) + elif line.startswith(self.SECONDARY_CONFIG_DESCRIPTION): + self.secondary_config = self._strip_prefix( + line, self.SECONDARY_CONFIG_DESCRIPTION) + else: + for stat_key, stat_description in self.STATS_KEYS: + if line.startswith(stat_description): + raw_value = self._strip_prefix(line, stat_description) + self._values[stat_key] = self._parse_value(raw_value) + break + else: + raise ValueError('Failed to parse ccache stats output: %s' % line) + + @staticmethod + def _strip_prefix(line, prefix): + return line[len(prefix):].strip() if line.startswith(prefix) else line + + @staticmethod + def _parse_value(raw_value): + value = raw_value.split() + unit = '' + if len(value) == 1: + numeric = value[0] + elif len(value) == 2: + numeric, unit = value + else: + raise ValueError('Failed to parse ccache stats value: %s' % raw_value) + + if '.' in numeric: + numeric = float(numeric) + else: + numeric = int(numeric) + + if unit in ('GB', 'Gbytes'): + unit = CCacheStats.GiB + elif unit in ('MB', 'Mbytes'): + unit = CCacheStats.MiB + elif unit in ('KB', 'Kbytes'): + unit = CCacheStats.KiB + else: + unit = 1 + + return int(numeric * unit) + + def hit_rate_message(self): + return 'ccache (direct) hit rate: {:.1%}; (preprocessed) hit rate: {:.1%}; miss rate: {:.1%}'.format(*self.hit_rates()) + + def hit_rates(self): + direct = self._values['cache_hit_direct'] + preprocessed = self._values['cache_hit_preprocessed'] + miss = self._values['cache_miss'] + total = float(direct + preprocessed + miss) + + if total > 0: + direct /= total + preprocessed /= total + miss /= total + + return (direct, preprocessed, miss) + + def __sub__(self, other): + result = CCacheStats() + result.cache_dir = self.cache_dir + + for k, prefix in self.STATS_KEYS: + if k not in self._values and k not in other._values: + continue + + our_value = self._values.get(k, 0) + other_value = other._values.get(k, 0) + + if k in self.ABSOLUTE_KEYS: + result._values[k] = our_value + else: + result._values[k] = our_value - other_value + + return result + + def __str__(self): + LEFT_ALIGN = 34 + lines = [] + + if self.cache_dir: + lines.append('%s%s' % (self.DIRECTORY_DESCRIPTION.ljust(LEFT_ALIGN), + self.cache_dir)) + + for stat_key, stat_description in self.STATS_KEYS: + if stat_key not in self._values: + continue + + value = self._values[stat_key] + + if stat_key in self.FORMAT_KEYS: + value = '%15s' % self._format_value(value) + else: + value = '%8u' % value + + lines.append('%s%s' % (stat_description.ljust(LEFT_ALIGN), value)) + + return '\n'.join(lines) + + def __nonzero__(self): + relative_values = [v for k, v in self._values.items() + if k not in self.ABSOLUTE_KEYS] + return (all(v >= 0 for v in relative_values) and + any(v > 0 for v in relative_values)) + + @staticmethod + def _format_value(v): + if v > CCacheStats.GiB: + return '%.1f Gbytes' % (float(v) / CCacheStats.GiB) + elif v > CCacheStats.MiB: + return '%.1f Mbytes' % (float(v) / CCacheStats.MiB) + else: + return '%.1f Kbytes' % (float(v) / CCacheStats.KiB) + + +class BuildDriver(MozbuildObject): + """Provides a high-level API for build actions.""" + + def install_tests(self, test_objs): + """Install test files.""" + + if self.is_clobber_needed(): + print(INSTALL_TESTS_CLOBBER.format( + clobber_file=os.path.join(self.topobjdir, 'CLOBBER'))) + sys.exit(1) + + if not test_objs: + # If we don't actually have a list of tests to install we install + # test and support files wholesale. + self._run_make(target='install-test-files', pass_thru=True, + print_directory=False) + else: + install_test_files(mozpath.normpath(self.topsrcdir), self.topobjdir, + '_tests', test_objs) diff --git a/python/mozbuild/mozbuild/controller/clobber.py b/python/mozbuild/mozbuild/controller/clobber.py new file mode 100644 index 000000000..02f75c6ad --- /dev/null +++ b/python/mozbuild/mozbuild/controller/clobber.py @@ -0,0 +1,237 @@ +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, You can obtain one at http://mozilla.org/MPL/2.0/. + +from __future__ import absolute_import, print_function + +r'''This module contains code for managing clobbering of the tree.''' + +import errno +import os +import subprocess +import sys + +from mozfile.mozfile import remove as mozfileremove +from textwrap import TextWrapper + + +CLOBBER_MESSAGE = ''.join([TextWrapper().fill(line) + '\n' for line in +''' +The CLOBBER file has been updated, indicating that an incremental build since \ +your last build will probably not work. A full/clobber build is required. + +The reason for the clobber is: + +{clobber_reason} + +Clobbering can be performed automatically. However, we didn't automatically \ +clobber this time because: + +{no_reason} + +The easiest and fastest way to clobber is to run: + + $ mach clobber + +If you know this clobber doesn't apply to you or you're feeling lucky -- \ +Well, are ya? -- you can ignore this clobber requirement by running: + + $ touch {clobber_file} +'''.splitlines()]) + +class Clobberer(object): + def __init__(self, topsrcdir, topobjdir): + """Create a new object to manage clobbering the tree. + + It is bound to a top source directory and to a specific object + directory. + """ + assert os.path.isabs(topsrcdir) + assert os.path.isabs(topobjdir) + + self.topsrcdir = os.path.normpath(topsrcdir) + self.topobjdir = os.path.normpath(topobjdir) + self.src_clobber = os.path.join(topsrcdir, 'CLOBBER') + self.obj_clobber = os.path.join(topobjdir, 'CLOBBER') + + # Try looking for mozilla/CLOBBER, for comm-central + if not os.path.isfile(self.src_clobber): + self.src_clobber = os.path.join(topsrcdir, 'mozilla', 'CLOBBER') + + assert os.path.isfile(self.src_clobber) + + def clobber_needed(self): + """Returns a bool indicating whether a tree clobber is required.""" + + # No object directory clobber file means we're good. + if not os.path.exists(self.obj_clobber): + return False + + # Object directory clobber older than current is fine. + if os.path.getmtime(self.src_clobber) <= \ + os.path.getmtime(self.obj_clobber): + + return False + + return True + + def clobber_cause(self): + """Obtain the cause why a clobber is required. + + This reads the cause from the CLOBBER file. + + This returns a list of lines describing why the clobber was required. + Each line is stripped of leading and trailing whitespace. + """ + with open(self.src_clobber, 'rt') as fh: + lines = [l.strip() for l in fh.readlines()] + return [l for l in lines if l and not l.startswith('#')] + + def have_winrm(self): + # `winrm -h` should print 'winrm version ...' and exit 1 + try: + p = subprocess.Popen(['winrm.exe', '-h'], + stdout=subprocess.PIPE, + stderr=subprocess.STDOUT) + return p.wait() == 1 and p.stdout.read().startswith('winrm') + except: + return False + + def remove_objdir(self, full=True): + """Remove the object directory. + + ``full`` controls whether to fully delete the objdir. If False, + some directories (e.g. Visual Studio Project Files) will not be + deleted. + """ + # Top-level files and directories to not clobber by default. + no_clobber = { + '.mozbuild', + 'msvc', + } + + if full: + # mozfile doesn't like unicode arguments (bug 818783). + paths = [self.topobjdir.encode('utf-8')] + else: + try: + paths = [] + for p in os.listdir(self.topobjdir): + if p not in no_clobber: + paths.append(os.path.join(self.topobjdir, p).encode('utf-8')) + except OSError as e: + if e.errno != errno.ENOENT: + raise + return + + procs = [] + for p in sorted(paths): + path = os.path.join(self.topobjdir, p) + if sys.platform.startswith('win') and self.have_winrm() and os.path.isdir(path): + procs.append(subprocess.Popen(['winrm', '-rf', path])) + else: + # We use mozfile because it is faster than shutil.rmtree(). + mozfileremove(path) + + for p in procs: + p.wait() + + def ensure_objdir_state(self): + """Ensure the CLOBBER file in the objdir exists. + + This is called as part of the build to ensure the clobber information + is configured properly for the objdir. + """ + if not os.path.exists(self.topobjdir): + os.makedirs(self.topobjdir) + + if not os.path.exists(self.obj_clobber): + # Simply touch the file. + with open(self.obj_clobber, 'a'): + pass + + def maybe_do_clobber(self, cwd, allow_auto=False, fh=sys.stderr): + """Perform a clobber if it is required. Maybe. + + This is the API the build system invokes to determine if a clobber + is needed and to automatically perform that clobber if we can. + + This returns a tuple of (bool, bool, str). The elements are: + + - Whether a clobber was/is required. + - Whether a clobber was performed. + - The reason why the clobber failed or could not be performed. This + will be None if no clobber is required or if we clobbered without + error. + """ + assert cwd + cwd = os.path.normpath(cwd) + + if not self.clobber_needed(): + print('Clobber not needed.', file=fh) + self.ensure_objdir_state() + return False, False, None + + # So a clobber is needed. We only perform a clobber if we are + # allowed to perform an automatic clobber (off by default) and if the + # current directory is not under the object directory. The latter is + # because operating systems, filesystems, and shell can throw fits + # if the current working directory is deleted from under you. While it + # can work in some scenarios, we take the conservative approach and + # never try. + if not allow_auto: + return True, False, \ + self._message('Automatic clobbering is not enabled\n' + ' (add "mk_add_options AUTOCLOBBER=1" to your ' + 'mozconfig).') + + if cwd.startswith(self.topobjdir) and cwd != self.topobjdir: + return True, False, self._message( + 'Cannot clobber while the shell is inside the object directory.') + + print('Automatically clobbering %s' % self.topobjdir, file=fh) + try: + self.remove_objdir(False) + self.ensure_objdir_state() + print('Successfully completed auto clobber.', file=fh) + return True, True, None + except (IOError) as error: + return True, False, self._message( + 'Error when automatically clobbering: ' + str(error)) + + def _message(self, reason): + lines = [' ' + line for line in self.clobber_cause()] + + return CLOBBER_MESSAGE.format(clobber_reason='\n'.join(lines), + no_reason=' ' + reason, clobber_file=self.obj_clobber) + + +def main(args, env, cwd, fh=sys.stderr): + if len(args) != 2: + print('Usage: clobber.py topsrcdir topobjdir', file=fh) + return 1 + + topsrcdir, topobjdir = args + + if not os.path.isabs(topsrcdir): + topsrcdir = os.path.abspath(topsrcdir) + + if not os.path.isabs(topobjdir): + topobjdir = os.path.abspath(topobjdir) + + auto = True if env.get('AUTOCLOBBER', False) else False + clobber = Clobberer(topsrcdir, topobjdir) + required, performed, message = clobber.maybe_do_clobber(cwd, auto, fh) + + if not required or performed: + if performed and env.get('TINDERBOX_OUTPUT'): + print('TinderboxPrint: auto clobber', file=fh) + return 0 + + print(message, file=fh) + return 1 + + +if __name__ == '__main__': + sys.exit(main(sys.argv[1:], os.environ, os.getcwd(), sys.stdout)) + diff --git a/python/mozbuild/mozbuild/doctor.py b/python/mozbuild/mozbuild/doctor.py new file mode 100644 index 000000000..2175042bf --- /dev/null +++ b/python/mozbuild/mozbuild/doctor.py @@ -0,0 +1,293 @@ +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, # You can obtain one at http://mozilla.org/MPL/2.0/. + +from __future__ import absolute_import + +import os +import subprocess +import sys + +import psutil + +from distutils.util import strtobool +from distutils.version import LooseVersion +import mozpack.path as mozpath + +# Minimum recommended logical processors in system. +PROCESSORS_THRESHOLD = 4 + +# Minimum recommended total system memory, in gigabytes. +MEMORY_THRESHOLD = 7.4 + +# Minimum recommended free space on each disk, in gigabytes. +FREESPACE_THRESHOLD = 10 + +# Latest MozillaBuild version +LATEST_MOZILLABUILD_VERSION = '1.11.0' + +DISABLE_LASTACCESS_WIN = ''' +Disable the last access time feature? +This improves the speed of file and +directory access by deferring Last Access Time modification on disk by up to an +hour. Backup programs that rely on this feature may be affected. +https://technet.microsoft.com/en-us/library/cc785435.aspx +''' + +class Doctor(object): + def __init__(self, srcdir, objdir, fix): + self.srcdir = mozpath.normpath(srcdir) + self.objdir = mozpath.normpath(objdir) + self.srcdir_mount = self.getmount(self.srcdir) + self.objdir_mount = self.getmount(self.objdir) + self.path_mounts = [ + ('srcdir', self.srcdir, self.srcdir_mount), + ('objdir', self.objdir, self.objdir_mount) + ] + self.fix = fix + self.results = [] + + def check_all(self): + checks = [ + 'cpu', + 'memory', + 'storage_freespace', + 'fs_lastaccess', + 'mozillabuild' + ] + for check in checks: + self.report(getattr(self, check)) + good = True + fixable = False + denied = False + for result in self.results: + if result.get('status') != 'GOOD': + good = False + if result.get('fixable', False): + fixable = True + if result.get('denied', False): + denied = True + if denied: + print('run "mach doctor --fix" AS ADMIN to re-attempt fixing your system') + elif False: # elif fixable: + print('run "mach doctor --fix" as admin to attempt fixing your system') + return int(not good) + + def getmount(self, path): + while path != '/' and not os.path.ismount(path): + path = mozpath.abspath(mozpath.join(path, os.pardir)) + return path + + def prompt_bool(self, prompt, limit=5): + ''' Prompts the user with prompt and requires a boolean value. ''' + valid = False + while not valid and limit > 0: + try: + choice = strtobool(raw_input(prompt + '[Y/N]\n')) + valid = True + except ValueError: + print("ERROR! Please enter a valid option!") + limit -= 1 + + if limit > 0: + return choice + else: + raise Exception("Error! Reached max attempts of entering option.") + + def report(self, results): + # Handle single dict result or list of results. + if isinstance(results, dict): + results = [results] + for result in results: + status = result.get('status', 'UNSURE') + if status == 'SKIPPED': + continue + self.results.append(result) + print('%s...\t%s\n' % ( + result.get('desc', ''), + status + ) + ).expandtabs(40) + + @property + def platform(self): + platform = getattr(self, '_platform', None) + if not platform: + platform = sys.platform + while platform[-1].isdigit(): + platform = platform[:-1] + setattr(self, '_platform', platform) + return platform + + @property + def cpu(self): + cpu_count = psutil.cpu_count() + if cpu_count < PROCESSORS_THRESHOLD: + status = 'BAD' + desc = '%d logical processors detected, <%d' % ( + cpu_count, PROCESSORS_THRESHOLD + ) + else: + status = 'GOOD' + desc = '%d logical processors detected, >=%d' % ( + cpu_count, PROCESSORS_THRESHOLD + ) + return {'status': status, 'desc': desc} + + @property + def memory(self): + memory = psutil.virtual_memory().total + # Convert to gigabytes. + memory_GB = memory / 1024**3.0 + if memory_GB < MEMORY_THRESHOLD: + status = 'BAD' + desc = '%.1fGB of physical memory, <%.1fGB' % ( + memory_GB, MEMORY_THRESHOLD + ) + else: + status = 'GOOD' + desc = '%.1fGB of physical memory, >%.1fGB' % ( + memory_GB, MEMORY_THRESHOLD + ) + return {'status': status, 'desc': desc} + + @property + def storage_freespace(self): + results = [] + desc = '' + mountpoint_line = self.srcdir_mount != self.objdir_mount + for (purpose, path, mount) in self.path_mounts: + desc += '%s = %s\n' % (purpose, path) + if not mountpoint_line: + mountpoint_line = True + continue + try: + usage = psutil.disk_usage(mount) + freespace, size = usage.free, usage.total + freespace_GB = freespace / 1024**3 + size_GB = size / 1024**3 + if freespace_GB < FREESPACE_THRESHOLD: + status = 'BAD' + desc += 'mountpoint = %s\n%dGB of %dGB free, <%dGB' % ( + mount, freespace_GB, size_GB, FREESPACE_THRESHOLD + ) + else: + status = 'GOOD' + desc += 'mountpoint = %s\n%dGB of %dGB free, >=%dGB' % ( + mount, freespace_GB, size_GB, FREESPACE_THRESHOLD + ) + except OSError: + status = 'UNSURE' + desc += 'path invalid' + results.append({'status': status, 'desc': desc}) + return results + + @property + def fs_lastaccess(self): + results = [] + if self.platform == 'win': + fixable = False + denied = False + # See 'fsutil behavior': + # https://technet.microsoft.com/en-us/library/cc785435.aspx + try: + command = 'fsutil behavior query disablelastaccess'.split(' ') + fsutil_output = subprocess.check_output(command) + disablelastaccess = int(fsutil_output.partition('=')[2][1]) + except subprocess.CalledProcessError: + disablelastaccess = -1 + status = 'UNSURE' + desc = 'unable to check lastaccess behavior' + if disablelastaccess == 1: + status = 'GOOD' + desc = 'lastaccess disabled systemwide' + elif disablelastaccess == 0: + if False: # if self.fix: + choice = self.prompt_bool(DISABLE_LASTACCESS_WIN) + if not choice: + return {'status': 'BAD, NOT FIXED', + 'desc': 'lastaccess enabled systemwide'} + try: + command = 'fsutil behavior set disablelastaccess 1'.split(' ') + fsutil_output = subprocess.check_output(command) + status = 'GOOD, FIXED' + desc = 'lastaccess disabled systemwide' + except subprocess.CalledProcessError, e: + desc = 'lastaccess enabled systemwide' + if e.output.find('denied') != -1: + status = 'BAD, FIX DENIED' + denied = True + else: + status = 'BAD, NOT FIXED' + else: + status = 'BAD, FIXABLE' + desc = 'lastaccess enabled' + fixable = True + results.append({'status': status, 'desc': desc, 'fixable': fixable, + 'denied': denied}) + elif self.platform in ['darwin', 'freebsd', 'linux', 'openbsd']: + common_mountpoint = self.srcdir_mount == self.objdir_mount + for (purpose, path, mount) in self.path_mounts: + results.append(self.check_mount_lastaccess(mount)) + if common_mountpoint: + break + else: + results.append({'status': 'SKIPPED'}) + return results + + def check_mount_lastaccess(self, mount): + partitions = psutil.disk_partitions() + atime_opts = {'atime', 'noatime', 'relatime', 'norelatime'} + option = '' + for partition in partitions: + if partition.mountpoint == mount: + mount_opts = set(partition.opts.split(',')) + intersection = list(atime_opts & mount_opts) + if len(intersection) == 1: + option = intersection[0] + break + if not option: + status = 'BAD' + if self.platform == 'linux': + option = 'noatime/relatime' + else: + option = 'noatime' + desc = '%s has no explicit %s mount option' % ( + mount, option + ) + elif option == 'atime' or option == 'norelatime': + status = 'BAD' + desc = '%s has %s mount option' % ( + mount, option + ) + elif option == 'noatime' or option == 'relatime': + status = 'GOOD' + desc = '%s has %s mount option' % ( + mount, option + ) + return {'status': status, 'desc': desc} + + @property + def mozillabuild(self): + if self.platform != 'win': + return {'status': 'SKIPPED'} + MOZILLABUILD = mozpath.normpath(os.environ.get('MOZILLABUILD', '')) + if not MOZILLABUILD or not os.path.exists(MOZILLABUILD): + return {'desc': 'not running under MozillaBuild'} + try: + with open(mozpath.join(MOZILLABUILD, 'VERSION'), 'r') as fh: + version = fh.readline() + if not version: + raise ValueError() + if LooseVersion(version) < LooseVersion(LATEST_MOZILLABUILD_VERSION): + status = 'BAD' + desc = 'MozillaBuild %s in use, <%s' % ( + version, LATEST_MOZILLABUILD_VERSION + ) + else: + status = 'GOOD' + desc = 'MozillaBuild %s in use' % version + except (IOError, ValueError): + status = 'UNSURE' + desc = 'MozillaBuild version not found' + return {'status': status, 'desc': desc} diff --git a/python/mozbuild/mozbuild/dotproperties.py b/python/mozbuild/mozbuild/dotproperties.py new file mode 100644 index 000000000..972ff2329 --- /dev/null +++ b/python/mozbuild/mozbuild/dotproperties.py @@ -0,0 +1,83 @@ +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this file, +# You can obtain one at http://mozilla.org/MPL/2.0/. + +# This file contains utility functions for reading .properties files, like +# region.properties. + +from __future__ import absolute_import, unicode_literals + +import codecs +import re +import sys + +if sys.version_info[0] == 3: + str_type = str +else: + str_type = basestring + +class DotProperties: + r'''A thin representation of a key=value .properties file.''' + + def __init__(self, file=None): + self._properties = {} + if file: + self.update(file) + + def update(self, file): + '''Updates properties from a file name or file-like object. + + Ignores empty lines and comment lines.''' + + if isinstance(file, str_type): + f = codecs.open(file, 'r', 'utf-8') + else: + f = file + + for l in f.readlines(): + line = l.strip() + if not line or line.startswith('#'): + continue + (k, v) = re.split('\s*=\s*', line, 1) + self._properties[k] = v + + def get(self, key, default=None): + return self._properties.get(key, default) + + def get_list(self, prefix): + '''Turns {'list.0':'foo', 'list.1':'bar'} into ['foo', 'bar']. + + Returns [] to indicate an empty or missing list.''' + + if not prefix.endswith('.'): + prefix = prefix + '.' + indexes = [] + for k, v in self._properties.iteritems(): + if not k.startswith(prefix): + continue + key = k[len(prefix):] + if '.' in key: + # We have something like list.sublist.0. + continue + indexes.append(int(key)) + return [self._properties[prefix + str(index)] for index in sorted(indexes)] + + def get_dict(self, prefix, required_keys=[]): + '''Turns {'foo.title':'title', ...} into {'title':'title', ...}. + + If |required_keys| is present, it must be an iterable of required key + names. If a required key is not present, ValueError is thrown. + + Returns {} to indicate an empty or missing dict.''' + + if not prefix.endswith('.'): + prefix = prefix + '.' + + D = dict((k[len(prefix):], v) for k, v in self._properties.iteritems() + if k.startswith(prefix) and '.' not in k[len(prefix):]) + + for required_key in required_keys: + if not required_key in D: + raise ValueError('Required key %s not present' % required_key) + + return D diff --git a/python/mozbuild/mozbuild/frontend/__init__.py b/python/mozbuild/mozbuild/frontend/__init__.py new file mode 100644 index 000000000..e69de29bb diff --git a/python/mozbuild/mozbuild/frontend/context.py b/python/mozbuild/mozbuild/frontend/context.py new file mode 100644 index 000000000..eb501dc66 --- /dev/null +++ b/python/mozbuild/mozbuild/frontend/context.py @@ -0,0 +1,2292 @@ +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, You can obtain one at http://mozilla.org/MPL/2.0/. + +###################################################################### +# DO NOT UPDATE THIS FILE WITHOUT SIGN-OFF FROM A BUILD MODULE PEER. # +###################################################################### + +r"""This module contains the data structure (context) holding the configuration +from a moz.build. The data emitted by the frontend derives from those contexts. + +It also defines the set of variables and functions available in moz.build. +If you are looking for the absolute authority on what moz.build files can +contain, you've come to the right place. +""" + +from __future__ import absolute_import, unicode_literals + +import os + +from collections import ( + Counter, + OrderedDict, +) +from mozbuild.util import ( + HierarchicalStringList, + KeyedDefaultDict, + List, + ListWithAction, + memoize, + memoized_property, + ReadOnlyKeyedDefaultDict, + StrictOrderingOnAppendList, + StrictOrderingOnAppendListWithAction, + StrictOrderingOnAppendListWithFlagsFactory, + TypedList, + TypedNamedTuple, +) + +from ..testing import ( + all_test_flavors, + read_manifestparser_manifest, + read_reftest_manifest, + read_wpt_manifest, +) + +import mozpack.path as mozpath +from types import FunctionType + +import itertools + + +class ContextDerivedValue(object): + """Classes deriving from this one receive a special treatment in a + Context. See Context documentation. + """ + __slots__ = () + + +class Context(KeyedDefaultDict): + """Represents a moz.build configuration context. + + Instances of this class are filled by the execution of sandboxes. + At the core, a Context is a dict, with a defined set of possible keys we'll + call variables. Each variable is associated with a type. + + When reading a value for a given key, we first try to read the existing + value. If a value is not found and it is defined in the allowed variables + set, we return a new instance of the class for that variable. We don't + assign default instances until they are accessed because this makes + debugging the end-result much simpler. Instead of a data structure with + lots of empty/default values, you have a data structure with only the + values that were read or touched. + + Instances of variables classes are created by invoking ``class_name()``, + except when class_name derives from ``ContextDerivedValue`` or + ``SubContext``, in which case ``class_name(instance_of_the_context)`` or + ``class_name(self)`` is invoked. A value is added to those calls when + instances are created during assignment (setitem). + + allowed_variables is a dict of the variables that can be set and read in + this context instance. Keys in this dict are the strings representing keys + in this context which are valid. Values are tuples of stored type, + assigned type, default value, a docstring describing the purpose of the + variable, and a tier indicator (see comment above the VARIABLES declaration + in this module). + + config is the ConfigEnvironment for this context. + """ + def __init__(self, allowed_variables={}, config=None, finder=None): + self._allowed_variables = allowed_variables + self.main_path = None + self.current_path = None + # There aren't going to be enough paths for the performance of scanning + # a list to be a problem. + self._all_paths = [] + self.config = config + self._sandbox = None + self._finder = finder + KeyedDefaultDict.__init__(self, self._factory) + + def push_source(self, path): + """Adds the given path as source of the data from this context and make + it the current path for the context.""" + assert os.path.isabs(path) + if not self.main_path: + self.main_path = path + else: + # Callers shouldn't push after main_path has been popped. + assert self.current_path + self.current_path = path + # The same file can be pushed twice, so don't remove any previous + # occurrence. + self._all_paths.append(path) + + def pop_source(self): + """Get back to the previous current path for the context.""" + assert self.main_path + assert self.current_path + last = self._all_paths.pop() + # Keep the popped path in the list of all paths, but before the main + # path so that it's not popped again. + self._all_paths.insert(0, last) + if last == self.main_path: + self.current_path = None + else: + self.current_path = self._all_paths[-1] + return last + + def add_source(self, path): + """Adds the given path as source of the data from this context.""" + assert os.path.isabs(path) + if not self.main_path: + self.main_path = self.current_path = path + # Insert at the beginning of the list so that it's always before the + # main path. + if path not in self._all_paths: + self._all_paths.insert(0, path) + + @property + def error_is_fatal(self): + """Returns True if the error function should be fatal.""" + return self.config and getattr(self.config, 'error_is_fatal', True) + + @property + def all_paths(self): + """Returns all paths ever added to the context.""" + return set(self._all_paths) + + @property + def source_stack(self): + """Returns the current stack of pushed sources.""" + if not self.current_path: + return [] + return self._all_paths[self._all_paths.index(self.main_path):] + + @memoized_property + def objdir(self): + return mozpath.join(self.config.topobjdir, self.relobjdir).rstrip('/') + + @memoize + def _srcdir(self, path): + return mozpath.join(self.config.topsrcdir, + self._relsrcdir(path)).rstrip('/') + + @property + def srcdir(self): + return self._srcdir(self.current_path or self.main_path) + + @memoize + def _relsrcdir(self, path): + return mozpath.relpath(mozpath.dirname(path), self.config.topsrcdir) + + @property + def relsrcdir(self): + assert self.main_path + return self._relsrcdir(self.current_path or self.main_path) + + @memoized_property + def relobjdir(self): + assert self.main_path + return mozpath.relpath(mozpath.dirname(self.main_path), + self.config.topsrcdir) + + def _factory(self, key): + """Function called when requesting a missing key.""" + defaults = self._allowed_variables.get(key) + if not defaults: + raise KeyError('global_ns', 'get_unknown', key) + + # If the default is specifically a lambda (or, rather, any function + # --but not a class that can be called), then it is actually a rule to + # generate the default that should be used. + default = defaults[0] + if issubclass(default, ContextDerivedValue): + return default(self) + else: + return default() + + def _validate(self, key, value, is_template=False): + """Validates whether the key is allowed and if the value's type + matches. + """ + stored_type, input_type, docs = \ + self._allowed_variables.get(key, (None, None, None)) + + if stored_type is None or not is_template and key in TEMPLATE_VARIABLES: + raise KeyError('global_ns', 'set_unknown', key, value) + + # If the incoming value is not the type we store, we try to convert + # it to that type. This relies on proper coercion rules existing. This + # is the responsibility of whoever defined the symbols: a type should + # not be in the allowed set if the constructor function for the stored + # type does not accept an instance of that type. + if not isinstance(value, (stored_type, input_type)): + raise ValueError('global_ns', 'set_type', key, value, input_type) + + return stored_type + + def __setitem__(self, key, value): + stored_type = self._validate(key, value) + + if not isinstance(value, stored_type): + if issubclass(stored_type, ContextDerivedValue): + value = stored_type(self, value) + else: + value = stored_type(value) + + return KeyedDefaultDict.__setitem__(self, key, value) + + def update(self, iterable={}, **kwargs): + """Like dict.update(), but using the context's setitem. + + This function is transactional: if setitem fails for one of the values, + the context is not updated at all.""" + if isinstance(iterable, dict): + iterable = iterable.items() + + update = {} + for key, value in itertools.chain(iterable, kwargs.items()): + stored_type = self._validate(key, value) + # Don't create an instance of stored_type if coercion is needed, + # until all values are validated. + update[key] = (value, stored_type) + for key, (value, stored_type) in update.items(): + if not isinstance(value, stored_type): + update[key] = stored_type(value) + else: + update[key] = value + KeyedDefaultDict.update(self, update) + + +class TemplateContext(Context): + def __init__(self, template=None, allowed_variables={}, config=None): + self.template = template + super(TemplateContext, self).__init__(allowed_variables, config) + + def _validate(self, key, value): + return Context._validate(self, key, value, True) + + +class SubContext(Context, ContextDerivedValue): + """A Context derived from another Context. + + Sub-contexts are intended to be used as context managers. + + Sub-contexts inherit paths and other relevant state from the parent + context. + """ + def __init__(self, parent): + assert isinstance(parent, Context) + + Context.__init__(self, allowed_variables=self.VARIABLES, + config=parent.config) + + # Copy state from parent. + for p in parent.source_stack: + self.push_source(p) + self._sandbox = parent._sandbox + + def __enter__(self): + if not self._sandbox or self._sandbox() is None: + raise Exception('a sandbox is required') + + self._sandbox().push_subcontext(self) + + def __exit__(self, exc_type, exc_value, traceback): + self._sandbox().pop_subcontext(self) + + +class InitializedDefines(ContextDerivedValue, OrderedDict): + def __init__(self, context, value=None): + OrderedDict.__init__(self) + for define in context.config.substs.get('MOZ_DEBUG_DEFINES', ()): + self[define] = 1 + if value: + self.update(value) + + +class FinalTargetValue(ContextDerivedValue, unicode): + def __new__(cls, context, value=""): + if not value: + value = 'dist/' + if context['XPI_NAME']: + value += 'xpi-stage/' + context['XPI_NAME'] + else: + value += 'bin' + if context['DIST_SUBDIR']: + value += '/' + context['DIST_SUBDIR'] + return unicode.__new__(cls, value) + + +def Enum(*values): + assert len(values) + default = values[0] + + class EnumClass(object): + def __new__(cls, value=None): + if value is None: + return default + if value in values: + return value + raise ValueError('Invalid value. Allowed values are: %s' + % ', '.join(repr(v) for v in values)) + return EnumClass + + +class PathMeta(type): + """Meta class for the Path family of classes. + + It handles calling __new__ and __init__ with the right arguments + in cases where a Path is instantiated with another instance of + Path instead of having received a context. + + It also makes Path(context, value) instantiate one of the + subclasses depending on the value, allowing callers to do + standard type checking (isinstance(path, ObjDirPath)) instead + of checking the value itself (path.startswith('!')). + """ + def __call__(cls, context, value=None): + if isinstance(context, Path): + assert value is None + value = context + context = context.context + else: + assert isinstance(context, Context) + if isinstance(value, Path): + context = value.context + if not issubclass(cls, (SourcePath, ObjDirPath, AbsolutePath)): + if value.startswith('!'): + cls = ObjDirPath + elif value.startswith('%'): + cls = AbsolutePath + else: + cls = SourcePath + return super(PathMeta, cls).__call__(context, value) + +class Path(ContextDerivedValue, unicode): + """Stores and resolves a source path relative to a given context + + This class is used as a backing type for some of the sandbox variables. + It expresses paths relative to a context. Supported paths are: + - '/topsrcdir/relative/paths' + - 'srcdir/relative/paths' + - '!/topobjdir/relative/paths' + - '!objdir/relative/paths' + - '%/filesystem/absolute/paths' + """ + __metaclass__ = PathMeta + + def __new__(cls, context, value=None): + return super(Path, cls).__new__(cls, value) + + def __init__(self, context, value=None): + # Only subclasses should be instantiated. + assert self.__class__ != Path + self.context = context + self.srcdir = context.srcdir + + def join(self, *p): + """ContextDerived equivalent of mozpath.join(self, *p), returning a + new Path instance. + """ + return Path(self.context, mozpath.join(self, *p)) + + def __cmp__(self, other): + if isinstance(other, Path) and self.srcdir != other.srcdir: + return cmp(self.full_path, other.full_path) + return cmp(unicode(self), other) + + # __cmp__ is not enough because unicode has __eq__, __ne__, etc. defined + # and __cmp__ is only used for those when they don't exist. + def __eq__(self, other): + return self.__cmp__(other) == 0 + + def __ne__(self, other): + return self.__cmp__(other) != 0 + + def __lt__(self, other): + return self.__cmp__(other) < 0 + + def __gt__(self, other): + return self.__cmp__(other) > 0 + + def __le__(self, other): + return self.__cmp__(other) <= 0 + + def __ge__(self, other): + return self.__cmp__(other) >= 0 + + def __repr__(self): + return '<%s (%s)%s>' % (self.__class__.__name__, self.srcdir, self) + + def __hash__(self): + return hash(self.full_path) + + @memoized_property + def target_basename(self): + return mozpath.basename(self.full_path) + + +class SourcePath(Path): + """Like Path, but limited to paths in the source directory.""" + def __init__(self, context, value): + if value.startswith('!'): + raise ValueError('Object directory paths are not allowed') + if value.startswith('%'): + raise ValueError('Filesystem absolute paths are not allowed') + super(SourcePath, self).__init__(context, value) + + if value.startswith('/'): + path = None + # If the path starts with a '/' and is actually relative to an + # external source dir, use that as base instead of topsrcdir. + if context.config.external_source_dir: + path = mozpath.join(context.config.external_source_dir, + value[1:]) + if not path or not os.path.exists(path): + path = mozpath.join(context.config.topsrcdir, + value[1:]) + else: + path = mozpath.join(self.srcdir, value) + self.full_path = mozpath.normpath(path) + + @memoized_property + def translated(self): + """Returns the corresponding path in the objdir. + + Ideally, we wouldn't need this function, but the fact that both source + path under topsrcdir and the external source dir end up mixed in the + objdir (aka pseudo-rework), this is needed. + """ + return ObjDirPath(self.context, '!%s' % self).full_path + + +class RenamedSourcePath(SourcePath): + """Like SourcePath, but with a different base name when installed. + + The constructor takes a tuple of (source, target_basename). + + This class is not meant to be exposed to moz.build sandboxes as of now, + and is not supported by the RecursiveMake backend. + """ + def __init__(self, context, value): + assert isinstance(value, tuple) + source, self._target_basename = value + super(RenamedSourcePath, self).__init__(context, source) + + @property + def target_basename(self): + return self._target_basename + + +class ObjDirPath(Path): + """Like Path, but limited to paths in the object directory.""" + def __init__(self, context, value=None): + if not value.startswith('!'): + raise ValueError('Object directory paths must start with ! prefix') + super(ObjDirPath, self).__init__(context, value) + + if value.startswith('!/'): + path = mozpath.join(context.config.topobjdir,value[2:]) + else: + path = mozpath.join(context.objdir, value[1:]) + self.full_path = mozpath.normpath(path) + + +class AbsolutePath(Path): + """Like Path, but allows arbitrary paths outside the source and object directories.""" + def __init__(self, context, value=None): + if not value.startswith('%'): + raise ValueError('Absolute paths must start with % prefix') + if not os.path.isabs(value[1:]): + raise ValueError('Path \'%s\' is not absolute' % value[1:]) + super(AbsolutePath, self).__init__(context, value) + + self.full_path = mozpath.normpath(value[1:]) + + +@memoize +def ContextDerivedTypedList(klass, base_class=List): + """Specialized TypedList for use with ContextDerivedValue types. + """ + assert issubclass(klass, ContextDerivedValue) + class _TypedList(ContextDerivedValue, TypedList(klass, base_class)): + def __init__(self, context, iterable=[]): + self.context = context + super(_TypedList, self).__init__(iterable) + + def normalize(self, e): + if not isinstance(e, klass): + e = klass(self.context, e) + return e + + return _TypedList + +@memoize +def ContextDerivedTypedListWithItems(type, base_class=List): + """Specialized TypedList for use with ContextDerivedValue types. + """ + class _TypedListWithItems(ContextDerivedTypedList(type, base_class)): + def __getitem__(self, name): + name = self.normalize(name) + return super(_TypedListWithItems, self).__getitem__(name) + + return _TypedListWithItems + + +@memoize +def ContextDerivedTypedRecord(*fields): + """Factory for objects with certain properties and dynamic + type checks. + + This API is extremely similar to the TypedNamedTuple API, + except that properties may be mutated. This supports syntax like: + + VARIABLE_NAME.property += [ + 'item1', + 'item2', + ] + """ + + class _TypedRecord(ContextDerivedValue): + __slots__ = tuple([name for name, _ in fields]) + + def __init__(self, context): + for fname, ftype in self._fields.items(): + if issubclass(ftype, ContextDerivedValue): + setattr(self, fname, self._fields[fname](context)) + else: + setattr(self, fname, self._fields[fname]()) + + def __setattr__(self, name, value): + if name in self._fields and not isinstance(value, self._fields[name]): + value = self._fields[name](value) + object.__setattr__(self, name, value) + + _TypedRecord._fields = dict(fields) + return _TypedRecord + + +@memoize +def ContextDerivedTypedHierarchicalStringList(type): + """Specialized HierarchicalStringList for use with ContextDerivedValue + types.""" + class _TypedListWithItems(ContextDerivedValue, HierarchicalStringList): + __slots__ = ('_strings', '_children', '_context') + + def __init__(self, context): + self._strings = ContextDerivedTypedList( + type, StrictOrderingOnAppendList)(context) + self._children = {} + self._context = context + + def _get_exportvariable(self, name): + child = self._children.get(name) + if not child: + child = self._children[name] = _TypedListWithItems( + self._context) + return child + + return _TypedListWithItems + +def OrderedListWithAction(action): + """Returns a class which behaves as a StrictOrderingOnAppendList, but + invokes the given callable with each input and a context as it is + read, storing a tuple including the result and the original item. + + This used to extend moz.build reading to make more data available in + filesystem-reading mode. + """ + class _OrderedListWithAction(ContextDerivedValue, + StrictOrderingOnAppendListWithAction): + def __init__(self, context, *args): + def _action(item): + return item, action(context, item) + super(_OrderedListWithAction, self).__init__(action=_action, *args) + + return _OrderedListWithAction + +def TypedListWithAction(typ, action): + """Returns a class which behaves as a TypedList with the provided type, but + invokes the given given callable with each input and a context as it is + read, storing a tuple including the result and the original item. + + This used to extend moz.build reading to make more data available in + filesystem-reading mode. + """ + class _TypedListWithAction(ContextDerivedValue, TypedList(typ), ListWithAction): + def __init__(self, context, *args): + def _action(item): + return item, action(context, item) + super(_TypedListWithAction, self).__init__(action=_action, *args) + return _TypedListWithAction + +WebPlatformTestManifest = TypedNamedTuple("WebPlatformTestManifest", + [("manifest_path", unicode), + ("test_root", unicode)]) +ManifestparserManifestList = OrderedListWithAction(read_manifestparser_manifest) +ReftestManifestList = OrderedListWithAction(read_reftest_manifest) +WptManifestList = TypedListWithAction(WebPlatformTestManifest, read_wpt_manifest) + +OrderedSourceList = ContextDerivedTypedList(SourcePath, StrictOrderingOnAppendList) +OrderedTestFlavorList = TypedList(Enum(*all_test_flavors()), + StrictOrderingOnAppendList) +OrderedStringList = TypedList(unicode, StrictOrderingOnAppendList) +DependentTestsEntry = ContextDerivedTypedRecord(('files', OrderedSourceList), + ('tags', OrderedStringList), + ('flavors', OrderedTestFlavorList)) +BugzillaComponent = TypedNamedTuple('BugzillaComponent', + [('product', unicode), ('component', unicode)]) + + +class Files(SubContext): + """Metadata attached to files. + + It is common to want to annotate files with metadata, such as which + Bugzilla component tracks issues with certain files. This sub-context is + where we stick that metadata. + + The argument to this sub-context is a file matching pattern that is applied + against the host file's directory. If the pattern matches a file whose info + is currently being sought, the metadata attached to this instance will be + applied to that file. + + Patterns are collections of filename characters with ``/`` used as the + directory separate (UNIX-style paths) and ``*`` and ``**`` used to denote + wildcard matching. + + Patterns without the ``*`` character are literal matches and will match at + most one entity. + + Patterns with ``*`` or ``**`` are wildcard matches. ``*`` matches files + at least within a single directory. ``**`` matches files across several + directories. + + ``foo.html`` + Will match only the ``foo.html`` file in the current directory. + ``*.jsm`` + Will match all ``.jsm`` files in the current directory. + ``**/*.cpp`` + Will match all ``.cpp`` files in this and all child directories. + ``foo/*.css`` + Will match all ``.css`` files in the ``foo/`` directory. + ``bar/*`` + Will match all files in the ``bar/`` directory and all of its + children directories. + ``bar/**`` + This is equivalent to ``bar/*`` above. + ``bar/**/foo`` + Will match all ``foo`` files in the ``bar/`` directory and all of its + children directories. + + The difference in behavior between ``*`` and ``**`` is only evident if + a pattern follows the ``*`` or ``**``. A pattern ending with ``*`` is + greedy. ``**`` is needed when you need an additional pattern after the + wildcard. e.g. ``**/foo``. + """ + + VARIABLES = { + 'BUG_COMPONENT': (BugzillaComponent, tuple, + """The bug component that tracks changes to these files. + + Values are a 2-tuple of unicode describing the Bugzilla product and + component. e.g. ``('Core', 'Build Config')``. + """), + + 'FINAL': (bool, bool, + """Mark variable assignments as finalized. + + During normal processing, values from newer Files contexts + overwrite previously set values. Last write wins. This behavior is + not always desired. ``FINAL`` provides a mechanism to prevent + further updates to a variable. + + When ``FINAL`` is set, the value of all variables defined in this + context are marked as frozen and all subsequent writes to them + are ignored during metadata reading. + + See :ref:`mozbuild_files_metadata_finalizing` for more info. + """), + 'IMPACTED_TESTS': (DependentTestsEntry, list, + """File patterns, tags, and flavors for tests relevant to these files. + + Maps source files to the tests potentially impacted by those files. + Tests can be specified by file pattern, tag, or flavor. + + For example: + + with Files('runtests.py'): + IMPACTED_TESTS.files += [ + '**', + ] + + in testing/mochitest/moz.build will suggest that any of the tests + under testing/mochitest may be impacted by a change to runtests.py. + + File patterns may be made relative to the topsrcdir with a leading + '/', so + + with Files('httpd.js'): + IMPACTED_TESTS.files += [ + '/testing/mochitest/tests/Harness_sanity/**', + ] + + in netwerk/test/httpserver/moz.build will suggest that any change to httpd.js + will be relevant to the mochitest sanity tests. + + Tags and flavors are sorted string lists (flavors are limited to valid + values). + + For example: + + with Files('toolkit/devtools/*'): + IMPACTED_TESTS.tags += [ + 'devtools', + ] + + in the root moz.build would suggest that any test tagged 'devtools' would + potentially be impacted by a change to a file under toolkit/devtools, and + + with Files('dom/base/nsGlobalWindow.cpp'): + IMPACTED_TESTS.flavors += [ + 'mochitest', + ] + + Would suggest that nsGlobalWindow.cpp is potentially relevant to + any plain mochitest. + """), + } + + def __init__(self, parent, pattern=None): + super(Files, self).__init__(parent) + self.pattern = pattern + self.finalized = set() + self.test_files = set() + self.test_tags = set() + self.test_flavors = set() + + def __iadd__(self, other): + assert isinstance(other, Files) + + self.test_files |= other.test_files + self.test_tags |= other.test_tags + self.test_flavors |= other.test_flavors + + for k, v in other.items(): + if k == 'IMPACTED_TESTS': + self.test_files |= set(mozpath.relpath(e.full_path, e.context.config.topsrcdir) + for e in v.files) + self.test_tags |= set(v.tags) + self.test_flavors |= set(v.flavors) + continue + + # Ignore updates to finalized flags. + if k in self.finalized: + continue + + # Only finalize variables defined in this instance. + if k == 'FINAL': + self.finalized |= set(other) - {'FINAL'} + continue + + self[k] = v + + return self + + def asdict(self): + """Return this instance as a dict with built-in data structures. + + Call this to obtain an object suitable for serializing. + """ + d = {} + if 'BUG_COMPONENT' in self: + bc = self['BUG_COMPONENT'] + d['bug_component'] = (bc.product, bc.component) + + return d + + @staticmethod + def aggregate(files): + """Given a mapping of path to Files, obtain aggregate results. + + Consumers may want to extract useful information from a collection of + Files describing paths. e.g. given the files info data for N paths, + recommend a single bug component based on the most frequent one. This + function provides logic for deriving aggregate knowledge from a + collection of path File metadata. + + Note: the intent of this function is to operate on the result of + :py:func:`mozbuild.frontend.reader.BuildReader.files_info`. The + :py:func:`mozbuild.frontend.context.Files` instances passed in are + thus the "collapsed" (``__iadd__``ed) results of all ``Files`` from all + moz.build files relevant to a specific path, not individual ``Files`` + instances from a single moz.build file. + """ + d = {} + + bug_components = Counter() + + for f in files.values(): + bug_component = f.get('BUG_COMPONENT') + if bug_component: + bug_components[bug_component] += 1 + + d['bug_component_counts'] = [] + for c, count in bug_components.most_common(): + component = (c.product, c.component) + d['bug_component_counts'].append((c, count)) + + if 'recommended_bug_component' not in d: + d['recommended_bug_component'] = component + recommended_count = count + elif count == recommended_count: + # Don't recommend a component if it doesn't have a clear lead. + d['recommended_bug_component'] = None + + # In case no bug components. + d.setdefault('recommended_bug_component', None) + + return d + + +# This defines functions that create sub-contexts. +# +# Values are classes that are SubContexts. The class name will be turned into +# a function that when called emits an instance of that class. +# +# Arbitrary arguments can be passed to the class constructor. The first +# argument is always the parent context. It is up to each class to perform +# argument validation. +SUBCONTEXTS = [ + Files, +] + +for cls in SUBCONTEXTS: + if not issubclass(cls, SubContext): + raise ValueError('SUBCONTEXTS entry not a SubContext class: %s' % cls) + + if not hasattr(cls, 'VARIABLES'): + raise ValueError('SUBCONTEXTS entry does not have VARIABLES: %s' % cls) + +SUBCONTEXTS = {cls.__name__: cls for cls in SUBCONTEXTS} + + +# This defines the set of mutable global variables. +# +# Each variable is a tuple of: +# +# (storage_type, input_types, docs) + +VARIABLES = { + 'ALLOW_COMPILER_WARNINGS': (bool, bool, + """Whether to allow compiler warnings (i.e. *not* treat them as + errors). + + This is commonplace (almost mandatory, in fact) in directories + containing third-party code that we regularly update from upstream and + thus do not control, but is otherwise discouraged. + """), + + # Variables controlling reading of other frontend files. + 'ANDROID_GENERATED_RESFILES': (StrictOrderingOnAppendList, list, + """Android resource files generated as part of the build. + + This variable contains a list of files that are expected to be + generated (often by preprocessing) into a 'res' directory as + part of the build process, and subsequently merged into an APK + file. + """), + + 'ANDROID_APK_NAME': (unicode, unicode, + """The name of an Android APK file to generate. + """), + + 'ANDROID_APK_PACKAGE': (unicode, unicode, + """The name of the Android package to generate R.java for, like org.mozilla.gecko. + """), + + 'ANDROID_EXTRA_PACKAGES': (StrictOrderingOnAppendList, list, + """The name of extra Android packages to generate R.java for, like ['org.mozilla.other']. + """), + + 'ANDROID_EXTRA_RES_DIRS': (ContextDerivedTypedListWithItems(Path, List), list, + """Android extra package resource directories. + + This variable contains a list of directories containing static files + to package into a 'res' directory and merge into an APK file. These + directories are packaged into the APK but are assumed to be static + unchecked dependencies that should not be otherwise re-distributed. + """), + + 'ANDROID_RES_DIRS': (ContextDerivedTypedListWithItems(Path, List), list, + """Android resource directories. + + This variable contains a list of directories containing static + files to package into a 'res' directory and merge into an APK + file. + """), + + 'ANDROID_ASSETS_DIRS': (ContextDerivedTypedListWithItems(Path, List), list, + """Android assets directories. + + This variable contains a list of directories containing static + files to package into an 'assets' directory and merge into an + APK file. + """), + + 'ANDROID_ECLIPSE_PROJECT_TARGETS': (dict, dict, + """Defines Android Eclipse project targets. + + This variable should not be populated directly. Instead, it should + populated by calling add_android_eclipse{_library}_project(). + """), + + 'SOURCES': (ContextDerivedTypedListWithItems(Path, StrictOrderingOnAppendListWithFlagsFactory({'no_pgo': bool, 'flags': List})), list, + """Source code files. + + This variable contains a list of source code files to compile. + Accepts assembler, C, C++, Objective C/C++. + """), + + 'FILES_PER_UNIFIED_FILE': (int, int, + """The number of source files to compile into each unified source file. + + """), + + 'IS_RUST_LIBRARY': (bool, bool, + """Whether the current library defined by this moz.build is built by Rust. + + The library defined by this moz.build should have a build definition in + a Cargo.toml file that exists in this moz.build's directory. + """), + + 'UNIFIED_SOURCES': (ContextDerivedTypedList(SourcePath, StrictOrderingOnAppendList), list, + """Source code files that can be compiled together. + + This variable contains a list of source code files to compile, + that can be concatenated all together and built as a single source + file. This can help make the build faster and reduce the debug info + size. + """), + + 'GENERATED_FILES': (StrictOrderingOnAppendListWithFlagsFactory({ + 'script': unicode, + 'inputs': list }), list, + """Generic generated files. + + This variable contains a list of files for the build system to + generate at export time. The generation method may be declared + with optional ``script`` and ``inputs`` flags on individual entries. + If the optional ``script`` flag is not present on an entry, it + is assumed that rules for generating the file are present in + the associated Makefile.in. + + Example:: + + GENERATED_FILES += ['bar.c', 'baz.c', 'foo.c'] + bar = GENERATED_FILES['bar.c'] + bar.script = 'generate.py' + bar.inputs = ['datafile-for-bar'] + foo = GENERATED_FILES['foo.c'] + foo.script = 'generate.py' + foo.inputs = ['datafile-for-foo'] + + This definition will generate bar.c by calling the main method of + generate.py with a open (for writing) file object for bar.c, and + the string ``datafile-for-bar``. In a similar fashion, the main + method of generate.py will also be called with an open + (for writing) file object for foo.c and the string + ``datafile-for-foo``. Please note that only string arguments are + supported for passing to scripts, and that all arguments provided + to the script should be filenames relative to the directory in which + the moz.build file is located. + + To enable using the same script for generating multiple files with + slightly different non-filename parameters, alternative entry points + into ``script`` can be specified:: + + GENERATED_FILES += ['bar.c'] + bar = GENERATED_FILES['bar.c'] + bar.script = 'generate.py:make_bar' + + The chosen script entry point may optionally return a set of strings, + indicating extra files the output depends on. + """), + + 'DEFINES': (InitializedDefines, dict, + """Dictionary of compiler defines to declare. + + These are passed in to the compiler as ``-Dkey='value'`` for string + values, ``-Dkey=value`` for numeric values, or ``-Dkey`` if the + value is True. Note that for string values, the outer-level of + single-quotes will be consumed by the shell. If you want to have + a string-literal in the program, the value needs to have + double-quotes. + + Example:: + + DEFINES['NS_NO_XPCOM'] = True + DEFINES['MOZ_EXTENSIONS_DB_SCHEMA'] = 15 + DEFINES['DLL_SUFFIX'] = '".so"' + + This will result in the compiler flags ``-DNS_NO_XPCOM``, + ``-DMOZ_EXTENSIONS_DB_SCHEMA=15``, and ``-DDLL_SUFFIX='".so"'``, + respectively. These could also be combined into a single + update:: + + DEFINES.update({ + 'NS_NO_XPCOM': True, + 'MOZ_EXTENSIONS_DB_SCHEMA': 15, + 'DLL_SUFFIX': '".so"', + }) + """), + + 'DELAYLOAD_DLLS': (List, list, + """Delay-loaded DLLs. + + This variable contains a list of DLL files which the module being linked + should load lazily. This only has an effect when building with MSVC. + """), + + 'DIRS': (ContextDerivedTypedList(SourcePath), list, + """Child directories to descend into looking for build frontend files. + + This works similarly to the ``DIRS`` variable in make files. Each str + value in the list is the name of a child directory. When this file is + done parsing, the build reader will descend into each listed directory + and read the frontend file there. If there is no frontend file, an error + is raised. + + Values are relative paths. They can be multiple directory levels + above or below. Use ``..`` for parent directories and ``/`` for path + delimiters. + """), + + 'HAS_MISC_RULE': (bool, bool, + """Whether this directory should be traversed in the ``misc`` tier. + + Many ``libs`` rules still exist in Makefile.in files. We highly prefer + that these rules exist in the ``misc`` tier/target so that they can be + executed concurrently during tier traversal (the ``misc`` tier is + fully concurrent). + + Presence of this variable indicates that this directory should be + traversed by the ``misc`` tier. + + Please note that converting ``libs`` rules to the ``misc`` tier must + be done with care, as there are many implicit dependencies that can + break the build in subtle ways. + """), + + 'FINAL_TARGET_FILES': (ContextDerivedTypedHierarchicalStringList(Path), list, + """List of files to be installed into the application directory. + + ``FINAL_TARGET_FILES`` will copy (or symlink, if the platform supports it) + the contents of its files to the directory specified by + ``FINAL_TARGET`` (typically ``dist/bin``). Files that are destined for a + subdirectory can be specified by accessing a field, or as a dict access. + For example, to export ``foo.png`` to the top-level directory and + ``bar.svg`` to the directory ``images/do-not-use``, append to + ``FINAL_TARGET_FILES`` like so:: + + FINAL_TARGET_FILES += ['foo.png'] + FINAL_TARGET_FILES.images['do-not-use'] += ['bar.svg'] + """), + + 'DISABLE_STL_WRAPPING': (bool, bool, + """Disable the wrappers for STL which allow it to work with C++ exceptions + disabled. + """), + + 'FINAL_TARGET_PP_FILES': (ContextDerivedTypedHierarchicalStringList(Path), list, + """Like ``FINAL_TARGET_FILES``, with preprocessing. + """), + + 'OBJDIR_FILES': (ContextDerivedTypedHierarchicalStringList(Path), list, + """List of files to be installed anywhere in the objdir. Use sparingly. + + ``OBJDIR_FILES`` is similar to FINAL_TARGET_FILES, but it allows copying + anywhere in the object directory. This is intended for various one-off + cases, not for general use. If you wish to add entries to OBJDIR_FILES, + please consult a build peer. + """), + + 'OBJDIR_PP_FILES': (ContextDerivedTypedHierarchicalStringList(Path), list, + """Like ``OBJDIR_FILES``, with preprocessing. Use sparingly. + """), + + 'FINAL_LIBRARY': (unicode, unicode, + """Library in which the objects of the current directory will be linked. + + This variable contains the name of a library, defined elsewhere with + ``LIBRARY_NAME``, in which the objects of the current directory will be + linked. + """), + + 'CPP_UNIT_TESTS': (StrictOrderingOnAppendList, list, + """Compile a list of C++ unit test names. + + Each name in this variable corresponds to an executable built from the + corresponding source file with the same base name. + + If the configuration token ``BIN_SUFFIX`` is set, its value will be + automatically appended to each name. If a name already ends with + ``BIN_SUFFIX``, the name will remain unchanged. + """), + + 'FORCE_SHARED_LIB': (bool, bool, + """Whether the library in this directory is a shared library. + """), + + 'FORCE_STATIC_LIB': (bool, bool, + """Whether the library in this directory is a static library. + """), + + 'USE_STATIC_LIBS': (bool, bool, + """Whether the code in this directory is a built against the static + runtime library. + + This variable only has an effect when building with MSVC. + """), + + 'HOST_SOURCES': (ContextDerivedTypedList(SourcePath, StrictOrderingOnAppendList), list, + """Source code files to compile with the host compiler. + + This variable contains a list of source code files to compile. + with the host compiler. + """), + + 'IS_COMPONENT': (bool, bool, + """Whether the library contains a binary XPCOM component manifest. + + Implies FORCE_SHARED_LIB. + """), + + 'PYTHON_UNIT_TESTS': (StrictOrderingOnAppendList, list, + """A list of python unit tests. + """), + + 'HOST_LIBRARY_NAME': (unicode, unicode, + """Name of target library generated when cross compiling. + """), + + 'JAVA_JAR_TARGETS': (dict, dict, + """Defines Java JAR targets to be built. + + This variable should not be populated directly. Instead, it should + populated by calling add_java_jar(). + """), + + 'LIBRARY_DEFINES': (OrderedDict, dict, + """Dictionary of compiler defines to declare for the entire library. + + This variable works like DEFINES, except that declarations apply to all + libraries that link into this library via FINAL_LIBRARY. + """), + + 'LIBRARY_NAME': (unicode, unicode, + """The code name of the library generated for a directory. + + By default STATIC_LIBRARY_NAME and SHARED_LIBRARY_NAME take this name. + In ``example/components/moz.build``,:: + + LIBRARY_NAME = 'xpcomsample' + + would generate ``example/components/libxpcomsample.so`` on Linux, or + ``example/components/xpcomsample.lib`` on Windows. + """), + + 'SHARED_LIBRARY_NAME': (unicode, unicode, + """The name of the static library generated for a directory, if it needs to + differ from the library code name. + + Implies FORCE_SHARED_LIB. + """), + + 'IS_FRAMEWORK': (bool, bool, + """Whether the library to build should be built as a framework on OSX. + + This implies the name of the library won't be prefixed nor suffixed. + Implies FORCE_SHARED_LIB. + """), + + 'STATIC_LIBRARY_NAME': (unicode, unicode, + """The name of the static library generated for a directory, if it needs to + differ from the library code name. + + Implies FORCE_STATIC_LIB. + """), + + 'USE_LIBS': (StrictOrderingOnAppendList, list, + """List of libraries to link to programs and libraries. + """), + + 'HOST_USE_LIBS': (StrictOrderingOnAppendList, list, + """List of libraries to link to host programs and libraries. + """), + + 'HOST_OS_LIBS': (List, list, + """List of system libraries for host programs and libraries. + """), + + 'LOCAL_INCLUDES': (ContextDerivedTypedList(Path, StrictOrderingOnAppendList), list, + """Additional directories to be searched for include files by the compiler. + """), + + 'NO_PGO': (bool, bool, + """Whether profile-guided optimization is disable in this directory. + """), + + 'NO_VISIBILITY_FLAGS': (bool, bool, + """Build sources listed in this file without VISIBILITY_FLAGS. + """), + + 'OS_LIBS': (List, list, + """System link libraries. + + This variable contains a list of system libaries to link against. + """), + 'RCFILE': (unicode, unicode, + """The program .rc file. + + This variable can only be used on Windows. + """), + + 'RESFILE': (unicode, unicode, + """The program .res file. + + This variable can only be used on Windows. + """), + + 'RCINCLUDE': (unicode, unicode, + """The resource script file to be included in the default .res file. + + This variable can only be used on Windows. + """), + + 'DEFFILE': (unicode, unicode, + """The program .def (module definition) file. + + This variable can only be used on Windows. + """), + + 'LD_VERSION_SCRIPT': (unicode, unicode, + """The linker version script for shared libraries. + + This variable can only be used on Linux. + """), + + 'SYMBOLS_FILE': (Path, unicode, + """A file containing a list of symbols to export from a shared library. + + The given file contains a list of symbols to be exported, and is + preprocessed. + A special marker "@DATA@" must be added after a symbol name if it + points to data instead of code, so that the Windows linker can treat + them correctly. + """), + + 'BRANDING_FILES': (ContextDerivedTypedHierarchicalStringList(Path), list, + """List of files to be installed into the branding directory. + + ``BRANDING_FILES`` will copy (or symlink, if the platform supports it) + the contents of its files to the ``dist/branding`` directory. Files that + are destined for a subdirectory can be specified by accessing a field. + For example, to export ``foo.png`` to the top-level directory and + ``bar.png`` to the directory ``images/subdir``, append to + ``BRANDING_FILES`` like so:: + + BRANDING_FILES += ['foo.png'] + BRANDING_FILES.images.subdir += ['bar.png'] + """), + + 'SDK_FILES': (ContextDerivedTypedHierarchicalStringList(Path), list, + """List of files to be installed into the sdk directory. + + ``SDK_FILES`` will copy (or symlink, if the platform supports it) + the contents of its files to the ``dist/sdk`` directory. Files that + are destined for a subdirectory can be specified by accessing a field. + For example, to export ``foo.py`` to the top-level directory and + ``bar.py`` to the directory ``subdir``, append to + ``SDK_FILES`` like so:: + + SDK_FILES += ['foo.py'] + SDK_FILES.subdir += ['bar.py'] + """), + + 'SDK_LIBRARY': (bool, bool, + """Whether the library built in the directory is part of the SDK. + + The library will be copied into ``SDK_LIB_DIR`` (``$DIST/sdk/lib``). + """), + + 'SIMPLE_PROGRAMS': (StrictOrderingOnAppendList, list, + """Compile a list of executable names. + + Each name in this variable corresponds to an executable built from the + corresponding source file with the same base name. + + If the configuration token ``BIN_SUFFIX`` is set, its value will be + automatically appended to each name. If a name already ends with + ``BIN_SUFFIX``, the name will remain unchanged. + """), + + 'SONAME': (unicode, unicode, + """The soname of the shared object currently being linked + + soname is the "logical name" of a shared object, often used to provide + version backwards compatibility. This variable makes sense only for + shared objects, and is supported only on some unix platforms. + """), + + 'HOST_SIMPLE_PROGRAMS': (StrictOrderingOnAppendList, list, + """Compile a list of host executable names. + + Each name in this variable corresponds to a hosst executable built + from the corresponding source file with the same base name. + + If the configuration token ``HOST_BIN_SUFFIX`` is set, its value will + be automatically appended to each name. If a name already ends with + ``HOST_BIN_SUFFIX``, the name will remain unchanged. + """), + + 'CONFIGURE_SUBST_FILES': (ContextDerivedTypedList(SourcePath, StrictOrderingOnAppendList), list, + """Output files that will be generated using configure-like substitution. + + This is a substitute for ``AC_OUTPUT`` in autoconf. For each path in this + list, we will search for a file in the srcdir having the name + ``{path}.in``. The contents of this file will be read and variable + patterns like ``@foo@`` will be substituted with the values of the + ``AC_SUBST`` variables declared during configure. + """), + + 'CONFIGURE_DEFINE_FILES': (ContextDerivedTypedList(SourcePath, StrictOrderingOnAppendList), list, + """Output files generated from configure/config.status. + + This is a substitute for ``AC_CONFIG_HEADER`` in autoconf. This is very + similar to ``CONFIGURE_SUBST_FILES`` except the generation logic takes + into account the values of ``AC_DEFINE`` instead of ``AC_SUBST``. + """), + + 'EXPORTS': (ContextDerivedTypedHierarchicalStringList(Path), list, + """List of files to be exported, and in which subdirectories. + + ``EXPORTS`` is generally used to list the include files to be exported to + ``dist/include``, but it can be used for other files as well. This variable + behaves as a list when appending filenames for export in the top-level + directory. Files can also be appended to a field to indicate which + subdirectory they should be exported to. For example, to export + ``foo.h`` to the top-level directory, and ``bar.h`` to ``mozilla/dom/``, + append to ``EXPORTS`` like so:: + + EXPORTS += ['foo.h'] + EXPORTS.mozilla.dom += ['bar.h'] + + Entries in ``EXPORTS`` are paths, so objdir paths may be used, but + any files listed from the objdir must also be listed in + ``GENERATED_FILES``. + """), + + 'PROGRAM' : (unicode, unicode, + """Compiled executable name. + + If the configuration token ``BIN_SUFFIX`` is set, its value will be + automatically appended to ``PROGRAM``. If ``PROGRAM`` already ends with + ``BIN_SUFFIX``, ``PROGRAM`` will remain unchanged. + """), + + 'HOST_PROGRAM' : (unicode, unicode, + """Compiled host executable name. + + If the configuration token ``HOST_BIN_SUFFIX`` is set, its value will be + automatically appended to ``HOST_PROGRAM``. If ``HOST_PROGRAM`` already + ends with ``HOST_BIN_SUFFIX``, ``HOST_PROGRAM`` will remain unchanged. + """), + + 'DIST_INSTALL': (Enum(None, False, True), bool, + """Whether to install certain files into the dist directory. + + By default, some files types are installed in the dist directory, and + some aren't. Set this variable to True to force the installation of + some files that wouldn't be installed by default. Set this variable to + False to force to not install some files that would be installed by + default. + + This is confusing for historical reasons, but eventually, the behavior + will be made explicit. + """), + + 'JAR_MANIFESTS': (ContextDerivedTypedList(SourcePath, StrictOrderingOnAppendList), list, + """JAR manifest files that should be processed as part of the build. + + JAR manifests are files in the tree that define how to package files + into JARs and how chrome registration is performed. For more info, + see :ref:`jar_manifests`. + """), + + # IDL Generation. + 'XPIDL_SOURCES': (StrictOrderingOnAppendList, list, + """XPCOM Interface Definition Files (xpidl). + + This is a list of files that define XPCOM interface definitions. + Entries must be files that exist. Entries are almost certainly ``.idl`` + files. + """), + + 'XPIDL_MODULE': (unicode, unicode, + """XPCOM Interface Definition Module Name. + + This is the name of the ``.xpt`` file that is created by linking + ``XPIDL_SOURCES`` together. If unspecified, it defaults to be the same + as ``MODULE``. + """), + + 'XPIDL_NO_MANIFEST': (bool, bool, + """Indicate that the XPIDL module should not be added to a manifest. + + This flag exists primarily to prevent test-only XPIDL modules from being + added to the application's chrome manifest. Most XPIDL modules should + not use this flag. + """), + + 'IPDL_SOURCES': (StrictOrderingOnAppendList, list, + """IPDL source files. + + These are ``.ipdl`` files that will be parsed and converted to + ``.cpp`` files. + """), + + 'WEBIDL_FILES': (StrictOrderingOnAppendList, list, + """WebIDL source files. + + These will be parsed and converted to ``.cpp`` and ``.h`` files. + """), + + 'GENERATED_EVENTS_WEBIDL_FILES': (StrictOrderingOnAppendList, list, + """WebIDL source files for generated events. + + These will be parsed and converted to ``.cpp`` and ``.h`` files. + """), + + 'TEST_WEBIDL_FILES': (StrictOrderingOnAppendList, list, + """Test WebIDL source files. + + These will be parsed and converted to ``.cpp`` and ``.h`` files + if tests are enabled. + """), + + 'GENERATED_WEBIDL_FILES': (StrictOrderingOnAppendList, list, + """Generated WebIDL source files. + + These will be generated from some other files. + """), + + 'PREPROCESSED_TEST_WEBIDL_FILES': (StrictOrderingOnAppendList, list, + """Preprocessed test WebIDL source files. + + These will be preprocessed, then parsed and converted to .cpp + and ``.h`` files if tests are enabled. + """), + + 'PREPROCESSED_WEBIDL_FILES': (StrictOrderingOnAppendList, list, + """Preprocessed WebIDL source files. + + These will be preprocessed before being parsed and converted. + """), + + 'WEBIDL_EXAMPLE_INTERFACES': (StrictOrderingOnAppendList, list, + """Names of example WebIDL interfaces to build as part of the build. + + Names in this list correspond to WebIDL interface names defined in + WebIDL files included in the build from one of the \*WEBIDL_FILES + variables. + """), + + # Test declaration. + 'A11Y_MANIFESTS': (ManifestparserManifestList, list, + """List of manifest files defining a11y tests. + """), + + 'BROWSER_CHROME_MANIFESTS': (ManifestparserManifestList, list, + """List of manifest files defining browser chrome tests. + """), + + 'JETPACK_PACKAGE_MANIFESTS': (ManifestparserManifestList, list, + """List of manifest files defining jetpack package tests. + """), + + 'JETPACK_ADDON_MANIFESTS': (ManifestparserManifestList, list, + """List of manifest files defining jetpack addon tests. + """), + + 'ANDROID_INSTRUMENTATION_MANIFESTS': (ManifestparserManifestList, list, + """List of manifest files defining Android instrumentation tests. + """), + + 'FIREFOX_UI_FUNCTIONAL_MANIFESTS': (ManifestparserManifestList, list, + """List of manifest files defining firefox-ui-functional tests. + """), + + 'FIREFOX_UI_UPDATE_MANIFESTS': (ManifestparserManifestList, list, + """List of manifest files defining firefox-ui-update tests. + """), + + 'PUPPETEER_FIREFOX_MANIFESTS': (ManifestparserManifestList, list, + """List of manifest files defining puppeteer unit tests for Firefox. + """), + + 'MARIONETTE_LAYOUT_MANIFESTS': (ManifestparserManifestList, list, + """List of manifest files defining marionette-layout tests. + """), + + 'MARIONETTE_UNIT_MANIFESTS': (ManifestparserManifestList, list, + """List of manifest files defining marionette-unit tests. + """), + + 'MARIONETTE_WEBAPI_MANIFESTS': (ManifestparserManifestList, list, + """List of manifest files defining marionette-webapi tests. + """), + + 'METRO_CHROME_MANIFESTS': (ManifestparserManifestList, list, + """List of manifest files defining metro browser chrome tests. + """), + + 'MOCHITEST_CHROME_MANIFESTS': (ManifestparserManifestList, list, + """List of manifest files defining mochitest chrome tests. + """), + + 'MOCHITEST_MANIFESTS': (ManifestparserManifestList, list, + """List of manifest files defining mochitest tests. + """), + + 'REFTEST_MANIFESTS': (ReftestManifestList, list, + """List of manifest files defining reftests. + + These are commonly named reftest.list. + """), + + 'CRASHTEST_MANIFESTS': (ReftestManifestList, list, + """List of manifest files defining crashtests. + + These are commonly named crashtests.list. + """), + + 'WEB_PLATFORM_TESTS_MANIFESTS': (WptManifestList, list, + """List of (manifest_path, test_path) defining web-platform-tests. + """), + + 'WEBRTC_SIGNALLING_TEST_MANIFESTS': (ManifestparserManifestList, list, + """List of manifest files defining WebRTC signalling tests. + """), + + 'XPCSHELL_TESTS_MANIFESTS': (ManifestparserManifestList, list, + """List of manifest files defining xpcshell tests. + """), + + # The following variables are used to control the target of installed files. + 'XPI_NAME': (unicode, unicode, + """The name of an extension XPI to generate. + + When this variable is present, the results of this directory will end up + being packaged into an extension instead of the main dist/bin results. + """), + + 'DIST_SUBDIR': (unicode, unicode, + """The name of an alternate directory to install files to. + + When this variable is present, the results of this directory will end up + being placed in the $(DIST_SUBDIR) subdirectory of where it would + otherwise be placed. + """), + + 'FINAL_TARGET': (FinalTargetValue, unicode, + """The name of the directory to install targets to. + + The directory is relative to the top of the object directory. The + default value is dependent on the values of XPI_NAME and DIST_SUBDIR. If + neither are present, the result is dist/bin. If XPI_NAME is present, the + result is dist/xpi-stage/$(XPI_NAME). If DIST_SUBDIR is present, then + the $(DIST_SUBDIR) directory of the otherwise default value is used. + """), + + 'USE_EXTENSION_MANIFEST': (bool, bool, + """Controls the name of the manifest for JAR files. + + By default, the name of the manifest is ${JAR_MANIFEST}.manifest. + Setting this variable to ``True`` changes the name of the manifest to + chrome.manifest. + """), + + 'NO_JS_MANIFEST': (bool, bool, + """Explicitly disclaims responsibility for manifest listing in EXTRA_COMPONENTS. + + Normally, if you have .js files listed in ``EXTRA_COMPONENTS`` or + ``EXTRA_PP_COMPONENTS``, you are expected to have a corresponding + .manifest file to go with those .js files. Setting ``NO_JS_MANIFEST`` + indicates that the relevant .manifest file and entries for those .js + files are elsehwere (jar.mn, for instance) and this state of affairs + is OK. + """), + + 'GYP_DIRS': (StrictOrderingOnAppendListWithFlagsFactory({ + 'variables': dict, + 'input': unicode, + 'sandbox_vars': dict, + 'non_unified_sources': StrictOrderingOnAppendList, + }), list, + """Defines a list of object directories handled by gyp configurations. + + Elements of this list give the relative object directory. For each + element of the list, GYP_DIRS may be accessed as a dictionary + (GYP_DIRS[foo]). The object this returns has attributes that need to be + set to further specify gyp processing: + - input, gives the path to the root gyp configuration file for that + object directory. + - variables, a dictionary containing variables and values to pass + to the gyp processor. + - sandbox_vars, a dictionary containing variables and values to + pass to the mozbuild processor on top of those derived from gyp + configuration. + - non_unified_sources, a list containing sources files, relative to + the current moz.build, that should be excluded from source file + unification. + + Typical use looks like: + GYP_DIRS += ['foo', 'bar'] + GYP_DIRS['foo'].input = 'foo/foo.gyp' + GYP_DIRS['foo'].variables = { + 'foo': 'bar', + (...) + } + (...) + """), + + 'SPHINX_TREES': (dict, dict, + """Describes what the Sphinx documentation tree will look like. + + Keys are relative directories inside the final Sphinx documentation + tree to install files into. Values are directories (relative to this + file) whose content to copy into the Sphinx documentation tree. + """), + + 'SPHINX_PYTHON_PACKAGE_DIRS': (StrictOrderingOnAppendList, list, + """Directories containing Python packages that Sphinx documents. + """), + + 'CFLAGS': (List, list, + """Flags passed to the C compiler for all of the C source files + declared in this directory. + + Note that the ordering of flags matters here, these flags will be + added to the compiler's command line in the same order as they + appear in the moz.build file. + """), + + 'CXXFLAGS': (List, list, + """Flags passed to the C++ compiler for all of the C++ source files + declared in this directory. + + Note that the ordering of flags matters here; these flags will be + added to the compiler's command line in the same order as they + appear in the moz.build file. + """), + + 'HOST_DEFINES': (InitializedDefines, dict, + """Dictionary of compiler defines to declare for host compilation. + See ``DEFINES`` for specifics. + """), + + 'CMFLAGS': (List, list, + """Flags passed to the Objective-C compiler for all of the Objective-C + source files declared in this directory. + + Note that the ordering of flags matters here; these flags will be + added to the compiler's command line in the same order as they + appear in the moz.build file. + """), + + 'CMMFLAGS': (List, list, + """Flags passed to the Objective-C++ compiler for all of the + Objective-C++ source files declared in this directory. + + Note that the ordering of flags matters here; these flags will be + added to the compiler's command line in the same order as they + appear in the moz.build file. + """), + + 'ASFLAGS': (List, list, + """Flags passed to the assembler for all of the assembly source files + declared in this directory. + + Note that the ordering of flags matters here; these flags will be + added to the assembler's command line in the same order as they + appear in the moz.build file. + """), + + 'HOST_CFLAGS': (List, list, + """Flags passed to the host C compiler for all of the C source files + declared in this directory. + + Note that the ordering of flags matters here, these flags will be + added to the compiler's command line in the same order as they + appear in the moz.build file. + """), + + 'HOST_CXXFLAGS': (List, list, + """Flags passed to the host C++ compiler for all of the C++ source files + declared in this directory. + + Note that the ordering of flags matters here; these flags will be + added to the compiler's command line in the same order as they + appear in the moz.build file. + """), + + 'LDFLAGS': (List, list, + """Flags passed to the linker when linking all of the libraries and + executables declared in this directory. + + Note that the ordering of flags matters here; these flags will be + added to the linker's command line in the same order as they + appear in the moz.build file. + """), + + 'EXTRA_DSO_LDOPTS': (List, list, + """Flags passed to the linker when linking a shared library. + + Note that the ordering of flags matter here, these flags will be + added to the linker's command line in the same order as they + appear in the moz.build file. + """), + + 'WIN32_EXE_LDFLAGS': (List, list, + """Flags passed to the linker when linking a Windows .exe executable + declared in this directory. + + Note that the ordering of flags matter here, these flags will be + added to the linker's command line in the same order as they + appear in the moz.build file. + + This variable only has an effect on Windows. + """), + + 'TEST_HARNESS_FILES': (ContextDerivedTypedHierarchicalStringList(Path), list, + """List of files to be installed for test harnesses. + + ``TEST_HARNESS_FILES`` can be used to install files to any directory + under $objdir/_tests. Files can be appended to a field to indicate + which subdirectory they should be exported to. For example, + to export ``foo.py`` to ``_tests/foo``, append to + ``TEST_HARNESS_FILES`` like so:: + TEST_HARNESS_FILES.foo += ['foo.py'] + + Files from topsrcdir and the objdir can also be installed by prefixing + the path(s) with a '/' character and a '!' character, respectively:: + TEST_HARNESS_FILES.path += ['/build/bar.py', '!quux.py'] + """), + + 'NO_EXPAND_LIBS': (bool, bool, + """Forces to build a real static library, and no corresponding fake + library. + """), + + 'NO_COMPONENTS_MANIFEST': (bool, bool, + """Do not create a binary-component manifest entry for the + corresponding XPCOMBinaryComponent. + """), + + 'USE_YASM': (bool, bool, + """Use the yasm assembler to assemble assembly files from SOURCES. + + By default, the build will use the toolchain assembler, $(AS), to + assemble source files in assembly language (.s or .asm files). Setting + this value to ``True`` will cause it to use yasm instead. + + If yasm is not available on this system, or does not support the + current target architecture, an error will be raised. + """), +} + +# Sanity check: we don't want any variable above to have a list as storage type. +for name, (storage_type, input_types, docs) in VARIABLES.items(): + if storage_type == list: + raise RuntimeError('%s has a "list" storage type. Use "List" instead.' + % name) + +# Set of variables that are only allowed in templates: +TEMPLATE_VARIABLES = { + 'CPP_UNIT_TESTS', + 'FORCE_SHARED_LIB', + 'HOST_PROGRAM', + 'HOST_LIBRARY_NAME', + 'HOST_SIMPLE_PROGRAMS', + 'IS_COMPONENT', + 'IS_FRAMEWORK', + 'LIBRARY_NAME', + 'PROGRAM', + 'SIMPLE_PROGRAMS', +} + +# Add a note to template variable documentation. +for name in TEMPLATE_VARIABLES: + if name not in VARIABLES: + raise RuntimeError('%s is in TEMPLATE_VARIABLES but not in VARIABLES.' + % name) + storage_type, input_types, docs = VARIABLES[name] + docs += 'This variable is only available in templates.\n' + VARIABLES[name] = (storage_type, input_types, docs) + + +# The set of functions exposed to the sandbox. +# +# Each entry is a tuple of: +# +# (function returning the corresponding function from a given sandbox, +# (argument types), docs) +# +# The first element is an attribute on Sandbox that should be a function type. +# +FUNCTIONS = { + 'include': (lambda self: self._include, (SourcePath,), + """Include another mozbuild file in the context of this one. + + This is similar to a ``#include`` in C languages. The filename passed to + the function will be read and its contents will be evaluated within the + context of the calling file. + + If a relative path is given, it is evaluated as relative to the file + currently being processed. If there is a chain of multiple include(), + the relative path computation is from the most recent/active file. + + If an absolute path is given, it is evaluated from ``TOPSRCDIR``. In + other words, ``include('/foo')`` references the path + ``TOPSRCDIR + '/foo'``. + + Example usage + ^^^^^^^^^^^^^ + + Include ``sibling.build`` from the current directory.:: + + include('sibling.build') + + Include ``foo.build`` from a path within the top source directory:: + + include('/elsewhere/foo.build') + """), + + 'add_java_jar': (lambda self: self._add_java_jar, (str,), + """Declare a Java JAR target to be built. + + This is the supported way to populate the JAVA_JAR_TARGETS + variable. + + The parameters are: + * dest - target name, without the trailing .jar. (required) + + This returns a rich Java JAR type, described at + :py:class:`mozbuild.frontend.data.JavaJarData`. + """), + + 'add_android_eclipse_project': ( + lambda self: self._add_android_eclipse_project, (str, str), + """Declare an Android Eclipse project. + + This is one of the supported ways to populate the + ANDROID_ECLIPSE_PROJECT_TARGETS variable. + + The parameters are: + * name - project name. + * manifest - path to AndroidManifest.xml. + + This returns a rich Android Eclipse project type, described at + :py:class:`mozbuild.frontend.data.AndroidEclipseProjectData`. + """), + + 'add_android_eclipse_library_project': ( + lambda self: self._add_android_eclipse_library_project, (str,), + """Declare an Android Eclipse library project. + + This is one of the supported ways to populate the + ANDROID_ECLIPSE_PROJECT_TARGETS variable. + + The parameters are: + * name - project name. + + This returns a rich Android Eclipse project type, described at + :py:class:`mozbuild.frontend.data.AndroidEclipseProjectData`. + """), + + 'export': (lambda self: self._export, (str,), + """Make the specified variable available to all child directories. + + The variable specified by the argument string is added to the + environment of all directories specified in the DIRS and TEST_DIRS + variables. If those directories themselves have child directories, + the variable will be exported to all of them. + + The value used for the variable is the final value at the end of the + moz.build file, so it is possible (but not recommended style) to place + the export before the definition of the variable. + + This function is limited to the upper-case variables that have special + meaning in moz.build files. + + NOTE: Please consult with a build peer before adding a new use of this + function. + + Example usage + ^^^^^^^^^^^^^ + + To make all children directories install as the given extension:: + + XPI_NAME = 'cool-extension' + export('XPI_NAME') + """), + + 'warning': (lambda self: self._warning, (str,), + """Issue a warning. + + Warnings are string messages that are printed during execution. + + Warnings are ignored during execution. + """), + + 'error': (lambda self: self._error, (str,), + """Issue a fatal error. + + If this function is called, processing is aborted immediately. + """), + + 'template': (lambda self: self._template_decorator, (FunctionType,), + """Decorator for template declarations. + + Templates are a special kind of functions that can be declared in + mozbuild files. Uppercase variables assigned in the function scope + are considered to be the result of the template. + + Contrary to traditional python functions: + - return values from template functions are ignored, + - template functions don't have access to the global scope. + + Example template + ^^^^^^^^^^^^^^^^ + + The following ``Program`` template sets two variables ``PROGRAM`` and + ``USE_LIBS``. ``PROGRAM`` is set to the argument given on the template + invocation, and ``USE_LIBS`` to contain "mozglue":: + + @template + def Program(name): + PROGRAM = name + USE_LIBS += ['mozglue'] + + Template invocation + ^^^^^^^^^^^^^^^^^^^ + + A template is invoked in the form of a function call:: + + Program('myprog') + + The result of the template, being all the uppercase variable it sets + is mixed to the existing set of variables defined in the mozbuild file + invoking the template:: + + FINAL_TARGET = 'dist/other' + USE_LIBS += ['mylib'] + Program('myprog') + USE_LIBS += ['otherlib'] + + The above mozbuild results in the following variables set: + + - ``FINAL_TARGET`` is 'dist/other' + - ``USE_LIBS`` is ['mylib', 'mozglue', 'otherlib'] + - ``PROGRAM`` is 'myprog' + + """), +} + + +TestDirsPlaceHolder = List() + + +# Special variables. These complement VARIABLES. +# +# Each entry is a tuple of: +# +# (function returning the corresponding value from a given context, type, docs) +# +SPECIAL_VARIABLES = { + 'TOPSRCDIR': (lambda context: context.config.topsrcdir, str, + """Constant defining the top source directory. + + The top source directory is the parent directory containing the source + code and all build files. It is typically the root directory of a + cloned repository. + """), + + 'TOPOBJDIR': (lambda context: context.config.topobjdir, str, + """Constant defining the top object directory. + + The top object directory is the parent directory which will contain + the output of the build. This is commonly referred to as "the object + directory." + """), + + 'RELATIVEDIR': (lambda context: context.relsrcdir, str, + """Constant defining the relative path of this file. + + The relative path is from ``TOPSRCDIR``. This is defined as relative + to the main file being executed, regardless of whether additional + files have been included using ``include()``. + """), + + 'SRCDIR': (lambda context: context.srcdir, str, + """Constant defining the source directory of this file. + + This is the path inside ``TOPSRCDIR`` where this file is located. It + is the same as ``TOPSRCDIR + RELATIVEDIR``. + """), + + 'OBJDIR': (lambda context: context.objdir, str, + """The path to the object directory for this file. + + Is is the same as ``TOPOBJDIR + RELATIVEDIR``. + """), + + 'CONFIG': (lambda context: ReadOnlyKeyedDefaultDict( + lambda key: context.config.substs_unicode.get(key)), dict, + """Dictionary containing the current configuration variables. + + All the variables defined by the configuration system are available + through this object. e.g. ``ENABLE_TESTS``, ``CFLAGS``, etc. + + Values in this container are read-only. Attempts at changing values + will result in a run-time error. + + Access to an unknown variable will return None. + """), + + 'EXTRA_COMPONENTS': (lambda context: context['FINAL_TARGET_FILES'].components._strings, list, + """Additional component files to distribute. + + This variable contains a list of files to copy into + ``$(FINAL_TARGET)/components/``. + """), + + 'EXTRA_PP_COMPONENTS': (lambda context: context['FINAL_TARGET_PP_FILES'].components._strings, list, + """Javascript XPCOM files. + + This variable contains a list of files to preprocess. Generated + files will be installed in the ``/components`` directory of the distribution. + """), + + 'JS_PREFERENCE_FILES': (lambda context: context['FINAL_TARGET_FILES'].defaults.pref._strings, list, + """Exported javascript files. + + A list of files copied into the dist directory for packaging and installation. + Path will be defined for gre or application prefs dir based on what is building. + """), + + 'JS_PREFERENCE_PP_FILES': (lambda context: context['FINAL_TARGET_PP_FILES'].defaults.pref._strings, list, + """Like JS_PREFERENCE_FILES, preprocessed.. + """), + + 'RESOURCE_FILES': (lambda context: context['FINAL_TARGET_FILES'].res, list, + """List of resources to be exported, and in which subdirectories. + + ``RESOURCE_FILES`` is used to list the resource files to be exported to + ``dist/bin/res``, but it can be used for other files as well. This variable + behaves as a list when appending filenames for resources in the top-level + directory. Files can also be appended to a field to indicate which + subdirectory they should be exported to. For example, to export + ``foo.res`` to the top-level directory, and ``bar.res`` to ``fonts/``, + append to ``RESOURCE_FILES`` like so:: + + RESOURCE_FILES += ['foo.res'] + RESOURCE_FILES.fonts += ['bar.res'] + """), + + 'EXTRA_JS_MODULES': (lambda context: context['FINAL_TARGET_FILES'].modules, list, + """Additional JavaScript files to distribute. + + This variable contains a list of files to copy into + ``$(FINAL_TARGET)/modules. + """), + + 'EXTRA_PP_JS_MODULES': (lambda context: context['FINAL_TARGET_PP_FILES'].modules, list, + """Additional JavaScript files to distribute. + + This variable contains a list of files to copy into + ``$(FINAL_TARGET)/modules``, after preprocessing. + """), + + 'TESTING_JS_MODULES': (lambda context: context['TEST_HARNESS_FILES'].modules, list, + """JavaScript modules to install in the test-only destination. + + Some JavaScript modules (JSMs) are test-only and not distributed + with Firefox. This variable defines them. + + To install modules in a subdirectory, use properties of this + variable to control the final destination. e.g. + + ``TESTING_JS_MODULES.foo += ['module.jsm']``. + """), + + 'TEST_DIRS': (lambda context: context['DIRS'] if context.config.substs.get('ENABLE_TESTS') + else TestDirsPlaceHolder, list, + """Like DIRS but only for directories that contain test-only code. + + If tests are not enabled, this variable will be ignored. + + This variable may go away once the transition away from Makefiles is + complete. + """), +} + +# Deprecation hints. +DEPRECATION_HINTS = { + 'CPP_UNIT_TESTS': ''' + Please use' + + CppUnitTests(['foo', 'bar']) + + instead of + + CPP_UNIT_TESTS += ['foo', 'bar'] + ''', + + 'HOST_PROGRAM': ''' + Please use + + HostProgram('foo') + + instead of + + HOST_PROGRAM = 'foo' + ''', + + 'HOST_LIBRARY_NAME': ''' + Please use + + HostLibrary('foo') + + instead of + + HOST_LIBRARY_NAME = 'foo' + ''', + + 'HOST_SIMPLE_PROGRAMS': ''' + Please use + + HostSimplePrograms(['foo', 'bar']) + + instead of + + HOST_SIMPLE_PROGRAMS += ['foo', 'bar']" + ''', + + 'LIBRARY_NAME': ''' + Please use + + Library('foo') + + instead of + + LIBRARY_NAME = 'foo' + ''', + + 'PROGRAM': ''' + Please use + + Program('foo') + + instead of + + PROGRAM = 'foo'" + ''', + + 'SIMPLE_PROGRAMS': ''' + Please use + + SimplePrograms(['foo', 'bar']) + + instead of + + SIMPLE_PROGRAMS += ['foo', 'bar']" + ''', + + 'FORCE_SHARED_LIB': ''' + Please use + + SharedLibrary('foo') + + instead of + + Library('foo') [ or LIBRARY_NAME = 'foo' ] + FORCE_SHARED_LIB = True + ''', + + 'IS_COMPONENT': ''' + Please use + + XPCOMBinaryComponent('foo') + + instead of + + Library('foo') [ or LIBRARY_NAME = 'foo' ] + IS_COMPONENT = True + ''', + + 'IS_FRAMEWORK': ''' + Please use + + Framework('foo') + + instead of + + Library('foo') [ or LIBRARY_NAME = 'foo' ] + IS_FRAMEWORK = True + ''', + + 'TOOL_DIRS': 'Please use the DIRS variable instead.', + + 'TEST_TOOL_DIRS': 'Please use the TEST_DIRS variable instead.', + + 'PARALLEL_DIRS': 'Please use the DIRS variable instead.', + + 'NO_DIST_INSTALL': ''' + Please use + + DIST_INSTALL = False + + instead of + + NO_DIST_INSTALL = True + ''', + + 'GENERATED_SOURCES': ''' + Please use + + SOURCES += [ '!foo.cpp' ] + + instead of + + GENERATED_SOURCES += [ 'foo.cpp'] + ''', + + 'GENERATED_INCLUDES': ''' + Please use + + LOCAL_INCLUDES += [ '!foo' ] + + instead of + + GENERATED_INCLUDES += [ 'foo' ] + ''', + + 'DIST_FILES': ''' + Please use + + FINAL_TARGET_PP_FILES += [ 'foo' ] + + instead of + + DIST_FILES += [ 'foo' ] + ''', +} + +# Make sure that all template variables have a deprecation hint. +for name in TEMPLATE_VARIABLES: + if name not in DEPRECATION_HINTS: + raise RuntimeError('Missing deprecation hint for %s' % name) diff --git a/python/mozbuild/mozbuild/frontend/data.py b/python/mozbuild/mozbuild/frontend/data.py new file mode 100644 index 000000000..fdf8cca17 --- /dev/null +++ b/python/mozbuild/mozbuild/frontend/data.py @@ -0,0 +1,1113 @@ +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, You can obtain one at http://mozilla.org/MPL/2.0/. + +r"""Data structures representing Mozilla's source tree. + +The frontend files are parsed into static data structures. These data +structures are defined in this module. + +All data structures of interest are children of the TreeMetadata class. + +Logic for populating these data structures is not defined in this class. +Instead, what we have here are dumb container classes. The emitter module +contains the code for converting executed mozbuild files into these data +structures. +""" + +from __future__ import absolute_import, unicode_literals + +from mozbuild.util import StrictOrderingOnAppendList +from mozpack.chrome.manifest import ManifestEntry + +import mozpack.path as mozpath +from .context import FinalTargetValue + +from ..util import ( + group_unified_files, +) + +from ..testing import ( + all_test_flavors, +) + + +class TreeMetadata(object): + """Base class for all data being captured.""" + __slots__ = () + + def to_dict(self): + return {k.lower(): getattr(self, k) for k in self.DICT_ATTRS} + + +class ContextDerived(TreeMetadata): + """Build object derived from a single Context instance. + + It holds fields common to all context derived classes. This class is likely + never instantiated directly but is instead derived from. + """ + + __slots__ = ( + 'context_main_path', + 'context_all_paths', + 'topsrcdir', + 'topobjdir', + 'relativedir', + 'srcdir', + 'objdir', + 'config', + '_context', + ) + + def __init__(self, context): + TreeMetadata.__init__(self) + + # Capture the files that were evaluated to fill this context. + self.context_main_path = context.main_path + self.context_all_paths = context.all_paths + + # Basic directory state. + self.topsrcdir = context.config.topsrcdir + self.topobjdir = context.config.topobjdir + + self.relativedir = context.relsrcdir + self.srcdir = context.srcdir + self.objdir = context.objdir + + self.config = context.config + + self._context = context + + @property + def install_target(self): + return self._context['FINAL_TARGET'] + + @property + def defines(self): + defines = self._context['DEFINES'] + return Defines(self._context, defines) if defines else None + + @property + def relobjdir(self): + return mozpath.relpath(self.objdir, self.topobjdir) + + +class HostMixin(object): + @property + def defines(self): + defines = self._context['HOST_DEFINES'] + return HostDefines(self._context, defines) if defines else None + + +class DirectoryTraversal(ContextDerived): + """Describes how directory traversal for building should work. + + This build object is likely only of interest to the recursive make backend. + Other build backends should (ideally) not attempt to mimic the behavior of + the recursive make backend. The only reason this exists is to support the + existing recursive make backend while the transition to mozbuild frontend + files is complete and we move to a more optimal build backend. + + Fields in this class correspond to similarly named variables in the + frontend files. + """ + __slots__ = ( + 'dirs', + ) + + def __init__(self, context): + ContextDerived.__init__(self, context) + + self.dirs = [] + + +class BaseConfigSubstitution(ContextDerived): + """Base class describing autogenerated files as part of config.status.""" + + __slots__ = ( + 'input_path', + 'output_path', + 'relpath', + ) + + def __init__(self, context): + ContextDerived.__init__(self, context) + + self.input_path = None + self.output_path = None + self.relpath = None + + +class ConfigFileSubstitution(BaseConfigSubstitution): + """Describes a config file that will be generated using substitutions.""" + + +class VariablePassthru(ContextDerived): + """A dict of variables to pass through to backend.mk unaltered. + + The purpose of this object is to facilitate rapid transitioning of + variables from Makefile.in to moz.build. In the ideal world, this class + does not exist and every variable has a richer class representing it. + As long as we rely on this class, we lose the ability to have flexibility + in our build backends since we will continue to be tied to our rules.mk. + """ + __slots__ = ('variables') + + def __init__(self, context): + ContextDerived.__init__(self, context) + self.variables = {} + +class XPIDLFile(ContextDerived): + """Describes an XPIDL file to be compiled.""" + + __slots__ = ( + 'source_path', + 'basename', + 'module', + 'add_to_manifest', + ) + + def __init__(self, context, source, module, add_to_manifest): + ContextDerived.__init__(self, context) + + self.source_path = source + self.basename = mozpath.basename(source) + self.module = module + self.add_to_manifest = add_to_manifest + +class BaseDefines(ContextDerived): + """Context derived container object for DEFINES/HOST_DEFINES, + which are OrderedDicts. + """ + __slots__ = ('defines') + + def __init__(self, context, defines): + ContextDerived.__init__(self, context) + self.defines = defines + + def get_defines(self): + for define, value in self.defines.iteritems(): + if value is True: + yield('-D%s' % define) + elif value is False: + yield('-U%s' % define) + else: + yield('-D%s=%s' % (define, value)) + + def update(self, more_defines): + if isinstance(more_defines, Defines): + self.defines.update(more_defines.defines) + else: + self.defines.update(more_defines) + +class Defines(BaseDefines): + pass + +class HostDefines(BaseDefines): + pass + +class IPDLFile(ContextDerived): + """Describes an individual .ipdl source file.""" + + __slots__ = ( + 'basename', + ) + + def __init__(self, context, path): + ContextDerived.__init__(self, context) + + self.basename = path + +class WebIDLFile(ContextDerived): + """Describes an individual .webidl source file.""" + + __slots__ = ( + 'basename', + ) + + def __init__(self, context, path): + ContextDerived.__init__(self, context) + + self.basename = path + +class GeneratedEventWebIDLFile(ContextDerived): + """Describes an individual .webidl source file.""" + + __slots__ = ( + 'basename', + ) + + def __init__(self, context, path): + ContextDerived.__init__(self, context) + + self.basename = path + +class TestWebIDLFile(ContextDerived): + """Describes an individual test-only .webidl source file.""" + + __slots__ = ( + 'basename', + ) + + def __init__(self, context, path): + ContextDerived.__init__(self, context) + + self.basename = path + +class PreprocessedTestWebIDLFile(ContextDerived): + """Describes an individual test-only .webidl source file that requires + preprocessing.""" + + __slots__ = ( + 'basename', + ) + + def __init__(self, context, path): + ContextDerived.__init__(self, context) + + self.basename = path + +class PreprocessedWebIDLFile(ContextDerived): + """Describes an individual .webidl source file that requires preprocessing.""" + + __slots__ = ( + 'basename', + ) + + def __init__(self, context, path): + ContextDerived.__init__(self, context) + + self.basename = path + +class GeneratedWebIDLFile(ContextDerived): + """Describes an individual .webidl source file that is generated from + build rules.""" + + __slots__ = ( + 'basename', + ) + + def __init__(self, context, path): + ContextDerived.__init__(self, context) + + self.basename = path + + +class ExampleWebIDLInterface(ContextDerived): + """An individual WebIDL interface to generate.""" + + __slots__ = ( + 'name', + ) + + def __init__(self, context, name): + ContextDerived.__init__(self, context) + + self.name = name + + +class LinkageWrongKindError(Exception): + """Error thrown when trying to link objects of the wrong kind""" + + +class LinkageMultipleRustLibrariesError(Exception): + """Error thrown when trying to link multiple Rust libraries to an object""" + + +class Linkable(ContextDerived): + """Generic context derived container object for programs and libraries""" + __slots__ = ( + 'cxx_link', + 'lib_defines', + 'linked_libraries', + 'linked_system_libs', + ) + + def __init__(self, context): + ContextDerived.__init__(self, context) + self.cxx_link = False + self.linked_libraries = [] + self.linked_system_libs = [] + self.lib_defines = Defines(context, {}) + + def link_library(self, obj): + assert isinstance(obj, BaseLibrary) + if isinstance(obj, SharedLibrary) and obj.variant == obj.COMPONENT: + raise LinkageWrongKindError( + 'Linkable.link_library() does not take components.') + if obj.KIND != self.KIND: + raise LinkageWrongKindError('%s != %s' % (obj.KIND, self.KIND)) + # Linking multiple Rust libraries into an object would result in + # multiple copies of the Rust standard library, as well as linking + # errors from duplicate symbols. + if isinstance(obj, RustLibrary) and any(isinstance(l, RustLibrary) + for l in self.linked_libraries): + raise LinkageMultipleRustLibrariesError("Cannot link multiple Rust libraries into %s", + self) + self.linked_libraries.append(obj) + if obj.cxx_link: + self.cxx_link = True + obj.refs.append(self) + + def link_system_library(self, lib): + # The '$' check is here as a special temporary rule, allowing the + # inherited use of make variables, most notably in TK_LIBS. + if not lib.startswith('$') and not lib.startswith('-'): + if self.config.substs.get('GNU_CC'): + lib = '-l%s' % lib + else: + lib = '%s%s%s' % ( + self.config.import_prefix, + lib, + self.config.import_suffix, + ) + self.linked_system_libs.append(lib) + +class BaseProgram(Linkable): + """Context derived container object for programs, which is a unicode + string. + + This class handles automatically appending a binary suffix to the program + name. + If the suffix is not defined, the program name is unchanged. + Otherwise, if the program name ends with the given suffix, it is unchanged + Otherwise, the suffix is appended to the program name. + """ + __slots__ = ('program') + + DICT_ATTRS = { + 'install_target', + 'KIND', + 'program', + 'relobjdir', + } + + def __init__(self, context, program, is_unit_test=False): + Linkable.__init__(self, context) + + bin_suffix = context.config.substs.get(self.SUFFIX_VAR, '') + if not program.endswith(bin_suffix): + program += bin_suffix + self.program = program + self.is_unit_test = is_unit_test + + def __repr__(self): + return '<%s: %s/%s>' % (type(self).__name__, self.relobjdir, self.program) + + +class Program(BaseProgram): + """Context derived container object for PROGRAM""" + SUFFIX_VAR = 'BIN_SUFFIX' + KIND = 'target' + + +class HostProgram(HostMixin, BaseProgram): + """Context derived container object for HOST_PROGRAM""" + SUFFIX_VAR = 'HOST_BIN_SUFFIX' + KIND = 'host' + + +class SimpleProgram(BaseProgram): + """Context derived container object for each program in SIMPLE_PROGRAMS""" + SUFFIX_VAR = 'BIN_SUFFIX' + KIND = 'target' + + +class HostSimpleProgram(HostMixin, BaseProgram): + """Context derived container object for each program in + HOST_SIMPLE_PROGRAMS""" + SUFFIX_VAR = 'HOST_BIN_SUFFIX' + KIND = 'host' + + +class BaseLibrary(Linkable): + """Generic context derived container object for libraries.""" + __slots__ = ( + 'basename', + 'lib_name', + 'import_name', + 'refs', + ) + + def __init__(self, context, basename): + Linkable.__init__(self, context) + + self.basename = self.lib_name = basename + if self.lib_name: + self.lib_name = '%s%s%s' % ( + context.config.lib_prefix, + self.lib_name, + context.config.lib_suffix + ) + self.import_name = self.lib_name + + self.refs = [] + + def __repr__(self): + return '<%s: %s/%s>' % (type(self).__name__, self.relobjdir, self.lib_name) + + +class Library(BaseLibrary): + """Context derived container object for a library""" + KIND = 'target' + __slots__ = ( + 'is_sdk', + ) + + def __init__(self, context, basename, real_name=None, is_sdk=False): + BaseLibrary.__init__(self, context, real_name or basename) + self.basename = basename + self.is_sdk = is_sdk + + +class StaticLibrary(Library): + """Context derived container object for a static library""" + __slots__ = ( + 'link_into', + 'no_expand_lib', + ) + + def __init__(self, context, basename, real_name=None, is_sdk=False, + link_into=None, no_expand_lib=False): + Library.__init__(self, context, basename, real_name, is_sdk) + self.link_into = link_into + self.no_expand_lib = no_expand_lib + + +class RustLibrary(StaticLibrary): + """Context derived container object for a static library""" + __slots__ = ( + 'cargo_file', + 'crate_type', + 'dependencies', + 'deps_path', + ) + + def __init__(self, context, basename, cargo_file, crate_type, dependencies, **args): + StaticLibrary.__init__(self, context, basename, **args) + self.cargo_file = cargo_file + self.crate_type = crate_type + # We need to adjust our naming here because cargo replaces '-' in + # package names defined in Cargo.toml with underscores in actual + # filenames. But we need to keep the basename consistent because + # many other things in the build system depend on that. + assert self.crate_type == 'staticlib' + self.lib_name = '%s%s%s' % (context.config.lib_prefix, + basename.replace('-', '_'), + context.config.lib_suffix) + self.dependencies = dependencies + # cargo creates several directories and places its build artifacts + # in those directories. The directory structure depends not only + # on the target, but also what sort of build we are doing. + rust_build_kind = 'release' + if context.config.substs.get('MOZ_DEBUG'): + rust_build_kind = 'debug' + build_dir = mozpath.join(context.config.substs['RUST_TARGET'], + rust_build_kind) + self.import_name = mozpath.join(build_dir, self.lib_name) + self.deps_path = mozpath.join(build_dir, 'deps') + + +class SharedLibrary(Library): + """Context derived container object for a shared library""" + __slots__ = ( + 'soname', + 'variant', + 'symbols_file', + ) + + DICT_ATTRS = { + 'basename', + 'import_name', + 'install_target', + 'lib_name', + 'relobjdir', + 'soname', + } + + FRAMEWORK = 1 + COMPONENT = 2 + MAX_VARIANT = 3 + + def __init__(self, context, basename, real_name=None, is_sdk=False, + soname=None, variant=None, symbols_file=False): + assert(variant in range(1, self.MAX_VARIANT) or variant is None) + Library.__init__(self, context, basename, real_name, is_sdk) + self.variant = variant + self.lib_name = real_name or basename + assert self.lib_name + + if variant == self.FRAMEWORK: + self.import_name = self.lib_name + else: + self.import_name = '%s%s%s' % ( + context.config.import_prefix, + self.lib_name, + context.config.import_suffix, + ) + self.lib_name = '%s%s%s' % ( + context.config.dll_prefix, + self.lib_name, + context.config.dll_suffix, + ) + if soname: + self.soname = '%s%s%s' % ( + context.config.dll_prefix, + soname, + context.config.dll_suffix, + ) + else: + self.soname = self.lib_name + + if symbols_file is False: + # No symbols file. + self.symbols_file = None + elif symbols_file is True: + # Symbols file with default name. + if context.config.substs['OS_TARGET'] == 'WINNT': + self.symbols_file = '%s.def' % self.lib_name + else: + self.symbols_file = '%s.symbols' % self.lib_name + else: + # Explicitly provided name. + self.symbols_file = symbols_file + + + +class ExternalLibrary(object): + """Empty mixin for libraries built by an external build system.""" + + +class ExternalStaticLibrary(StaticLibrary, ExternalLibrary): + """Context derived container for static libraries built by an external + build system.""" + + +class ExternalSharedLibrary(SharedLibrary, ExternalLibrary): + """Context derived container for shared libraries built by an external + build system.""" + + +class HostLibrary(HostMixin, BaseLibrary): + """Context derived container object for a host library""" + KIND = 'host' + + +class TestManifest(ContextDerived): + """Represents a manifest file containing information about tests.""" + + __slots__ = ( + # The type of test manifest this is. + 'flavor', + + # Maps source filename to destination filename. The destination + # path is relative from the tests root directory. Values are 2-tuples + # of (destpath, is_test_file) where the 2nd item is True if this + # item represents a test file (versus a support file). + 'installs', + + # A list of pattern matching installs to perform. Entries are + # (base, pattern, dest). + 'pattern_installs', + + # Where all files for this manifest flavor are installed in the unified + # test package directory. + 'install_prefix', + + # Set of files provided by an external mechanism. + 'external_installs', + + # Set of files required by multiple test directories, whose installation + # will be resolved when running tests. + 'deferred_installs', + + # The full path of this manifest file. + 'path', + + # The directory where this manifest is defined. + 'directory', + + # The parsed manifestparser.TestManifest instance. + 'manifest', + + # List of tests. Each element is a dict of metadata. + 'tests', + + # The relative path of the parsed manifest within the srcdir. + 'manifest_relpath', + + # The relative path of the parsed manifest within the objdir. + 'manifest_obj_relpath', + + # If this manifest is a duplicate of another one, this is the + # manifestparser.TestManifest of the other one. + 'dupe_manifest', + ) + + def __init__(self, context, path, manifest, flavor=None, + install_prefix=None, relpath=None, dupe_manifest=False): + ContextDerived.__init__(self, context) + + assert flavor in all_test_flavors() + + self.path = path + self.directory = mozpath.dirname(path) + self.manifest = manifest + self.flavor = flavor + self.install_prefix = install_prefix + self.manifest_relpath = relpath + self.manifest_obj_relpath = relpath + self.dupe_manifest = dupe_manifest + self.installs = {} + self.pattern_installs = [] + self.tests = [] + self.external_installs = set() + self.deferred_installs = set() + + +class LocalInclude(ContextDerived): + """Describes an individual local include path.""" + + __slots__ = ( + 'path', + ) + + def __init__(self, context, path): + ContextDerived.__init__(self, context) + + self.path = path + + +class PerSourceFlag(ContextDerived): + """Describes compiler flags specified for individual source files.""" + + __slots__ = ( + 'file_name', + 'flags', + ) + + def __init__(self, context, file_name, flags): + ContextDerived.__init__(self, context) + + self.file_name = file_name + self.flags = flags + + +class JARManifest(ContextDerived): + """Describes an individual JAR manifest file and how to process it. + + This class isn't very useful for optimizing backends yet because we don't + capture defines. We can't capture defines safely until all of them are + defined in moz.build and not Makefile.in files. + """ + __slots__ = ( + 'path', + ) + + def __init__(self, context, path): + ContextDerived.__init__(self, context) + + self.path = path + + +class ContextWrapped(ContextDerived): + """Generic context derived container object for a wrapped rich object. + + Use this wrapper class to shuttle a rich build system object + completely defined in moz.build files through the tree metadata + emitter to the build backend for processing as-is. + """ + + __slots__ = ( + 'wrapped', + ) + + def __init__(self, context, wrapped): + ContextDerived.__init__(self, context) + + self.wrapped = wrapped + + +class JavaJarData(object): + """Represents a Java JAR file. + + A Java JAR has the following members: + * sources - strictly ordered list of input java sources + * generated_sources - strictly ordered list of generated input + java sources + * extra_jars - list of JAR file dependencies to include on the + javac compiler classpath + * javac_flags - list containing extra flags passed to the + javac compiler + """ + + __slots__ = ( + 'name', + 'sources', + 'generated_sources', + 'extra_jars', + 'javac_flags', + ) + + def __init__(self, name, sources=[], generated_sources=[], + extra_jars=[], javac_flags=[]): + self.name = name + self.sources = StrictOrderingOnAppendList(sources) + self.generated_sources = StrictOrderingOnAppendList(generated_sources) + self.extra_jars = list(extra_jars) + self.javac_flags = list(javac_flags) + + +class BaseSources(ContextDerived): + """Base class for files to be compiled during the build.""" + + __slots__ = ( + 'files', + 'canonical_suffix', + ) + + def __init__(self, context, files, canonical_suffix): + ContextDerived.__init__(self, context) + + self.files = files + self.canonical_suffix = canonical_suffix + + +class Sources(BaseSources): + """Represents files to be compiled during the build.""" + + def __init__(self, context, files, canonical_suffix): + BaseSources.__init__(self, context, files, canonical_suffix) + + +class GeneratedSources(BaseSources): + """Represents generated files to be compiled during the build.""" + + def __init__(self, context, files, canonical_suffix): + BaseSources.__init__(self, context, files, canonical_suffix) + + +class HostSources(HostMixin, BaseSources): + """Represents files to be compiled for the host during the build.""" + + def __init__(self, context, files, canonical_suffix): + BaseSources.__init__(self, context, files, canonical_suffix) + + +class UnifiedSources(BaseSources): + """Represents files to be compiled in a unified fashion during the build.""" + + __slots__ = ( + 'have_unified_mapping', + 'unified_source_mapping' + ) + + def __init__(self, context, files, canonical_suffix, files_per_unified_file=16): + BaseSources.__init__(self, context, files, canonical_suffix) + + self.have_unified_mapping = files_per_unified_file > 1 + + if self.have_unified_mapping: + # Sorted so output is consistent and we don't bump mtimes. + source_files = list(sorted(self.files)) + + # On Windows, path names have a maximum length of 255 characters, + # so avoid creating extremely long path names. + unified_prefix = context.relsrcdir + if len(unified_prefix) > 20: + unified_prefix = unified_prefix[-20:].split('/', 1)[-1] + unified_prefix = unified_prefix.replace('/', '_') + + suffix = self.canonical_suffix[1:] + unified_prefix='Unified_%s_%s' % (suffix, unified_prefix) + self.unified_source_mapping = list(group_unified_files(source_files, + unified_prefix=unified_prefix, + unified_suffix=suffix, + files_per_unified_file=files_per_unified_file)) + + +class InstallationTarget(ContextDerived): + """Describes the rules that affect where files get installed to.""" + + __slots__ = ( + 'xpiname', + 'subdir', + 'target', + 'enabled' + ) + + def __init__(self, context): + ContextDerived.__init__(self, context) + + self.xpiname = context.get('XPI_NAME', '') + self.subdir = context.get('DIST_SUBDIR', '') + self.target = context['FINAL_TARGET'] + self.enabled = context['DIST_INSTALL'] is not False + + def is_custom(self): + """Returns whether or not the target is not derived from the default + given xpiname and subdir.""" + + return FinalTargetValue(dict( + XPI_NAME=self.xpiname, + DIST_SUBDIR=self.subdir)) == self.target + + +class FinalTargetFiles(ContextDerived): + """Sandbox container object for FINAL_TARGET_FILES, which is a + HierarchicalStringList. + + We need an object derived from ContextDerived for use in the backend, so + this object fills that role. It just has a reference to the underlying + HierarchicalStringList, which is created when parsing FINAL_TARGET_FILES. + """ + __slots__ = ('files') + + def __init__(self, sandbox, files): + ContextDerived.__init__(self, sandbox) + self.files = files + + +class FinalTargetPreprocessedFiles(ContextDerived): + """Sandbox container object for FINAL_TARGET_PP_FILES, which is a + HierarchicalStringList. + + We need an object derived from ContextDerived for use in the backend, so + this object fills that role. It just has a reference to the underlying + HierarchicalStringList, which is created when parsing + FINAL_TARGET_PP_FILES. + """ + __slots__ = ('files') + + def __init__(self, sandbox, files): + ContextDerived.__init__(self, sandbox) + self.files = files + + +class ObjdirFiles(ContextDerived): + """Sandbox container object for OBJDIR_FILES, which is a + HierarchicalStringList. + """ + __slots__ = ('files') + + def __init__(self, sandbox, files): + ContextDerived.__init__(self, sandbox) + self.files = files + + @property + def install_target(self): + return '' + + +class ObjdirPreprocessedFiles(ContextDerived): + """Sandbox container object for OBJDIR_PP_FILES, which is a + HierarchicalStringList. + """ + __slots__ = ('files') + + def __init__(self, sandbox, files): + ContextDerived.__init__(self, sandbox) + self.files = files + + @property + def install_target(self): + return '' + + +class TestHarnessFiles(FinalTargetFiles): + """Sandbox container object for TEST_HARNESS_FILES, + which is a HierarchicalStringList. + """ + @property + def install_target(self): + return '_tests' + + +class Exports(FinalTargetFiles): + """Context derived container object for EXPORTS, which is a + HierarchicalStringList. + + We need an object derived from ContextDerived for use in the backend, so + this object fills that role. It just has a reference to the underlying + HierarchicalStringList, which is created when parsing EXPORTS. + """ + @property + def install_target(self): + return 'dist/include' + + +class BrandingFiles(FinalTargetFiles): + """Sandbox container object for BRANDING_FILES, which is a + HierarchicalStringList. + + We need an object derived from ContextDerived for use in the backend, so + this object fills that role. It just has a reference to the underlying + HierarchicalStringList, which is created when parsing BRANDING_FILES. + """ + @property + def install_target(self): + return 'dist/branding' + + +class SdkFiles(FinalTargetFiles): + """Sandbox container object for SDK_FILES, which is a + HierarchicalStringList. + + We need an object derived from ContextDerived for use in the backend, so + this object fills that role. It just has a reference to the underlying + HierarchicalStringList, which is created when parsing SDK_FILES. + """ + @property + def install_target(self): + return 'dist/sdk' + + +class GeneratedFile(ContextDerived): + """Represents a generated file.""" + + __slots__ = ( + 'script', + 'method', + 'outputs', + 'inputs', + 'flags', + ) + + def __init__(self, context, script, method, outputs, inputs, flags=()): + ContextDerived.__init__(self, context) + self.script = script + self.method = method + self.outputs = outputs if isinstance(outputs, tuple) else (outputs,) + self.inputs = inputs + self.flags = flags + + +class ClassPathEntry(object): + """Represents a classpathentry in an Android Eclipse project.""" + + __slots__ = ( + 'dstdir', + 'srcdir', + 'path', + 'exclude_patterns', + 'ignore_warnings', + ) + + def __init__(self): + self.dstdir = None + self.srcdir = None + self.path = None + self.exclude_patterns = [] + self.ignore_warnings = False + + +class AndroidEclipseProjectData(object): + """Represents an Android Eclipse project.""" + + __slots__ = ( + 'name', + 'package_name', + 'is_library', + 'res', + 'assets', + 'libs', + 'manifest', + 'recursive_make_targets', + 'extra_jars', + 'included_projects', + 'referenced_projects', + '_classpathentries', + 'filtered_resources', + ) + + def __init__(self, name): + self.name = name + self.is_library = False + self.manifest = None + self.res = None + self.assets = None + self.libs = [] + self.recursive_make_targets = [] + self.extra_jars = [] + self.included_projects = [] + self.referenced_projects = [] + self._classpathentries = [] + self.filtered_resources = [] + + def add_classpathentry(self, path, srcdir, dstdir, exclude_patterns=[], ignore_warnings=False): + cpe = ClassPathEntry() + cpe.srcdir = srcdir + cpe.dstdir = dstdir + cpe.path = path + cpe.exclude_patterns = list(exclude_patterns) + cpe.ignore_warnings = ignore_warnings + self._classpathentries.append(cpe) + return cpe + + +class AndroidResDirs(ContextDerived): + """Represents Android resource directories.""" + + __slots__ = ( + 'paths', + ) + + def __init__(self, context, paths): + ContextDerived.__init__(self, context) + self.paths = paths + +class AndroidAssetsDirs(ContextDerived): + """Represents Android assets directories.""" + + __slots__ = ( + 'paths', + ) + + def __init__(self, context, paths): + ContextDerived.__init__(self, context) + self.paths = paths + +class AndroidExtraResDirs(ContextDerived): + """Represents Android extra resource directories. + + Extra resources are resources provided by libraries and including in a + packaged APK, but not otherwise redistributed. In practice, this means + resources included in Fennec but not in GeckoView. + """ + + __slots__ = ( + 'paths', + ) + + def __init__(self, context, paths): + ContextDerived.__init__(self, context) + self.paths = paths + +class AndroidExtraPackages(ContextDerived): + """Represents Android extra packages.""" + + __slots__ = ( + 'packages', + ) + + def __init__(self, context, packages): + ContextDerived.__init__(self, context) + self.packages = packages + +class ChromeManifestEntry(ContextDerived): + """Represents a chrome.manifest entry.""" + + __slots__ = ( + 'path', + 'entry', + ) + + def __init__(self, context, manifest_path, entry): + ContextDerived.__init__(self, context) + assert isinstance(entry, ManifestEntry) + self.path = mozpath.join(self.install_target, manifest_path) + # Ensure the entry is relative to the directory containing the + # manifest path. + entry = entry.rebase(mozpath.dirname(manifest_path)) + # Then add the install_target to the entry base directory. + self.entry = entry.move(mozpath.dirname(self.path)) diff --git a/python/mozbuild/mozbuild/frontend/emitter.py b/python/mozbuild/mozbuild/frontend/emitter.py new file mode 100644 index 000000000..52f571867 --- /dev/null +++ b/python/mozbuild/mozbuild/frontend/emitter.py @@ -0,0 +1,1416 @@ +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, You can obtain one at http://mozilla.org/MPL/2.0/. + +from __future__ import absolute_import, unicode_literals + +import itertools +import logging +import os +import traceback +import sys +import time + +from collections import defaultdict, OrderedDict +from mach.mixin.logging import LoggingMixin +from mozbuild.util import ( + memoize, + OrderedDefaultDict, +) + +import mozpack.path as mozpath +import mozinfo +import pytoml + +from .data import ( + AndroidAssetsDirs, + AndroidExtraPackages, + AndroidExtraResDirs, + AndroidResDirs, + BaseSources, + BrandingFiles, + ChromeManifestEntry, + ConfigFileSubstitution, + ContextWrapped, + Defines, + DirectoryTraversal, + Exports, + FinalTargetFiles, + FinalTargetPreprocessedFiles, + GeneratedEventWebIDLFile, + GeneratedFile, + GeneratedSources, + GeneratedWebIDLFile, + ExampleWebIDLInterface, + ExternalStaticLibrary, + ExternalSharedLibrary, + HostDefines, + HostLibrary, + HostProgram, + HostSimpleProgram, + HostSources, + InstallationTarget, + IPDLFile, + JARManifest, + Library, + Linkable, + LocalInclude, + ObjdirFiles, + ObjdirPreprocessedFiles, + PerSourceFlag, + PreprocessedTestWebIDLFile, + PreprocessedWebIDLFile, + Program, + RustLibrary, + SdkFiles, + SharedLibrary, + SimpleProgram, + Sources, + StaticLibrary, + TestHarnessFiles, + TestWebIDLFile, + TestManifest, + UnifiedSources, + VariablePassthru, + WebIDLFile, + XPIDLFile, +) +from mozpack.chrome.manifest import ( + ManifestBinaryComponent, + Manifest, +) + +from .reader import SandboxValidationError + +from ..testing import ( + TEST_MANIFESTS, + REFTEST_FLAVORS, + WEB_PLATFORM_TESTS_FLAVORS, + SupportFilesConverter, +) + +from .context import ( + Context, + SourcePath, + ObjDirPath, + Path, + SubContext, + TemplateContext, +) + +from mozbuild.base import ExecutionSummary + + +class TreeMetadataEmitter(LoggingMixin): + """Converts the executed mozbuild files into data structures. + + This is a bridge between reader.py and data.py. It takes what was read by + reader.BuildReader and converts it into the classes defined in the data + module. + """ + + def __init__(self, config): + self.populate_logger() + + self.config = config + + mozinfo.find_and_update_from_json(config.topobjdir) + + # Python 2.6 doesn't allow unicode keys to be used for keyword + # arguments. This gross hack works around the problem until we + # rid ourselves of 2.6. + self.info = {} + for k, v in mozinfo.info.items(): + if isinstance(k, unicode): + k = k.encode('ascii') + self.info[k] = v + + self._libs = OrderedDefaultDict(list) + self._binaries = OrderedDict() + self._linkage = [] + self._static_linking_shared = set() + self._crate_verified_local = set() + self._crate_directories = dict() + + # Keep track of external paths (third party build systems), starting + # from what we run a subconfigure in. We'll eliminate some directories + # as we traverse them with moz.build (e.g. js/src). + subconfigures = os.path.join(self.config.topobjdir, 'subconfigures') + paths = [] + if os.path.exists(subconfigures): + paths = open(subconfigures).read().splitlines() + self._external_paths = set(mozpath.normsep(d) for d in paths) + # Add security/nss manually, since it doesn't have a subconfigure. + self._external_paths.add('security/nss') + + self._emitter_time = 0.0 + self._object_count = 0 + self._test_files_converter = SupportFilesConverter() + + def summary(self): + return ExecutionSummary( + 'Processed into {object_count:d} build config descriptors in ' + '{execution_time:.2f}s', + execution_time=self._emitter_time, + object_count=self._object_count) + + def emit(self, output): + """Convert the BuildReader output into data structures. + + The return value from BuildReader.read_topsrcdir() (a generator) is + typically fed into this function. + """ + contexts = {} + + def emit_objs(objs): + for o in objs: + self._object_count += 1 + yield o + + for out in output: + # Nothing in sub-contexts is currently of interest to us. Filter + # them all out. + if isinstance(out, SubContext): + continue + + if isinstance(out, Context): + # Keep all contexts around, we will need them later. + contexts[out.objdir] = out + + start = time.time() + # We need to expand the generator for the timings to work. + objs = list(self.emit_from_context(out)) + self._emitter_time += time.time() - start + + for o in emit_objs(objs): yield o + + else: + raise Exception('Unhandled output type: %s' % type(out)) + + # Don't emit Linkable objects when COMPILE_ENVIRONMENT is not set + if self.config.substs.get('COMPILE_ENVIRONMENT'): + start = time.time() + objs = list(self._emit_libs_derived(contexts)) + self._emitter_time += time.time() - start + + for o in emit_objs(objs): yield o + + def _emit_libs_derived(self, contexts): + # First do FINAL_LIBRARY linkage. + for lib in (l for libs in self._libs.values() for l in libs): + if not isinstance(lib, (StaticLibrary, RustLibrary)) or not lib.link_into: + continue + if lib.link_into not in self._libs: + raise SandboxValidationError( + 'FINAL_LIBRARY ("%s") does not match any LIBRARY_NAME' + % lib.link_into, contexts[lib.objdir]) + candidates = self._libs[lib.link_into] + + # When there are multiple candidates, but all are in the same + # directory and have a different type, we want all of them to + # have the library linked. The typical usecase is when building + # both a static and a shared library in a directory, and having + # that as a FINAL_LIBRARY. + if len(set(type(l) for l in candidates)) == len(candidates) and \ + len(set(l.objdir for l in candidates)) == 1: + for c in candidates: + c.link_library(lib) + else: + raise SandboxValidationError( + 'FINAL_LIBRARY ("%s") matches a LIBRARY_NAME defined in ' + 'multiple places:\n %s' % (lib.link_into, + '\n '.join(l.objdir for l in candidates)), + contexts[lib.objdir]) + + # Next, USE_LIBS linkage. + for context, obj, variable in self._linkage: + self._link_libraries(context, obj, variable) + + def recurse_refs(lib): + for o in lib.refs: + yield o + if isinstance(o, StaticLibrary): + for q in recurse_refs(o): + yield q + + # Check that all static libraries refering shared libraries in + # USE_LIBS are linked into a shared library or program. + for lib in self._static_linking_shared: + if all(isinstance(o, StaticLibrary) for o in recurse_refs(lib)): + shared_libs = sorted(l.basename for l in lib.linked_libraries + if isinstance(l, SharedLibrary)) + raise SandboxValidationError( + 'The static "%s" library is not used in a shared library ' + 'or a program, but USE_LIBS contains the following shared ' + 'library names:\n %s\n\nMaybe you can remove the ' + 'static "%s" library?' % (lib.basename, + '\n '.join(shared_libs), lib.basename), + contexts[lib.objdir]) + + # Propagate LIBRARY_DEFINES to all child libraries recursively. + def propagate_defines(outerlib, defines): + outerlib.lib_defines.update(defines) + for lib in outerlib.linked_libraries: + # Propagate defines only along FINAL_LIBRARY paths, not USE_LIBS + # paths. + if (isinstance(lib, StaticLibrary) and + lib.link_into == outerlib.basename): + propagate_defines(lib, defines) + + for lib in (l for libs in self._libs.values() for l in libs): + if isinstance(lib, Library): + propagate_defines(lib, lib.lib_defines) + yield lib + + for obj in self._binaries.values(): + yield obj + + LIBRARY_NAME_VAR = { + 'host': 'HOST_LIBRARY_NAME', + 'target': 'LIBRARY_NAME', + } + + def _link_libraries(self, context, obj, variable): + """Add linkage declarations to a given object.""" + assert isinstance(obj, Linkable) + + for path in context.get(variable, []): + force_static = path.startswith('static:') and obj.KIND == 'target' + if force_static: + path = path[7:] + name = mozpath.basename(path) + dir = mozpath.dirname(path) + candidates = [l for l in self._libs[name] if l.KIND == obj.KIND] + if dir: + if dir.startswith('/'): + dir = mozpath.normpath( + mozpath.join(obj.topobjdir, dir[1:])) + else: + dir = mozpath.normpath( + mozpath.join(obj.objdir, dir)) + dir = mozpath.relpath(dir, obj.topobjdir) + candidates = [l for l in candidates if l.relobjdir == dir] + if not candidates: + # If the given directory is under one of the external + # (third party) paths, use a fake library reference to + # there. + for d in self._external_paths: + if dir.startswith('%s/' % d): + candidates = [self._get_external_library(dir, name, + force_static)] + break + + if not candidates: + raise SandboxValidationError( + '%s contains "%s", but there is no "%s" %s in %s.' + % (variable, path, name, + self.LIBRARY_NAME_VAR[obj.KIND], dir), context) + + if len(candidates) > 1: + # If there's more than one remaining candidate, it could be + # that there are instances for the same library, in static and + # shared form. + libs = {} + for l in candidates: + key = mozpath.join(l.relobjdir, l.basename) + if force_static: + if isinstance(l, StaticLibrary): + libs[key] = l + else: + if key in libs and isinstance(l, SharedLibrary): + libs[key] = l + if key not in libs: + libs[key] = l + candidates = libs.values() + if force_static and not candidates: + if dir: + raise SandboxValidationError( + '%s contains "static:%s", but there is no static ' + '"%s" %s in %s.' % (variable, path, name, + self.LIBRARY_NAME_VAR[obj.KIND], dir), context) + raise SandboxValidationError( + '%s contains "static:%s", but there is no static "%s" ' + '%s in the tree' % (variable, name, name, + self.LIBRARY_NAME_VAR[obj.KIND]), context) + + if not candidates: + raise SandboxValidationError( + '%s contains "%s", which does not match any %s in the tree.' + % (variable, path, self.LIBRARY_NAME_VAR[obj.KIND]), + context) + + elif len(candidates) > 1: + paths = (mozpath.join(l.relativedir, 'moz.build') + for l in candidates) + raise SandboxValidationError( + '%s contains "%s", which matches a %s defined in multiple ' + 'places:\n %s' % (variable, path, + self.LIBRARY_NAME_VAR[obj.KIND], + '\n '.join(paths)), context) + + elif force_static and not isinstance(candidates[0], StaticLibrary): + raise SandboxValidationError( + '%s contains "static:%s", but there is only a shared "%s" ' + 'in %s. You may want to add FORCE_STATIC_LIB=True in ' + '%s/moz.build, or remove "static:".' % (variable, path, + name, candidates[0].relobjdir, candidates[0].relobjdir), + context) + + elif isinstance(obj, StaticLibrary) and isinstance(candidates[0], + SharedLibrary): + self._static_linking_shared.add(obj) + obj.link_library(candidates[0]) + + # Link system libraries from OS_LIBS/HOST_OS_LIBS. + for lib in context.get(variable.replace('USE', 'OS'), []): + obj.link_system_library(lib) + + @memoize + def _get_external_library(self, dir, name, force_static): + # Create ExternalStaticLibrary or ExternalSharedLibrary object with a + # context more or less truthful about where the external library is. + context = Context(config=self.config) + context.add_source(mozpath.join(self.config.topsrcdir, dir, 'dummy')) + if force_static: + return ExternalStaticLibrary(context, name) + else: + return ExternalSharedLibrary(context, name) + + def _parse_cargo_file(self, toml_file): + """Parse toml_file and return a Python object representation of it.""" + with open(toml_file, 'r') as f: + return pytoml.load(f) + + def _verify_deps(self, context, crate_dir, crate_name, dependencies, description='Dependency'): + """Verify that a crate's dependencies all specify local paths.""" + for dep_crate_name, values in dependencies.iteritems(): + # A simple version number. + if isinstance(values, (str, unicode)): + raise SandboxValidationError( + '%s %s of crate %s does not list a path' % (description, dep_crate_name, crate_name), + context) + + dep_path = values.get('path', None) + if not dep_path: + raise SandboxValidationError( + '%s %s of crate %s does not list a path' % (description, dep_crate_name, crate_name), + context) + + # Try to catch the case where somebody listed a + # local path for development. + if os.path.isabs(dep_path): + raise SandboxValidationError( + '%s %s of crate %s has a non-relative path' % (description, dep_crate_name, crate_name), + context) + + if not os.path.exists(mozpath.join(context.config.topsrcdir, crate_dir, dep_path)): + raise SandboxValidationError( + '%s %s of crate %s refers to a non-existent path' % (description, dep_crate_name, crate_name), + context) + + def _rust_library(self, context, libname, static_args): + # We need to note any Rust library for linking purposes. + cargo_file = mozpath.join(context.srcdir, 'Cargo.toml') + if not os.path.exists(cargo_file): + raise SandboxValidationError( + 'No Cargo.toml file found in %s' % cargo_file, context) + + config = self._parse_cargo_file(cargo_file) + crate_name = config['package']['name'] + + if crate_name != libname: + raise SandboxValidationError( + 'library %s does not match Cargo.toml-defined package %s' % (libname, crate_name), + context) + + # Check that the [lib.crate-type] field is correct + lib_section = config.get('lib', None) + if not lib_section: + raise SandboxValidationError( + 'Cargo.toml for %s has no [lib] section' % libname, + context) + + crate_type = lib_section.get('crate-type', None) + if not crate_type: + raise SandboxValidationError( + 'Can\'t determine a crate-type for %s from Cargo.toml' % libname, + context) + + crate_type = crate_type[0] + if crate_type != 'staticlib': + raise SandboxValidationError( + 'crate-type %s is not permitted for %s' % (crate_type, libname), + context) + + # Check that the [profile.{dev,release}.panic] field is "abort" + profile_section = config.get('profile', None) + if not profile_section: + raise SandboxValidationError( + 'Cargo.toml for %s has no [profile] section' % libname, + context) + + for profile_name in ['dev', 'release']: + profile = profile_section.get(profile_name, None) + if not profile: + raise SandboxValidationError( + 'Cargo.toml for %s has no [profile.%s] section' % (libname, profile_name), + context) + + panic = profile.get('panic', None) + if panic != 'abort': + raise SandboxValidationError( + ('Cargo.toml for %s does not specify `panic = "abort"`' + ' in [profile.%s] section') % (libname, profile_name), + context) + + dependencies = set(config.get('dependencies', {}).iterkeys()) + + return RustLibrary(context, libname, cargo_file, crate_type, + dependencies, **static_args) + + def _handle_linkables(self, context, passthru, generated_files): + linkables = [] + host_linkables = [] + def add_program(prog, var): + if var.startswith('HOST_'): + host_linkables.append(prog) + else: + linkables.append(prog) + + for kind, cls in [('PROGRAM', Program), ('HOST_PROGRAM', HostProgram)]: + program = context.get(kind) + if program: + if program in self._binaries: + raise SandboxValidationError( + 'Cannot use "%s" as %s name, ' + 'because it is already used in %s' % (program, kind, + self._binaries[program].relativedir), context) + self._binaries[program] = cls(context, program) + self._linkage.append((context, self._binaries[program], + kind.replace('PROGRAM', 'USE_LIBS'))) + add_program(self._binaries[program], kind) + + for kind, cls in [ + ('SIMPLE_PROGRAMS', SimpleProgram), + ('CPP_UNIT_TESTS', SimpleProgram), + ('HOST_SIMPLE_PROGRAMS', HostSimpleProgram)]: + for program in context[kind]: + if program in self._binaries: + raise SandboxValidationError( + 'Cannot use "%s" in %s, ' + 'because it is already used in %s' % (program, kind, + self._binaries[program].relativedir), context) + self._binaries[program] = cls(context, program, + is_unit_test=kind == 'CPP_UNIT_TESTS') + self._linkage.append((context, self._binaries[program], + 'HOST_USE_LIBS' if kind == 'HOST_SIMPLE_PROGRAMS' + else 'USE_LIBS')) + add_program(self._binaries[program], kind) + + host_libname = context.get('HOST_LIBRARY_NAME') + libname = context.get('LIBRARY_NAME') + + if host_libname: + if host_libname == libname: + raise SandboxValidationError('LIBRARY_NAME and ' + 'HOST_LIBRARY_NAME must have a different value', context) + lib = HostLibrary(context, host_libname) + self._libs[host_libname].append(lib) + self._linkage.append((context, lib, 'HOST_USE_LIBS')) + host_linkables.append(lib) + + final_lib = context.get('FINAL_LIBRARY') + if not libname and final_lib: + # If no LIBRARY_NAME is given, create one. + libname = context.relsrcdir.replace('/', '_') + + static_lib = context.get('FORCE_STATIC_LIB') + shared_lib = context.get('FORCE_SHARED_LIB') + + static_name = context.get('STATIC_LIBRARY_NAME') + shared_name = context.get('SHARED_LIBRARY_NAME') + + is_framework = context.get('IS_FRAMEWORK') + is_component = context.get('IS_COMPONENT') + + soname = context.get('SONAME') + + lib_defines = context.get('LIBRARY_DEFINES') + + shared_args = {} + static_args = {} + + if final_lib: + if static_lib: + raise SandboxValidationError( + 'FINAL_LIBRARY implies FORCE_STATIC_LIB. ' + 'Please remove the latter.', context) + if shared_lib: + raise SandboxValidationError( + 'FINAL_LIBRARY conflicts with FORCE_SHARED_LIB. ' + 'Please remove one.', context) + if is_framework: + raise SandboxValidationError( + 'FINAL_LIBRARY conflicts with IS_FRAMEWORK. ' + 'Please remove one.', context) + if is_component: + raise SandboxValidationError( + 'FINAL_LIBRARY conflicts with IS_COMPONENT. ' + 'Please remove one.', context) + static_args['link_into'] = final_lib + static_lib = True + + if libname: + if is_component: + if static_lib: + raise SandboxValidationError( + 'IS_COMPONENT conflicts with FORCE_STATIC_LIB. ' + 'Please remove one.', context) + shared_lib = True + shared_args['variant'] = SharedLibrary.COMPONENT + + if is_framework: + if soname: + raise SandboxValidationError( + 'IS_FRAMEWORK conflicts with SONAME. ' + 'Please remove one.', context) + shared_lib = True + shared_args['variant'] = SharedLibrary.FRAMEWORK + + if not static_lib and not shared_lib: + static_lib = True + + if static_name: + if not static_lib: + raise SandboxValidationError( + 'STATIC_LIBRARY_NAME requires FORCE_STATIC_LIB', + context) + static_args['real_name'] = static_name + + if shared_name: + if not shared_lib: + raise SandboxValidationError( + 'SHARED_LIBRARY_NAME requires FORCE_SHARED_LIB', + context) + shared_args['real_name'] = shared_name + + if soname: + if not shared_lib: + raise SandboxValidationError( + 'SONAME requires FORCE_SHARED_LIB', context) + shared_args['soname'] = soname + + # If both a shared and a static library are created, only the + # shared library is meant to be a SDK library. + if context.get('SDK_LIBRARY'): + if shared_lib: + shared_args['is_sdk'] = True + elif static_lib: + static_args['is_sdk'] = True + + if context.get('NO_EXPAND_LIBS'): + if not static_lib: + raise SandboxValidationError( + 'NO_EXPAND_LIBS can only be set for static libraries.', + context) + static_args['no_expand_lib'] = True + + if shared_lib and static_lib: + if not static_name and not shared_name: + raise SandboxValidationError( + 'Both FORCE_STATIC_LIB and FORCE_SHARED_LIB are True, ' + 'but neither STATIC_LIBRARY_NAME or ' + 'SHARED_LIBRARY_NAME is set. At least one is required.', + context) + if static_name and not shared_name and static_name == libname: + raise SandboxValidationError( + 'Both FORCE_STATIC_LIB and FORCE_SHARED_LIB are True, ' + 'but STATIC_LIBRARY_NAME is the same as LIBRARY_NAME, ' + 'and SHARED_LIBRARY_NAME is unset. Please either ' + 'change STATIC_LIBRARY_NAME or LIBRARY_NAME, or set ' + 'SHARED_LIBRARY_NAME.', context) + if shared_name and not static_name and shared_name == libname: + raise SandboxValidationError( + 'Both FORCE_STATIC_LIB and FORCE_SHARED_LIB are True, ' + 'but SHARED_LIBRARY_NAME is the same as LIBRARY_NAME, ' + 'and STATIC_LIBRARY_NAME is unset. Please either ' + 'change SHARED_LIBRARY_NAME or LIBRARY_NAME, or set ' + 'STATIC_LIBRARY_NAME.', context) + if shared_name and static_name and shared_name == static_name: + raise SandboxValidationError( + 'Both FORCE_STATIC_LIB and FORCE_SHARED_LIB are True, ' + 'but SHARED_LIBRARY_NAME is the same as ' + 'STATIC_LIBRARY_NAME. Please change one of them.', + context) + + symbols_file = context.get('SYMBOLS_FILE') + if symbols_file: + if not shared_lib: + raise SandboxValidationError( + 'SYMBOLS_FILE can only be used with a SHARED_LIBRARY.', + context) + if context.get('DEFFILE') or context.get('LD_VERSION_SCRIPT'): + raise SandboxValidationError( + 'SYMBOLS_FILE cannot be used along DEFFILE or ' + 'LD_VERSION_SCRIPT.', context) + if isinstance(symbols_file, SourcePath): + if not os.path.exists(symbols_file.full_path): + raise SandboxValidationError( + 'Path specified in SYMBOLS_FILE does not exist: %s ' + '(resolved to %s)' % (symbols_file, + symbols_file.full_path), context) + shared_args['symbols_file'] = True + else: + if symbols_file.target_basename not in generated_files: + raise SandboxValidationError( + ('Objdir file specified in SYMBOLS_FILE not in ' + + 'GENERATED_FILES: %s') % (symbols_file,), context) + shared_args['symbols_file'] = symbols_file.target_basename + + if shared_lib: + lib = SharedLibrary(context, libname, **shared_args) + self._libs[libname].append(lib) + self._linkage.append((context, lib, 'USE_LIBS')) + linkables.append(lib) + generated_files.add(lib.lib_name) + if is_component and not context['NO_COMPONENTS_MANIFEST']: + yield ChromeManifestEntry(context, + 'components/components.manifest', + ManifestBinaryComponent('components', lib.lib_name)) + if symbols_file and isinstance(symbols_file, SourcePath): + script = mozpath.join( + mozpath.dirname(mozpath.dirname(__file__)), + 'action', 'generate_symbols_file.py') + defines = () + if lib.defines: + defines = lib.defines.get_defines() + yield GeneratedFile(context, script, + 'generate_symbols_file', lib.symbols_file, + [symbols_file], defines) + if static_lib: + is_rust_library = context.get('IS_RUST_LIBRARY') + if is_rust_library: + lib = self._rust_library(context, libname, static_args) + else: + lib = StaticLibrary(context, libname, **static_args) + self._libs[libname].append(lib) + self._linkage.append((context, lib, 'USE_LIBS')) + linkables.append(lib) + + if lib_defines: + if not libname: + raise SandboxValidationError('LIBRARY_DEFINES needs a ' + 'LIBRARY_NAME to take effect', context) + lib.lib_defines.update(lib_defines) + + # Only emit sources if we have linkables defined in the same context. + # Note the linkables are not emitted in this function, but much later, + # after aggregation (because of e.g. USE_LIBS processing). + if not (linkables or host_linkables): + return + + sources = defaultdict(list) + gen_sources = defaultdict(list) + all_flags = {} + for symbol in ('SOURCES', 'HOST_SOURCES', 'UNIFIED_SOURCES'): + srcs = sources[symbol] + gen_srcs = gen_sources[symbol] + context_srcs = context.get(symbol, []) + for f in context_srcs: + full_path = f.full_path + if isinstance(f, SourcePath): + srcs.append(full_path) + else: + assert isinstance(f, Path) + gen_srcs.append(full_path) + if symbol == 'SOURCES': + flags = context_srcs[f] + if flags: + all_flags[full_path] = flags + + if isinstance(f, SourcePath) and not os.path.exists(full_path): + raise SandboxValidationError('File listed in %s does not ' + 'exist: \'%s\'' % (symbol, full_path), context) + + # HOST_SOURCES and UNIFIED_SOURCES only take SourcePaths, so + # there should be no generated source in here + assert not gen_sources['HOST_SOURCES'] + assert not gen_sources['UNIFIED_SOURCES'] + + no_pgo = context.get('NO_PGO') + no_pgo_sources = [f for f, flags in all_flags.iteritems() + if flags.no_pgo] + if no_pgo: + if no_pgo_sources: + raise SandboxValidationError('NO_PGO and SOURCES[...].no_pgo ' + 'cannot be set at the same time', context) + passthru.variables['NO_PROFILE_GUIDED_OPTIMIZE'] = no_pgo + if no_pgo_sources: + passthru.variables['NO_PROFILE_GUIDED_OPTIMIZE'] = no_pgo_sources + + # A map from "canonical suffixes" for a particular source file + # language to the range of suffixes associated with that language. + # + # We deliberately don't list the canonical suffix in the suffix list + # in the definition; we'll add it in programmatically after defining + # things. + suffix_map = { + '.s': set(['.asm']), + '.c': set(), + '.m': set(), + '.mm': set(), + '.cpp': set(['.cc', '.cxx']), + '.S': set(), + } + + # The inverse of the above, mapping suffixes to their canonical suffix. + canonicalized_suffix_map = {} + for suffix, alternatives in suffix_map.iteritems(): + alternatives.add(suffix) + for a in alternatives: + canonicalized_suffix_map[a] = suffix + + def canonical_suffix_for_file(f): + return canonicalized_suffix_map[mozpath.splitext(f)[1]] + + # A map from moz.build variables to the canonical suffixes of file + # kinds that can be listed therein. + all_suffixes = list(suffix_map.keys()) + varmap = dict( + SOURCES=(Sources, GeneratedSources, all_suffixes), + HOST_SOURCES=(HostSources, None, ['.c', '.mm', '.cpp']), + UNIFIED_SOURCES=(UnifiedSources, None, ['.c', '.mm', '.cpp']), + ) + # Track whether there are any C++ source files. + # Technically this won't do the right thing for SIMPLE_PROGRAMS in + # a directory with mixed C and C++ source, but it's not that important. + cxx_sources = defaultdict(bool) + + for variable, (klass, gen_klass, suffixes) in varmap.items(): + allowed_suffixes = set().union(*[suffix_map[s] for s in suffixes]) + + # First ensure that we haven't been given filetypes that we don't + # recognize. + for f in itertools.chain(sources[variable], gen_sources[variable]): + ext = mozpath.splitext(f)[1] + if ext not in allowed_suffixes: + raise SandboxValidationError( + '%s has an unknown file type.' % f, context) + + for srcs, cls in ((sources[variable], klass), + (gen_sources[variable], gen_klass)): + # Now sort the files to let groupby work. + sorted_files = sorted(srcs, key=canonical_suffix_for_file) + for canonical_suffix, files in itertools.groupby( + sorted_files, canonical_suffix_for_file): + if canonical_suffix in ('.cpp', '.mm'): + cxx_sources[variable] = True + arglist = [context, list(files), canonical_suffix] + if (variable.startswith('UNIFIED_') and + 'FILES_PER_UNIFIED_FILE' in context): + arglist.append(context['FILES_PER_UNIFIED_FILE']) + obj = cls(*arglist) + yield obj + + for f, flags in all_flags.iteritems(): + if flags.flags: + ext = mozpath.splitext(f)[1] + yield PerSourceFlag(context, f, flags.flags) + + # If there are any C++ sources, set all the linkables defined here + # to require the C++ linker. + for vars, linkable_items in ((('SOURCES', 'UNIFIED_SOURCES'), linkables), + (('HOST_SOURCES',), host_linkables)): + for var in vars: + if cxx_sources[var]: + for l in linkable_items: + l.cxx_link = True + break + + + def emit_from_context(self, context): + """Convert a Context to tree metadata objects. + + This is a generator of mozbuild.frontend.data.ContextDerived instances. + """ + + # We only want to emit an InstallationTarget if one of the consulted + # variables is defined. Later on, we look up FINAL_TARGET, which has + # the side-effect of populating it. So, we need to do this lookup + # early. + if any(k in context for k in ('FINAL_TARGET', 'XPI_NAME', 'DIST_SUBDIR')): + yield InstallationTarget(context) + + # We always emit a directory traversal descriptor. This is needed by + # the recursive make backend. + for o in self._emit_directory_traversal_from_context(context): yield o + + for obj in self._process_xpidl(context): + yield obj + + # Proxy some variables as-is until we have richer classes to represent + # them. We should aim to keep this set small because it violates the + # desired abstraction of the build definition away from makefiles. + passthru = VariablePassthru(context) + varlist = [ + 'ALLOW_COMPILER_WARNINGS', + 'ANDROID_APK_NAME', + 'ANDROID_APK_PACKAGE', + 'ANDROID_GENERATED_RESFILES', + 'DISABLE_STL_WRAPPING', + 'EXTRA_DSO_LDOPTS', + 'PYTHON_UNIT_TESTS', + 'RCFILE', + 'RESFILE', + 'RCINCLUDE', + 'DEFFILE', + 'WIN32_EXE_LDFLAGS', + 'LD_VERSION_SCRIPT', + 'USE_EXTENSION_MANIFEST', + 'NO_JS_MANIFEST', + 'HAS_MISC_RULE', + ] + for v in varlist: + if v in context and context[v]: + passthru.variables[v] = context[v] + + if context.config.substs.get('OS_TARGET') == 'WINNT' and \ + context['DELAYLOAD_DLLS']: + context['LDFLAGS'].extend([('-DELAYLOAD:%s' % dll) + for dll in context['DELAYLOAD_DLLS']]) + context['OS_LIBS'].append('delayimp') + + for v in ['CFLAGS', 'CXXFLAGS', 'CMFLAGS', 'CMMFLAGS', 'ASFLAGS', + 'LDFLAGS', 'HOST_CFLAGS', 'HOST_CXXFLAGS']: + if v in context and context[v]: + passthru.variables['MOZBUILD_' + v] = context[v] + + # NO_VISIBILITY_FLAGS is slightly different + if context['NO_VISIBILITY_FLAGS']: + passthru.variables['VISIBILITY_FLAGS'] = '' + + if isinstance(context, TemplateContext) and context.template == 'Gyp': + passthru.variables['IS_GYP_DIR'] = True + + dist_install = context['DIST_INSTALL'] + if dist_install is True: + passthru.variables['DIST_INSTALL'] = True + elif dist_install is False: + passthru.variables['NO_DIST_INSTALL'] = True + + # Ideally, this should be done in templates, but this is difficult at + # the moment because USE_STATIC_LIBS can be set after a template + # returns. Eventually, with context-based templates, it will be + # possible. + if (context.config.substs.get('OS_ARCH') == 'WINNT' and + not context.config.substs.get('GNU_CC')): + use_static_lib = (context.get('USE_STATIC_LIBS') and + not context.config.substs.get('MOZ_ASAN')) + rtl_flag = '-MT' if use_static_lib else '-MD' + if (context.config.substs.get('MOZ_DEBUG') and + not context.config.substs.get('MOZ_NO_DEBUG_RTL')): + rtl_flag += 'd' + # Use a list, like MOZBUILD_*FLAGS variables + passthru.variables['RTL_FLAGS'] = [rtl_flag] + + generated_files = set() + for obj in self._process_generated_files(context): + for f in obj.outputs: + generated_files.add(f) + yield obj + + for path in context['CONFIGURE_SUBST_FILES']: + sub = self._create_substitution(ConfigFileSubstitution, context, + path) + generated_files.add(str(sub.relpath)) + yield sub + + defines = context.get('DEFINES') + if defines: + yield Defines(context, defines) + + host_defines = context.get('HOST_DEFINES') + if host_defines: + yield HostDefines(context, host_defines) + + simple_lists = [ + ('GENERATED_EVENTS_WEBIDL_FILES', GeneratedEventWebIDLFile), + ('GENERATED_WEBIDL_FILES', GeneratedWebIDLFile), + ('IPDL_SOURCES', IPDLFile), + ('PREPROCESSED_TEST_WEBIDL_FILES', PreprocessedTestWebIDLFile), + ('PREPROCESSED_WEBIDL_FILES', PreprocessedWebIDLFile), + ('TEST_WEBIDL_FILES', TestWebIDLFile), + ('WEBIDL_FILES', WebIDLFile), + ('WEBIDL_EXAMPLE_INTERFACES', ExampleWebIDLInterface), + ] + for context_var, klass in simple_lists: + for name in context.get(context_var, []): + yield klass(context, name) + + for local_include in context.get('LOCAL_INCLUDES', []): + if (not isinstance(local_include, ObjDirPath) and + not os.path.exists(local_include.full_path)): + raise SandboxValidationError('Path specified in LOCAL_INCLUDES ' + 'does not exist: %s (resolved to %s)' % (local_include, + local_include.full_path), context) + yield LocalInclude(context, local_include) + + for obj in self._handle_linkables(context, passthru, generated_files): + yield obj + + generated_files.update(['%s%s' % (k, self.config.substs.get('BIN_SUFFIX', '')) for k in self._binaries.keys()]) + + components = [] + for var, cls in ( + ('BRANDING_FILES', BrandingFiles), + ('EXPORTS', Exports), + ('FINAL_TARGET_FILES', FinalTargetFiles), + ('FINAL_TARGET_PP_FILES', FinalTargetPreprocessedFiles), + ('OBJDIR_FILES', ObjdirFiles), + ('OBJDIR_PP_FILES', ObjdirPreprocessedFiles), + ('SDK_FILES', SdkFiles), + ('TEST_HARNESS_FILES', TestHarnessFiles), + ): + all_files = context.get(var) + if not all_files: + continue + if dist_install is False and var != 'TEST_HARNESS_FILES': + raise SandboxValidationError( + '%s cannot be used with DIST_INSTALL = False' % var, + context) + has_prefs = False + has_resources = False + for base, files in all_files.walk(): + if var == 'TEST_HARNESS_FILES' and not base: + raise SandboxValidationError( + 'Cannot install files to the root of TEST_HARNESS_FILES', context) + if base == 'components': + components.extend(files) + if base == 'defaults/pref': + has_prefs = True + if mozpath.split(base)[0] == 'res': + has_resources = True + for f in files: + if ((var == 'FINAL_TARGET_PP_FILES' or + var == 'OBJDIR_PP_FILES') and + not isinstance(f, SourcePath)): + raise SandboxValidationError( + ('Only source directory paths allowed in ' + + '%s: %s') + % (var, f,), context) + if not isinstance(f, ObjDirPath): + path = f.full_path + if '*' not in path and not os.path.exists(path): + raise SandboxValidationError( + 'File listed in %s does not exist: %s' + % (var, path), context) + else: + # TODO: Bug 1254682 - The '/' check is to allow + # installing files generated from other directories, + # which is done occasionally for tests. However, it + # means we don't fail early if the file isn't actually + # created by the other moz.build file. + if f.target_basename not in generated_files and '/' not in f: + raise SandboxValidationError( + ('Objdir file listed in %s not in ' + + 'GENERATED_FILES: %s') % (var, f), context) + + # Addons (when XPI_NAME is defined) and Applications (when + # DIST_SUBDIR is defined) use a different preferences directory + # (default/preferences) from the one the GRE uses (defaults/pref). + # Hence, we move the files from the latter to the former in that + # case. + if has_prefs and (context.get('XPI_NAME') or + context.get('DIST_SUBDIR')): + all_files.defaults.preferences += all_files.defaults.pref + del all_files.defaults._children['pref'] + + if has_resources and (context.get('DIST_SUBDIR') or + context.get('XPI_NAME')): + raise SandboxValidationError( + 'RESOURCES_FILES cannot be used with DIST_SUBDIR or ' + 'XPI_NAME.', context) + + yield cls(context, all_files) + + # Check for manifest declarations in EXTRA_{PP_,}COMPONENTS. + if any(e.endswith('.js') for e in components) and \ + not any(e.endswith('.manifest') for e in components) and \ + not context.get('NO_JS_MANIFEST', False): + raise SandboxValidationError('A .js component was specified in EXTRA_COMPONENTS ' + 'or EXTRA_PP_COMPONENTS without a matching ' + '.manifest file. See ' + 'https://developer.mozilla.org/en/XPCOM/XPCOM_changes_in_Gecko_2.0 .', + context); + + for c in components: + if c.endswith('.manifest'): + yield ChromeManifestEntry(context, 'chrome.manifest', + Manifest('components', + mozpath.basename(c))) + + for obj in self._process_test_manifests(context): + yield obj + + for obj in self._process_jar_manifests(context): + yield obj + + for name, jar in context.get('JAVA_JAR_TARGETS', {}).items(): + yield ContextWrapped(context, jar) + + for name, data in context.get('ANDROID_ECLIPSE_PROJECT_TARGETS', {}).items(): + yield ContextWrapped(context, data) + + if context.get('USE_YASM') is True: + yasm = context.config.substs.get('YASM') + if not yasm: + raise SandboxValidationError('yasm is not available', context) + passthru.variables['AS'] = yasm + passthru.variables['ASFLAGS'] = context.config.substs.get('YASM_ASFLAGS') + passthru.variables['AS_DASH_C_FLAG'] = '' + + for (symbol, cls) in [ + ('ANDROID_RES_DIRS', AndroidResDirs), + ('ANDROID_EXTRA_RES_DIRS', AndroidExtraResDirs), + ('ANDROID_ASSETS_DIRS', AndroidAssetsDirs)]: + paths = context.get(symbol) + if not paths: + continue + for p in paths: + if isinstance(p, SourcePath) and not os.path.isdir(p.full_path): + raise SandboxValidationError('Directory listed in ' + '%s is not a directory: \'%s\'' % + (symbol, p.full_path), context) + yield cls(context, paths) + + android_extra_packages = context.get('ANDROID_EXTRA_PACKAGES') + if android_extra_packages: + yield AndroidExtraPackages(context, android_extra_packages) + + if passthru.variables: + yield passthru + + def _create_substitution(self, cls, context, path): + sub = cls(context) + sub.input_path = '%s.in' % path.full_path + sub.output_path = path.translated + sub.relpath = path + + return sub + + def _process_xpidl(self, context): + # XPIDL source files get processed and turned into .h and .xpt files. + # If there are multiple XPIDL files in a directory, they get linked + # together into a final .xpt, which has the name defined by + # XPIDL_MODULE. + xpidl_module = context['XPIDL_MODULE'] + + if context['XPIDL_SOURCES'] and not xpidl_module: + raise SandboxValidationError('XPIDL_MODULE must be defined if ' + 'XPIDL_SOURCES is defined.', context) + + if xpidl_module and not context['XPIDL_SOURCES']: + raise SandboxValidationError('XPIDL_MODULE cannot be defined ' + 'unless there are XPIDL_SOURCES', context) + + if context['XPIDL_SOURCES'] and context['DIST_INSTALL'] is False: + self.log(logging.WARN, 'mozbuild_warning', dict( + path=context.main_path), + '{path}: DIST_INSTALL = False has no effect on XPIDL_SOURCES.') + + for idl in context['XPIDL_SOURCES']: + yield XPIDLFile(context, mozpath.join(context.srcdir, idl), + xpidl_module, add_to_manifest=not context['XPIDL_NO_MANIFEST']) + + def _process_generated_files(self, context): + for path in context['CONFIGURE_DEFINE_FILES']: + script = mozpath.join(mozpath.dirname(mozpath.dirname(__file__)), + 'action', 'process_define_files.py') + yield GeneratedFile(context, script, 'process_define_file', + unicode(path), + [Path(context, path + '.in')]) + + generated_files = context.get('GENERATED_FILES') + if not generated_files: + return + + for f in generated_files: + flags = generated_files[f] + outputs = f + inputs = [] + if flags.script: + method = "main" + script = SourcePath(context, flags.script).full_path + + # Deal with cases like "C:\\path\\to\\script.py:function". + if '.py:' in script: + script, method = script.rsplit('.py:', 1) + script += '.py' + + if not os.path.exists(script): + raise SandboxValidationError( + 'Script for generating %s does not exist: %s' + % (f, script), context) + if os.path.splitext(script)[1] != '.py': + raise SandboxValidationError( + 'Script for generating %s does not end in .py: %s' + % (f, script), context) + + for i in flags.inputs: + p = Path(context, i) + if (isinstance(p, SourcePath) and + not os.path.exists(p.full_path)): + raise SandboxValidationError( + 'Input for generating %s does not exist: %s' + % (f, p.full_path), context) + inputs.append(p) + else: + script = None + method = None + yield GeneratedFile(context, script, method, outputs, inputs) + + def _process_test_manifests(self, context): + for prefix, info in TEST_MANIFESTS.items(): + for path, manifest in context.get('%s_MANIFESTS' % prefix, []): + for obj in self._process_test_manifest(context, info, path, manifest): + yield obj + + for flavor in REFTEST_FLAVORS: + for path, manifest in context.get('%s_MANIFESTS' % flavor.upper(), []): + for obj in self._process_reftest_manifest(context, flavor, path, manifest): + yield obj + + for flavor in WEB_PLATFORM_TESTS_FLAVORS: + for path, manifest in context.get("%s_MANIFESTS" % flavor.upper().replace('-', '_'), []): + for obj in self._process_web_platform_tests_manifest(context, path, manifest): + yield obj + + python_tests = context.get('PYTHON_UNIT_TESTS') + if python_tests: + for obj in self._process_python_tests(context, python_tests): + yield obj + + def _process_test_manifest(self, context, info, manifest_path, mpmanifest): + flavor, install_root, install_subdir, package_tests = info + + path = mozpath.normpath(mozpath.join(context.srcdir, manifest_path)) + manifest_dir = mozpath.dirname(path) + manifest_reldir = mozpath.dirname(mozpath.relpath(path, + context.config.topsrcdir)) + install_prefix = mozpath.join(install_root, install_subdir) + + try: + if not mpmanifest.tests: + raise SandboxValidationError('Empty test manifest: %s' + % path, context) + + defaults = mpmanifest.manifest_defaults[os.path.normpath(path)] + obj = TestManifest(context, path, mpmanifest, flavor=flavor, + install_prefix=install_prefix, + relpath=mozpath.join(manifest_reldir, mozpath.basename(path)), + dupe_manifest='dupe-manifest' in defaults) + + filtered = mpmanifest.tests + + # Jetpack add-on tests are expected to be generated during the + # build process so they won't exist here. + if flavor != 'jetpack-addon': + missing = [t['name'] for t in filtered if not os.path.exists(t['path'])] + if missing: + raise SandboxValidationError('Test manifest (%s) lists ' + 'test that does not exist: %s' % ( + path, ', '.join(missing)), context) + + out_dir = mozpath.join(install_prefix, manifest_reldir) + if 'install-to-subdir' in defaults: + # This is terrible, but what are you going to do? + out_dir = mozpath.join(out_dir, defaults['install-to-subdir']) + obj.manifest_obj_relpath = mozpath.join(manifest_reldir, + defaults['install-to-subdir'], + mozpath.basename(path)) + + def process_support_files(test): + install_info = self._test_files_converter.convert_support_files( + test, install_root, manifest_dir, out_dir) + + obj.pattern_installs.extend(install_info.pattern_installs) + for source, dest in install_info.installs: + obj.installs[source] = (dest, False) + obj.external_installs |= install_info.external_installs + for install_path in install_info.deferred_installs: + if all(['*' not in install_path, + not os.path.isfile(mozpath.join(context.config.topsrcdir, + install_path[2:])), + install_path not in install_info.external_installs]): + raise SandboxValidationError('Error processing test ' + 'manifest %s: entry in support-files not present ' + 'in the srcdir: %s' % (path, install_path), context) + + obj.deferred_installs |= install_info.deferred_installs + + for test in filtered: + obj.tests.append(test) + + # Some test files are compiled and should not be copied into the + # test package. They function as identifiers rather than files. + if package_tests: + manifest_relpath = mozpath.relpath(test['path'], + mozpath.dirname(test['manifest'])) + obj.installs[mozpath.normpath(test['path'])] = \ + ((mozpath.join(out_dir, manifest_relpath)), True) + + process_support_files(test) + + for path, m_defaults in mpmanifest.manifest_defaults.items(): + process_support_files(m_defaults) + + # We also copy manifests into the output directory, + # including manifests from [include:foo] directives. + for mpath in mpmanifest.manifests(): + mpath = mozpath.normpath(mpath) + out_path = mozpath.join(out_dir, mozpath.basename(mpath)) + obj.installs[mpath] = (out_path, False) + + # Some manifests reference files that are auto generated as + # part of the build or shouldn't be installed for some + # reason. Here, we prune those files from the install set. + # FUTURE we should be able to detect autogenerated files from + # other build metadata. Once we do that, we can get rid of this. + for f in defaults.get('generated-files', '').split(): + # We re-raise otherwise the stack trace isn't informative. + try: + del obj.installs[mozpath.join(manifest_dir, f)] + except KeyError: + raise SandboxValidationError('Error processing test ' + 'manifest %s: entry in generated-files not present ' + 'elsewhere in manifest: %s' % (path, f), context) + + yield obj + except (AssertionError, Exception): + raise SandboxValidationError('Error processing test ' + 'manifest file %s: %s' % (path, + '\n'.join(traceback.format_exception(*sys.exc_info()))), + context) + + def _process_reftest_manifest(self, context, flavor, manifest_path, manifest): + manifest_full_path = mozpath.normpath(mozpath.join( + context.srcdir, manifest_path)) + manifest_reldir = mozpath.dirname(mozpath.relpath(manifest_full_path, + context.config.topsrcdir)) + + # reftest manifests don't come from manifest parser. But they are + # similar enough that we can use the same emitted objects. Note + # that we don't perform any installs for reftests. + obj = TestManifest(context, manifest_full_path, manifest, + flavor=flavor, install_prefix='%s/' % flavor, + relpath=mozpath.join(manifest_reldir, + mozpath.basename(manifest_path))) + + for test, source_manifest in sorted(manifest.tests): + obj.tests.append({ + 'path': test, + 'here': mozpath.dirname(test), + 'manifest': source_manifest, + 'name': mozpath.basename(test), + 'head': '', + 'tail': '', + 'support-files': '', + 'subsuite': '', + }) + + yield obj + + def _process_web_platform_tests_manifest(self, context, paths, manifest): + manifest_path, tests_root = paths + manifest_full_path = mozpath.normpath(mozpath.join( + context.srcdir, manifest_path)) + manifest_reldir = mozpath.dirname(mozpath.relpath(manifest_full_path, + context.config.topsrcdir)) + tests_root = mozpath.normpath(mozpath.join(context.srcdir, tests_root)) + + # Create a equivalent TestManifest object + obj = TestManifest(context, manifest_full_path, manifest, + flavor="web-platform-tests", + relpath=mozpath.join(manifest_reldir, + mozpath.basename(manifest_path)), + install_prefix="web-platform/") + + + for path, tests in manifest: + path = mozpath.join(tests_root, path) + for test in tests: + if test.item_type not in ["testharness", "reftest"]: + continue + + obj.tests.append({ + 'path': path, + 'here': mozpath.dirname(path), + 'manifest': manifest_path, + 'name': test.id, + 'head': '', + 'tail': '', + 'support-files': '', + 'subsuite': '', + }) + + yield obj + + def _process_python_tests(self, context, python_tests): + manifest_full_path = context.main_path + manifest_reldir = mozpath.dirname(mozpath.relpath(manifest_full_path, + context.config.topsrcdir)) + + obj = TestManifest(context, manifest_full_path, + mozpath.basename(manifest_full_path), + flavor='python', install_prefix='python/', + relpath=mozpath.join(manifest_reldir, + mozpath.basename(manifest_full_path))) + + for test in python_tests: + test = mozpath.normpath(mozpath.join(context.srcdir, test)) + if not os.path.isfile(test): + raise SandboxValidationError('Path specified in ' + 'PYTHON_UNIT_TESTS does not exist: %s' % test, + context) + obj.tests.append({ + 'path': test, + 'here': mozpath.dirname(test), + 'manifest': manifest_full_path, + 'name': mozpath.basename(test), + 'head': '', + 'tail': '', + 'support-files': '', + 'subsuite': '', + }) + + yield obj + + def _process_jar_manifests(self, context): + jar_manifests = context.get('JAR_MANIFESTS', []) + if len(jar_manifests) > 1: + raise SandboxValidationError('While JAR_MANIFESTS is a list, ' + 'it is currently limited to one value.', context) + + for path in jar_manifests: + yield JARManifest(context, path) + + # Temporary test to look for jar.mn files that creep in without using + # the new declaration. Before, we didn't require jar.mn files to + # declared anywhere (they were discovered). This will detect people + # relying on the old behavior. + if os.path.exists(os.path.join(context.srcdir, 'jar.mn')): + if 'jar.mn' not in jar_manifests: + raise SandboxValidationError('A jar.mn exists but it ' + 'is not referenced in the moz.build file. ' + 'Please define JAR_MANIFESTS.', context) + + def _emit_directory_traversal_from_context(self, context): + o = DirectoryTraversal(context) + o.dirs = context.get('DIRS', []) + + # Some paths have a subconfigure, yet also have a moz.build. Those + # shouldn't end up in self._external_paths. + if o.objdir: + self._external_paths -= { o.relobjdir } + + yield o diff --git a/python/mozbuild/mozbuild/frontend/gyp_reader.py b/python/mozbuild/mozbuild/frontend/gyp_reader.py new file mode 100644 index 000000000..459c553c3 --- /dev/null +++ b/python/mozbuild/mozbuild/frontend/gyp_reader.py @@ -0,0 +1,248 @@ +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, You can obtain one at http://mozilla.org/MPL/2.0/. + +from __future__ import absolute_import, unicode_literals + +import gyp +import sys +import os +import types +import mozpack.path as mozpath +from mozpack.files import FileFinder +from .sandbox import alphabetical_sorted +from .context import ( + SourcePath, + TemplateContext, + VARIABLES, +) +from mozbuild.util import ( + expand_variables, + List, + memoize, +) +from .reader import SandboxValidationError + +# Define this module as gyp.generator.mozbuild so that gyp can use it +# as a generator under the name "mozbuild". +sys.modules['gyp.generator.mozbuild'] = sys.modules[__name__] + +# build/gyp_chromium does this: +# script_dir = os.path.dirname(os.path.realpath(__file__)) +# chrome_src = os.path.abspath(os.path.join(script_dir, os.pardir)) +# sys.path.insert(0, os.path.join(chrome_src, 'tools', 'gyp', 'pylib')) +# We're not importing gyp_chromium, but we want both script_dir and +# chrome_src for the default includes, so go backwards from the pylib +# directory, which is the parent directory of gyp module. +chrome_src = mozpath.abspath(mozpath.join(mozpath.dirname(gyp.__file__), + '../../../..')) +script_dir = mozpath.join(chrome_src, 'build') + +# Default variables gyp uses when evaluating gyp files. +generator_default_variables = { +} +for dirname in ['INTERMEDIATE_DIR', 'SHARED_INTERMEDIATE_DIR', 'PRODUCT_DIR', + 'LIB_DIR', 'SHARED_LIB_DIR']: + # Some gyp steps fail if these are empty(!). + generator_default_variables[dirname] = b'dir' + +for unused in ['RULE_INPUT_PATH', 'RULE_INPUT_ROOT', 'RULE_INPUT_NAME', + 'RULE_INPUT_DIRNAME', 'RULE_INPUT_EXT', + 'EXECUTABLE_PREFIX', 'EXECUTABLE_SUFFIX', + 'STATIC_LIB_PREFIX', 'STATIC_LIB_SUFFIX', + 'SHARED_LIB_PREFIX', 'SHARED_LIB_SUFFIX', + 'LINKER_SUPPORTS_ICF']: + generator_default_variables[unused] = b'' + + +class GypContext(TemplateContext): + """Specialized Context for use with data extracted from Gyp. + + config is the ConfigEnvironment for this context. + relobjdir is the object directory that will be used for this context, + relative to the topobjdir defined in the ConfigEnvironment. + """ + def __init__(self, config, relobjdir): + self._relobjdir = relobjdir + TemplateContext.__init__(self, template='Gyp', + allowed_variables=VARIABLES, config=config) + + +def encode(value): + if isinstance(value, unicode): + return value.encode('utf-8') + return value + + +def read_from_gyp(config, path, output, vars, non_unified_sources = set()): + """Read a gyp configuration and emits GypContexts for the backend to + process. + + config is a ConfigEnvironment, path is the path to a root gyp configuration + file, output is the base path under which the objdir for the various gyp + dependencies will be, and vars a dict of variables to pass to the gyp + processor. + """ + + # gyp expects plain str instead of unicode. The frontend code gives us + # unicode strings, so convert them. + path = encode(path) + str_vars = dict((name, encode(value)) for name, value in vars.items()) + + params = { + b'parallel': False, + b'generator_flags': {}, + b'build_files': [path], + b'root_targets': None, + } + + # Files that gyp_chromium always includes + includes = [encode(mozpath.join(script_dir, 'common.gypi'))] + finder = FileFinder(chrome_src, find_executables=False) + includes.extend(encode(mozpath.join(chrome_src, name)) + for name, _ in finder.find('*/supplement.gypi')) + + # Read the given gyp file and its dependencies. + generator, flat_list, targets, data = \ + gyp.Load([path], format=b'mozbuild', + default_variables=str_vars, + includes=includes, + depth=encode(chrome_src), + params=params) + + # Process all targets from the given gyp files and its dependencies. + # The path given to AllTargets needs to use os.sep, while the frontend code + # gives us paths normalized with forward slash separator. + for target in gyp.common.AllTargets(flat_list, targets, path.replace(b'/', os.sep)): + build_file, target_name, toolset = gyp.common.ParseQualifiedTarget(target) + + # Each target is given its own objdir. The base of that objdir + # is derived from the relative path from the root gyp file path + # to the current build_file, placed under the given output + # directory. Since several targets can be in a given build_file, + # separate them in subdirectories using the build_file basename + # and the target_name. + reldir = mozpath.relpath(mozpath.dirname(build_file), + mozpath.dirname(path)) + subdir = '%s_%s' % ( + mozpath.splitext(mozpath.basename(build_file))[0], + target_name, + ) + # Emit a context for each target. + context = GypContext(config, mozpath.relpath( + mozpath.join(output, reldir, subdir), config.topobjdir)) + context.add_source(mozpath.abspath(build_file)) + # The list of included files returned by gyp are relative to build_file + for f in data[build_file]['included_files']: + context.add_source(mozpath.abspath(mozpath.join( + mozpath.dirname(build_file), f))) + + spec = targets[target] + + # Derive which gyp configuration to use based on MOZ_DEBUG. + c = 'Debug' if config.substs['MOZ_DEBUG'] else 'Release' + if c not in spec['configurations']: + raise RuntimeError('Missing %s gyp configuration for target %s ' + 'in %s' % (c, target_name, build_file)) + target_conf = spec['configurations'][c] + + if spec['type'] == 'none': + continue + elif spec['type'] == 'static_library': + # Remove leading 'lib' from the target_name if any, and use as + # library name. + name = spec['target_name'] + if name.startswith('lib'): + name = name[3:] + # The context expects an unicode string. + context['LIBRARY_NAME'] = name.decode('utf-8') + # gyp files contain headers and asm sources in sources lists. + sources = [] + unified_sources = [] + extensions = set() + for f in spec.get('sources', []): + ext = mozpath.splitext(f)[-1] + extensions.add(ext) + s = SourcePath(context, f) + if ext == '.h': + continue + if ext != '.S' and s not in non_unified_sources: + unified_sources.append(s) + else: + sources.append(s) + + # The context expects alphabetical order when adding sources + context['SOURCES'] = alphabetical_sorted(sources) + context['UNIFIED_SOURCES'] = alphabetical_sorted(unified_sources) + + for define in target_conf.get('defines', []): + if '=' in define: + name, value = define.split('=', 1) + context['DEFINES'][name] = value + else: + context['DEFINES'][define] = True + + for include in target_conf.get('include_dirs', []): + # moz.build expects all LOCAL_INCLUDES to exist, so ensure they do. + # + # NB: gyp files sometimes have actual absolute paths (e.g. + # /usr/include32) and sometimes paths that moz.build considers + # absolute, i.e. starting from topsrcdir. There's no good way + # to tell them apart here, and the actual absolute paths are + # likely bogus. In any event, actual absolute paths will be + # filtered out by trying to find them in topsrcdir. + if include.startswith('/'): + resolved = mozpath.abspath(mozpath.join(config.topsrcdir, include[1:])) + else: + resolved = mozpath.abspath(mozpath.join(mozpath.dirname(build_file), include)) + if not os.path.exists(resolved): + continue + context['LOCAL_INCLUDES'] += [include] + + context['ASFLAGS'] = target_conf.get('asflags_mozilla', []) + flags = target_conf.get('cflags_mozilla', []) + if flags: + suffix_map = { + '.c': 'CFLAGS', + '.cpp': 'CXXFLAGS', + '.cc': 'CXXFLAGS', + '.m': 'CMFLAGS', + '.mm': 'CMMFLAGS', + } + variables = ( + suffix_map[e] + for e in extensions if e in suffix_map + ) + for var in variables: + for f in flags: + # We may be getting make variable references out of the + # gyp data, and we don't want those in emitted data, so + # substitute them with their actual value. + f = expand_variables(f, config.substs).split() + if not f: + continue + # the result may be a string or a list. + if isinstance(f, types.StringTypes): + context[var].append(f) + else: + context[var].extend(f) + else: + # Ignore other types than static_library because we don't have + # anything using them, and we're not testing them. They can be + # added when that becomes necessary. + raise NotImplementedError('Unsupported gyp target type: %s' % spec['type']) + + # Add some features to all contexts. Put here in case LOCAL_INCLUDES + # order matters. + context['LOCAL_INCLUDES'] += [ + '!/ipc/ipdl/_ipdlheaders', + '/ipc/chromium/src', + '/ipc/glue', + ] + # These get set via VC project file settings for normal GYP builds. + if config.substs['OS_TARGET'] == 'WINNT': + context['DEFINES']['UNICODE'] = True + context['DEFINES']['_UNICODE'] = True + context['DISABLE_STL_WRAPPING'] = True + + yield context diff --git a/python/mozbuild/mozbuild/frontend/mach_commands.py b/python/mozbuild/mozbuild/frontend/mach_commands.py new file mode 100644 index 000000000..cbecc1137 --- /dev/null +++ b/python/mozbuild/mozbuild/frontend/mach_commands.py @@ -0,0 +1,218 @@ +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, You can obtain one at http://mozilla.org/MPL/2.0/. + +from __future__ import absolute_import, print_function, unicode_literals + +from collections import defaultdict +import os + +from mach.decorators import ( + CommandArgument, + CommandProvider, + Command, + SubCommand, +) + +from mozbuild.base import MachCommandBase +import mozpack.path as mozpath + + +class InvalidPathException(Exception): + """Represents an error due to an invalid path.""" + + +@CommandProvider +class MozbuildFileCommands(MachCommandBase): + @Command('mozbuild-reference', category='build-dev', + description='View reference documentation on mozbuild files.') + @CommandArgument('symbol', default=None, nargs='*', + help='Symbol to view help on. If not specified, all will be shown.') + @CommandArgument('--name-only', '-n', default=False, action='store_true', + help='Print symbol names only.') + def reference(self, symbol, name_only=False): + # mozbuild.sphinx imports some Sphinx modules, so we need to be sure + # the optional Sphinx package is installed. + self._activate_virtualenv() + self.virtualenv_manager.install_pip_package('Sphinx==1.1.3') + + from mozbuild.sphinx import ( + format_module, + function_reference, + special_reference, + variable_reference, + ) + + import mozbuild.frontend.context as m + + if name_only: + for s in sorted(m.VARIABLES.keys()): + print(s) + + for s in sorted(m.FUNCTIONS.keys()): + print(s) + + for s in sorted(m.SPECIAL_VARIABLES.keys()): + print(s) + + return 0 + + if len(symbol): + for s in symbol: + if s in m.VARIABLES: + for line in variable_reference(s, *m.VARIABLES[s]): + print(line) + continue + elif s in m.FUNCTIONS: + for line in function_reference(s, *m.FUNCTIONS[s]): + print(line) + continue + elif s in m.SPECIAL_VARIABLES: + for line in special_reference(s, *m.SPECIAL_VARIABLES[s]): + print(line) + continue + + print('Could not find symbol: %s' % s) + return 1 + + return 0 + + for line in format_module(m): + print(line) + + return 0 + + @Command('file-info', category='build-dev', + description='Query for metadata about files.') + def file_info(self): + """Show files metadata derived from moz.build files. + + moz.build files contain "Files" sub-contexts for declaring metadata + against file patterns. This command suite is used to query that data. + """ + + @SubCommand('file-info', 'bugzilla-component', + 'Show Bugzilla component info for files listed.') + @CommandArgument('-r', '--rev', + help='Version control revision to look up info from') + @CommandArgument('paths', nargs='+', + help='Paths whose data to query') + def file_info_bugzilla(self, paths, rev=None): + """Show Bugzilla component for a set of files. + + Given a requested set of files (which can be specified using + wildcards), print the Bugzilla component for each file. + """ + components = defaultdict(set) + try: + for p, m in self._get_files_info(paths, rev=rev).items(): + components[m.get('BUG_COMPONENT')].add(p) + except InvalidPathException as e: + print(e.message) + return 1 + + for component, files in sorted(components.items(), key=lambda x: (x is None, x)): + print('%s :: %s' % (component.product, component.component) if component else 'UNKNOWN') + for f in sorted(files): + print(' %s' % f) + + @SubCommand('file-info', 'missing-bugzilla', + 'Show files missing Bugzilla component info') + @CommandArgument('-r', '--rev', + help='Version control revision to look up info from') + @CommandArgument('paths', nargs='+', + help='Paths whose data to query') + def file_info_missing_bugzilla(self, paths, rev=None): + try: + for p, m in sorted(self._get_files_info(paths, rev=rev).items()): + if 'BUG_COMPONENT' not in m: + print(p) + except InvalidPathException as e: + print(e.message) + return 1 + + @SubCommand('file-info', 'dep-tests', + 'Show test files marked as dependencies of these source files.') + @CommandArgument('-r', '--rev', + help='Version control revision to look up info from') + @CommandArgument('paths', nargs='+', + help='Paths whose data to query') + def file_info_test_deps(self, paths, rev=None): + try: + for p, m in self._get_files_info(paths, rev=rev).items(): + print('%s:' % mozpath.relpath(p, self.topsrcdir)) + if m.test_files: + print('\tTest file patterns:') + for p in m.test_files: + print('\t\t%s' % p) + if m.test_tags: + print('\tRelevant tags:') + for p in m.test_tags: + print('\t\t%s' % p) + if m.test_flavors: + print('\tRelevant flavors:') + for p in m.test_flavors: + print('\t\t%s' % p) + + except InvalidPathException as e: + print(e.message) + return 1 + + + def _get_reader(self, finder): + from mozbuild.frontend.reader import ( + BuildReader, + EmptyConfig, + ) + + config = EmptyConfig(self.topsrcdir) + return BuildReader(config, finder=finder) + + def _get_files_info(self, paths, rev=None): + from mozbuild.frontend.reader import default_finder + from mozpack.files import FileFinder, MercurialRevisionFinder + + # Normalize to relative from topsrcdir. + relpaths = [] + for p in paths: + a = mozpath.abspath(p) + if not mozpath.basedir(a, [self.topsrcdir]): + raise InvalidPathException('path is outside topsrcdir: %s' % p) + + relpaths.append(mozpath.relpath(a, self.topsrcdir)) + + repo = None + if rev: + hg_path = os.path.join(self.topsrcdir, '.hg') + if not os.path.exists(hg_path): + raise InvalidPathException('a Mercurial repo is required ' + 'when specifying a revision') + + repo = self.topsrcdir + + # We need two finders because the reader's finder operates on + # absolute paths. + finder = FileFinder(self.topsrcdir, find_executables=False) + if repo: + reader_finder = MercurialRevisionFinder(repo, rev=rev, + recognize_repo_paths=True) + else: + reader_finder = default_finder + + # Expand wildcards. + allpaths = [] + for p in relpaths: + if '*' not in p: + if p not in allpaths: + allpaths.append(p) + continue + + if repo: + raise InvalidPathException('cannot use wildcard in version control mode') + + for path, f in finder.find(p): + if path not in allpaths: + allpaths.append(path) + + reader = self._get_reader(finder=reader_finder) + return reader.files_info(allpaths) diff --git a/python/mozbuild/mozbuild/frontend/reader.py b/python/mozbuild/mozbuild/frontend/reader.py new file mode 100644 index 000000000..8192b1ec6 --- /dev/null +++ b/python/mozbuild/mozbuild/frontend/reader.py @@ -0,0 +1,1408 @@ +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, You can obtain one at http://mozilla.org/MPL/2.0/. + +# This file contains code for reading metadata from the build system into +# data structures. + +r"""Read build frontend files into data structures. + +In terms of code architecture, the main interface is BuildReader. BuildReader +starts with a root mozbuild file. It creates a new execution environment for +this file, which is represented by the Sandbox class. The Sandbox class is used +to fill a Context, representing the output of an individual mozbuild file. The + +The BuildReader contains basic logic for traversing a tree of mozbuild files. +It does this by examining specific variables populated during execution. +""" + +from __future__ import absolute_import, print_function, unicode_literals + +import ast +import inspect +import logging +import os +import sys +import textwrap +import time +import traceback +import types + +from collections import ( + defaultdict, + OrderedDict, +) +from io import StringIO + +from mozbuild.util import ( + EmptyValue, + HierarchicalStringList, + memoize, + ReadOnlyDefaultDict, +) + +from mozbuild.testing import ( + TEST_MANIFESTS, + REFTEST_FLAVORS, + WEB_PLATFORM_TESTS_FLAVORS, +) + +from mozbuild.backend.configenvironment import ConfigEnvironment + +from mozpack.files import FileFinder +import mozpack.path as mozpath + +from .data import ( + AndroidEclipseProjectData, + JavaJarData, +) + +from .sandbox import ( + default_finder, + SandboxError, + SandboxExecutionError, + SandboxLoadError, + Sandbox, +) + +from .context import ( + Context, + ContextDerivedValue, + Files, + FUNCTIONS, + VARIABLES, + DEPRECATION_HINTS, + SourcePath, + SPECIAL_VARIABLES, + SUBCONTEXTS, + SubContext, + TemplateContext, +) + +from mozbuild.base import ExecutionSummary + + +if sys.version_info.major == 2: + text_type = unicode + type_type = types.TypeType +else: + text_type = str + type_type = type + + +def log(logger, level, action, params, formatter): + logger.log(level, formatter, extra={'action': action, 'params': params}) + + +class EmptyConfig(object): + """A config object that is empty. + + This config object is suitable for using with a BuildReader on a vanilla + checkout, without any existing configuration. The config is simply + bootstrapped from a top source directory path. + """ + class PopulateOnGetDict(ReadOnlyDefaultDict): + """A variation on ReadOnlyDefaultDict that populates during .get(). + + This variation is needed because CONFIG uses .get() to access members. + Without it, None (instead of our EmptyValue types) would be returned. + """ + def get(self, key, default=None): + return self[key] + + def __init__(self, topsrcdir): + self.topsrcdir = topsrcdir + self.topobjdir = '' + + self.substs = self.PopulateOnGetDict(EmptyValue, { + # These 2 variables are used semi-frequently and it isn't worth + # changing all the instances. + b'MOZ_APP_NAME': b'empty', + b'MOZ_CHILD_PROCESS_NAME': b'empty', + # Set manipulations are performed within the moz.build files. But + # set() is not an exposed symbol, so we can't create an empty set. + b'NECKO_PROTOCOLS': set(), + # Needed to prevent js/src's config.status from loading. + b'JS_STANDALONE': b'1', + }) + udict = {} + for k, v in self.substs.items(): + if isinstance(v, str): + udict[k.decode('utf-8')] = v.decode('utf-8') + else: + udict[k] = v + self.substs_unicode = self.PopulateOnGetDict(EmptyValue, udict) + self.defines = self.substs + self.external_source_dir = None + self.error_is_fatal = False + + +def is_read_allowed(path, config): + """Whether we are allowed to load a mozbuild file at the specified path. + + This is used as cheap security to ensure the build is isolated to known + source directories. + + We are allowed to read from the main source directory and any defined + external source directories. The latter is to allow 3rd party applications + to hook into our build system. + """ + assert os.path.isabs(path) + assert os.path.isabs(config.topsrcdir) + + path = mozpath.normpath(path) + topsrcdir = mozpath.normpath(config.topsrcdir) + + if mozpath.basedir(path, [topsrcdir]): + return True + + if config.external_source_dir: + external_dir = os.path.normcase(config.external_source_dir) + norm_path = os.path.normcase(path) + if mozpath.basedir(norm_path, [external_dir]): + return True + + return False + + +class SandboxCalledError(SandboxError): + """Represents an error resulting from calling the error() function.""" + + def __init__(self, file_stack, message): + SandboxError.__init__(self, file_stack) + self.message = message + + +class MozbuildSandbox(Sandbox): + """Implementation of a Sandbox tailored for mozbuild files. + + We expose a few useful functions and expose the set of variables defining + Mozilla's build system. + + context is a Context instance. + + metadata is a dict of metadata that can be used during the sandbox + evaluation. + """ + def __init__(self, context, metadata={}, finder=default_finder): + assert isinstance(context, Context) + + Sandbox.__init__(self, context, finder=finder) + + self._log = logging.getLogger(__name__) + + self.metadata = dict(metadata) + exports = self.metadata.get('exports', {}) + self.exports = set(exports.keys()) + context.update(exports) + self.templates = self.metadata.setdefault('templates', {}) + self.special_variables = self.metadata.setdefault('special_variables', + SPECIAL_VARIABLES) + self.functions = self.metadata.setdefault('functions', FUNCTIONS) + self.subcontext_types = self.metadata.setdefault('subcontexts', + SUBCONTEXTS) + + def __getitem__(self, key): + if key in self.special_variables: + return self.special_variables[key][0](self._context) + if key in self.functions: + return self._create_function(self.functions[key]) + if key in self.subcontext_types: + return self._create_subcontext(self.subcontext_types[key]) + if key in self.templates: + return self._create_template_wrapper(self.templates[key]) + return Sandbox.__getitem__(self, key) + + def __contains__(self, key): + if any(key in d for d in (self.special_variables, self.functions, + self.subcontext_types, self.templates)): + return True + + return Sandbox.__contains__(self, key) + + def __setitem__(self, key, value): + if key in self.special_variables and value is self[key]: + return + if key in self.special_variables or key in self.functions or key in self.subcontext_types: + raise KeyError('Cannot set "%s" because it is a reserved keyword' + % key) + if key in self.exports: + self._context[key] = value + self.exports.remove(key) + return + Sandbox.__setitem__(self, key, value) + + def exec_file(self, path): + """Override exec_file to normalize paths and restrict file loading. + + Paths will be rejected if they do not fall under topsrcdir or one of + the external roots. + """ + + # realpath() is needed for true security. But, this isn't for security + # protection, so it is omitted. + if not is_read_allowed(path, self._context.config): + raise SandboxLoadError(self._context.source_stack, + sys.exc_info()[2], illegal_path=path) + + Sandbox.exec_file(self, path) + + def _add_java_jar(self, name): + """Add a Java JAR build target.""" + if not name: + raise Exception('Java JAR cannot be registered without a name') + + if '/' in name or '\\' in name or '.jar' in name: + raise Exception('Java JAR names must not include slashes or' + ' .jar: %s' % name) + + if name in self['JAVA_JAR_TARGETS']: + raise Exception('Java JAR has already been registered: %s' % name) + + jar = JavaJarData(name) + self['JAVA_JAR_TARGETS'][name] = jar + return jar + + # Not exposed to the sandbox. + def add_android_eclipse_project_helper(self, name): + """Add an Android Eclipse project target.""" + if not name: + raise Exception('Android Eclipse project cannot be registered without a name') + + if name in self['ANDROID_ECLIPSE_PROJECT_TARGETS']: + raise Exception('Android Eclipse project has already been registered: %s' % name) + + data = AndroidEclipseProjectData(name) + self['ANDROID_ECLIPSE_PROJECT_TARGETS'][name] = data + return data + + def _add_android_eclipse_project(self, name, manifest): + if not manifest: + raise Exception('Android Eclipse project must specify a manifest') + + data = self.add_android_eclipse_project_helper(name) + data.manifest = manifest + data.is_library = False + return data + + def _add_android_eclipse_library_project(self, name): + data = self.add_android_eclipse_project_helper(name) + data.manifest = None + data.is_library = True + return data + + def _export(self, varname): + """Export the variable to all subdirectories of the current path.""" + + exports = self.metadata.setdefault('exports', dict()) + if varname in exports: + raise Exception('Variable has already been exported: %s' % varname) + + try: + # Doing a regular self._context[varname] causes a set as a side + # effect. By calling the dict method instead, we don't have any + # side effects. + exports[varname] = dict.__getitem__(self._context, varname) + except KeyError: + self.last_name_error = KeyError('global_ns', 'get_unknown', varname) + raise self.last_name_error + + def recompute_exports(self): + """Recompute the variables to export to subdirectories with the current + values in the subdirectory.""" + + if 'exports' in self.metadata: + for key in self.metadata['exports']: + self.metadata['exports'][key] = self[key] + + def _include(self, path): + """Include and exec another file within the context of this one.""" + + # path is a SourcePath + self.exec_file(path.full_path) + + def _warning(self, message): + # FUTURE consider capturing warnings in a variable instead of printing. + print('WARNING: %s' % message, file=sys.stderr) + + def _error(self, message): + if self._context.error_is_fatal: + raise SandboxCalledError(self._context.source_stack, message) + else: + self._warning(message) + + def _template_decorator(self, func): + """Registers a template function.""" + + if not inspect.isfunction(func): + raise Exception('`template` is a function decorator. You must ' + 'use it as `@template` preceding a function declaration.') + + name = func.func_name + + if name in self.templates: + raise KeyError( + 'A template named "%s" was already declared in %s.' % (name, + self.templates[name].path)) + + if name.islower() or name.isupper() or name[0].islower(): + raise NameError('Template function names must be CamelCase.') + + self.templates[name] = TemplateFunction(func, self) + + @memoize + def _create_subcontext(self, cls): + """Return a function object that creates SubContext instances.""" + def fn(*args, **kwargs): + return cls(self._context, *args, **kwargs) + + return fn + + @memoize + def _create_function(self, function_def): + """Returns a function object for use within the sandbox for the given + function definition. + + The wrapper function does type coercion on the function arguments + """ + func, args_def, doc = function_def + def function(*args): + def coerce(arg, type): + if not isinstance(arg, type): + if issubclass(type, ContextDerivedValue): + arg = type(self._context, arg) + else: + arg = type(arg) + return arg + args = [coerce(arg, type) for arg, type in zip(args, args_def)] + return func(self)(*args) + + return function + + @memoize + def _create_template_wrapper(self, template): + """Returns a function object for use within the sandbox for the given + TemplateFunction instance.. + + When a moz.build file contains a reference to a template call, the + sandbox needs a function to execute. This is what this method returns. + That function creates a new sandbox for execution of the template. + After the template is executed, the data from its execution is merged + with the context of the calling sandbox. + """ + def template_wrapper(*args, **kwargs): + context = TemplateContext( + template=template.name, + allowed_variables=self._context._allowed_variables, + config=self._context.config) + context.add_source(self._context.current_path) + for p in self._context.all_paths: + context.add_source(p) + + sandbox = MozbuildSandbox(context, metadata={ + # We should arguably set these defaults to something else. + # Templates, for example, should arguably come from the state + # of the sandbox from when the template was declared, not when + # it was instantiated. Bug 1137319. + 'functions': self.metadata.get('functions', {}), + 'special_variables': self.metadata.get('special_variables', {}), + 'subcontexts': self.metadata.get('subcontexts', {}), + 'templates': self.metadata.get('templates', {}) + }, finder=self._finder) + + template.exec_in_sandbox(sandbox, *args, **kwargs) + + # This is gross, but allows the merge to happen. Eventually, the + # merging will go away and template contexts emitted independently. + klass = self._context.__class__ + self._context.__class__ = TemplateContext + # The sandbox will do all the necessary checks for these merges. + for key, value in context.items(): + if isinstance(value, dict): + self[key].update(value) + elif isinstance(value, (list, HierarchicalStringList)): + self[key] += value + else: + self[key] = value + self._context.__class__ = klass + + for p in context.all_paths: + self._context.add_source(p) + + return template_wrapper + + +class TemplateFunction(object): + def __init__(self, func, sandbox): + self.path = func.func_code.co_filename + self.name = func.func_name + + code = func.func_code + firstlineno = code.co_firstlineno + lines = sandbox._current_source.splitlines(True) + lines = inspect.getblock(lines[firstlineno - 1:]) + + # The code lines we get out of inspect.getsourcelines look like + # @template + # def Template(*args, **kwargs): + # VAR = 'value' + # ... + func_ast = ast.parse(''.join(lines), self.path) + # Remove decorators + func_ast.body[0].decorator_list = [] + # Adjust line numbers accordingly + ast.increment_lineno(func_ast, firstlineno - 1) + + # When using a custom dictionary for function globals/locals, Cpython + # actually never calls __getitem__ and __setitem__, so we need to + # modify the AST so that accesses to globals are properly directed + # to a dict. + self._global_name = b'_data' # AST wants str for this, not unicode + # In case '_data' is a name used for a variable in the function code, + # prepend more underscores until we find an unused name. + while (self._global_name in code.co_names or + self._global_name in code.co_varnames): + self._global_name += '_' + func_ast = self.RewriteName(sandbox, self._global_name).visit(func_ast) + + # Execute the rewritten code. That code now looks like: + # def Template(*args, **kwargs): + # _data['VAR'] = 'value' + # ... + # The result of executing this code is the creation of a 'Template' + # function object in the global namespace. + glob = {'__builtins__': sandbox._builtins} + func = types.FunctionType( + compile(func_ast, self.path, 'exec'), + glob, + self.name, + func.func_defaults, + func.func_closure, + ) + func() + + self._func = glob[self.name] + + def exec_in_sandbox(self, sandbox, *args, **kwargs): + """Executes the template function in the given sandbox.""" + # Create a new function object associated with the execution sandbox + glob = { + self._global_name: sandbox, + '__builtins__': sandbox._builtins + } + func = types.FunctionType( + self._func.func_code, + glob, + self.name, + self._func.func_defaults, + self._func.func_closure + ) + sandbox.exec_function(func, args, kwargs, self.path, + becomes_current_path=False) + + class RewriteName(ast.NodeTransformer): + """AST Node Transformer to rewrite variable accesses to go through + a dict. + """ + def __init__(self, sandbox, global_name): + self._sandbox = sandbox + self._global_name = global_name + + def visit_Str(self, node): + # String nodes we got from the AST parser are str, but we want + # unicode literals everywhere, so transform them. + node.s = unicode(node.s) + return node + + def visit_Name(self, node): + # Modify uppercase variable references and names known to the + # sandbox as if they were retrieved from a dict instead. + if not node.id.isupper() and node.id not in self._sandbox: + return node + + def c(new_node): + return ast.copy_location(new_node, node) + + return c(ast.Subscript( + value=c(ast.Name(id=self._global_name, ctx=ast.Load())), + slice=c(ast.Index(value=c(ast.Str(s=node.id)))), + ctx=node.ctx + )) + + +class SandboxValidationError(Exception): + """Represents an error encountered when validating sandbox results.""" + def __init__(self, message, context): + Exception.__init__(self, message) + self.context = context + + def __str__(self): + s = StringIO() + + delim = '=' * 30 + s.write('\n%s\nERROR PROCESSING MOZBUILD FILE\n%s\n\n' % (delim, delim)) + + s.write('The error occurred while processing the following file or ') + s.write('one of the files it includes:\n') + s.write('\n') + s.write(' %s/moz.build\n' % self.context.srcdir) + s.write('\n') + + s.write('The error occurred when validating the result of ') + s.write('the execution. The reported error is:\n') + s.write('\n') + s.write(''.join(' %s\n' % l + for l in self.message.splitlines())) + s.write('\n') + + return s.getvalue() + + +class BuildReaderError(Exception): + """Represents errors encountered during BuildReader execution. + + The main purpose of this class is to facilitate user-actionable error + messages. Execution errors should say: + + - Why they failed + - Where they failed + - What can be done to prevent the error + + A lot of the code in this class should arguably be inside sandbox.py. + However, extraction is somewhat difficult given the additions + MozbuildSandbox has over Sandbox (e.g. the concept of included files - + which affect error messages, of course). + """ + def __init__(self, file_stack, trace, sandbox_exec_error=None, + sandbox_load_error=None, validation_error=None, other_error=None, + sandbox_called_error=None): + + self.file_stack = file_stack + self.trace = trace + self.sandbox_called_error = sandbox_called_error + self.sandbox_exec = sandbox_exec_error + self.sandbox_load = sandbox_load_error + self.validation_error = validation_error + self.other = other_error + + @property + def main_file(self): + return self.file_stack[-1] + + @property + def actual_file(self): + # We report the file that called out to the file that couldn't load. + if self.sandbox_load is not None: + if len(self.sandbox_load.file_stack) > 1: + return self.sandbox_load.file_stack[-2] + + if len(self.file_stack) > 1: + return self.file_stack[-2] + + if self.sandbox_error is not None and \ + len(self.sandbox_error.file_stack): + return self.sandbox_error.file_stack[-1] + + return self.file_stack[-1] + + @property + def sandbox_error(self): + return self.sandbox_exec or self.sandbox_load or \ + self.sandbox_called_error + + def __str__(self): + s = StringIO() + + delim = '=' * 30 + s.write('\n%s\nERROR PROCESSING MOZBUILD FILE\n%s\n\n' % (delim, delim)) + + s.write('The error occurred while processing the following file:\n') + s.write('\n') + s.write(' %s\n' % self.actual_file) + s.write('\n') + + if self.actual_file != self.main_file and not self.sandbox_load: + s.write('This file was included as part of processing:\n') + s.write('\n') + s.write(' %s\n' % self.main_file) + s.write('\n') + + if self.sandbox_error is not None: + self._print_sandbox_error(s) + elif self.validation_error is not None: + s.write('The error occurred when validating the result of ') + s.write('the execution. The reported error is:\n') + s.write('\n') + s.write(''.join(' %s\n' % l + for l in self.validation_error.message.splitlines())) + s.write('\n') + else: + s.write('The error appears to be part of the %s ' % __name__) + s.write('Python module itself! It is possible you have stumbled ') + s.write('across a legitimate bug.\n') + s.write('\n') + + for l in traceback.format_exception(type(self.other), self.other, + self.trace): + s.write(unicode(l)) + + return s.getvalue() + + def _print_sandbox_error(self, s): + # Try to find the frame of the executed code. + script_frame = None + + # We don't currently capture the trace for SandboxCalledError. + # Therefore, we don't get line numbers from the moz.build file. + # FUTURE capture this. + trace = getattr(self.sandbox_error, 'trace', None) + frames = [] + if trace: + frames = traceback.extract_tb(trace) + for frame in frames: + if frame[0] == self.actual_file: + script_frame = frame + + # Reset if we enter a new execution context. This prevents errors + # in this module from being attributes to a script. + elif frame[0] == __file__ and frame[2] == 'exec_function': + script_frame = None + + if script_frame is not None: + s.write('The error was triggered on line %d ' % script_frame[1]) + s.write('of this file:\n') + s.write('\n') + s.write(' %s\n' % script_frame[3]) + s.write('\n') + + if self.sandbox_called_error is not None: + self._print_sandbox_called_error(s) + return + + if self.sandbox_load is not None: + self._print_sandbox_load_error(s) + return + + self._print_sandbox_exec_error(s) + + def _print_sandbox_called_error(self, s): + assert self.sandbox_called_error is not None + + s.write('A moz.build file called the error() function.\n') + s.write('\n') + s.write('The error it encountered is:\n') + s.write('\n') + s.write(' %s\n' % self.sandbox_called_error.message) + s.write('\n') + s.write('Correct the error condition and try again.\n') + + def _print_sandbox_load_error(self, s): + assert self.sandbox_load is not None + + if self.sandbox_load.illegal_path is not None: + s.write('The underlying problem is an illegal file access. ') + s.write('This is likely due to trying to access a file ') + s.write('outside of the top source directory.\n') + s.write('\n') + s.write('The path whose access was denied is:\n') + s.write('\n') + s.write(' %s\n' % self.sandbox_load.illegal_path) + s.write('\n') + s.write('Modify the script to not access this file and ') + s.write('try again.\n') + return + + if self.sandbox_load.read_error is not None: + if not os.path.exists(self.sandbox_load.read_error): + s.write('The underlying problem is we referenced a path ') + s.write('that does not exist. That path is:\n') + s.write('\n') + s.write(' %s\n' % self.sandbox_load.read_error) + s.write('\n') + s.write('Either create the file if it needs to exist or ') + s.write('do not reference it.\n') + else: + s.write('The underlying problem is a referenced path could ') + s.write('not be read. The trouble path is:\n') + s.write('\n') + s.write(' %s\n' % self.sandbox_load.read_error) + s.write('\n') + s.write('It is possible the path is not correct. Is it ') + s.write('pointing to a directory? It could also be a file ') + s.write('permissions issue. Ensure that the file is ') + s.write('readable.\n') + + return + + # This module is buggy if you see this. + raise AssertionError('SandboxLoadError with unhandled properties!') + + def _print_sandbox_exec_error(self, s): + assert self.sandbox_exec is not None + + inner = self.sandbox_exec.exc_value + + if isinstance(inner, SyntaxError): + s.write('The underlying problem is a Python syntax error ') + s.write('on line %d:\n' % inner.lineno) + s.write('\n') + s.write(' %s\n' % inner.text) + if inner.offset: + s.write((' ' * (inner.offset + 4)) + '^\n') + s.write('\n') + s.write('Fix the syntax error and try again.\n') + return + + if isinstance(inner, KeyError): + self._print_keyerror(inner, s) + elif isinstance(inner, ValueError): + self._print_valueerror(inner, s) + else: + self._print_exception(inner, s) + + def _print_keyerror(self, inner, s): + if not inner.args or inner.args[0] not in ('global_ns', 'local_ns'): + self._print_exception(inner, s) + return + + if inner.args[0] == 'global_ns': + import difflib + + verb = None + if inner.args[1] == 'get_unknown': + verb = 'read' + elif inner.args[1] == 'set_unknown': + verb = 'write' + elif inner.args[1] == 'reassign': + s.write('The underlying problem is an attempt to reassign ') + s.write('a reserved UPPERCASE variable.\n') + s.write('\n') + s.write('The reassigned variable causing the error is:\n') + s.write('\n') + s.write(' %s\n' % inner.args[2]) + s.write('\n') + s.write('Maybe you meant "+=" instead of "="?\n') + return + else: + raise AssertionError('Unhandled global_ns: %s' % inner.args[1]) + + s.write('The underlying problem is an attempt to %s ' % verb) + s.write('a reserved UPPERCASE variable that does not exist.\n') + s.write('\n') + s.write('The variable %s causing the error is:\n' % verb) + s.write('\n') + s.write(' %s\n' % inner.args[2]) + s.write('\n') + close_matches = difflib.get_close_matches(inner.args[2], + VARIABLES.keys(), 2) + if close_matches: + s.write('Maybe you meant %s?\n' % ' or '.join(close_matches)) + s.write('\n') + + if inner.args[2] in DEPRECATION_HINTS: + s.write('%s\n' % + textwrap.dedent(DEPRECATION_HINTS[inner.args[2]]).strip()) + return + + s.write('Please change the file to not use this variable.\n') + s.write('\n') + s.write('For reference, the set of valid variables is:\n') + s.write('\n') + s.write(', '.join(sorted(VARIABLES.keys())) + '\n') + return + + s.write('The underlying problem is a reference to an undefined ') + s.write('local variable:\n') + s.write('\n') + s.write(' %s\n' % inner.args[2]) + s.write('\n') + s.write('Please change the file to not reference undefined ') + s.write('variables and try again.\n') + + def _print_valueerror(self, inner, s): + if not inner.args or inner.args[0] not in ('global_ns', 'local_ns'): + self._print_exception(inner, s) + return + + assert inner.args[1] == 'set_type' + + s.write('The underlying problem is an attempt to write an illegal ') + s.write('value to a special variable.\n') + s.write('\n') + s.write('The variable whose value was rejected is:\n') + s.write('\n') + s.write(' %s' % inner.args[2]) + s.write('\n') + s.write('The value being written to it was of the following type:\n') + s.write('\n') + s.write(' %s\n' % type(inner.args[3]).__name__) + s.write('\n') + s.write('This variable expects the following type(s):\n') + s.write('\n') + if type(inner.args[4]) == type_type: + s.write(' %s\n' % inner.args[4].__name__) + else: + for t in inner.args[4]: + s.write( ' %s\n' % t.__name__) + s.write('\n') + s.write('Change the file to write a value of the appropriate type ') + s.write('and try again.\n') + + def _print_exception(self, e, s): + s.write('An error was encountered as part of executing the file ') + s.write('itself. The error appears to be the fault of the script.\n') + s.write('\n') + s.write('The error as reported by Python is:\n') + s.write('\n') + s.write(' %s\n' % traceback.format_exception_only(type(e), e)) + + +class BuildReader(object): + """Read a tree of mozbuild files into data structures. + + This is where the build system starts. You give it a tree configuration + (the output of configuration) and it executes the moz.build files and + collects the data they define. + + The reader can optionally call a callable after each sandbox is evaluated + but before its evaluated content is processed. This gives callers the + opportunity to modify contexts before side-effects occur from their + content. This callback receives the ``Context`` containing the result of + each sandbox evaluation. Its return value is ignored. + """ + + def __init__(self, config, finder=default_finder): + self.config = config + + self._log = logging.getLogger(__name__) + self._read_files = set() + self._execution_stack = [] + self._finder = finder + + self._execution_time = 0.0 + self._file_count = 0 + + def summary(self): + return ExecutionSummary( + 'Finished reading {file_count:d} moz.build files in ' + '{execution_time:.2f}s', + file_count=self._file_count, + execution_time=self._execution_time) + + def read_topsrcdir(self): + """Read the tree of linked moz.build files. + + This starts with the tree's top-most moz.build file and descends into + all linked moz.build files until all relevant files have been evaluated. + + This is a generator of Context instances. As each moz.build file is + read, a new Context is created and emitted. + """ + path = mozpath.join(self.config.topsrcdir, 'moz.build') + return self.read_mozbuild(path, self.config) + + def all_mozbuild_paths(self): + """Iterator over all available moz.build files. + + This method has little to do with the reader. It should arguably belong + elsewhere. + """ + # In the future, we may traverse moz.build files by looking + # for DIRS references in the AST, even if a directory is added behind + # a conditional. For now, just walk the filesystem. + ignore = { + # Ignore fake moz.build files used for testing moz.build. + 'python/mozbuild/mozbuild/test', + + # Ignore object directories. + 'obj*', + } + + finder = FileFinder(self.config.topsrcdir, find_executables=False, + ignore=ignore) + + # The root doesn't get picked up by FileFinder. + yield 'moz.build' + + for path, f in finder.find('**/moz.build'): + yield path + + def find_sphinx_variables(self): + """This function finds all assignments of Sphinx documentation variables. + + This is a generator of tuples of (moz.build path, var, key, value). For + variables that assign to keys in objects, key will be defined. + + With a little work, this function could be made more generic. But if we + end up writing a lot of ast code, it might be best to import a + high-level AST manipulation library into the tree. + """ + # This function looks for assignments to SPHINX_TREES and + # SPHINX_PYTHON_PACKAGE_DIRS variables. + # + # SPHINX_TREES is a dict. Keys and values should both be strings. The + # target of the assignment should be a Subscript node. The value + # assigned should be a Str node. e.g. + # + # SPHINX_TREES['foo'] = 'bar' + # + # This is an Assign node with a Subscript target. The Subscript's value + # is a Name node with id "SPHINX_TREES." The slice of this target + # is an Index node and its value is a Str with value "foo." + # + # SPHINX_PYTHON_PACKAGE_DIRS is a simple list. The target of the + # assignment should be a Name node. Values should be a List node, whose + # elements are Str nodes. e.g. + # + # SPHINX_PYTHON_PACKAGE_DIRS += ['foo'] + # + # This is an AugAssign node with a Name target with id + # "SPHINX_PYTHON_PACKAGE_DIRS." The value is a List node containing 1 + # Str elt whose value is "foo." + relevant = [ + 'SPHINX_TREES', + 'SPHINX_PYTHON_PACKAGE_DIRS', + ] + + def assigned_variable(node): + # This is not correct, but we don't care yet. + if hasattr(node, 'targets'): + # Nothing in moz.build does multi-assignment (yet). So error if + # we see it. + assert len(node.targets) == 1 + + target = node.targets[0] + else: + target = node.target + + if isinstance(target, ast.Subscript): + if not isinstance(target.value, ast.Name): + return None, None + name = target.value.id + elif isinstance(target, ast.Name): + name = target.id + else: + return None, None + + if name not in relevant: + return None, None + + key = None + if isinstance(target, ast.Subscript): + assert isinstance(target.slice, ast.Index) + assert isinstance(target.slice.value, ast.Str) + key = target.slice.value.s + + return name, key + + def assigned_values(node): + value = node.value + if isinstance(value, ast.List): + for v in value.elts: + assert isinstance(v, ast.Str) + yield v.s + else: + assert isinstance(value, ast.Str) + yield value.s + + assignments = [] + + class Visitor(ast.NodeVisitor): + def helper(self, node): + name, key = assigned_variable(node) + if not name: + return + + for v in assigned_values(node): + assignments.append((name, key, v)) + + def visit_Assign(self, node): + self.helper(node) + + def visit_AugAssign(self, node): + self.helper(node) + + for p in self.all_mozbuild_paths(): + assignments[:] = [] + full = os.path.join(self.config.topsrcdir, p) + + with open(full, 'rb') as fh: + source = fh.read() + + tree = ast.parse(source, full) + Visitor().visit(tree) + + for name, key, value in assignments: + yield p, name, key, value + + def read_mozbuild(self, path, config, descend=True, metadata={}): + """Read and process a mozbuild file, descending into children. + + This starts with a single mozbuild file, executes it, and descends into + other referenced files per our traversal logic. + + The traversal logic is to iterate over the *DIRS variables, treating + each element as a relative directory path. For each encountered + directory, we will open the moz.build file located in that + directory in a new Sandbox and process it. + + If descend is True (the default), we will descend into child + directories and files per variable values. + + Arbitrary metadata in the form of a dict can be passed into this + function. This feature is intended to facilitate the build reader + injecting state and annotations into moz.build files that is + independent of the sandbox's execution context. + + Traversal is performed depth first (for no particular reason). + """ + self._execution_stack.append(path) + try: + for s in self._read_mozbuild(path, config, descend=descend, + metadata=metadata): + yield s + + except BuildReaderError as bre: + raise bre + + except SandboxCalledError as sce: + raise BuildReaderError(list(self._execution_stack), + sys.exc_info()[2], sandbox_called_error=sce) + + except SandboxExecutionError as se: + raise BuildReaderError(list(self._execution_stack), + sys.exc_info()[2], sandbox_exec_error=se) + + except SandboxLoadError as sle: + raise BuildReaderError(list(self._execution_stack), + sys.exc_info()[2], sandbox_load_error=sle) + + except SandboxValidationError as ve: + raise BuildReaderError(list(self._execution_stack), + sys.exc_info()[2], validation_error=ve) + + except Exception as e: + raise BuildReaderError(list(self._execution_stack), + sys.exc_info()[2], other_error=e) + + def _read_mozbuild(self, path, config, descend, metadata): + path = mozpath.normpath(path) + log(self._log, logging.DEBUG, 'read_mozbuild', {'path': path}, + 'Reading file: {path}') + + if path in self._read_files: + log(self._log, logging.WARNING, 'read_already', {'path': path}, + 'File already read. Skipping: {path}') + return + + self._read_files.add(path) + + time_start = time.time() + + topobjdir = config.topobjdir + + if not mozpath.basedir(path, [config.topsrcdir]): + external = config.external_source_dir + if external and mozpath.basedir(path, [external]): + config = ConfigEnvironment.from_config_status( + mozpath.join(topobjdir, 'config.status')) + config.topsrcdir = external + config.external_source_dir = None + + relpath = mozpath.relpath(path, config.topsrcdir) + reldir = mozpath.dirname(relpath) + + if mozpath.dirname(relpath) == 'js/src' and \ + not config.substs.get('JS_STANDALONE'): + config = ConfigEnvironment.from_config_status( + mozpath.join(topobjdir, reldir, 'config.status')) + config.topobjdir = topobjdir + config.external_source_dir = None + + context = Context(VARIABLES, config, self._finder) + sandbox = MozbuildSandbox(context, metadata=metadata, + finder=self._finder) + sandbox.exec_file(path) + self._execution_time += time.time() - time_start + self._file_count += len(context.all_paths) + + # Yield main context before doing any processing. This gives immediate + # consumers an opportunity to change state before our remaining + # processing is performed. + yield context + + # We need the list of directories pre-gyp processing for later. + dirs = list(context.get('DIRS', [])) + + curdir = mozpath.dirname(path) + + gyp_contexts = [] + for target_dir in context.get('GYP_DIRS', []): + gyp_dir = context['GYP_DIRS'][target_dir] + for v in ('input', 'variables'): + if not getattr(gyp_dir, v): + raise SandboxValidationError('Missing value for ' + 'GYP_DIRS["%s"].%s' % (target_dir, v), context) + + # The make backend assumes contexts for sub-directories are + # emitted after their parent, so accumulate the gyp contexts. + # We could emit the parent context before processing gyp + # configuration, but we need to add the gyp objdirs to that context + # first. + from .gyp_reader import read_from_gyp + non_unified_sources = set() + for s in gyp_dir.non_unified_sources: + source = SourcePath(context, s) + if not self._finder.get(source.full_path): + raise SandboxValidationError('Cannot find %s.' % source, + context) + non_unified_sources.add(source) + time_start = time.time() + for gyp_context in read_from_gyp(context.config, + mozpath.join(curdir, gyp_dir.input), + mozpath.join(context.objdir, + target_dir), + gyp_dir.variables, + non_unified_sources = non_unified_sources): + gyp_context.update(gyp_dir.sandbox_vars) + gyp_contexts.append(gyp_context) + self._file_count += len(gyp_context.all_paths) + self._execution_time += time.time() - time_start + + for gyp_context in gyp_contexts: + context['DIRS'].append(mozpath.relpath(gyp_context.objdir, context.objdir)) + sandbox.subcontexts.append(gyp_context) + + for subcontext in sandbox.subcontexts: + yield subcontext + + # Traverse into referenced files. + + # It's very tempting to use a set here. Unfortunately, the recursive + # make backend needs order preserved. Once we autogenerate all backend + # files, we should be able to convert this to a set. + recurse_info = OrderedDict() + for d in dirs: + if d in recurse_info: + raise SandboxValidationError( + 'Directory (%s) registered multiple times' % ( + mozpath.relpath(d.full_path, context.srcdir)), + context) + + recurse_info[d] = {} + for key in sandbox.metadata: + if key == 'exports': + sandbox.recompute_exports() + + recurse_info[d][key] = dict(sandbox.metadata[key]) + + for path, child_metadata in recurse_info.items(): + child_path = path.join('moz.build').full_path + + # Ensure we don't break out of the topsrcdir. We don't do realpath + # because it isn't necessary. If there are symlinks in the srcdir, + # that's not our problem. We're not a hosted application: we don't + # need to worry about security too much. + if not is_read_allowed(child_path, context.config): + raise SandboxValidationError( + 'Attempting to process file outside of allowed paths: %s' % + child_path, context) + + if not descend: + continue + + for res in self.read_mozbuild(child_path, context.config, + metadata=child_metadata): + yield res + + self._execution_stack.pop() + + def _find_relevant_mozbuilds(self, paths): + """Given a set of filesystem paths, find all relevant moz.build files. + + We assume that a moz.build file in the directory ancestry of a given path + is relevant to that path. Let's say we have the following files on disk:: + + moz.build + foo/moz.build + foo/baz/moz.build + foo/baz/file1 + other/moz.build + other/file2 + + If ``foo/baz/file1`` is passed in, the relevant moz.build files are + ``moz.build``, ``foo/moz.build``, and ``foo/baz/moz.build``. For + ``other/file2``, the relevant moz.build files are ``moz.build`` and + ``other/moz.build``. + + Returns a dict of input paths to a list of relevant moz.build files. + The root moz.build file is first and the leaf-most moz.build is last. + """ + root = self.config.topsrcdir + result = {} + + @memoize + def exists(path): + return self._finder.get(path) is not None + + def itermozbuild(path): + subpath = '' + yield 'moz.build' + for part in mozpath.split(path): + subpath = mozpath.join(subpath, part) + yield mozpath.join(subpath, 'moz.build') + + for path in sorted(paths): + path = mozpath.normpath(path) + if os.path.isabs(path): + if not mozpath.basedir(path, [root]): + raise Exception('Path outside topsrcdir: %s' % path) + path = mozpath.relpath(path, root) + + result[path] = [p for p in itermozbuild(path) + if exists(mozpath.join(root, p))] + + return result + + def read_relevant_mozbuilds(self, paths): + """Read and process moz.build files relevant for a set of paths. + + For an iterable of relative-to-root filesystem paths ``paths``, + find all moz.build files that may apply to them based on filesystem + hierarchy and read those moz.build files. + + The return value is a 2-tuple. The first item is a dict mapping each + input filesystem path to a list of Context instances that are relevant + to that path. The second item is a list of all Context instances. Each + Context instance is in both data structures. + """ + relevants = self._find_relevant_mozbuilds(paths) + + topsrcdir = self.config.topsrcdir + + # Source moz.build file to directories to traverse. + dirs = defaultdict(set) + # Relevant path to absolute paths of relevant contexts. + path_mozbuilds = {} + + # There is room to improve this code (and the code in + # _find_relevant_mozbuilds) to better handle multiple files in the same + # directory. Bug 1136966 tracks. + for path, mbpaths in relevants.items(): + path_mozbuilds[path] = [mozpath.join(topsrcdir, p) for p in mbpaths] + + for i, mbpath in enumerate(mbpaths[0:-1]): + source_dir = mozpath.dirname(mbpath) + target_dir = mozpath.dirname(mbpaths[i + 1]) + + d = mozpath.normpath(mozpath.join(topsrcdir, mbpath)) + dirs[d].add(mozpath.relpath(target_dir, source_dir)) + + # Exporting doesn't work reliably in tree traversal mode. Override + # the function to no-op. + functions = dict(FUNCTIONS) + def export(sandbox): + return lambda varname: None + functions['export'] = tuple([export] + list(FUNCTIONS['export'][1:])) + + metadata = { + 'functions': functions, + } + + contexts = defaultdict(list) + all_contexts = [] + for context in self.read_mozbuild(mozpath.join(topsrcdir, 'moz.build'), + self.config, metadata=metadata): + # Explicitly set directory traversal variables to override default + # traversal rules. + if not isinstance(context, SubContext): + for v in ('DIRS', 'GYP_DIRS'): + context[v][:] = [] + + context['DIRS'] = sorted(dirs[context.main_path]) + + contexts[context.main_path].append(context) + all_contexts.append(context) + + result = {} + for path, paths in path_mozbuilds.items(): + result[path] = reduce(lambda x, y: x + y, (contexts[p] for p in paths), []) + + return result, all_contexts + + def files_info(self, paths): + """Obtain aggregate data from Files for a set of files. + + Given a set of input paths, determine which moz.build files may + define metadata for them, evaluate those moz.build files, and + apply file metadata rules defined within to determine metadata + values for each file requested. + + Essentially, for each input path: + + 1. Determine the set of moz.build files relevant to that file by + looking for moz.build files in ancestor directories. + 2. Evaluate moz.build files starting with the most distant. + 3. Iterate over Files sub-contexts. + 4. If the file pattern matches the file we're seeking info on, + apply attribute updates. + 5. Return the most recent value of attributes. + """ + paths, _ = self.read_relevant_mozbuilds(paths) + + r = {} + + for path, ctxs in paths.items(): + flags = Files(Context()) + + for ctx in ctxs: + if not isinstance(ctx, Files): + continue + + relpath = mozpath.relpath(path, ctx.relsrcdir) + pattern = ctx.pattern + + # Only do wildcard matching if the '*' character is present. + # Otherwise, mozpath.match will match directories, which we've + # arbitrarily chosen to not allow. + if pattern == relpath or \ + ('*' in pattern and mozpath.match(relpath, pattern)): + flags += ctx + + if not any([flags.test_tags, flags.test_files, flags.test_flavors]): + flags += self.test_defaults_for_path(ctxs) + + r[path] = flags + + return r + + def test_defaults_for_path(self, ctxs): + # This names the context keys that will end up emitting a test + # manifest. + test_manifest_contexts = set( + ['%s_MANIFESTS' % key for key in TEST_MANIFESTS] + + ['%s_MANIFESTS' % flavor.upper() for flavor in REFTEST_FLAVORS] + + ['%s_MANIFESTS' % flavor.upper().replace('-', '_') for flavor in WEB_PLATFORM_TESTS_FLAVORS] + ) + + result_context = Files(Context()) + for ctx in ctxs: + for key in ctx: + if key not in test_manifest_contexts: + continue + for paths, obj in ctx[key]: + if isinstance(paths, tuple): + path, tests_root = paths + tests_root = mozpath.join(ctx.relsrcdir, tests_root) + for t in (mozpath.join(tests_root, path) for path, _ in obj): + result_context.test_files.add(mozpath.dirname(t) + '/**') + else: + for t in obj.tests: + if isinstance(t, tuple): + path, _ = t + relpath = mozpath.relpath(path, + self.config.topsrcdir) + else: + relpath = t['relpath'] + result_context.test_files.add(mozpath.dirname(relpath) + '/**') + return result_context diff --git a/python/mozbuild/mozbuild/frontend/sandbox.py b/python/mozbuild/mozbuild/frontend/sandbox.py new file mode 100644 index 000000000..0bf1599f2 --- /dev/null +++ b/python/mozbuild/mozbuild/frontend/sandbox.py @@ -0,0 +1,308 @@ +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, You can obtain one at http://mozilla.org/MPL/2.0/. + +r"""Python sandbox implementation for build files. + +This module contains classes for Python sandboxes that execute in a +highly-controlled environment. + +The main class is `Sandbox`. This provides an execution environment for Python +code and is used to fill a Context instance for the takeaway information from +the execution. + +Code in this module takes a different approach to exception handling compared +to what you'd see elsewhere in Python. Arguments to built-in exceptions like +KeyError are machine parseable. This machine-friendly data is used to present +user-friendly error messages in the case of errors. +""" + +from __future__ import absolute_import, unicode_literals + +import os +import sys +import weakref + +from mozbuild.util import ( + exec_, + ReadOnlyDict, +) +from .context import Context +from mozpack.files import FileFinder + + +default_finder = FileFinder('/', find_executables=False) + + +def alphabetical_sorted(iterable, cmp=None, key=lambda x: x.lower(), + reverse=False): + """sorted() replacement for the sandbox, ordering alphabetically by + default. + """ + return sorted(iterable, cmp, key, reverse) + + +class SandboxError(Exception): + def __init__(self, file_stack): + self.file_stack = file_stack + + +class SandboxExecutionError(SandboxError): + """Represents errors encountered during execution of a Sandbox. + + This is a simple container exception. It's purpose is to capture state + so something else can report on it. + """ + def __init__(self, file_stack, exc_type, exc_value, trace): + SandboxError.__init__(self, file_stack) + + self.exc_type = exc_type + self.exc_value = exc_value + self.trace = trace + + +class SandboxLoadError(SandboxError): + """Represents errors encountered when loading a file for execution. + + This exception represents errors in a Sandbox that occurred as part of + loading a file. The error could have occurred in the course of executing + a file. If so, the file_stack will be non-empty and the file that caused + the load will be on top of the stack. + """ + def __init__(self, file_stack, trace, illegal_path=None, read_error=None): + SandboxError.__init__(self, file_stack) + + self.trace = trace + self.illegal_path = illegal_path + self.read_error = read_error + + +class Sandbox(dict): + """Represents a sandbox for executing Python code. + + This class provides a sandbox for execution of a single mozbuild frontend + file. The results of that execution is stored in the Context instance given + as the ``context`` argument. + + Sandbox is effectively a glorified wrapper around compile() + exec(). You + point it at some Python code and it executes it. The main difference from + executing Python code like normal is that the executed code is very limited + in what it can do: the sandbox only exposes a very limited set of Python + functionality. Only specific types and functions are available. This + prevents executed code from doing things like import modules, open files, + etc. + + Sandbox instances act as global namespace for the sandboxed execution + itself. They shall not be used to access the results of the execution. + Those results are available in the given Context instance after execution. + + The Sandbox itself is responsible for enforcing rules such as forbidding + reassignment of variables. + + Implementation note: Sandbox derives from dict because exec() insists that + what it is given for namespaces is a dict. + """ + # The default set of builtins. + BUILTINS = ReadOnlyDict({ + # Only real Python built-ins should go here. + 'None': None, + 'False': False, + 'True': True, + 'sorted': alphabetical_sorted, + 'int': int, + }) + + def __init__(self, context, builtins=None, finder=default_finder): + """Initialize a Sandbox ready for execution. + """ + self._builtins = builtins or self.BUILTINS + dict.__setitem__(self, '__builtins__', self._builtins) + + assert isinstance(self._builtins, ReadOnlyDict) + assert isinstance(context, Context) + + # Contexts are modeled as a stack because multiple context managers + # may be active. + self._active_contexts = [context] + + # Seen sub-contexts. Will be populated with other Context instances + # that were related to execution of this instance. + self.subcontexts = [] + + # We need to record this because it gets swallowed as part of + # evaluation. + self._last_name_error = None + + # Current literal source being executed. + self._current_source = None + + self._finder = finder + + @property + def _context(self): + return self._active_contexts[-1] + + def exec_file(self, path): + """Execute code at a path in the sandbox. + + The path must be absolute. + """ + assert os.path.isabs(path) + + try: + source = self._finder.get(path).read() + except Exception as e: + raise SandboxLoadError(self._context.source_stack, + sys.exc_info()[2], read_error=path) + + self.exec_source(source, path) + + def exec_source(self, source, path=''): + """Execute Python code within a string. + + The passed string should contain Python code to be executed. The string + will be compiled and executed. + + You should almost always go through exec_file() because exec_source() + does not perform extra path normalization. This can cause relative + paths to behave weirdly. + """ + def execute(): + # compile() inherits the __future__ from the module by default. We + # do want Unicode literals. + code = compile(source, path, 'exec') + # We use ourself as the global namespace for the execution. There + # is no need for a separate local namespace as moz.build execution + # is flat, namespace-wise. + old_source = self._current_source + self._current_source = source + try: + exec_(code, self) + finally: + self._current_source = old_source + + self.exec_function(execute, path=path) + + def exec_function(self, func, args=(), kwargs={}, path='', + becomes_current_path=True): + """Execute function with the given arguments in the sandbox. + """ + if path and becomes_current_path: + self._context.push_source(path) + + old_sandbox = self._context._sandbox + self._context._sandbox = weakref.ref(self) + + # We don't have to worry about bytecode generation here because we are + # too low-level for that. However, we could add bytecode generation via + # the marshall module if parsing performance were ever an issue. + + old_source = self._current_source + self._current_source = None + try: + func(*args, **kwargs) + except SandboxError as e: + raise e + except NameError as e: + # A NameError is raised when a variable could not be found. + # The original KeyError has been dropped by the interpreter. + # However, we should have it cached in our instance! + + # Unless a script is doing something wonky like catching NameError + # itself (that would be silly), if there is an exception on the + # global namespace, that's our error. + actual = e + + if self._last_name_error is not None: + actual = self._last_name_error + source_stack = self._context.source_stack + if not becomes_current_path: + # Add current file to the stack because it wasn't added before + # sandbox execution. + source_stack.append(path) + raise SandboxExecutionError(source_stack, type(actual), actual, + sys.exc_info()[2]) + + except Exception as e: + # Need to copy the stack otherwise we get a reference and that is + # mutated during the finally. + exc = sys.exc_info() + source_stack = self._context.source_stack + if not becomes_current_path: + # Add current file to the stack because it wasn't added before + # sandbox execution. + source_stack.append(path) + raise SandboxExecutionError(source_stack, exc[0], exc[1], exc[2]) + finally: + self._current_source = old_source + self._context._sandbox = old_sandbox + if path and becomes_current_path: + self._context.pop_source() + + def push_subcontext(self, context): + """Push a SubContext onto the execution stack. + + When called, the active context will be set to the specified context, + meaning all variable accesses will go through it. We also record this + SubContext as having been executed as part of this sandbox. + """ + self._active_contexts.append(context) + if context not in self.subcontexts: + self.subcontexts.append(context) + + def pop_subcontext(self, context): + """Pop a SubContext off the execution stack. + + SubContexts must be pushed and popped in opposite order. This is + validated as part of the function call to ensure proper consumer API + use. + """ + popped = self._active_contexts.pop() + assert popped == context + + def __getitem__(self, key): + if key.isupper(): + try: + return self._context[key] + except Exception as e: + self._last_name_error = e + raise + + return dict.__getitem__(self, key) + + def __setitem__(self, key, value): + if key in self._builtins or key == '__builtins__': + raise KeyError('Cannot reassign builtins') + + if key.isupper(): + # Forbid assigning over a previously set value. Interestingly, when + # doing FOO += ['bar'], python actually does something like: + # foo = namespace.__getitem__('FOO') + # foo.__iadd__(['bar']) + # namespace.__setitem__('FOO', foo) + # This means __setitem__ is called with the value that is already + # in the dict, when doing +=, which is permitted. + if key in self._context and self._context[key] is not value: + raise KeyError('global_ns', 'reassign', key) + + if (key not in self._context and isinstance(value, (list, dict)) + and not value): + raise KeyError('Variable %s assigned an empty value.' % key) + + self._context[key] = value + else: + dict.__setitem__(self, key, value) + + def get(self, key, default=None): + raise NotImplementedError('Not supported') + + def __len__(self): + raise NotImplementedError('Not supported') + + def __iter__(self): + raise NotImplementedError('Not supported') + + def __contains__(self, key): + if key.isupper(): + return key in self._context + return dict.__contains__(self, key) diff --git a/python/mozbuild/mozbuild/html_build_viewer.py b/python/mozbuild/mozbuild/html_build_viewer.py new file mode 100644 index 000000000..5151f04a4 --- /dev/null +++ b/python/mozbuild/mozbuild/html_build_viewer.py @@ -0,0 +1,120 @@ +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, You can obtain one at http://mozilla.org/MPL/2.0/. + +# This module contains code for running an HTTP server to view build info. + +from __future__ import absolute_import, unicode_literals + +import BaseHTTPServer +import json +import os + +import requests + + +class HTTPHandler(BaseHTTPServer.BaseHTTPRequestHandler): + def do_GET(self): + s = self.server.wrapper + p = self.path + + if p == '/list': + self.send_response(200) + self.send_header('Content-Type', 'application/json; charset=utf-8') + self.end_headers() + + keys = sorted(s.json_files.keys()) + json.dump({'files': keys}, self.wfile) + return + + if p.startswith('/resources/'): + key = p[len('/resources/'):] + + if key not in s.json_files: + self.send_error(404) + return + + self.send_response(200) + self.send_header('Content-Type', 'application/json; charset=utf-8') + self.end_headers() + + self.wfile.write(s.json_files[key]) + return + + if p == '/': + p = '/index.html' + + self.serve_docroot(s.doc_root, p[1:]) + + def do_POST(self): + if self.path == '/shutdown': + self.server.wrapper.do_shutdown = True + self.send_response(200) + return + + self.send_error(404) + + def serve_docroot(self, root, path): + local_path = os.path.normpath(os.path.join(root, path)) + + # Cheap security. This doesn't resolve symlinks, etc. But, it should be + # acceptable since this server only runs locally. + if not local_path.startswith(root): + self.send_error(404) + + if not os.path.exists(local_path): + self.send_error(404) + return + + if os.path.isdir(local_path): + self.send_error(500) + return + + self.send_response(200) + ct = 'text/plain' + if path.endswith('.html'): + ct = 'text/html' + + self.send_header('Content-Type', ct) + self.end_headers() + + with open(local_path, 'rb') as fh: + self.wfile.write(fh.read()) + + +class BuildViewerServer(object): + def __init__(self, address='localhost', port=0): + # TODO use pkg_resources to obtain HTML resources. + pkg_dir = os.path.dirname(os.path.abspath(__file__)) + doc_root = os.path.join(pkg_dir, 'resources', 'html-build-viewer') + assert os.path.isdir(doc_root) + + self.doc_root = doc_root + self.json_files = {} + + self.server = BaseHTTPServer.HTTPServer((address, port), HTTPHandler) + self.server.wrapper = self + self.do_shutdown = False + + @property + def url(self): + hostname, port = self.server.server_address + return 'http://%s:%d/' % (hostname, port) + + def add_resource_json_file(self, key, path): + """Register a resource JSON file with the server. + + The file will be made available under the name/key specified.""" + with open(path, 'rb') as fh: + self.json_files[key] = fh.read() + + def add_resource_json_url(self, key, url): + """Register a resource JSON file at a URL.""" + r = requests.get(url) + if r.status_code != 200: + raise Exception('Non-200 HTTP response code') + self.json_files[key] = r.text + + def run(self): + while not self.do_shutdown: + self.server.handle_request() diff --git a/python/mozbuild/mozbuild/jar.py b/python/mozbuild/mozbuild/jar.py new file mode 100644 index 000000000..d40751b69 --- /dev/null +++ b/python/mozbuild/mozbuild/jar.py @@ -0,0 +1,597 @@ +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, You can obtain one at http://mozilla.org/MPL/2.0/. + +'''jarmaker.py provides a python class to package up chrome content by +processing jar.mn files. + +See the documentation for jar.mn on MDC for further details on the format. +''' + +from __future__ import absolute_import + +import sys +import os +import errno +import re +import logging +from time import localtime +from MozZipFile import ZipFile +from cStringIO import StringIO +from collections import defaultdict + +from mozbuild.preprocessor import Preprocessor +from mozbuild.action.buildlist import addEntriesToListFile +from mozpack.files import FileFinder +import mozpack.path as mozpath +if sys.platform == 'win32': + from ctypes import windll, WinError + CreateHardLink = windll.kernel32.CreateHardLinkA + +__all__ = ['JarMaker'] + + +class ZipEntry(object): + '''Helper class for jar output. + + This class defines a simple file-like object for a zipfile.ZipEntry + so that we can consecutively write to it and then close it. + This methods hooks into ZipFile.writestr on close(). + ''' + + def __init__(self, name, zipfile): + self._zipfile = zipfile + self._name = name + self._inner = StringIO() + + def write(self, content): + '''Append the given content to this zip entry''' + + self._inner.write(content) + return + + def close(self): + '''The close method writes the content back to the zip file.''' + + self._zipfile.writestr(self._name, self._inner.getvalue()) + + +def getModTime(aPath): + if not os.path.isfile(aPath): + return 0 + mtime = os.stat(aPath).st_mtime + return localtime(mtime) + + +class JarManifestEntry(object): + def __init__(self, output, source, is_locale=False, preprocess=False): + self.output = output + self.source = source + self.is_locale = is_locale + self.preprocess = preprocess + + +class JarInfo(object): + def __init__(self, base_or_jarinfo, name=None): + if name is None: + assert isinstance(base_or_jarinfo, JarInfo) + self.base = base_or_jarinfo.base + self.name = base_or_jarinfo.name + else: + assert not isinstance(base_or_jarinfo, JarInfo) + self.base = base_or_jarinfo or '' + self.name = name + # For compatibility with existing jar.mn files, if there is no + # base, the jar name is under chrome/ + if not self.base: + self.name = mozpath.join('chrome', self.name) + self.relativesrcdir = None + self.chrome_manifests = [] + self.entries = [] + + +class DeprecatedJarManifest(Exception): pass + + +class JarManifestParser(object): + + ignore = re.compile('\s*(\#.*)?$') + jarline = re.compile(''' + (?: + (?:\[(?P[\w\d.\-\_\\\/{}@]+)\]\s*)? # optional [base/path] + (?P[\w\d.\-\_\\\/{}]+).jar\: # filename.jar: + | + (?:\s*(\#.*)?) # comment + )\s*$ # whitespaces + ''', re.VERBOSE) + relsrcline = re.compile('relativesrcdir\s+(?P.+?):') + regline = re.compile('\%\s+(.*)$') + entryre = '(?P\*)?(?P\+?)\s+' + entryline = re.compile(entryre + + '(?P[\w\d.\-\_\\\/\+\@]+)\s*(\((?P\%?)(?P[\w\d.\-\_\\\/\@\*]+)\))?\s*$' + ) + + def __init__(self): + self._current_jar = None + self._jars = [] + + def write(self, line): + # A Preprocessor instance feeds the parser through calls to this method. + + # Ignore comments and empty lines + if self.ignore.match(line): + return + + # A jar manifest file can declare several different sections, each of + # which applies to a given "jar file". Each of those sections starts + # with ".jar:", in which case the path is assumed relative to + # a "chrome" directory, or "[] .jar:", where + # a base directory is given (usually pointing at the root of the + # application or addon) and the jar path is given relative to the base + # directory. + if self._current_jar is None: + m = self.jarline.match(line) + if not m: + raise RuntimeError(line) + if m.group('jarfile'): + self._current_jar = JarInfo(m.group('base'), + m.group('jarfile')) + self._jars.append(self._current_jar) + return + + # Within each section, there can be three different types of entries: + + # - indications of the relative source directory we pretend to be in + # when considering localization files, in the following form; + # "relativesrcdir :" + m = self.relsrcline.match(line) + if m: + if self._current_jar.chrome_manifests or self._current_jar.entries: + self._current_jar = JarInfo(self._current_jar) + self._jars.append(self._current_jar) + self._current_jar.relativesrcdir = m.group('relativesrcdir') + return + + # - chrome manifest entries, prefixed with "%". + m = self.regline.match(line) + if m: + rline = ' '.join(m.group(1).split()) + if rline not in self._current_jar.chrome_manifests: + self._current_jar.chrome_manifests.append(rline) + return + + # - entries indicating files to be part of the given jar. They are + # formed thusly: + # "" + # or + # " ()" + # The is where the file(s) will be put in the chrome jar. + # The is where the file(s) can be found in the source + # directory. The may start with a "%" for files part + # of a localization directory, in which case the "%" counts as the + # locale. + # Each entry can be prefixed with "*" for preprocessing. + m = self.entryline.match(line) + if m: + if m.group('optOverwrite'): + raise DeprecatedJarManifest( + 'The "+" prefix is not supported anymore') + self._current_jar.entries.append(JarManifestEntry( + m.group('output'), + m.group('source') or mozpath.basename(m.group('output')), + is_locale=bool(m.group('locale')), + preprocess=bool(m.group('optPreprocess')), + )) + return + + self._current_jar = None + self.write(line) + + def __iter__(self): + return iter(self._jars) + + +class JarMaker(object): + '''JarMaker reads jar.mn files and process those into jar files or + flat directories, along with chrome.manifest files. + ''' + + def __init__(self, outputFormat='flat', useJarfileManifest=True, + useChromeManifest=False): + + self.outputFormat = outputFormat + self.useJarfileManifest = useJarfileManifest + self.useChromeManifest = useChromeManifest + self.pp = Preprocessor() + self.topsourcedir = None + self.sourcedirs = [] + self.localedirs = None + self.l10nbase = None + self.l10nmerge = None + self.relativesrcdir = None + self.rootManifestAppId = None + self._seen_output = set() + + def getCommandLineParser(self): + '''Get a optparse.OptionParser for jarmaker. + + This OptionParser has the options for jarmaker as well as + the options for the inner PreProcessor. + ''' + + # HACK, we need to unescape the string variables we get, + # the perl versions didn't grok strings right + + p = self.pp.getCommandLineParser(unescapeDefines=True) + p.add_option('-f', type='choice', default='jar', + choices=('jar', 'flat', 'symlink'), + help='fileformat used for output', + metavar='[jar, flat, symlink]', + ) + p.add_option('-v', action='store_true', dest='verbose', + help='verbose output') + p.add_option('-q', action='store_false', dest='verbose', + help='verbose output') + p.add_option('-e', action='store_true', + help='create chrome.manifest instead of jarfile.manifest' + ) + p.add_option('-s', type='string', action='append', default=[], + help='source directory') + p.add_option('-t', type='string', help='top source directory') + p.add_option('-c', '--l10n-src', type='string', action='append' + , help='localization directory') + p.add_option('--l10n-base', type='string', action='store', + help='base directory to be used for localization (requires relativesrcdir)' + ) + p.add_option('--locale-mergedir', type='string', action='store' + , + help='base directory to be used for l10n-merge (requires l10n-base and relativesrcdir)' + ) + p.add_option('--relativesrcdir', type='string', + help='relativesrcdir to be used for localization') + p.add_option('-d', type='string', help='base directory') + p.add_option('--root-manifest-entry-appid', type='string', + help='add an app id specific root chrome manifest entry.' + ) + return p + + def finalizeJar(self, jardir, jarbase, jarname, chromebasepath, register, doZip=True): + '''Helper method to write out the chrome registration entries to + jarfile.manifest or chrome.manifest, or both. + + The actual file processing is done in updateManifest. + ''' + + # rewrite the manifest, if entries given + if not register: + return + + chromeManifest = os.path.join(jardir, jarbase, 'chrome.manifest') + + if self.useJarfileManifest: + self.updateManifest(os.path.join(jardir, jarbase, + jarname + '.manifest'), + chromebasepath.format(''), register) + if jarname != 'chrome': + addEntriesToListFile(chromeManifest, + ['manifest {0}.manifest'.format(jarname)]) + if self.useChromeManifest: + chromebase = os.path.dirname(jarname) + '/' + self.updateManifest(chromeManifest, + chromebasepath.format(chromebase), register) + + # If requested, add a root chrome manifest entry (assumed to be in the parent directory + # of chromeManifest) with the application specific id. In cases where we're building + # lang packs, the root manifest must know about application sub directories. + + if self.rootManifestAppId: + rootChromeManifest = \ + os.path.join(os.path.normpath(os.path.dirname(chromeManifest)), + '..', 'chrome.manifest') + rootChromeManifest = os.path.normpath(rootChromeManifest) + chromeDir = \ + os.path.basename(os.path.dirname(os.path.normpath(chromeManifest))) + logging.info("adding '%s' entry to root chrome manifest appid=%s" + % (chromeDir, self.rootManifestAppId)) + addEntriesToListFile(rootChromeManifest, + ['manifest %s/chrome.manifest application=%s' + % (chromeDir, + self.rootManifestAppId)]) + + def updateManifest(self, manifestPath, chromebasepath, register): + '''updateManifest replaces the % in the chrome registration entries + with the given chrome base path, and updates the given manifest file. + ''' + myregister = dict.fromkeys(map(lambda s: s.replace('%', + chromebasepath), register)) + addEntriesToListFile(manifestPath, myregister.iterkeys()) + + def makeJar(self, infile, jardir): + '''makeJar is the main entry point to JarMaker. + + It takes the input file, the output directory, the source dirs and the + top source dir as argument, and optionally the l10n dirs. + ''' + + # making paths absolute, guess srcdir if file and add to sourcedirs + _normpath = lambda p: os.path.normpath(os.path.abspath(p)) + self.topsourcedir = _normpath(self.topsourcedir) + self.sourcedirs = [_normpath(p) for p in self.sourcedirs] + if self.localedirs: + self.localedirs = [_normpath(p) for p in self.localedirs] + elif self.relativesrcdir: + self.localedirs = \ + self.generateLocaleDirs(self.relativesrcdir) + if isinstance(infile, basestring): + logging.info('processing ' + infile) + self.sourcedirs.append(_normpath(os.path.dirname(infile))) + pp = self.pp.clone() + pp.out = JarManifestParser() + pp.do_include(infile) + + for info in pp.out: + self.processJarSection(info, jardir) + + def generateLocaleDirs(self, relativesrcdir): + if os.path.basename(relativesrcdir) == 'locales': + # strip locales + l10nrelsrcdir = os.path.dirname(relativesrcdir) + else: + l10nrelsrcdir = relativesrcdir + locdirs = [] + + # generate locales dirs, merge, l10nbase, en-US + if self.l10nmerge: + locdirs.append(os.path.join(self.l10nmerge, l10nrelsrcdir)) + if self.l10nbase: + locdirs.append(os.path.join(self.l10nbase, l10nrelsrcdir)) + if self.l10nmerge or not self.l10nbase: + # add en-US if we merge, or if it's not l10n + locdirs.append(os.path.join(self.topsourcedir, + relativesrcdir, 'en-US')) + return locdirs + + def processJarSection(self, jarinfo, jardir): + '''Internal method called by makeJar to actually process a section + of a jar.mn file. + ''' + + # chromebasepath is used for chrome registration manifests + # {0} is getting replaced with chrome/ for chrome.manifest, and with + # an empty string for jarfile.manifest + + chromebasepath = '{0}' + os.path.basename(jarinfo.name) + if self.outputFormat == 'jar': + chromebasepath = 'jar:' + chromebasepath + '.jar!' + chromebasepath += '/' + + jarfile = os.path.join(jardir, jarinfo.base, jarinfo.name) + jf = None + if self.outputFormat == 'jar': + # jar + jarfilepath = jarfile + '.jar' + try: + os.makedirs(os.path.dirname(jarfilepath)) + except OSError, error: + if error.errno != errno.EEXIST: + raise + jf = ZipFile(jarfilepath, 'a', lock=True) + outHelper = self.OutputHelper_jar(jf) + else: + outHelper = getattr(self, 'OutputHelper_' + + self.outputFormat)(jarfile) + + if jarinfo.relativesrcdir: + self.localedirs = self.generateLocaleDirs(jarinfo.relativesrcdir) + + for e in jarinfo.entries: + self._processEntryLine(e, outHelper, jf) + + self.finalizeJar(jardir, jarinfo.base, jarinfo.name, chromebasepath, + jarinfo.chrome_manifests) + if jf is not None: + jf.close() + + def _processEntryLine(self, e, outHelper, jf): + out = e.output + src = e.source + + # pick the right sourcedir -- l10n, topsrc or src + + if e.is_locale: + src_base = self.localedirs + elif src.startswith('/'): + # path/in/jar/file_name.xul (/path/in/sourcetree/file_name.xul) + # refers to a path relative to topsourcedir, use that as base + # and strip the leading '/' + src_base = [self.topsourcedir] + src = src[1:] + else: + # use srcdirs and the objdir (current working dir) for relative paths + src_base = self.sourcedirs + [os.getcwd()] + + if '*' in src: + def _prefix(s): + for p in s.split('/'): + if '*' not in p: + yield p + '/' + prefix = ''.join(_prefix(src)) + emitted = set() + for _srcdir in src_base: + finder = FileFinder(_srcdir, find_executables=False) + for path, _ in finder.find(src): + # If the path was already seen in one of the other source + # directories, skip it. That matches the non-wildcard case + # below, where we pick the first existing file. + reduced_path = path[len(prefix):] + if reduced_path in emitted: + continue + emitted.add(reduced_path) + e = JarManifestEntry( + mozpath.join(out, reduced_path), + path, + is_locale=e.is_locale, + preprocess=e.preprocess, + ) + self._processEntryLine(e, outHelper, jf) + return + + # check if the source file exists + realsrc = None + for _srcdir in src_base: + if os.path.isfile(os.path.join(_srcdir, src)): + realsrc = os.path.join(_srcdir, src) + break + if realsrc is None: + if jf is not None: + jf.close() + raise RuntimeError('File "{0}" not found in {1}'.format(src, + ', '.join(src_base))) + + if out in self._seen_output: + raise RuntimeError('%s already added' % out) + self._seen_output.add(out) + + if e.preprocess: + outf = outHelper.getOutput(out) + inf = open(realsrc) + pp = self.pp.clone() + if src[-4:] == '.css': + pp.setMarker('%') + pp.out = outf + pp.do_include(inf) + pp.failUnused(realsrc) + outf.close() + inf.close() + return + + # copy or symlink if newer + + if getModTime(realsrc) > outHelper.getDestModTime(e.output): + if self.outputFormat == 'symlink': + outHelper.symlink(realsrc, out) + return + outf = outHelper.getOutput(out) + + # open in binary mode, this can be images etc + + inf = open(realsrc, 'rb') + outf.write(inf.read()) + outf.close() + inf.close() + + class OutputHelper_jar(object): + '''Provide getDestModTime and getOutput for a given jarfile.''' + + def __init__(self, jarfile): + self.jarfile = jarfile + + def getDestModTime(self, aPath): + try: + info = self.jarfile.getinfo(aPath) + return info.date_time + except: + return 0 + + def getOutput(self, name): + return ZipEntry(name, self.jarfile) + + class OutputHelper_flat(object): + '''Provide getDestModTime and getOutput for a given flat + output directory. The helper method ensureDirFor is used by + the symlink subclass. + ''' + + def __init__(self, basepath): + self.basepath = basepath + + def getDestModTime(self, aPath): + return getModTime(os.path.join(self.basepath, aPath)) + + def getOutput(self, name): + out = self.ensureDirFor(name) + + # remove previous link or file + try: + os.remove(out) + except OSError, e: + if e.errno != errno.ENOENT: + raise + return open(out, 'wb') + + def ensureDirFor(self, name): + out = os.path.join(self.basepath, name) + outdir = os.path.dirname(out) + if not os.path.isdir(outdir): + try: + os.makedirs(outdir) + except OSError, error: + if error.errno != errno.EEXIST: + raise + return out + + class OutputHelper_symlink(OutputHelper_flat): + '''Subclass of OutputHelper_flat that provides a helper for + creating a symlink including creating the parent directories. + ''' + + def symlink(self, src, dest): + out = self.ensureDirFor(dest) + + # remove previous link or file + try: + os.remove(out) + except OSError, e: + if e.errno != errno.ENOENT: + raise + if sys.platform != 'win32': + os.symlink(src, out) + else: + # On Win32, use ctypes to create a hardlink + rv = CreateHardLink(out, src, None) + if rv == 0: + raise WinError() + + +def main(args=None): + args = args or sys.argv + jm = JarMaker() + p = jm.getCommandLineParser() + (options, args) = p.parse_args(args) + jm.outputFormat = options.f + jm.sourcedirs = options.s + jm.topsourcedir = options.t + if options.e: + jm.useChromeManifest = True + jm.useJarfileManifest = False + if options.l10n_base: + if not options.relativesrcdir: + p.error('relativesrcdir required when using l10n-base') + if options.l10n_src: + p.error('both l10n-src and l10n-base are not supported') + jm.l10nbase = options.l10n_base + jm.relativesrcdir = options.relativesrcdir + jm.l10nmerge = options.locale_mergedir + if jm.l10nmerge and not os.path.isdir(jm.l10nmerge): + logging.warning("WARNING: --locale-mergedir passed, but '%s' does not exist. " + "Ignore this message if the locale is complete." % jm.l10nmerge) + elif options.locale_mergedir: + p.error('l10n-base required when using locale-mergedir') + jm.localedirs = options.l10n_src + if options.root_manifest_entry_appid: + jm.rootManifestAppId = options.root_manifest_entry_appid + noise = logging.INFO + if options.verbose is not None: + noise = options.verbose and logging.DEBUG or logging.WARN + if sys.version_info[:2] > (2, 3): + logging.basicConfig(format='%(message)s') + else: + logging.basicConfig() + logging.getLogger().setLevel(noise) + topsrc = options.t + topsrc = os.path.normpath(os.path.abspath(topsrc)) + if not args: + infile = sys.stdin + else: + (infile, ) = args + jm.makeJar(infile, options.d) diff --git a/python/mozbuild/mozbuild/locale/en-US/LC_MESSAGES/mozbuild.mo b/python/mozbuild/mozbuild/locale/en-US/LC_MESSAGES/mozbuild.mo new file mode 100644 index 000000000..be7711cb2 Binary files /dev/null and b/python/mozbuild/mozbuild/locale/en-US/LC_MESSAGES/mozbuild.mo differ diff --git a/python/mozbuild/mozbuild/locale/en-US/LC_MESSAGES/mozbuild.po b/python/mozbuild/mozbuild/locale/en-US/LC_MESSAGES/mozbuild.po new file mode 100644 index 000000000..fbdfabd83 --- /dev/null +++ b/python/mozbuild/mozbuild/locale/en-US/LC_MESSAGES/mozbuild.po @@ -0,0 +1,8 @@ +msgid "build.threads.short" +msgstr "Thread Count" + +msgid "build.threads.full" +msgstr "The number of threads to use when performing CPU intensive tasks. " + "This constrols the level of parallelization. The default value is " + "the number of cores in your machine." + diff --git a/python/mozbuild/mozbuild/mach_commands.py b/python/mozbuild/mozbuild/mach_commands.py new file mode 100644 index 000000000..b6802a47c --- /dev/null +++ b/python/mozbuild/mozbuild/mach_commands.py @@ -0,0 +1,1603 @@ +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, # You can obtain one at http://mozilla.org/MPL/2.0/. + +from __future__ import absolute_import, print_function, unicode_literals + +import argparse +import errno +import itertools +import json +import logging +import operator +import os +import subprocess +import sys + +import mozpack.path as mozpath + +from mach.decorators import ( + CommandArgument, + CommandArgumentGroup, + CommandProvider, + Command, + SubCommand, +) + +from mach.mixin.logging import LoggingMixin + +from mozbuild.base import ( + BuildEnvironmentNotFoundException, + MachCommandBase, + MachCommandConditions as conditions, + MozbuildObject, + MozconfigFindException, + MozconfigLoadException, + ObjdirMismatchException, +) + +from mozbuild.backend import ( + backends, + get_backend_class, +) +from mozbuild.shellutil import quote as shell_quote + + +BUILD_WHAT_HELP = ''' +What to build. Can be a top-level make target or a relative directory. If +multiple options are provided, they will be built serially. Takes dependency +information from `topsrcdir/build/dumbmake-dependencies` to build additional +targets as needed. BUILDING ONLY PARTS OF THE TREE CAN RESULT IN BAD TREE +STATE. USE AT YOUR OWN RISK. +'''.strip() + +FINDER_SLOW_MESSAGE = ''' +=================== +PERFORMANCE WARNING + +The OS X Finder application (file indexing used by Spotlight) used a lot of CPU +during the build - an average of %f%% (100%% is 1 core). This made your build +slower. + +Consider adding ".noindex" to the end of your object directory name to have +Finder ignore it. Or, add an indexing exclusion through the Spotlight System +Preferences. +=================== +'''.strip() + +EXCESSIVE_SWAP_MESSAGE = ''' +=================== +PERFORMANCE WARNING + +Your machine experienced a lot of swap activity during the build. This is +possibly a sign that your machine doesn't have enough physical memory or +not enough available memory to perform the build. It's also possible some +other system activity during the build is to blame. + +If you feel this message is not appropriate for your machine configuration, +please file a Core :: Build Config bug at +https://bugzilla.mozilla.org/enter_bug.cgi?product=Core&component=Build%20Config +and tell us about your machine and build configuration so we can adjust the +warning heuristic. +=================== +''' + + +class TerminalLoggingHandler(logging.Handler): + """Custom logging handler that works with terminal window dressing. + + This class should probably live elsewhere, like the mach core. Consider + this a proving ground for its usefulness. + """ + def __init__(self): + logging.Handler.__init__(self) + + self.fh = sys.stdout + self.footer = None + + def flush(self): + self.acquire() + + try: + self.fh.flush() + finally: + self.release() + + def emit(self, record): + msg = self.format(record) + + self.acquire() + + try: + if self.footer: + self.footer.clear() + + self.fh.write(msg) + self.fh.write('\n') + + if self.footer: + self.footer.draw() + + # If we don't flush, the footer may not get drawn. + self.fh.flush() + finally: + self.release() + + +class BuildProgressFooter(object): + """Handles display of a build progress indicator in a terminal. + + When mach builds inside a blessings-supported terminal, it will render + progress information collected from a BuildMonitor. This class converts the + state of BuildMonitor into terminal output. + """ + + def __init__(self, terminal, monitor): + # terminal is a blessings.Terminal. + self._t = terminal + self._fh = sys.stdout + self.tiers = monitor.tiers.tier_status.viewitems() + + def clear(self): + """Removes the footer from the current terminal.""" + self._fh.write(self._t.move_x(0)) + self._fh.write(self._t.clear_eos()) + + def draw(self): + """Draws this footer in the terminal.""" + + if not self.tiers: + return + + # The drawn terminal looks something like: + # TIER: base nspr nss js platform app SUBTIER: static export libs tools DIRECTORIES: 06/09 (memory) + + # This is a list of 2-tuples of (encoding function, input). None means + # no encoding. For a full reason on why we do things this way, read the + # big comment below. + parts = [('bold', 'TIER:')] + append = parts.append + for tier, status in self.tiers: + if status is None: + append(tier) + elif status == 'finished': + append(('green', tier)) + else: + append(('underline_yellow', tier)) + + # We don't want to write more characters than the current width of the + # terminal otherwise wrapping may result in weird behavior. We can't + # simply truncate the line at terminal width characters because a) + # non-viewable escape characters count towards the limit and b) we + # don't want to truncate in the middle of an escape sequence because + # subsequent output would inherit the escape sequence. + max_width = self._t.width + written = 0 + write_pieces = [] + for part in parts: + try: + func, part = part + encoded = getattr(self._t, func)(part) + except ValueError: + encoded = part + + len_part = len(part) + len_spaces = len(write_pieces) + if written + len_part + len_spaces > max_width: + write_pieces.append(part[0:max_width - written - len_spaces]) + written += len_part + break + + write_pieces.append(encoded) + written += len_part + + with self._t.location(): + self._t.move(self._t.height-1,0) + self._fh.write(' '.join(write_pieces)) + + +class BuildOutputManager(LoggingMixin): + """Handles writing build output to a terminal, to logs, etc.""" + + def __init__(self, log_manager, monitor): + self.populate_logger() + + self.monitor = monitor + self.footer = None + + terminal = log_manager.terminal + + # TODO convert terminal footer to config file setting. + if not terminal or os.environ.get('MACH_NO_TERMINAL_FOOTER', None): + return + + self.t = terminal + self.footer = BuildProgressFooter(terminal, monitor) + + self._handler = TerminalLoggingHandler() + self._handler.setFormatter(log_manager.terminal_formatter) + self._handler.footer = self.footer + + old = log_manager.replace_terminal_handler(self._handler) + self._handler.level = old.level + + def __enter__(self): + return self + + def __exit__(self, exc_type, exc_value, traceback): + if self.footer: + self.footer.clear() + # Prevents the footer from being redrawn if logging occurs. + self._handler.footer = None + + # Ensure the resource monitor is stopped because leaving it running + # could result in the process hanging on exit because the resource + # collection child process hasn't been told to stop. + self.monitor.stop_resource_recording() + + def write_line(self, line): + if self.footer: + self.footer.clear() + + print(line) + + if self.footer: + self.footer.draw() + + def refresh(self): + if not self.footer: + return + + self.footer.clear() + self.footer.draw() + + def on_line(self, line): + warning, state_changed, relevant = self.monitor.on_line(line) + + if warning: + self.log(logging.INFO, 'compiler_warning', warning, + 'Warning: {flag} in {filename}: {message}') + + if relevant: + self.log(logging.INFO, 'build_output', {'line': line}, '{line}') + elif state_changed: + have_handler = hasattr(self, 'handler') + if have_handler: + self.handler.acquire() + try: + self.refresh() + finally: + if have_handler: + self.handler.release() + + +@CommandProvider +class Build(MachCommandBase): + """Interface to build the tree.""" + + @Command('build', category='build', description='Build the tree.') + @CommandArgument('--jobs', '-j', default='0', metavar='jobs', type=int, + help='Number of concurrent jobs to run. Default is the number of CPUs.') + @CommandArgument('-C', '--directory', default=None, + help='Change to a subdirectory of the build directory first.') + @CommandArgument('what', default=None, nargs='*', help=BUILD_WHAT_HELP) + @CommandArgument('-X', '--disable-extra-make-dependencies', + default=False, action='store_true', + help='Do not add extra make dependencies.') + @CommandArgument('-v', '--verbose', action='store_true', + help='Verbose output for what commands the build is running.') + def build(self, what=None, disable_extra_make_dependencies=None, jobs=0, + directory=None, verbose=False): + """Build the source tree. + + With no arguments, this will perform a full build. + + Positional arguments define targets to build. These can be make targets + or patterns like "/" to indicate a make target within a + directory. + + There are a few special targets that can be used to perform a partial + build faster than what `mach build` would perform: + + * binaries - compiles and links all C/C++ sources and produces shared + libraries and executables (binaries). + + * faster - builds JavaScript, XUL, CSS, etc files. + + "binaries" and "faster" almost fully complement each other. However, + there are build actions not captured by either. If things don't appear to + be rebuilding, perform a vanilla `mach build` to rebuild the world. + """ + import which + from mozbuild.controller.building import BuildMonitor + from mozbuild.util import ( + mkdir, + resolve_target_to_make, + ) + + self.log_manager.register_structured_logger(logging.getLogger('mozbuild')) + + warnings_path = self._get_state_filename('warnings.json') + monitor = self._spawn(BuildMonitor) + monitor.init(warnings_path) + ccache_start = monitor.ccache_stats() + + # Disable indexing in objdir because it is not necessary and can slow + # down builds. + mkdir(self.topobjdir, not_indexed=True) + + with BuildOutputManager(self.log_manager, monitor) as output: + monitor.start() + + if directory is not None and not what: + print('Can only use -C/--directory with an explicit target ' + 'name.') + return 1 + + if directory is not None: + disable_extra_make_dependencies=True + directory = mozpath.normsep(directory) + if directory.startswith('/'): + directory = directory[1:] + + status = None + monitor.start_resource_recording() + if what: + top_make = os.path.join(self.topobjdir, 'Makefile') + if not os.path.exists(top_make): + print('Your tree has not been configured yet. Please run ' + '|mach build| with no arguments.') + return 1 + + # Collect target pairs. + target_pairs = [] + for target in what: + path_arg = self._wrap_path_argument(target) + + if directory is not None: + make_dir = os.path.join(self.topobjdir, directory) + make_target = target + else: + make_dir, make_target = \ + resolve_target_to_make(self.topobjdir, + path_arg.relpath()) + + if make_dir is None and make_target is None: + return 1 + + # See bug 886162 - we don't want to "accidentally" build + # the entire tree (if that's really the intent, it's + # unlikely they would have specified a directory.) + if not make_dir and not make_target: + print("The specified directory doesn't contain a " + "Makefile and the first parent with one is the " + "root of the tree. Please specify a directory " + "with a Makefile or run |mach build| if you " + "want to build the entire tree.") + return 1 + + target_pairs.append((make_dir, make_target)) + + # Possibly add extra make depencies using dumbmake. + if not disable_extra_make_dependencies: + from dumbmake.dumbmake import (dependency_map, + add_extra_dependencies) + depfile = os.path.join(self.topsrcdir, 'build', + 'dumbmake-dependencies') + with open(depfile) as f: + dm = dependency_map(f.readlines()) + new_pairs = list(add_extra_dependencies(target_pairs, dm)) + self.log(logging.DEBUG, 'dumbmake', + {'target_pairs': target_pairs, + 'new_pairs': new_pairs}, + 'Added extra dependencies: will build {new_pairs} ' + + 'instead of {target_pairs}.') + target_pairs = new_pairs + + # Ensure build backend is up to date. The alternative is to + # have rules in the invoked Makefile to rebuild the build + # backend. But that involves make reinvoking itself and there + # are undesired side-effects of this. See bug 877308 for a + # comprehensive history lesson. + self._run_make(directory=self.topobjdir, target='backend', + line_handler=output.on_line, log=False, + print_directory=False) + + # Build target pairs. + for make_dir, make_target in target_pairs: + # We don't display build status messages during partial + # tree builds because they aren't reliable there. This + # could potentially be fixed if the build monitor were more + # intelligent about encountering undefined state. + status = self._run_make(directory=make_dir, target=make_target, + line_handler=output.on_line, log=False, print_directory=False, + ensure_exit_code=False, num_jobs=jobs, silent=not verbose, + append_env={b'NO_BUILDSTATUS_MESSAGES': b'1'}) + + if status != 0: + break + else: + # Try to call the default backend's build() method. This will + # run configure to determine BUILD_BACKENDS if it hasn't run + # yet. + config = None + try: + config = self.config_environment + except Exception: + config_rc = self.configure(buildstatus_messages=True, + line_handler=output.on_line) + if config_rc != 0: + return config_rc + + # Even if configure runs successfully, we may have trouble + # getting the config_environment for some builds, such as + # OSX Universal builds. These have to go through client.mk + # regardless. + try: + config = self.config_environment + except Exception: + pass + + if config: + active_backend = config.substs.get('BUILD_BACKENDS', [None])[0] + if active_backend: + backend_cls = get_backend_class(active_backend)(config) + status = backend_cls.build(self, output, jobs, verbose) + + # If the backend doesn't specify a build() method, then just + # call client.mk directly. + if status is None: + status = self._run_make(srcdir=True, filename='client.mk', + line_handler=output.on_line, log=False, print_directory=False, + allow_parallel=False, ensure_exit_code=False, num_jobs=jobs, + silent=not verbose) + + self.log(logging.WARNING, 'warning_summary', + {'count': len(monitor.warnings_database)}, + '{count} compiler warnings present.') + + monitor.finish(record_usage=status==0) + + high_finder, finder_percent = monitor.have_high_finder_usage() + if high_finder: + print(FINDER_SLOW_MESSAGE % finder_percent) + + ccache_end = monitor.ccache_stats() + + ccache_diff = None + if ccache_start and ccache_end: + ccache_diff = ccache_end - ccache_start + if ccache_diff: + self.log(logging.INFO, 'ccache', + {'msg': ccache_diff.hit_rate_message()}, "{msg}") + + notify_minimum_time = 300 + try: + notify_minimum_time = int(os.environ.get('MACH_NOTIFY_MINTIME', '300')) + except ValueError: + # Just stick with the default + pass + + if monitor.elapsed > notify_minimum_time: + # Display a notification when the build completes. + self.notify('Build complete' if not status else 'Build failed') + + if status: + return status + + long_build = monitor.elapsed > 600 + + if long_build: + output.on_line('We know it took a while, but your build finally finished successfully!') + else: + output.on_line('Your build was successful!') + + if monitor.have_resource_usage: + excessive, swap_in, swap_out = monitor.have_excessive_swapping() + # if excessive: + # print(EXCESSIVE_SWAP_MESSAGE) + + print('To view resource usage of the build, run |mach ' + 'resource-usage|.') + + telemetry_handler = getattr(self._mach_context, + 'telemetry_handler', None) + telemetry_data = monitor.get_resource_usage() + + # Record build configuration data. For now, we cherry pick + # items we need rather than grabbing everything, in order + # to avoid accidentally disclosing PII. + telemetry_data['substs'] = {} + try: + for key in ['MOZ_ARTIFACT_BUILDS', 'MOZ_USING_CCACHE']: + value = self.substs.get(key, False) + telemetry_data['substs'][key] = value + except BuildEnvironmentNotFoundException: + pass + + # Grab ccache stats if available. We need to be careful not + # to capture information that can potentially identify the + # user (such as the cache location) + if ccache_diff: + telemetry_data['ccache'] = {} + for key in [key[0] for key in ccache_diff.STATS_KEYS]: + try: + telemetry_data['ccache'][key] = ccache_diff._values[key] + except KeyError: + pass + + telemetry_handler(self._mach_context, telemetry_data) + + # Only for full builds because incremental builders likely don't + # need to be burdened with this. + if not what: + try: + # Fennec doesn't have useful output from just building. We should + # arguably make the build action useful for Fennec. Another day... + if self.substs['MOZ_BUILD_APP'] != 'mobile/android': + print('To take your build for a test drive, run: |mach run|') + app = self.substs['MOZ_BUILD_APP'] + if app in ('browser', 'mobile/android'): + print('For more information on what to do now, see ' + 'https://developer.mozilla.org/docs/Developer_Guide/So_You_Just_Built_Firefox') + except Exception: + # Ignore Exceptions in case we can't find config.status (such + # as when doing OSX Universal builds) + pass + + return status + + @Command('configure', category='build', + description='Configure the tree (run configure and config.status).') + @CommandArgument('options', default=None, nargs=argparse.REMAINDER, + help='Configure options') + def configure(self, options=None, buildstatus_messages=False, line_handler=None): + def on_line(line): + self.log(logging.INFO, 'build_output', {'line': line}, '{line}') + + line_handler = line_handler or on_line + + options = ' '.join(shell_quote(o) for o in options or ()) + append_env = {b'CONFIGURE_ARGS': options.encode('utf-8')} + + # Only print build status messages when we have an active + # monitor. + if not buildstatus_messages: + append_env[b'NO_BUILDSTATUS_MESSAGES'] = b'1' + status = self._run_make(srcdir=True, filename='client.mk', + target='configure', line_handler=line_handler, log=False, + print_directory=False, allow_parallel=False, ensure_exit_code=False, + append_env=append_env) + + if not status: + print('Configure complete!') + print('Be sure to run |mach build| to pick up any changes'); + + return status + + @Command('resource-usage', category='post-build', + description='Show information about system resource usage for a build.') + @CommandArgument('--address', default='localhost', + help='Address the HTTP server should listen on.') + @CommandArgument('--port', type=int, default=0, + help='Port number the HTTP server should listen on.') + @CommandArgument('--browser', default='firefox', + help='Web browser to automatically open. See webbrowser Python module.') + @CommandArgument('--url', + help='URL of JSON document to display') + def resource_usage(self, address=None, port=None, browser=None, url=None): + import webbrowser + from mozbuild.html_build_viewer import BuildViewerServer + + server = BuildViewerServer(address, port) + + if url: + server.add_resource_json_url('url', url) + else: + last = self._get_state_filename('build_resources.json') + if not os.path.exists(last): + print('Build resources not available. If you have performed a ' + 'build and receive this message, the psutil Python package ' + 'likely failed to initialize properly.') + return 1 + + server.add_resource_json_file('last', last) + try: + webbrowser.get(browser).open_new_tab(server.url) + except Exception: + print('Cannot get browser specified, trying the default instead.') + try: + browser = webbrowser.get().open_new_tab(server.url) + except Exception: + print('Please open %s in a browser.' % server.url) + + print('Hit CTRL+c to stop server.') + server.run() + + @Command('build-backend', category='build', + description='Generate a backend used to build the tree.') + @CommandArgument('-d', '--diff', action='store_true', + help='Show a diff of changes.') + # It would be nice to filter the choices below based on + # conditions, but that is for another day. + @CommandArgument('-b', '--backend', nargs='+', choices=sorted(backends), + help='Which backend to build.') + @CommandArgument('-v', '--verbose', action='store_true', + help='Verbose output.') + @CommandArgument('-n', '--dry-run', action='store_true', + help='Do everything except writing files out.') + def build_backend(self, backend, diff=False, verbose=False, dry_run=False): + python = self.virtualenv_manager.python_path + config_status = os.path.join(self.topobjdir, 'config.status') + + if not os.path.exists(config_status): + print('config.status not found. Please run |mach configure| ' + 'or |mach build| prior to building the %s build backend.' + % backend) + return 1 + + args = [python, config_status] + if backend: + args.append('--backend') + args.extend(backend) + if diff: + args.append('--diff') + if verbose: + args.append('--verbose') + if dry_run: + args.append('--dry-run') + + return self._run_command_in_objdir(args=args, pass_thru=True, + ensure_exit_code=False) + +@CommandProvider +class Doctor(MachCommandBase): + """Provide commands for diagnosing common build environment problems""" + @Command('doctor', category='devenv', + description='') + @CommandArgument('--fix', default=None, action='store_true', + help='Attempt to fix found problems.') + def doctor(self, fix=None): + self._activate_virtualenv() + from mozbuild.doctor import Doctor + doctor = Doctor(self.topsrcdir, self.topobjdir, fix) + return doctor.check_all() + +@CommandProvider +class Clobber(MachCommandBase): + NO_AUTO_LOG = True + CLOBBER_CHOICES = ['objdir', 'python'] + @Command('clobber', category='build', + description='Clobber the tree (delete the object directory).') + @CommandArgument('what', default=['objdir'], nargs='*', + help='Target to clobber, must be one of {{{}}} (default objdir).'.format( + ', '.join(CLOBBER_CHOICES))) + @CommandArgument('--full', action='store_true', + help='Perform a full clobber') + def clobber(self, what, full=False): + invalid = set(what) - set(self.CLOBBER_CHOICES) + if invalid: + print('Unknown clobber target(s): {}'.format(', '.join(invalid))) + return 1 + + ret = 0 + if 'objdir' in what: + from mozbuild.controller.clobber import Clobberer + try: + Clobberer(self.topsrcdir, self.topobjdir).remove_objdir(full) + except OSError as e: + if sys.platform.startswith('win'): + if isinstance(e, WindowsError) and e.winerror in (5,32): + self.log(logging.ERROR, 'file_access_error', {'error': e}, + "Could not clobber because a file was in use. If the " + "application is running, try closing it. {error}") + return 1 + raise + + if 'python' in what: + if os.path.isdir(mozpath.join(self.topsrcdir, '.hg')): + cmd = ['hg', 'purge', '--all', '-I', 'glob:**.py[co]'] + elif os.path.isdir(mozpath.join(self.topsrcdir, '.git')): + cmd = ['git', 'clean', '-f', '-x', '*.py[co]'] + else: + cmd = ['find', '.', '-type', 'f', '-name', '*.py[co]', '-delete'] + ret = subprocess.call(cmd, cwd=self.topsrcdir) + return ret + +@CommandProvider +class Logs(MachCommandBase): + """Provide commands to read mach logs.""" + NO_AUTO_LOG = True + + @Command('show-log', category='post-build', + description='Display mach logs') + @CommandArgument('log_file', nargs='?', type=argparse.FileType('rb'), + help='Filename to read log data from. Defaults to the log of the last ' + 'mach command.') + def show_log(self, log_file=None): + if not log_file: + path = self._get_state_filename('last_log.json') + log_file = open(path, 'rb') + + if os.isatty(sys.stdout.fileno()): + env = dict(os.environ) + if 'LESS' not in env: + # Sensible default flags if none have been set in the user + # environment. + env[b'LESS'] = b'FRX' + less = subprocess.Popen(['less'], stdin=subprocess.PIPE, env=env) + # Various objects already have a reference to sys.stdout, so we + # can't just change it, we need to change the file descriptor under + # it to redirect to less's input. + # First keep a copy of the sys.stdout file descriptor. + output_fd = os.dup(sys.stdout.fileno()) + os.dup2(less.stdin.fileno(), sys.stdout.fileno()) + + startTime = 0 + for line in log_file: + created, action, params = json.loads(line) + if not startTime: + startTime = created + self.log_manager.terminal_handler.formatter.start_time = \ + created + if 'line' in params: + record = logging.makeLogRecord({ + 'created': created, + 'name': self._logger.name, + 'levelno': logging.INFO, + 'msg': '{line}', + 'params': params, + 'action': action, + }) + self._logger.handle(record) + + if self.log_manager.terminal: + # Close less's input so that it knows that we're done sending data. + less.stdin.close() + # Since the less's input file descriptor is now also the stdout + # file descriptor, we still actually have a non-closed system file + # descriptor for less's input. Replacing sys.stdout's file + # descriptor with what it was before we replaced it will properly + # close less's input. + os.dup2(output_fd, sys.stdout.fileno()) + less.wait() + + +@CommandProvider +class Warnings(MachCommandBase): + """Provide commands for inspecting warnings.""" + + @property + def database_path(self): + return self._get_state_filename('warnings.json') + + @property + def database(self): + from mozbuild.compilation.warnings import WarningsDatabase + + path = self.database_path + + database = WarningsDatabase() + + if os.path.exists(path): + database.load_from_file(path) + + return database + + @Command('warnings-summary', category='post-build', + description='Show a summary of compiler warnings.') + @CommandArgument('-C', '--directory', default=None, + help='Change to a subdirectory of the build directory first.') + @CommandArgument('report', default=None, nargs='?', + help='Warnings report to display. If not defined, show the most ' + 'recent report.') + def summary(self, directory=None, report=None): + database = self.database + + if directory: + dirpath = self.join_ensure_dir(self.topsrcdir, directory) + if not dirpath: + return 1 + else: + dirpath = None + + type_counts = database.type_counts(dirpath) + sorted_counts = sorted(type_counts.iteritems(), + key=operator.itemgetter(1)) + + total = 0 + for k, v in sorted_counts: + print('%d\t%s' % (v, k)) + total += v + + print('%d\tTotal' % total) + + @Command('warnings-list', category='post-build', + description='Show a list of compiler warnings.') + @CommandArgument('-C', '--directory', default=None, + help='Change to a subdirectory of the build directory first.') + @CommandArgument('--flags', default=None, nargs='+', + help='Which warnings flags to match.') + @CommandArgument('report', default=None, nargs='?', + help='Warnings report to display. If not defined, show the most ' + 'recent report.') + def list(self, directory=None, flags=None, report=None): + database = self.database + + by_name = sorted(database.warnings) + + topsrcdir = mozpath.normpath(self.topsrcdir) + + if directory: + directory = mozpath.normsep(directory) + dirpath = self.join_ensure_dir(topsrcdir, directory) + if not dirpath: + return 1 + + if flags: + # Flatten lists of flags. + flags = set(itertools.chain(*[flaglist.split(',') for flaglist in flags])) + + for warning in by_name: + filename = mozpath.normsep(warning['filename']) + + if filename.startswith(topsrcdir): + filename = filename[len(topsrcdir) + 1:] + + if directory and not filename.startswith(directory): + continue + + if flags and warning['flag'] not in flags: + continue + + if warning['column'] is not None: + print('%s:%d:%d [%s] %s' % (filename, warning['line'], + warning['column'], warning['flag'], warning['message'])) + else: + print('%s:%d [%s] %s' % (filename, warning['line'], + warning['flag'], warning['message'])) + + def join_ensure_dir(self, dir1, dir2): + dir1 = mozpath.normpath(dir1) + dir2 = mozpath.normsep(dir2) + joined_path = mozpath.join(dir1, dir2) + if os.path.isdir(joined_path): + return joined_path + else: + print('Specified directory not found.') + return None + +@CommandProvider +class GTestCommands(MachCommandBase): + @Command('gtest', category='testing', + description='Run GTest unit tests (C++ tests).') + @CommandArgument('gtest_filter', default=b"*", nargs='?', metavar='gtest_filter', + help="test_filter is a ':'-separated list of wildcard patterns (called the positive patterns)," + "optionally followed by a '-' and another ':'-separated pattern list (called the negative patterns).") + @CommandArgument('--jobs', '-j', default='1', nargs='?', metavar='jobs', type=int, + help='Run the tests in parallel using multiple processes.') + @CommandArgument('--tbpl-parser', '-t', action='store_true', + help='Output test results in a format that can be parsed by TBPL.') + @CommandArgument('--shuffle', '-s', action='store_true', + help='Randomize the execution order of tests.') + + @CommandArgumentGroup('debugging') + @CommandArgument('--debug', action='store_true', group='debugging', + help='Enable the debugger. Not specifying a --debugger option will result in the default debugger being used.') + @CommandArgument('--debugger', default=None, type=str, group='debugging', + help='Name of debugger to use.') + @CommandArgument('--debugger-args', default=None, metavar='params', type=str, + group='debugging', + help='Command-line arguments to pass to the debugger itself; split as the Bourne shell would.') + + def gtest(self, shuffle, jobs, gtest_filter, tbpl_parser, debug, debugger, + debugger_args): + + # We lazy build gtest because it's slow to link + self._run_make(directory="testing/gtest", target='gtest', + print_directory=False, ensure_exit_code=True) + + app_path = self.get_binary_path('app') + args = [app_path, '-unittest']; + + if debug or debugger or debugger_args: + args = self.prepend_debugger_args(args, debugger, debugger_args) + + cwd = os.path.join(self.topobjdir, '_tests', 'gtest') + + if not os.path.isdir(cwd): + os.makedirs(cwd) + + # Use GTest environment variable to control test execution + # For details see: + # https://code.google.com/p/googletest/wiki/AdvancedGuide#Running_Test_Programs:_Advanced_Options + gtest_env = {b'GTEST_FILTER': gtest_filter} + + # Note: we must normalize the path here so that gtest on Windows sees + # a MOZ_GMP_PATH which has only Windows dir seperators, because + # nsILocalFile cannot open the paths with non-Windows dir seperators. + xre_path = os.path.join(os.path.normpath(self.topobjdir), "dist", "bin") + gtest_env["MOZ_XRE_DIR"] = xre_path + gtest_env["MOZ_GMP_PATH"] = os.pathsep.join( + os.path.join(xre_path, p, "1.0") + for p in ('gmp-fake', 'gmp-fakeopenh264') + ) + + gtest_env[b"MOZ_RUN_GTEST"] = b"True" + + if shuffle: + gtest_env[b"GTEST_SHUFFLE"] = b"True" + + if tbpl_parser: + gtest_env[b"MOZ_TBPL_PARSER"] = b"True" + + if jobs == 1: + return self.run_process(args=args, + append_env=gtest_env, + cwd=cwd, + ensure_exit_code=False, + pass_thru=True) + + from mozprocess import ProcessHandlerMixin + import functools + def handle_line(job_id, line): + # Prepend the jobId + line = '[%d] %s' % (job_id + 1, line.strip()) + self.log(logging.INFO, "GTest", {'line': line}, '{line}') + + gtest_env["GTEST_TOTAL_SHARDS"] = str(jobs) + processes = {} + for i in range(0, jobs): + gtest_env["GTEST_SHARD_INDEX"] = str(i) + processes[i] = ProcessHandlerMixin([app_path, "-unittest"], + cwd=cwd, + env=gtest_env, + processOutputLine=[functools.partial(handle_line, i)], + universal_newlines=True) + processes[i].run() + + exit_code = 0 + for process in processes.values(): + status = process.wait() + if status: + exit_code = status + + # Clamp error code to 255 to prevent overflowing multiple of + # 256 into 0 + if exit_code > 255: + exit_code = 255 + + return exit_code + + def prepend_debugger_args(self, args, debugger, debugger_args): + ''' + Given an array with program arguments, prepend arguments to run it under a + debugger. + + :param args: The executable and arguments used to run the process normally. + :param debugger: The debugger to use, or empty to use the default debugger. + :param debugger_args: Any additional parameters to pass to the debugger. + ''' + + import mozdebug + + if not debugger: + # No debugger name was provided. Look for the default ones on + # current OS. + debugger = mozdebug.get_default_debugger_name(mozdebug.DebuggerSearch.KeepLooking) + + if debugger: + debuggerInfo = mozdebug.get_debugger_info(debugger, debugger_args) + if not debuggerInfo: + print("Could not find a suitable debugger in your PATH.") + return 1 + + # Parameters come from the CLI. We need to convert them before + # their use. + if debugger_args: + from mozbuild import shellutil + try: + debugger_args = shellutil.split(debugger_args) + except shellutil.MetaCharacterException as e: + print("The --debugger_args you passed require a real shell to parse them.") + print("(We can't handle the %r character.)" % e.char) + return 1 + + # Prepend the debugger args. + args = [debuggerInfo.path] + debuggerInfo.args + args + return args + +@CommandProvider +class ClangCommands(MachCommandBase): + @Command('clang-complete', category='devenv', + description='Generate a .clang_complete file.') + def clang_complete(self): + import shlex + + build_vars = {} + + def on_line(line): + elements = [s.strip() for s in line.split('=', 1)] + + if len(elements) != 2: + return + + build_vars[elements[0]] = elements[1] + + try: + old_logger = self.log_manager.replace_terminal_handler(None) + self._run_make(target='showbuild', log=False, line_handler=on_line) + finally: + self.log_manager.replace_terminal_handler(old_logger) + + def print_from_variable(name): + if name not in build_vars: + return + + value = build_vars[name] + + value = value.replace('-I.', '-I%s' % self.topobjdir) + value = value.replace(' .', ' %s' % self.topobjdir) + value = value.replace('-I..', '-I%s/..' % self.topobjdir) + value = value.replace(' ..', ' %s/..' % self.topobjdir) + + args = shlex.split(value) + for i in range(0, len(args) - 1): + arg = args[i] + + if arg.startswith(('-I', '-D')): + print(arg) + continue + + if arg.startswith('-include'): + print(arg + ' ' + args[i + 1]) + continue + + print_from_variable('COMPILE_CXXFLAGS') + + print('-I%s/ipc/chromium/src' % self.topsrcdir) + print('-I%s/ipc/glue' % self.topsrcdir) + print('-I%s/ipc/ipdl/_ipdlheaders' % self.topobjdir) + + +@CommandProvider +class Package(MachCommandBase): + """Package the built product for distribution.""" + + @Command('package', category='post-build', + description='Package the built product for distribution as an APK, DMG, etc.') + @CommandArgument('-v', '--verbose', action='store_true', + help='Verbose output for what commands the packaging process is running.') + def package(self, verbose=False): + ret = self._run_make(directory=".", target='package', + silent=not verbose, ensure_exit_code=False) + if ret == 0: + self.notify('Packaging complete') + return ret + +@CommandProvider +class Install(MachCommandBase): + """Install a package.""" + + @Command('install', category='post-build', + description='Install the package on the machine, or on a device.') + @CommandArgument('--verbose', '-v', action='store_true', + help='Print verbose output when installing to an Android emulator.') + def install(self, verbose=False): + if conditions.is_android(self): + from mozrunner.devices.android_device import verify_android_device + verify_android_device(self, verbose=verbose) + ret = self._run_make(directory=".", target='install', ensure_exit_code=False) + if ret == 0: + self.notify('Install complete') + return ret + +@CommandProvider +class RunProgram(MachCommandBase): + """Run the compiled program.""" + + prog_group = 'the compiled program' + + @Command('run', category='post-build', + description='Run the compiled program, possibly under a debugger or DMD.') + @CommandArgument('params', nargs='...', group=prog_group, + help='Command-line arguments to be passed through to the program. Not specifying a --profile or -P option will result in a temporary profile being used.') + @CommandArgumentGroup(prog_group) + @CommandArgument('--remote', '-r', action='store_true', group=prog_group, + help='Do not pass the --no-remote argument by default.') + @CommandArgument('--background', '-b', action='store_true', group=prog_group, + help='Do not pass the --foreground argument by default on Mac.') + @CommandArgument('--noprofile', '-n', action='store_true', group=prog_group, + help='Do not pass the --profile argument by default.') + @CommandArgument('--disable-e10s', action='store_true', group=prog_group, + help='Run the program with electrolysis disabled.') + + @CommandArgumentGroup('debugging') + @CommandArgument('--debug', action='store_true', group='debugging', + help='Enable the debugger. Not specifying a --debugger option will result in the default debugger being used.') + @CommandArgument('--debugger', default=None, type=str, group='debugging', + help='Name of debugger to use.') + @CommandArgument('--debugparams', default=None, metavar='params', type=str, + group='debugging', + help='Command-line arguments to pass to the debugger itself; split as the Bourne shell would.') + # Bug 933807 introduced JS_DISABLE_SLOW_SCRIPT_SIGNALS to avoid clever + # segfaults induced by the slow-script-detecting logic for Ion/Odin JITted + # code. If we don't pass this, the user will need to periodically type + # "continue" to (safely) resume execution. There are ways to implement + # automatic resuming; see the bug. + @CommandArgument('--slowscript', action='store_true', group='debugging', + help='Do not set the JS_DISABLE_SLOW_SCRIPT_SIGNALS env variable; when not set, recoverable but misleading SIGSEGV instances may occur in Ion/Odin JIT code.') + + @CommandArgumentGroup('DMD') + @CommandArgument('--dmd', action='store_true', group='DMD', + help='Enable DMD. The following arguments have no effect without this.') + @CommandArgument('--mode', choices=['live', 'dark-matter', 'cumulative', 'scan'], group='DMD', + help='Profiling mode. The default is \'dark-matter\'.') + @CommandArgument('--stacks', choices=['partial', 'full'], group='DMD', + help='Allocation stack trace coverage. The default is \'partial\'.') + @CommandArgument('--show-dump-stats', action='store_true', group='DMD', + help='Show stats when doing dumps.') + def run(self, params, remote, background, noprofile, disable_e10s, debug, + debugger, debugparams, slowscript, dmd, mode, stacks, show_dump_stats): + + if conditions.is_android(self): + # Running Firefox for Android is completely different + if dmd: + print("DMD is not supported for Firefox for Android") + return 1 + from mozrunner.devices.android_device import verify_android_device, run_firefox_for_android + if not (debug or debugger or debugparams): + verify_android_device(self, install=True) + return run_firefox_for_android(self, params) + verify_android_device(self, install=True, debugger=True) + args = [''] + + else: + + try: + binpath = self.get_binary_path('app') + except Exception as e: + print("It looks like your program isn't built.", + "You can run |mach build| to build it.") + print(e) + return 1 + + args = [binpath] + + if params: + args.extend(params) + + if not remote: + args.append('-no-remote') + + if not background and sys.platform == 'darwin': + args.append('-foreground') + + no_profile_option_given = \ + all(p not in params for p in ['-profile', '--profile', '-P']) + if no_profile_option_given and not noprofile: + path = os.path.join(self.topobjdir, 'tmp', 'scratch_user') + if not os.path.isdir(path): + os.makedirs(path) + args.append('-profile') + args.append(path) + + extra_env = {'MOZ_CRASHREPORTER_DISABLE': '1'} + if disable_e10s: + extra_env['MOZ_FORCE_DISABLE_E10S'] = '1' + + if debug or debugger or debugparams: + if 'INSIDE_EMACS' in os.environ: + self.log_manager.terminal_handler.setLevel(logging.WARNING) + + import mozdebug + if not debugger: + # No debugger name was provided. Look for the default ones on + # current OS. + debugger = mozdebug.get_default_debugger_name(mozdebug.DebuggerSearch.KeepLooking) + + if debugger: + self.debuggerInfo = mozdebug.get_debugger_info(debugger, debugparams) + if not self.debuggerInfo: + print("Could not find a suitable debugger in your PATH.") + return 1 + + # Parameters come from the CLI. We need to convert them before + # their use. + if debugparams: + from mozbuild import shellutil + try: + debugparams = shellutil.split(debugparams) + except shellutil.MetaCharacterException as e: + print("The --debugparams you passed require a real shell to parse them.") + print("(We can't handle the %r character.)" % e.char) + return 1 + + if not slowscript: + extra_env['JS_DISABLE_SLOW_SCRIPT_SIGNALS'] = '1' + + # Prepend the debugger args. + args = [self.debuggerInfo.path] + self.debuggerInfo.args + args + + if dmd: + dmd_params = [] + + if mode: + dmd_params.append('--mode=' + mode) + if stacks: + dmd_params.append('--stacks=' + stacks) + if show_dump_stats: + dmd_params.append('--show-dump-stats=yes') + + bin_dir = os.path.dirname(binpath) + lib_name = self.substs['DLL_PREFIX'] + 'dmd' + self.substs['DLL_SUFFIX'] + dmd_lib = os.path.join(bin_dir, lib_name) + if not os.path.exists(dmd_lib): + print("Please build with |--enable-dmd| to use DMD.") + return 1 + + env_vars = { + "Darwin": { + "DYLD_INSERT_LIBRARIES": dmd_lib, + "LD_LIBRARY_PATH": bin_dir, + }, + "Linux": { + "LD_PRELOAD": dmd_lib, + "LD_LIBRARY_PATH": bin_dir, + }, + "WINNT": { + "MOZ_REPLACE_MALLOC_LIB": dmd_lib, + }, + } + + arch = self.substs['OS_ARCH'] + + if dmd_params: + env_vars[arch]["DMD"] = " ".join(dmd_params) + + extra_env.update(env_vars.get(arch, {})) + + return self.run_process(args=args, ensure_exit_code=False, + pass_thru=True, append_env=extra_env) + +@CommandProvider +class Buildsymbols(MachCommandBase): + """Produce a package of debug symbols suitable for use with Breakpad.""" + + @Command('buildsymbols', category='post-build', + description='Produce a package of Breakpad-format symbols.') + def buildsymbols(self): + return self._run_make(directory=".", target='buildsymbols', ensure_exit_code=False) + +@CommandProvider +class Makefiles(MachCommandBase): + @Command('empty-makefiles', category='build-dev', + description='Find empty Makefile.in in the tree.') + def empty(self): + import pymake.parser + import pymake.parserdata + + IGNORE_VARIABLES = { + 'DEPTH': ('@DEPTH@',), + 'topsrcdir': ('@top_srcdir@',), + 'srcdir': ('@srcdir@',), + 'relativesrcdir': ('@relativesrcdir@',), + 'VPATH': ('@srcdir@',), + } + + IGNORE_INCLUDES = [ + 'include $(DEPTH)/config/autoconf.mk', + 'include $(topsrcdir)/config/config.mk', + 'include $(topsrcdir)/config/rules.mk', + ] + + def is_statement_relevant(s): + if isinstance(s, pymake.parserdata.SetVariable): + exp = s.vnameexp + if not exp.is_static_string: + return True + + if exp.s not in IGNORE_VARIABLES: + return True + + return s.value not in IGNORE_VARIABLES[exp.s] + + if isinstance(s, pymake.parserdata.Include): + if s.to_source() in IGNORE_INCLUDES: + return False + + return True + + for path in self._makefile_ins(): + relpath = os.path.relpath(path, self.topsrcdir) + try: + statements = [s for s in pymake.parser.parsefile(path) + if is_statement_relevant(s)] + + if not statements: + print(relpath) + except pymake.parser.SyntaxError: + print('Warning: Could not parse %s' % relpath, file=sys.stderr) + + def _makefile_ins(self): + for root, dirs, files in os.walk(self.topsrcdir): + for f in files: + if f == 'Makefile.in': + yield os.path.join(root, f) + +@CommandProvider +class MachDebug(MachCommandBase): + @Command('environment', category='build-dev', + description='Show info about the mach and build environment.') + @CommandArgument('--format', default='pretty', + choices=['pretty', 'client.mk', 'configure', 'json'], + help='Print data in the given format.') + @CommandArgument('--output', '-o', type=str, + help='Output to the given file.') + @CommandArgument('--verbose', '-v', action='store_true', + help='Print verbose output.') + def environment(self, format, output=None, verbose=False): + func = getattr(self, '_environment_%s' % format.replace('.', '_')) + + if output: + # We want to preserve mtimes if the output file already exists + # and the content hasn't changed. + from mozbuild.util import FileAvoidWrite + with FileAvoidWrite(output) as out: + return func(out, verbose) + return func(sys.stdout, verbose) + + def _environment_pretty(self, out, verbose): + state_dir = self._mach_context.state_dir + import platform + print('platform:\n\t%s' % platform.platform(), file=out) + print('python version:\n\t%s' % sys.version, file=out) + print('python prefix:\n\t%s' % sys.prefix, file=out) + print('mach cwd:\n\t%s' % self._mach_context.cwd, file=out) + print('os cwd:\n\t%s' % os.getcwd(), file=out) + print('mach directory:\n\t%s' % self._mach_context.topdir, file=out) + print('state directory:\n\t%s' % state_dir, file=out) + + print('object directory:\n\t%s' % self.topobjdir, file=out) + + if self.mozconfig['path']: + print('mozconfig path:\n\t%s' % self.mozconfig['path'], file=out) + if self.mozconfig['configure_args']: + print('mozconfig configure args:', file=out) + for arg in self.mozconfig['configure_args']: + print('\t%s' % arg, file=out) + + if self.mozconfig['make_extra']: + print('mozconfig extra make args:', file=out) + for arg in self.mozconfig['make_extra']: + print('\t%s' % arg, file=out) + + if self.mozconfig['make_flags']: + print('mozconfig make flags:', file=out) + for arg in self.mozconfig['make_flags']: + print('\t%s' % arg, file=out) + + config = None + + try: + config = self.config_environment + + except Exception: + pass + + if config: + print('config topsrcdir:\n\t%s' % config.topsrcdir, file=out) + print('config topobjdir:\n\t%s' % config.topobjdir, file=out) + + if verbose: + print('config substitutions:', file=out) + for k in sorted(config.substs): + print('\t%s: %s' % (k, config.substs[k]), file=out) + + print('config defines:', file=out) + for k in sorted(config.defines): + print('\t%s' % k, file=out) + + def _environment_client_mk(self, out, verbose): + if self.mozconfig['make_extra']: + for arg in self.mozconfig['make_extra']: + print(arg, file=out) + if self.mozconfig['make_flags']: + print('MOZ_MAKE_FLAGS=%s' % ' '.join(self.mozconfig['make_flags'])) + objdir = mozpath.normsep(self.topobjdir) + print('MOZ_OBJDIR=%s' % objdir, file=out) + if 'MOZ_CURRENT_PROJECT' in os.environ: + objdir = mozpath.join(objdir, os.environ['MOZ_CURRENT_PROJECT']) + print('OBJDIR=%s' % objdir, file=out) + if self.mozconfig['path']: + print('FOUND_MOZCONFIG=%s' % mozpath.normsep(self.mozconfig['path']), + file=out) + + def _environment_json(self, out, verbose): + import json + class EnvironmentEncoder(json.JSONEncoder): + def default(self, obj): + if isinstance(obj, MozbuildObject): + result = { + 'topsrcdir': obj.topsrcdir, + 'topobjdir': obj.topobjdir, + 'mozconfig': obj.mozconfig, + } + if verbose: + result['substs'] = obj.substs + result['defines'] = obj.defines + return result + elif isinstance(obj, set): + return list(obj) + return json.JSONEncoder.default(self, obj) + json.dump(self, cls=EnvironmentEncoder, sort_keys=True, fp=out) + +class ArtifactSubCommand(SubCommand): + def __call__(self, func): + after = SubCommand.__call__(self, func) + jobchoices = { + 'android-api-15', + 'android-x86', + 'linux', + 'linux64', + 'macosx64', + 'win32', + 'win64' + } + args = [ + CommandArgument('--tree', metavar='TREE', type=str, + help='Firefox tree.'), + CommandArgument('--job', metavar='JOB', choices=jobchoices, + help='Build job.'), + CommandArgument('--verbose', '-v', action='store_true', + help='Print verbose output.'), + ] + for arg in args: + after = arg(after) + return after + + +@CommandProvider +class PackageFrontend(MachCommandBase): + """Fetch and install binary artifacts from Mozilla automation.""" + + @Command('artifact', category='post-build', + description='Use pre-built artifacts to build Firefox.') + def artifact(self): + '''Download, cache, and install pre-built binary artifacts to build Firefox. + + Use |mach build| as normal to freshen your installed binary libraries: + artifact builds automatically download, cache, and install binary + artifacts from Mozilla automation, replacing whatever may be in your + object directory. Use |mach artifact last| to see what binary artifacts + were last used. + + Never build libxul again! + + ''' + pass + + def _set_log_level(self, verbose): + self.log_manager.terminal_handler.setLevel(logging.INFO if not verbose else logging.DEBUG) + + def _install_pip_package(self, package): + if os.environ.get('MOZ_AUTOMATION'): + self.virtualenv_manager._run_pip([ + 'install', + package, + '--no-index', + '--find-links', + 'http://pypi.pub.build.mozilla.org/pub', + '--trusted-host', + 'pypi.pub.build.mozilla.org', + ]) + return + self.virtualenv_manager.install_pip_package(package) + + def _make_artifacts(self, tree=None, job=None, skip_cache=False): + # Undo PATH munging that will be done by activating the virtualenv, + # so that invoked subprocesses expecting to find system python + # (git cinnabar, in particular), will not find virtualenv python. + original_path = os.environ.get('PATH', '') + self._activate_virtualenv() + os.environ['PATH'] = original_path + + for package in ('taskcluster==0.0.32', + 'mozregression==1.0.2'): + self._install_pip_package(package) + + state_dir = self._mach_context.state_dir + cache_dir = os.path.join(state_dir, 'package-frontend') + + try: + os.makedirs(cache_dir) + except OSError as e: + if e.errno != errno.EEXIST: + raise + + import which + + here = os.path.abspath(os.path.dirname(__file__)) + build_obj = MozbuildObject.from_environment(cwd=here) + + hg = None + if conditions.is_hg(build_obj): + if self._is_windows(): + hg = which.which('hg.exe') + else: + hg = which.which('hg') + + git = None + if conditions.is_git(build_obj): + if self._is_windows(): + git = which.which('git.exe') + else: + git = which.which('git') + + # Absolutely must come after the virtualenv is populated! + from mozbuild.artifacts import Artifacts + artifacts = Artifacts(tree, self.substs, self.defines, job, + log=self.log, cache_dir=cache_dir, + skip_cache=skip_cache, hg=hg, git=git, + topsrcdir=self.topsrcdir) + return artifacts + + @ArtifactSubCommand('artifact', 'install', + 'Install a good pre-built artifact.') + @CommandArgument('source', metavar='SRC', nargs='?', type=str, + help='Where to fetch and install artifacts from. Can be omitted, in ' + 'which case the current hg repository is inspected; an hg revision; ' + 'a remote URL; or a local file.', + default=None) + @CommandArgument('--skip-cache', action='store_true', + help='Skip all local caches to force re-fetching remote artifacts.', + default=False) + def artifact_install(self, source=None, skip_cache=False, tree=None, job=None, verbose=False): + self._set_log_level(verbose) + artifacts = self._make_artifacts(tree=tree, job=job, skip_cache=skip_cache) + + return artifacts.install_from(source, self.distdir) + + @ArtifactSubCommand('artifact', 'last', + 'Print the last pre-built artifact installed.') + def artifact_print_last(self, tree=None, job=None, verbose=False): + self._set_log_level(verbose) + artifacts = self._make_artifacts(tree=tree, job=job) + artifacts.print_last() + return 0 + + @ArtifactSubCommand('artifact', 'print-cache', + 'Print local artifact cache for debugging.') + def artifact_print_cache(self, tree=None, job=None, verbose=False): + self._set_log_level(verbose) + artifacts = self._make_artifacts(tree=tree, job=job) + artifacts.print_cache() + return 0 + + @ArtifactSubCommand('artifact', 'clear-cache', + 'Delete local artifacts and reset local artifact cache.') + def artifact_clear_cache(self, tree=None, job=None, verbose=False): + self._set_log_level(verbose) + artifacts = self._make_artifacts(tree=tree, job=job) + artifacts.clear_cache() + return 0 + +@CommandProvider +class Vendor(MachCommandBase): + """Vendor third-party dependencies into the source repository.""" + + @Command('vendor', category='misc', + description='Vendor third-party dependencies into the source repository.') + def vendor(self): + self.parser.print_usage() + sys.exit(1) + + @SubCommand('vendor', 'rust', + description='Vendor rust crates from crates.io into third_party/rust') + @CommandArgument('--ignore-modified', action='store_true', + help='Ignore modified files in current checkout', + default=False) + def vendor_rust(self, **kwargs): + from mozbuild.vendor_rust import VendorRust + vendor_command = self._spawn(VendorRust) + vendor_command.vendor(**kwargs) diff --git a/python/mozbuild/mozbuild/makeutil.py b/python/mozbuild/mozbuild/makeutil.py new file mode 100644 index 000000000..fcd45bed2 --- /dev/null +++ b/python/mozbuild/mozbuild/makeutil.py @@ -0,0 +1,186 @@ +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, You can obtain one at http://mozilla.org/MPL/2.0/. + +from __future__ import absolute_import + +import os +import re +from types import StringTypes +from collections import Iterable + + +class Makefile(object): + '''Provides an interface for writing simple makefiles + + Instances of this class are created, populated with rules, then + written. + ''' + + def __init__(self): + self._statements = [] + + def create_rule(self, targets=[]): + ''' + Create a new rule in the makefile for the given targets. + Returns the corresponding Rule instance. + ''' + rule = Rule(targets) + self._statements.append(rule) + return rule + + def add_statement(self, statement): + ''' + Add a raw statement in the makefile. Meant to be used for + simple variable assignments. + ''' + self._statements.append(statement) + + def dump(self, fh, removal_guard=True): + ''' + Dump all the rules to the given file handle. Optionally (and by + default), add guard rules for file removals (empty rules for other + rules' dependencies) + ''' + all_deps = set() + all_targets = set() + for statement in self._statements: + if isinstance(statement, Rule): + statement.dump(fh) + all_deps.update(statement.dependencies()) + all_targets.update(statement.targets()) + else: + fh.write('%s\n' % statement) + if removal_guard: + guard = Rule(sorted(all_deps - all_targets)) + guard.dump(fh) + + +class _SimpleOrderedSet(object): + ''' + Simple ordered set, specialized for used in Rule below only. + It doesn't expose a complete API, and normalizes path separators + at insertion. + ''' + def __init__(self): + self._list = [] + self._set = set() + + def __nonzero__(self): + return bool(self._set) + + def __iter__(self): + return iter(self._list) + + def __contains__(self, key): + return key in self._set + + def update(self, iterable): + def _add(iterable): + emitted = set() + for i in iterable: + i = i.replace(os.sep, '/') + if i not in self._set and i not in emitted: + yield i + emitted.add(i) + added = list(_add(iterable)) + self._set.update(added) + self._list.extend(added) + + +class Rule(object): + '''Class handling simple rules in the form: + target1 target2 ... : dep1 dep2 ... + command1 + command2 + ... + ''' + def __init__(self, targets=[]): + self._targets = _SimpleOrderedSet() + self._dependencies = _SimpleOrderedSet() + self._commands = [] + self.add_targets(targets) + + def add_targets(self, targets): + '''Add additional targets to the rule.''' + assert isinstance(targets, Iterable) and not isinstance(targets, StringTypes) + self._targets.update(targets) + return self + + def add_dependencies(self, deps): + '''Add dependencies to the rule.''' + assert isinstance(deps, Iterable) and not isinstance(deps, StringTypes) + self._dependencies.update(deps) + return self + + def add_commands(self, commands): + '''Add commands to the rule.''' + assert isinstance(commands, Iterable) and not isinstance(commands, StringTypes) + self._commands.extend(commands) + return self + + def targets(self): + '''Return an iterator on the rule targets.''' + # Ensure the returned iterator is actually just that, an iterator. + # Avoids caller fiddling with the set itself. + return iter(self._targets) + + def dependencies(self): + '''Return an iterator on the rule dependencies.''' + return iter(d for d in self._dependencies if not d in self._targets) + + def commands(self): + '''Return an iterator on the rule commands.''' + return iter(self._commands) + + def dump(self, fh): + ''' + Dump the rule to the given file handle. + ''' + if not self._targets: + return + fh.write('%s:' % ' '.join(self._targets)) + if self._dependencies: + fh.write(' %s' % ' '.join(self.dependencies())) + fh.write('\n') + for cmd in self._commands: + fh.write('\t%s\n' % cmd) + + +# colon followed by anything except a slash (Windows path detection) +_depfilesplitter = re.compile(r':(?![\\/])') + + +def read_dep_makefile(fh): + """ + Read the file handler containing a dep makefile (simple makefile only + containing dependencies) and returns an iterator of the corresponding Rules + it contains. Ignores removal guard rules. + """ + + rule = '' + for line in fh.readlines(): + assert not line.startswith('\t') + line = line.strip() + if line.endswith('\\'): + rule += line[:-1] + else: + rule += line + split_rule = _depfilesplitter.split(rule, 1) + if len(split_rule) > 1 and split_rule[1].strip(): + yield Rule(split_rule[0].strip().split()) \ + .add_dependencies(split_rule[1].strip().split()) + rule = '' + + if rule: + raise Exception('Makefile finishes with a backslash. Expected more input.') + +def write_dep_makefile(fh, target, deps): + ''' + Write a Makefile containing only target's dependencies to the file handle + specified. + ''' + mk = Makefile() + rule = mk.create_rule(targets=[target]) + rule.add_dependencies(deps) + mk.dump(fh, removal_guard=True) diff --git a/python/mozbuild/mozbuild/milestone.py b/python/mozbuild/mozbuild/milestone.py new file mode 100644 index 000000000..c2aa78fcd --- /dev/null +++ b/python/mozbuild/mozbuild/milestone.py @@ -0,0 +1,75 @@ +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, You can obtain one at http://mozilla.org/MPL/2.0/. + +from __future__ import absolute_import, print_function, unicode_literals + +import argparse +import os +import re +import sys + + +def get_milestone_ab_with_num(milestone): + """ + Returns the alpha and beta tag with its number (a1, a2, b3, ...). + """ + + match = re.search(r"([ab]\d+)", milestone) + if match: + return match.group(1) + + return "" + + +def get_official_milestone(path): + """ + Returns the contents of the first line in `path` that starts with a digit. + """ + + with open(path) as fp: + for line in fp: + line = line.strip() + if line[:1].isdigit(): + return line + + raise Exception("Didn't find a line that starts with a digit.") + + +def get_milestone_major(milestone): + """ + Returns the major (first) part of the milestone. + """ + + return milestone.split('.')[0] + + +def main(args): + parser = argparse.ArgumentParser() + parser.add_argument('--uaversion', default=False, action='store_true') + parser.add_argument('--symbolversion', default=False, action='store_true') + parser.add_argument('--topsrcdir', metavar='TOPSRCDIR', required=True) + options = parser.parse_args(args) + + milestone_file = os.path.join(options.topsrcdir, 'config', 'milestone.txt') + + milestone = get_official_milestone(milestone_file) + + if options.uaversion: + # Only expose the major milestone in the UA string, hide the patch + # level (bugs 572659 and 870868). + uaversion = "%s.0" % (get_milestone_major(milestone),) + print(uaversion) + + elif options.symbolversion: + # Only expose major milestone and alpha version. Used for symbol + # versioning on Linux. + symbolversion = "%s%s" % (get_milestone_major(milestone), + get_milestone_ab_with_num(milestone)) + print(symbolversion) + else: + print(milestone) + + +if __name__ == '__main__': + main(sys.argv[1:]) diff --git a/python/mozbuild/mozbuild/mozconfig.py b/python/mozbuild/mozbuild/mozconfig.py new file mode 100644 index 000000000..71267c1be --- /dev/null +++ b/python/mozbuild/mozbuild/mozconfig.py @@ -0,0 +1,485 @@ +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, You can obtain one at http://mozilla.org/MPL/2.0/. + +from __future__ import absolute_import, unicode_literals + +import filecmp +import os +import re +import sys +import subprocess +import traceback + +from collections import defaultdict +from mozpack import path as mozpath + + +MOZ_MYCONFIG_ERROR = ''' +The MOZ_MYCONFIG environment variable to define the location of mozconfigs +is deprecated. If you wish to define the mozconfig path via an environment +variable, use MOZCONFIG instead. +'''.strip() + +MOZCONFIG_LEGACY_PATH = ''' +You currently have a mozconfig at %s. This implicit location is no longer +supported. Please move it to %s/.mozconfig or set an explicit path +via the $MOZCONFIG environment variable. +'''.strip() + +MOZCONFIG_BAD_EXIT_CODE = ''' +Evaluation of your mozconfig exited with an error. This could be triggered +by a command inside your mozconfig failing. Please change your mozconfig +to not error and/or to catch errors in executed commands. +'''.strip() + +MOZCONFIG_BAD_OUTPUT = ''' +Evaluation of your mozconfig produced unexpected output. This could be +triggered by a command inside your mozconfig failing or producing some warnings +or error messages. Please change your mozconfig to not error and/or to catch +errors in executed commands. +'''.strip() + + +class MozconfigFindException(Exception): + """Raised when a mozconfig location is not defined properly.""" + + +class MozconfigLoadException(Exception): + """Raised when a mozconfig could not be loaded properly. + + This typically indicates a malformed or misbehaving mozconfig file. + """ + + def __init__(self, path, message, output=None): + self.path = path + self.output = output + Exception.__init__(self, message) + + +class MozconfigLoader(object): + """Handles loading and parsing of mozconfig files.""" + + RE_MAKE_VARIABLE = re.compile(''' + ^\s* # Leading whitespace + (?P[a-zA-Z_0-9]+) # Variable name + \s* [?:]?= \s* # Assignment operator surrounded by optional + # spaces + (?P.*$)''', # Everything else (likely the value) + re.VERBOSE) + + # Default mozconfig files in the topsrcdir. + DEFAULT_TOPSRCDIR_PATHS = ('.mozconfig', 'mozconfig') + + DEPRECATED_TOPSRCDIR_PATHS = ('mozconfig.sh', 'myconfig.sh') + DEPRECATED_HOME_PATHS = ('.mozconfig', '.mozconfig.sh', '.mozmyconfig.sh') + + IGNORE_SHELL_VARIABLES = {'_'} + + ENVIRONMENT_VARIABLES = { + 'CC', 'CXX', 'CFLAGS', 'CXXFLAGS', 'LDFLAGS', 'MOZ_OBJDIR', + } + + AUTODETECT = object() + + def __init__(self, topsrcdir): + self.topsrcdir = topsrcdir + + @property + def _loader_script(self): + our_dir = os.path.abspath(os.path.dirname(__file__)) + + return os.path.join(our_dir, 'mozconfig_loader') + + def find_mozconfig(self, env=os.environ): + """Find the active mozconfig file for the current environment. + + This emulates the logic in mozconfig-find. + + 1) If ENV[MOZCONFIG] is set, use that + 2) If $TOPSRCDIR/mozconfig or $TOPSRCDIR/.mozconfig exists, use it. + 3) If both exist or if there are legacy locations detected, error out. + + The absolute path to the found mozconfig will be returned on success. + None will be returned if no mozconfig could be found. A + MozconfigFindException will be raised if there is a bad state, + including conditions from #3 above. + """ + # Check for legacy methods first. + + if 'MOZ_MYCONFIG' in env: + raise MozconfigFindException(MOZ_MYCONFIG_ERROR) + + env_path = env.get('MOZCONFIG', None) or None + if env_path is not None: + if not os.path.isabs(env_path): + potential_roots = [self.topsrcdir, os.getcwd()] + # Attempt to eliminate duplicates for e.g. + # self.topsrcdir == os.curdir. + potential_roots = set(os.path.abspath(p) for p in potential_roots) + existing = [root for root in potential_roots + if os.path.exists(os.path.join(root, env_path))] + if len(existing) > 1: + # There are multiple files, but we might have a setup like: + # + # somedirectory/ + # srcdir/ + # objdir/ + # + # MOZCONFIG=../srcdir/some/path/to/mozconfig + # + # and be configuring from the objdir. So even though we + # have multiple existing files, they are actually the same + # file. + mozconfigs = [os.path.join(root, env_path) + for root in existing] + if not all(map(lambda p1, p2: filecmp.cmp(p1, p2, shallow=False), + mozconfigs[:-1], mozconfigs[1:])): + raise MozconfigFindException( + 'MOZCONFIG environment variable refers to a path that ' + + 'exists in more than one of ' + ', '.join(potential_roots) + + '. Remove all but one.') + elif not existing: + raise MozconfigFindException( + 'MOZCONFIG environment variable refers to a path that ' + + 'does not exist in any of ' + ', '.join(potential_roots)) + + env_path = os.path.join(existing[0], env_path) + elif not os.path.exists(env_path): # non-relative path + raise MozconfigFindException( + 'MOZCONFIG environment variable refers to a path that ' + 'does not exist: ' + env_path) + + if not os.path.isfile(env_path): + raise MozconfigFindException( + 'MOZCONFIG environment variable refers to a ' + 'non-file: ' + env_path) + + srcdir_paths = [os.path.join(self.topsrcdir, p) for p in + self.DEFAULT_TOPSRCDIR_PATHS] + existing = [p for p in srcdir_paths if os.path.isfile(p)] + + if env_path is None and len(existing) > 1: + raise MozconfigFindException('Multiple default mozconfig files ' + 'present. Remove all but one. ' + ', '.join(existing)) + + path = None + + if env_path is not None: + path = env_path + elif len(existing): + assert len(existing) == 1 + path = existing[0] + + if path is not None: + return os.path.abspath(path) + + deprecated_paths = [os.path.join(self.topsrcdir, s) for s in + self.DEPRECATED_TOPSRCDIR_PATHS] + + home = env.get('HOME', None) + if home is not None: + deprecated_paths.extend([os.path.join(home, s) for s in + self.DEPRECATED_HOME_PATHS]) + + for path in deprecated_paths: + if os.path.exists(path): + raise MozconfigFindException( + MOZCONFIG_LEGACY_PATH % (path, self.topsrcdir)) + + return None + + def read_mozconfig(self, path=None, moz_build_app=None): + """Read the contents of a mozconfig into a data structure. + + This takes the path to a mozconfig to load. If the given path is + AUTODETECT, will try to find a mozconfig from the environment using + find_mozconfig(). + + mozconfig files are shell scripts. So, we can't just parse them. + Instead, we run the shell script in a wrapper which allows us to record + state from execution. Thus, the output from a mozconfig is a friendly + static data structure. + """ + if path is self.AUTODETECT: + path = self.find_mozconfig() + + result = { + 'path': path, + 'topobjdir': None, + 'configure_args': None, + 'make_flags': None, + 'make_extra': None, + 'env': None, + 'vars': None, + } + + if path is None: + return result + + path = mozpath.normsep(path) + + result['configure_args'] = [] + result['make_extra'] = [] + result['make_flags'] = [] + + env = dict(os.environ) + + # Since mozconfig_loader is a shell script, running it "normally" + # actually leads to two shell executions on Windows. Avoid this by + # directly calling sh mozconfig_loader. + shell = 'sh' + if 'MOZILLABUILD' in os.environ: + shell = os.environ['MOZILLABUILD'] + '/msys/bin/sh' + if sys.platform == 'win32': + shell = shell + '.exe' + + command = [shell, mozpath.normsep(self._loader_script), + mozpath.normsep(self.topsrcdir), path, sys.executable, + mozpath.join(mozpath.dirname(self._loader_script), + 'action', 'dump_env.py')] + + try: + # We need to capture stderr because that's where the shell sends + # errors if execution fails. + output = subprocess.check_output(command, stderr=subprocess.STDOUT, + cwd=self.topsrcdir, env=env) + except subprocess.CalledProcessError as e: + lines = e.output.splitlines() + + # Output before actual execution shouldn't be relevant. + try: + index = lines.index('------END_BEFORE_SOURCE') + lines = lines[index + 1:] + except ValueError: + pass + + raise MozconfigLoadException(path, MOZCONFIG_BAD_EXIT_CODE, lines) + + try: + parsed = self._parse_loader_output(output) + except AssertionError: + # _parse_loader_output uses assertions to verify the + # well-formedness of the shell output; when these fail, it + # generally means there was a problem with the output, but we + # include the assertion traceback just to be sure. + print('Assertion failed in _parse_loader_output:') + traceback.print_exc() + raise MozconfigLoadException(path, MOZCONFIG_BAD_OUTPUT, + output.splitlines()) + + def diff_vars(vars_before, vars_after): + set1 = set(vars_before.keys()) - self.IGNORE_SHELL_VARIABLES + set2 = set(vars_after.keys()) - self.IGNORE_SHELL_VARIABLES + added = set2 - set1 + removed = set1 - set2 + maybe_modified = set1 & set2 + changed = { + 'added': {}, + 'removed': {}, + 'modified': {}, + 'unmodified': {}, + } + + for key in added: + changed['added'][key] = vars_after[key] + + for key in removed: + changed['removed'][key] = vars_before[key] + + for key in maybe_modified: + if vars_before[key] != vars_after[key]: + changed['modified'][key] = ( + vars_before[key], vars_after[key]) + elif key in self.ENVIRONMENT_VARIABLES: + # In order for irrelevant environment variable changes not + # to incur in re-running configure, only a set of + # environment variables are stored when they are + # unmodified. Otherwise, changes such as using a different + # terminal window, or even rebooting, would trigger + # reconfigures. + changed['unmodified'][key] = vars_after[key] + + return changed + + result['env'] = diff_vars(parsed['env_before'], parsed['env_after']) + + # Environment variables also appear as shell variables, but that's + # uninteresting duplication of information. Filter them out. + filt = lambda x, y: {k: v for k, v in x.items() if k not in y} + result['vars'] = diff_vars( + filt(parsed['vars_before'], parsed['env_before']), + filt(parsed['vars_after'], parsed['env_after']) + ) + + result['configure_args'] = [self._expand(o) for o in parsed['ac']] + + if moz_build_app is not None: + result['configure_args'].extend(self._expand(o) for o in + parsed['ac_app'][moz_build_app]) + + if 'MOZ_OBJDIR' in parsed['env_before']: + result['topobjdir'] = parsed['env_before']['MOZ_OBJDIR'] + + mk = [self._expand(o) for o in parsed['mk']] + + for o in mk: + match = self.RE_MAKE_VARIABLE.match(o) + + if match is None: + result['make_extra'].append(o) + continue + + name, value = match.group('var'), match.group('value') + + if name == 'MOZ_MAKE_FLAGS': + result['make_flags'] = value.split() + continue + + if name == 'MOZ_OBJDIR': + result['topobjdir'] = value + continue + + result['make_extra'].append(o) + + return result + + def _parse_loader_output(self, output): + mk_options = [] + ac_options = [] + ac_app_options = defaultdict(list) + before_source = {} + after_source = {} + env_before_source = {} + env_after_source = {} + + current = None + current_type = None + in_variable = None + + for line in output.splitlines(): + + # XXX This is an ugly hack. Data may be lost from things + # like environment variable values. + # See https://bugzilla.mozilla.org/show_bug.cgi?id=831381 + line = line.decode('mbcs' if sys.platform == 'win32' else 'utf-8', + 'ignore') + + if not line: + continue + + if line.startswith('------BEGIN_'): + assert current_type is None + assert current is None + assert not in_variable + current_type = line[len('------BEGIN_'):] + current = [] + continue + + if line.startswith('------END_'): + assert not in_variable + section = line[len('------END_'):] + assert current_type == section + + if current_type == 'AC_OPTION': + ac_options.append('\n'.join(current)) + elif current_type == 'MK_OPTION': + mk_options.append('\n'.join(current)) + elif current_type == 'AC_APP_OPTION': + app = current.pop(0) + ac_app_options[app].append('\n'.join(current)) + + current = None + current_type = None + continue + + assert current_type is not None + + vars_mapping = { + 'BEFORE_SOURCE': before_source, + 'AFTER_SOURCE': after_source, + 'ENV_BEFORE_SOURCE': env_before_source, + 'ENV_AFTER_SOURCE': env_after_source, + } + + if current_type in vars_mapping: + # mozconfigs are sourced using the Bourne shell (or at least + # in Bourne shell mode). This means |set| simply lists + # variables from the current shell (not functions). (Note that + # if Bash is installed in /bin/sh it acts like regular Bourne + # and doesn't print functions.) So, lines should have the + # form: + # + # key='value' + # key=value + # + # The only complication is multi-line variables. Those have the + # form: + # + # key='first + # second' + + # TODO Bug 818377 Properly handle multi-line variables of form: + # $ foo="a='b' + # c='d'" + # $ set + # foo='a='"'"'b'"'"' + # c='"'"'d'"'" + + name = in_variable + value = None + if in_variable: + # Reached the end of a multi-line variable. + if line.endswith("'") and not line.endswith("\\'"): + current.append(line[:-1]) + value = '\n'.join(current) + in_variable = None + else: + current.append(line) + continue + else: + equal_pos = line.find('=') + + if equal_pos < 1: + # TODO log warning? + continue + + name = line[0:equal_pos] + value = line[equal_pos + 1:] + + if len(value): + has_quote = value[0] == "'" + + if has_quote: + value = value[1:] + + # Lines with a quote not ending in a quote are multi-line. + if has_quote and not value.endswith("'"): + in_variable = name + current.append(value) + continue + else: + value = value[:-1] if has_quote else value + + assert name is not None + + vars_mapping[current_type][name] = value + + current = [] + + continue + + current.append(line) + + return { + 'mk': mk_options, + 'ac': ac_options, + 'ac_app': ac_app_options, + 'vars_before': before_source, + 'vars_after': after_source, + 'env_before': env_before_source, + 'env_after': env_after_source, + } + + def _expand(self, s): + return s.replace('@TOPSRCDIR@', self.topsrcdir) diff --git a/python/mozbuild/mozbuild/mozconfig_loader b/python/mozbuild/mozbuild/mozconfig_loader new file mode 100755 index 000000000..6b1e05dce --- /dev/null +++ b/python/mozbuild/mozbuild/mozconfig_loader @@ -0,0 +1,80 @@ +#!/bin/sh +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, You can obtain one at http://mozilla.org/MPL/2.0/. + +# This script provides an execution environment for mozconfig scripts. +# This script is not meant to be called by users. Instead, some +# higher-level driver invokes it and parses the machine-tailored output. + +set -e + +ac_add_options() { + local opt + for opt; do + case "$opt" in + --target=*) + echo "------BEGIN_MK_OPTION" + echo $opt | sed s/--target/CONFIG_GUESS/ + echo "------END_MK_OPTION" + ;; + esac + echo "------BEGIN_AC_OPTION" + echo $opt + echo "------END_AC_OPTION" + done +} + +ac_add_app_options() { + local app + app=$1 + shift + echo "------BEGIN_AC_APP_OPTION" + echo $app + echo "$*" + echo "------END_AC_APP_OPTION" +} + +mk_add_options() { + local opt name op value + for opt; do + echo "------BEGIN_MK_OPTION" + echo $opt + # Remove any leading "export" + opt=${opt#export} + case "$opt" in + *\?=*) op="?=" ;; + *:=*) op=":=" ;; + *+=*) op="+=" ;; + *=*) op="=" ;; + esac + # Remove the operator and the value that follows + name=${opt%%${op}*} + # Note: $(echo ${name}) strips the variable from any leading and trailing + # whitespaces. + eval "$(echo ${name})_IS_SET=1" + echo "------END_MK_OPTION" + done +} + +echo "------BEGIN_ENV_BEFORE_SOURCE" +$3 $4 +echo "------END_ENV_BEFORE_SOURCE" + +echo "------BEGIN_BEFORE_SOURCE" +set +echo "------END_BEFORE_SOURCE" + +topsrcdir=$1 + +. $2 + +unset topsrcdir + +echo "------BEGIN_AFTER_SOURCE" +set +echo "------END_AFTER_SOURCE" + +echo "------BEGIN_ENV_AFTER_SOURCE" +$3 $4 +echo "------END_ENV_AFTER_SOURCE" diff --git a/python/mozbuild/mozbuild/mozinfo.py b/python/mozbuild/mozbuild/mozinfo.py new file mode 100755 index 000000000..f0b0df9bb --- /dev/null +++ b/python/mozbuild/mozbuild/mozinfo.py @@ -0,0 +1,160 @@ +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, You can obtain one at http://mozilla.org/MPL/2.0/. + +# This module produces a JSON file that provides basic build info and +# configuration metadata. + +from __future__ import absolute_import + +import os +import re +import json + + +def build_dict(config, env=os.environ): + """ + Build a dict containing data about the build configuration from + the environment. + """ + substs = config.substs + + # Check that all required variables are present first. + required = ["TARGET_CPU", "OS_TARGET"] + missing = [r for r in required if r not in substs] + if missing: + raise Exception("Missing required environment variables: %s" % + ', '.join(missing)) + + d = {} + d['topsrcdir'] = config.topsrcdir + + if config.mozconfig: + d['mozconfig'] = config.mozconfig + + # os + o = substs["OS_TARGET"] + known_os = {"Linux": "linux", + "WINNT": "win", + "Darwin": "mac", + "Android": "b2g" if substs.get("MOZ_WIDGET_TOOLKIT") == "gonk" else "android"} + if o in known_os: + d["os"] = known_os[o] + else: + # Allow unknown values, just lowercase them. + d["os"] = o.lower() + + # Widget toolkit, just pass the value directly through. + d["toolkit"] = substs.get("MOZ_WIDGET_TOOLKIT") + + # Application name + if 'MOZ_APP_NAME' in substs: + d["appname"] = substs["MOZ_APP_NAME"] + + # Build app name + if 'MOZ_MULET' in substs and substs.get('MOZ_MULET') == "1": + d["buildapp"] = "mulet" + elif 'MOZ_BUILD_APP' in substs: + d["buildapp"] = substs["MOZ_BUILD_APP"] + + # processor + p = substs["TARGET_CPU"] + # for universal mac builds, put in a special value + if d["os"] == "mac" and "UNIVERSAL_BINARY" in substs and substs["UNIVERSAL_BINARY"] == "1": + p = "universal-x86-x86_64" + else: + # do some slight massaging for some values + #TODO: retain specific values in case someone wants them? + if p.startswith("arm"): + p = "arm" + elif re.match("i[3-9]86", p): + p = "x86" + d["processor"] = p + # hardcoded list of 64-bit CPUs + if p in ["x86_64", "ppc64"]: + d["bits"] = 64 + # hardcoded list of known 32-bit CPUs + elif p in ["x86", "arm", "ppc"]: + d["bits"] = 32 + # other CPUs will wind up with unknown bits + + d['debug'] = substs.get('MOZ_DEBUG') == '1' + d['nightly_build'] = substs.get('NIGHTLY_BUILD') == '1' + d['release_or_beta'] = substs.get('RELEASE_OR_BETA') == '1' + d['pgo'] = substs.get('MOZ_PGO') == '1' + d['crashreporter'] = bool(substs.get('MOZ_CRASHREPORTER')) + d['datareporting'] = bool(substs.get('MOZ_DATA_REPORTING')) + d['healthreport'] = substs.get('MOZ_SERVICES_HEALTHREPORT') == '1' + d['sync'] = substs.get('MOZ_SERVICES_SYNC') == '1' + d['asan'] = substs.get('MOZ_ASAN') == '1' + d['tsan'] = substs.get('MOZ_TSAN') == '1' + d['telemetry'] = substs.get('MOZ_TELEMETRY_REPORTING') == '1' + d['tests_enabled'] = substs.get('ENABLE_TESTS') == "1" + d['bin_suffix'] = substs.get('BIN_SUFFIX', '') + d['addon_signing'] = substs.get('MOZ_ADDON_SIGNING') == '1' + d['require_signing'] = substs.get('MOZ_REQUIRE_SIGNING') == '1' + d['official'] = bool(substs.get('MOZILLA_OFFICIAL')) + d['sm_promise'] = bool(substs.get('SPIDERMONKEY_PROMISE')) + + def guess_platform(): + if d['buildapp'] in ('browser', 'mulet'): + p = d['os'] + if p == 'mac': + p = 'macosx64' + elif d['bits'] == 64: + p = '{}64'.format(p) + elif p in ('win',): + p = '{}32'.format(p) + + if d['buildapp'] == 'mulet': + p = '{}-mulet'.format(p) + + if d['asan']: + p = '{}-asan'.format(p) + + return p + + if d['buildapp'] == 'b2g': + if d['toolkit'] == 'gonk': + return 'emulator' + + if d['bits'] == 64: + return 'linux64_gecko' + return 'linux32_gecko' + + if d['buildapp'] == 'mobile/android': + if d['processor'] == 'x86': + return 'android-x86' + return 'android-arm' + + def guess_buildtype(): + if d['debug']: + return 'debug' + if d['pgo']: + return 'pgo' + return 'opt' + + # if buildapp or bits are unknown, we don't have a configuration similar to + # any in automation and the guesses are useless. + if 'buildapp' in d and (d['os'] == 'mac' or 'bits' in d): + d['platform_guess'] = guess_platform() + d['buildtype_guess'] = guess_buildtype() + + if 'buildapp' in d and d['buildapp'] == 'mobile/android' and 'MOZ_ANDROID_MIN_SDK_VERSION' in substs: + d['android_min_sdk'] = substs['MOZ_ANDROID_MIN_SDK_VERSION'] + + return d + + +def write_mozinfo(file, config, env=os.environ): + """Write JSON data about the configuration specified in config and an + environment variable dict to |file|, which may be a filename or file-like + object. + See build_dict for information about what environment variables are used, + and what keys are produced. + """ + build_conf = build_dict(config, env) + if isinstance(file, basestring): + file = open(file, 'wb') + + json.dump(build_conf, file, sort_keys=True, indent=4) diff --git a/python/mozbuild/mozbuild/preprocessor.py b/python/mozbuild/mozbuild/preprocessor.py new file mode 100644 index 000000000..e8aac7057 --- /dev/null +++ b/python/mozbuild/mozbuild/preprocessor.py @@ -0,0 +1,805 @@ +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, You can obtain one at http://mozilla.org/MPL/2.0/. +""" +This is a very primitive line based preprocessor, for times when using +a C preprocessor isn't an option. + +It currently supports the following grammar for expressions, whitespace is +ignored: + +expression : + and_cond ( '||' expression ) ? ; +and_cond: + test ( '&&' and_cond ) ? ; +test: + unary ( ( '==' | '!=' ) unary ) ? ; +unary : + '!'? value ; +value : + [0-9]+ # integer + | 'defined(' \w+ ')' + | \w+ # string identifier or value; +""" + +import sys +import os +import re +from optparse import OptionParser +import errno +from makeutil import Makefile + +# hack around win32 mangling our line endings +# http://aspn.activestate.com/ASPN/Cookbook/Python/Recipe/65443 +if sys.platform == "win32": + import msvcrt + msvcrt.setmode(sys.stdout.fileno(), os.O_BINARY) + os.linesep = '\n' + + +__all__ = [ + 'Context', + 'Expression', + 'Preprocessor', + 'preprocess' +] + + +class Expression: + def __init__(self, expression_string): + """ + Create a new expression with this string. + The expression will already be parsed into an Abstract Syntax Tree. + """ + self.content = expression_string + self.offset = 0 + self.__ignore_whitespace() + self.e = self.__get_logical_or() + if self.content: + raise Expression.ParseError, self + + def __get_logical_or(self): + """ + Production: and_cond ( '||' expression ) ? + """ + if not len(self.content): + return None + rv = Expression.__AST("logical_op") + # test + rv.append(self.__get_logical_and()) + self.__ignore_whitespace() + if self.content[:2] != '||': + # no logical op needed, short cut to our prime element + return rv[0] + # append operator + rv.append(Expression.__ASTLeaf('op', self.content[:2])) + self.__strip(2) + self.__ignore_whitespace() + rv.append(self.__get_logical_or()) + self.__ignore_whitespace() + return rv + + def __get_logical_and(self): + """ + Production: test ( '&&' and_cond ) ? + """ + if not len(self.content): + return None + rv = Expression.__AST("logical_op") + # test + rv.append(self.__get_equality()) + self.__ignore_whitespace() + if self.content[:2] != '&&': + # no logical op needed, short cut to our prime element + return rv[0] + # append operator + rv.append(Expression.__ASTLeaf('op', self.content[:2])) + self.__strip(2) + self.__ignore_whitespace() + rv.append(self.__get_logical_and()) + self.__ignore_whitespace() + return rv + + def __get_equality(self): + """ + Production: unary ( ( '==' | '!=' ) unary ) ? + """ + if not len(self.content): + return None + rv = Expression.__AST("equality") + # unary + rv.append(self.__get_unary()) + self.__ignore_whitespace() + if not re.match('[=!]=', self.content): + # no equality needed, short cut to our prime unary + return rv[0] + # append operator + rv.append(Expression.__ASTLeaf('op', self.content[:2])) + self.__strip(2) + self.__ignore_whitespace() + rv.append(self.__get_unary()) + self.__ignore_whitespace() + return rv + + def __get_unary(self): + """ + Production: '!'? value + """ + # eat whitespace right away, too + not_ws = re.match('!\s*', self.content) + if not not_ws: + return self.__get_value() + rv = Expression.__AST('not') + self.__strip(not_ws.end()) + rv.append(self.__get_value()) + self.__ignore_whitespace() + return rv + + def __get_value(self): + """ + Production: ( [0-9]+ | 'defined(' \w+ ')' | \w+ ) + Note that the order is important, and the expression is kind-of + ambiguous as \w includes 0-9. One could make it unambiguous by + removing 0-9 from the first char of a string literal. + """ + rv = None + m = re.match('defined\s*\(\s*(\w+)\s*\)', self.content) + if m: + word_len = m.end() + rv = Expression.__ASTLeaf('defined', m.group(1)) + else: + word_len = re.match('[0-9]*', self.content).end() + if word_len: + value = int(self.content[:word_len]) + rv = Expression.__ASTLeaf('int', value) + else: + word_len = re.match('\w*', self.content).end() + if word_len: + rv = Expression.__ASTLeaf('string', self.content[:word_len]) + else: + raise Expression.ParseError, self + self.__strip(word_len) + self.__ignore_whitespace() + return rv + + def __ignore_whitespace(self): + ws_len = re.match('\s*', self.content).end() + self.__strip(ws_len) + return + + def __strip(self, length): + """ + Remove a given amount of chars from the input and update + the offset. + """ + self.content = self.content[length:] + self.offset += length + + def evaluate(self, context): + """ + Evaluate the expression with the given context + """ + + # Helper function to evaluate __get_equality results + def eval_equality(tok): + left = opmap[tok[0].type](tok[0]) + right = opmap[tok[2].type](tok[2]) + rv = left == right + if tok[1].value == '!=': + rv = not rv + return rv + # Helper function to evaluate __get_logical_and and __get_logical_or results + def eval_logical_op(tok): + left = opmap[tok[0].type](tok[0]) + right = opmap[tok[2].type](tok[2]) + if tok[1].value == '&&': + return left and right + elif tok[1].value == '||': + return left or right + raise Expression.ParseError, self + + # Mapping from token types to evaluator functions + # Apart from (non-)equality, all these can be simple lambda forms. + opmap = { + 'logical_op': eval_logical_op, + 'equality': eval_equality, + 'not': lambda tok: not opmap[tok[0].type](tok[0]), + 'string': lambda tok: context[tok.value], + 'defined': lambda tok: tok.value in context, + 'int': lambda tok: tok.value} + + return opmap[self.e.type](self.e); + + class __AST(list): + """ + Internal class implementing Abstract Syntax Tree nodes + """ + def __init__(self, type): + self.type = type + super(self.__class__, self).__init__(self) + + class __ASTLeaf: + """ + Internal class implementing Abstract Syntax Tree leafs + """ + def __init__(self, type, value): + self.value = value + self.type = type + def __str__(self): + return self.value.__str__() + def __repr__(self): + return self.value.__repr__() + + class ParseError(StandardError): + """ + Error raised when parsing fails. + It has two members, offset and content, which give the offset of the + error and the offending content. + """ + def __init__(self, expression): + self.offset = expression.offset + self.content = expression.content[:3] + def __str__(self): + return 'Unexpected content at offset {0}, "{1}"'.format(self.offset, + self.content) + +class Context(dict): + """ + This class holds variable values by subclassing dict, and while it + truthfully reports True and False on + + name in context + + it returns the variable name itself on + + context["name"] + + to reflect the ambiguity between string literals and preprocessor + variables. + """ + def __getitem__(self, key): + if key in self: + return super(self.__class__, self).__getitem__(key) + return key + + +class Preprocessor: + """ + Class for preprocessing text files. + """ + class Error(RuntimeError): + def __init__(self, cpp, MSG, context): + self.file = cpp.context['FILE'] + self.line = cpp.context['LINE'] + self.key = MSG + RuntimeError.__init__(self, (self.file, self.line, self.key, context)) + + def __init__(self, defines=None, marker='#'): + self.context = Context() + for k,v in {'FILE': '', + 'LINE': 0, + 'DIRECTORY': os.path.abspath('.')}.iteritems(): + self.context[k] = v + self.actionLevel = 0 + self.disableLevel = 0 + # ifStates can be + # 0: hadTrue + # 1: wantsTrue + # 2: #else found + self.ifStates = [] + self.checkLineNumbers = False + self.filters = [] + self.cmds = {} + for cmd, level in {'define': 0, + 'undef': 0, + 'if': sys.maxint, + 'ifdef': sys.maxint, + 'ifndef': sys.maxint, + 'else': 1, + 'elif': 1, + 'elifdef': 1, + 'elifndef': 1, + 'endif': sys.maxint, + 'expand': 0, + 'literal': 0, + 'filter': 0, + 'unfilter': 0, + 'include': 0, + 'includesubst': 0, + 'error': 0}.iteritems(): + self.cmds[cmd] = (level, getattr(self, 'do_' + cmd)) + self.out = sys.stdout + self.setMarker(marker) + self.varsubst = re.compile('@(?P\w+)@', re.U) + self.includes = set() + self.silenceMissingDirectiveWarnings = False + if defines: + self.context.update(defines) + + def failUnused(self, file): + msg = None + if self.actionLevel == 0 and not self.silenceMissingDirectiveWarnings: + msg = 'no preprocessor directives found' + elif self.actionLevel == 1: + msg = 'no useful preprocessor directives found' + if msg: + class Fake(object): pass + fake = Fake() + fake.context = { + 'FILE': file, + 'LINE': None, + } + raise Preprocessor.Error(fake, msg, None) + + def setMarker(self, aMarker): + """ + Set the marker to be used for processing directives. + Used for handling CSS files, with pp.setMarker('%'), for example. + The given marker may be None, in which case no markers are processed. + """ + self.marker = aMarker + if aMarker: + self.instruction = re.compile('{0}(?P[a-z]+)(?:\s(?P.*))?$' + .format(aMarker), + re.U) + self.comment = re.compile(aMarker, re.U) + else: + class NoMatch(object): + def match(self, *args): + return False + self.instruction = self.comment = NoMatch() + + def setSilenceDirectiveWarnings(self, value): + """ + Sets whether missing directive warnings are silenced, according to + ``value``. The default behavior of the preprocessor is to emit + such warnings. + """ + self.silenceMissingDirectiveWarnings = value + + def addDefines(self, defines): + """ + Adds the specified defines to the preprocessor. + ``defines`` may be a dictionary object or an iterable of key/value pairs + (as tuples or other iterables of length two) + """ + self.context.update(defines) + + def clone(self): + """ + Create a clone of the current processor, including line ending + settings, marker, variable definitions, output stream. + """ + rv = Preprocessor() + rv.context.update(self.context) + rv.setMarker(self.marker) + rv.out = self.out + return rv + + def processFile(self, input, output, depfile=None): + """ + Preprocesses the contents of the ``input`` stream and writes the result + to the ``output`` stream. If ``depfile`` is set, the dependencies of + ``output`` file are written to ``depfile`` in Makefile format. + """ + self.out = output + + self.do_include(input, False) + self.failUnused(input.name) + + if depfile: + mk = Makefile() + mk.create_rule([output.name]).add_dependencies(self.includes) + mk.dump(depfile) + + def computeDependencies(self, input): + """ + Reads the ``input`` stream, and computes the dependencies for that input. + """ + try: + old_out = self.out + self.out = None + self.do_include(input, False) + + return self.includes + finally: + self.out = old_out + + def applyFilters(self, aLine): + for f in self.filters: + aLine = f[1](aLine) + return aLine + + def noteLineInfo(self): + # Record the current line and file. Called once before transitioning + # into or out of an included file and after writing each line. + self.line_info = self.context['FILE'], self.context['LINE'] + + def write(self, aLine): + """ + Internal method for handling output. + """ + if not self.out: + return + + next_line, next_file = self.context['LINE'], self.context['FILE'] + if self.checkLineNumbers: + expected_file, expected_line = self.line_info + expected_line += 1 + if (expected_line != next_line or + expected_file and expected_file != next_file): + self.out.write('//@line {line} "{file}"\n'.format(line=next_line, + file=next_file)) + self.noteLineInfo() + + filteredLine = self.applyFilters(aLine) + if filteredLine != aLine: + self.actionLevel = 2 + self.out.write(filteredLine) + + def handleCommandLine(self, args, defaultToStdin = False): + """ + Parse a commandline into this parser. + Uses OptionParser internally, no args mean sys.argv[1:]. + """ + def get_output_file(path): + dir = os.path.dirname(path) + if dir: + try: + os.makedirs(dir) + except OSError as error: + if error.errno != errno.EEXIST: + raise + return open(path, 'wb') + + p = self.getCommandLineParser() + options, args = p.parse_args(args=args) + out = self.out + depfile = None + + if options.output: + out = get_output_file(options.output) + if defaultToStdin and len(args) == 0: + args = [sys.stdin] + if options.depend: + raise Preprocessor.Error(self, "--depend doesn't work with stdin", + None) + if options.depend: + if not options.output: + raise Preprocessor.Error(self, "--depend doesn't work with stdout", + None) + try: + from makeutil import Makefile + except: + raise Preprocessor.Error(self, "--depend requires the " + "mozbuild.makeutil module", None) + depfile = get_output_file(options.depend) + + if args: + for f in args: + with open(f, 'rU') as input: + self.processFile(input=input, output=out) + if depfile: + mk = Makefile() + mk.create_rule([options.output]).add_dependencies(self.includes) + mk.dump(depfile) + depfile.close() + + if options.output: + out.close() + + def getCommandLineParser(self, unescapeDefines = False): + escapedValue = re.compile('".*"$') + numberValue = re.compile('\d+$') + def handleD(option, opt, value, parser): + vals = value.split('=', 1) + if len(vals) == 1: + vals.append(1) + elif unescapeDefines and escapedValue.match(vals[1]): + # strip escaped string values + vals[1] = vals[1][1:-1] + elif numberValue.match(vals[1]): + vals[1] = int(vals[1]) + self.context[vals[0]] = vals[1] + def handleU(option, opt, value, parser): + del self.context[value] + def handleF(option, opt, value, parser): + self.do_filter(value) + def handleMarker(option, opt, value, parser): + self.setMarker(value) + def handleSilenceDirectiveWarnings(option, opt, value, parse): + self.setSilenceDirectiveWarnings(True) + p = OptionParser() + p.add_option('-D', action='callback', callback=handleD, type="string", + metavar="VAR[=VAL]", help='Define a variable') + p.add_option('-U', action='callback', callback=handleU, type="string", + metavar="VAR", help='Undefine a variable') + p.add_option('-F', action='callback', callback=handleF, type="string", + metavar="FILTER", help='Enable the specified filter') + p.add_option('-o', '--output', type="string", default=None, + metavar="FILENAME", help='Output to the specified file '+ + 'instead of stdout') + p.add_option('--depend', type="string", default=None, metavar="FILENAME", + help='Generate dependencies in the given file') + p.add_option('--marker', action='callback', callback=handleMarker, + type="string", + help='Use the specified marker instead of #') + p.add_option('--silence-missing-directive-warnings', action='callback', + callback=handleSilenceDirectiveWarnings, + help='Don\'t emit warnings about missing directives') + return p + + def handleLine(self, aLine): + """ + Handle a single line of input (internal). + """ + if self.actionLevel == 0 and self.comment.match(aLine): + self.actionLevel = 1 + m = self.instruction.match(aLine) + if m: + args = None + cmd = m.group('cmd') + try: + args = m.group('args') + except IndexError: + pass + if cmd not in self.cmds: + raise Preprocessor.Error(self, 'INVALID_CMD', aLine) + level, cmd = self.cmds[cmd] + if (level >= self.disableLevel): + cmd(args) + if cmd != 'literal': + self.actionLevel = 2 + elif self.disableLevel == 0 and not self.comment.match(aLine): + self.write(aLine) + + # Instruction handlers + # These are named do_'instruction name' and take one argument + + # Variables + def do_define(self, args): + m = re.match('(?P\w+)(?:\s(?P.*))?', args, re.U) + if not m: + raise Preprocessor.Error(self, 'SYNTAX_DEF', args) + val = '' + if m.group('value'): + val = self.applyFilters(m.group('value')) + try: + val = int(val) + except: + pass + self.context[m.group('name')] = val + def do_undef(self, args): + m = re.match('(?P\w+)$', args, re.U) + if not m: + raise Preprocessor.Error(self, 'SYNTAX_DEF', args) + if args in self.context: + del self.context[args] + # Logic + def ensure_not_else(self): + if len(self.ifStates) == 0 or self.ifStates[-1] == 2: + sys.stderr.write('WARNING: bad nesting of #else\n') + def do_if(self, args, replace=False): + if self.disableLevel and not replace: + self.disableLevel += 1 + return + val = None + try: + e = Expression(args) + val = e.evaluate(self.context) + except Exception: + # XXX do real error reporting + raise Preprocessor.Error(self, 'SYNTAX_ERR', args) + if type(val) == str: + # we're looking for a number value, strings are false + val = False + if not val: + self.disableLevel = 1 + if replace: + if val: + self.disableLevel = 0 + self.ifStates[-1] = self.disableLevel + else: + self.ifStates.append(self.disableLevel) + pass + def do_ifdef(self, args, replace=False): + if self.disableLevel and not replace: + self.disableLevel += 1 + return + if re.match('\W', args, re.U): + raise Preprocessor.Error(self, 'INVALID_VAR', args) + if args not in self.context: + self.disableLevel = 1 + if replace: + if args in self.context: + self.disableLevel = 0 + self.ifStates[-1] = self.disableLevel + else: + self.ifStates.append(self.disableLevel) + pass + def do_ifndef(self, args, replace=False): + if self.disableLevel and not replace: + self.disableLevel += 1 + return + if re.match('\W', args, re.U): + raise Preprocessor.Error(self, 'INVALID_VAR', args) + if args in self.context: + self.disableLevel = 1 + if replace: + if args not in self.context: + self.disableLevel = 0 + self.ifStates[-1] = self.disableLevel + else: + self.ifStates.append(self.disableLevel) + pass + def do_else(self, args, ifState = 2): + self.ensure_not_else() + hadTrue = self.ifStates[-1] == 0 + self.ifStates[-1] = ifState # in-else + if hadTrue: + self.disableLevel = 1 + return + self.disableLevel = 0 + def do_elif(self, args): + if self.disableLevel == 1: + if self.ifStates[-1] == 1: + self.do_if(args, replace=True) + else: + self.do_else(None, self.ifStates[-1]) + def do_elifdef(self, args): + if self.disableLevel == 1: + if self.ifStates[-1] == 1: + self.do_ifdef(args, replace=True) + else: + self.do_else(None, self.ifStates[-1]) + def do_elifndef(self, args): + if self.disableLevel == 1: + if self.ifStates[-1] == 1: + self.do_ifndef(args, replace=True) + else: + self.do_else(None, self.ifStates[-1]) + def do_endif(self, args): + if self.disableLevel > 0: + self.disableLevel -= 1 + if self.disableLevel == 0: + self.ifStates.pop() + # output processing + def do_expand(self, args): + lst = re.split('__(\w+)__', args, re.U) + do_replace = False + def vsubst(v): + if v in self.context: + return str(self.context[v]) + return '' + for i in range(1, len(lst), 2): + lst[i] = vsubst(lst[i]) + lst.append('\n') # add back the newline + self.write(reduce(lambda x, y: x+y, lst, '')) + def do_literal(self, args): + self.write(args + '\n') + def do_filter(self, args): + filters = [f for f in args.split(' ') if hasattr(self, 'filter_' + f)] + if len(filters) == 0: + return + current = dict(self.filters) + for f in filters: + current[f] = getattr(self, 'filter_' + f) + filterNames = current.keys() + filterNames.sort() + self.filters = [(fn, current[fn]) for fn in filterNames] + return + def do_unfilter(self, args): + filters = args.split(' ') + current = dict(self.filters) + for f in filters: + if f in current: + del current[f] + filterNames = current.keys() + filterNames.sort() + self.filters = [(fn, current[fn]) for fn in filterNames] + return + # Filters + # + # emptyLines + # Strips blank lines from the output. + def filter_emptyLines(self, aLine): + if aLine == '\n': + return '' + return aLine + # slashslash + # Strips everything after // + def filter_slashslash(self, aLine): + if (aLine.find('//') == -1): + return aLine + [aLine, rest] = aLine.split('//', 1) + if rest: + aLine += '\n' + return aLine + # spaces + # Collapses sequences of spaces into a single space + def filter_spaces(self, aLine): + return re.sub(' +', ' ', aLine).strip(' ') + # substition + # helper to be used by both substition and attemptSubstitution + def filter_substitution(self, aLine, fatal=True): + def repl(matchobj): + varname = matchobj.group('VAR') + if varname in self.context: + return str(self.context[varname]) + if fatal: + raise Preprocessor.Error(self, 'UNDEFINED_VAR', varname) + return matchobj.group(0) + return self.varsubst.sub(repl, aLine) + def filter_attemptSubstitution(self, aLine): + return self.filter_substitution(aLine, fatal=False) + # File ops + def do_include(self, args, filters=True): + """ + Preprocess a given file. + args can either be a file name, or a file-like object. + Files should be opened, and will be closed after processing. + """ + isName = type(args) == str or type(args) == unicode + oldCheckLineNumbers = self.checkLineNumbers + self.checkLineNumbers = False + if isName: + try: + args = str(args) + if filters: + args = self.applyFilters(args) + if not os.path.isabs(args): + args = os.path.join(self.context['DIRECTORY'], args) + args = open(args, 'rU') + except Preprocessor.Error: + raise + except: + raise Preprocessor.Error(self, 'FILE_NOT_FOUND', str(args)) + self.checkLineNumbers = bool(re.search('\.(js|jsm|java)(?:\.in)?$', args.name)) + oldFile = self.context['FILE'] + oldLine = self.context['LINE'] + oldDir = self.context['DIRECTORY'] + self.noteLineInfo() + + if args.isatty(): + # we're stdin, use '-' and '' for file and dir + self.context['FILE'] = '-' + self.context['DIRECTORY'] = '' + else: + abspath = os.path.abspath(args.name) + self.includes.add(abspath) + self.context['FILE'] = abspath + self.context['DIRECTORY'] = os.path.dirname(abspath) + self.context['LINE'] = 0 + + for l in args: + self.context['LINE'] += 1 + self.handleLine(l) + if isName: + args.close() + + self.context['FILE'] = oldFile + self.checkLineNumbers = oldCheckLineNumbers + self.context['LINE'] = oldLine + self.context['DIRECTORY'] = oldDir + def do_includesubst(self, args): + args = self.filter_substitution(args) + self.do_include(args) + def do_error(self, args): + raise Preprocessor.Error(self, 'Error: ', str(args)) + + +def preprocess(includes=[sys.stdin], defines={}, + output = sys.stdout, + marker='#'): + pp = Preprocessor(defines=defines, + marker=marker) + for f in includes: + with open(f, 'rU') as input: + pp.processFile(input=input, output=output) + return pp.includes + + +# Keep this module independently executable. +if __name__ == "__main__": + pp = Preprocessor() + pp.handleCommandLine(None, True) diff --git a/python/mozbuild/mozbuild/pythonutil.py b/python/mozbuild/mozbuild/pythonutil.py new file mode 100644 index 000000000..3dba25691 --- /dev/null +++ b/python/mozbuild/mozbuild/pythonutil.py @@ -0,0 +1,25 @@ +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, You can obtain one at http://mozilla.org/MPL/2.0/. + +from __future__ import absolute_import + +import os +import sys + + +def iter_modules_in_path(*paths): + paths = [os.path.abspath(os.path.normcase(p)) + os.sep + for p in paths] + for name, module in sys.modules.items(): + if not hasattr(module, '__file__'): + continue + + path = module.__file__ + + if path.endswith('.pyc'): + path = path[:-1] + path = os.path.abspath(os.path.normcase(path)) + + if any(path.startswith(p) for p in paths): + yield path diff --git a/python/mozbuild/mozbuild/resources/html-build-viewer/index.html b/python/mozbuild/mozbuild/resources/html-build-viewer/index.html new file mode 100644 index 000000000..fe7512188 --- /dev/null +++ b/python/mozbuild/mozbuild/resources/html-build-viewer/index.html @@ -0,0 +1,475 @@ + + + + + Build System Resource Usage + + + + + + + +

Build Resource Usage Report

+ + + + + +
+
+ + + + + + + + + +
Wall Time (s)
Start Date
End Date
CPU %
Write Bytes
Read Bytes
Write Time
Read Time
+
+ + diff --git a/python/mozbuild/mozbuild/shellutil.py b/python/mozbuild/mozbuild/shellutil.py new file mode 100644 index 000000000..185a970ee --- /dev/null +++ b/python/mozbuild/mozbuild/shellutil.py @@ -0,0 +1,209 @@ +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this file, +# You can obtain one at http://mozilla.org/MPL/2.0/. + +import re + + +def _tokens2re(**tokens): + # Create a pattern for non-escaped tokens, in the form: + # (?pattern) + # which matches the pattern and captures it in a named match group. + # The group names and patterns are given as arguments. + all_tokens = '|'.join('(?P<%s>%s)' % (name, value) + for name, value in tokens.iteritems()) + nonescaped = r'(?\\\\)')) + +UNQUOTED_TOKENS_RE = _tokens2re( + whitespace=r'[\t\r\n ]+', + quote=r'[\'"]', + comment='#', + special=r'[<>&|`~(){}$;\*\?]', + backslashed=r'\\[^\\]', +) + +DOUBLY_QUOTED_TOKENS_RE = _tokens2re( + quote='"', + backslashedquote=r'\\"', + special='\$', + backslashed=r'\\[^\\"]', +) + +ESCAPED_NEWLINES_RE = re.compile(r'\\\n') + +# This regexp contains the same characters as all those listed in +# UNQUOTED_TOKENS_RE. Please keep in sync. +SHELL_QUOTE_RE = re.compile(r'[\\\t\r\n \'\"#<>&|`~(){}$;\*\?]') + + +class MetaCharacterException(Exception): + def __init__(self, char): + self.char = char + + +class _ClineSplitter(object): + ''' + Parses a given command line string and creates a list of command + and arguments, with wildcard expansion. + ''' + def __init__(self, cline): + self.arg = None + self.cline = cline + self.result = [] + self._parse_unquoted() + + def _push(self, str): + ''' + Push the given string as part of the current argument + ''' + if self.arg is None: + self.arg = '' + self.arg += str + + def _next(self): + ''' + Finalize current argument, effectively adding it to the list. + ''' + if self.arg is None: + return + self.result.append(self.arg) + self.arg = None + + def _parse_unquoted(self): + ''' + Parse command line remainder in the context of an unquoted string. + ''' + while self.cline: + # Find the next token + m = UNQUOTED_TOKENS_RE.search(self.cline) + # If we find none, the remainder of the string can be pushed to + # the current argument and the argument finalized + if not m: + self._push(self.cline) + break + # The beginning of the string, up to the found token, is part of + # the current argument + if m.start(): + self._push(self.cline[:m.start()]) + self.cline = self.cline[m.end():] + + match = {name: value + for name, value in m.groupdict().items() if value} + if 'quote' in match: + # " or ' start a quoted string + if match['quote'] == '"': + self._parse_doubly_quoted() + else: + self._parse_quoted() + elif 'comment' in match: + # Comments are ignored. The current argument can be finalized, + # and parsing stopped. + break + elif 'special' in match: + # Unquoted, non-escaped special characters need to be sent to a + # shell. + raise MetaCharacterException(match['special']) + elif 'whitespace' in match: + # Whitespaces terminate current argument. + self._next() + elif 'escape' in match: + # Escaped backslashes turn into a single backslash + self._push('\\') + elif 'backslashed' in match: + # Backslashed characters are unbackslashed + # e.g. echo \a -> a + self._push(match['backslashed'][1]) + else: + raise Exception("Shouldn't reach here") + if self.arg: + self._next() + + def _parse_quoted(self): + # Single quoted strings are preserved, except for the final quote + index = self.cline.find("'") + if index == -1: + raise Exception('Unterminated quoted string in command') + self._push(self.cline[:index]) + self.cline = self.cline[index+1:] + + def _parse_doubly_quoted(self): + if not self.cline: + raise Exception('Unterminated quoted string in command') + while self.cline: + m = DOUBLY_QUOTED_TOKENS_RE.search(self.cline) + if not m: + raise Exception('Unterminated quoted string in command') + self._push(self.cline[:m.start()]) + self.cline = self.cline[m.end():] + match = {name: value + for name, value in m.groupdict().items() if value} + if 'quote' in match: + # a double quote ends the quoted string, so go back to + # unquoted parsing + return + elif 'special' in match: + # Unquoted, non-escaped special characters in a doubly quoted + # string still have a special meaning and need to be sent to a + # shell. + raise MetaCharacterException(match['special']) + elif 'escape' in match: + # Escaped backslashes turn into a single backslash + self._push('\\') + elif 'backslashedquote' in match: + # Backslashed double quotes are un-backslashed + self._push('"') + elif 'backslashed' in match: + # Backslashed characters are kept backslashed + self._push(match['backslashed']) + + +def split(cline): + ''' + Split the given command line string. + ''' + s = ESCAPED_NEWLINES_RE.sub('', cline) + return _ClineSplitter(s).result + + +def _quote(s): + '''Given a string, returns a version that can be used literally on a shell + command line, enclosing it with single quotes if necessary. + + As a special case, if given an int, returns a string containing the int, + not enclosed in quotes. + ''' + if type(s) == int: + return '%d' % s + + # Empty strings need to be quoted to have any significance + if s and not SHELL_QUOTE_RE.search(s): + return s + + # Single quoted strings can contain any characters unescaped except the + # single quote itself, which can't even be escaped, so the string needs to + # be closed, an escaped single quote added, and reopened. + t = type(s) + return t("'%s'") % s.replace(t("'"), t("'\\''")) + + +def quote(*strings): + '''Given one or more strings, returns a quoted string that can be used + literally on a shell command line. + + >>> quote('a', 'b') + "a b" + >>> quote('a b', 'c') + "'a b' c" + ''' + return ' '.join(_quote(s) for s in strings) + + +__all__ = ['MetaCharacterException', 'split', 'quote'] diff --git a/python/mozbuild/mozbuild/sphinx.py b/python/mozbuild/mozbuild/sphinx.py new file mode 100644 index 000000000..0f8e22ca1 --- /dev/null +++ b/python/mozbuild/mozbuild/sphinx.py @@ -0,0 +1,200 @@ +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, You can obtain one at http://mozilla.org/MPL/2.0/. + +from __future__ import absolute_import + +import importlib +import os +import sys + +from sphinx.util.compat import Directive +from sphinx.util.docstrings import prepare_docstring + + +def function_reference(f, attr, args, doc): + lines = [] + + lines.extend([ + f, + '-' * len(f), + '', + ]) + + docstring = prepare_docstring(doc) + + lines.extend([ + docstring[0], + '', + ]) + + arg_types = [] + + for t in args: + if isinstance(t, list): + inner_types = [t2.__name__ for t2 in t] + arg_types.append(' | ' .join(inner_types)) + continue + + arg_types.append(t.__name__) + + arg_s = '(%s)' % ', '.join(arg_types) + + lines.extend([ + ':Arguments: %s' % arg_s, + '', + ]) + + lines.extend(docstring[1:]) + lines.append('') + + return lines + + +def variable_reference(v, st_type, in_type, doc): + lines = [ + v, + '-' * len(v), + '', + ] + + docstring = prepare_docstring(doc) + + lines.extend([ + docstring[0], + '', + ]) + + lines.extend([ + ':Storage Type: ``%s``' % st_type.__name__, + ':Input Type: ``%s``' % in_type.__name__, + '', + ]) + + lines.extend(docstring[1:]) + lines.append('') + + return lines + + +def special_reference(v, func, typ, doc): + lines = [ + v, + '-' * len(v), + '', + ] + + docstring = prepare_docstring(doc) + + lines.extend([ + docstring[0], + '', + ':Type: ``%s``' % typ.__name__, + '', + ]) + + lines.extend(docstring[1:]) + lines.append('') + + return lines + + +def format_module(m): + lines = [] + + for subcontext, cls in sorted(m.SUBCONTEXTS.items()): + lines.extend([ + '.. _mozbuild_subcontext_%s:' % subcontext, + '', + 'Sub-Context: %s' % subcontext, + '=============' + '=' * len(subcontext), + '', + ]) + lines.extend(prepare_docstring(cls.__doc__)) + if lines[-1]: + lines.append('') + + for k, v in sorted(cls.VARIABLES.items()): + lines.extend(variable_reference(k, *v)) + + lines.extend([ + 'Variables', + '=========', + '', + ]) + + for v in sorted(m.VARIABLES): + lines.extend(variable_reference(v, *m.VARIABLES[v])) + + lines.extend([ + 'Functions', + '=========', + '', + ]) + + for func in sorted(m.FUNCTIONS): + lines.extend(function_reference(func, *m.FUNCTIONS[func])) + + lines.extend([ + 'Special Variables', + '=================', + '', + ]) + + for v in sorted(m.SPECIAL_VARIABLES): + lines.extend(special_reference(v, *m.SPECIAL_VARIABLES[v])) + + return lines + + +class MozbuildSymbols(Directive): + """Directive to insert mozbuild sandbox symbol information.""" + + required_arguments = 1 + + def run(self): + module = importlib.import_module(self.arguments[0]) + fname = module.__file__ + if fname.endswith('.pyc'): + fname = fname[0:-1] + + self.state.document.settings.record_dependencies.add(fname) + + # We simply format out the documentation as rst then feed it back + # into the parser for conversion. We don't even emit ourselves, so + # there's no record of us. + self.state_machine.insert_input(format_module(module), fname) + + return [] + + +def setup(app): + app.add_directive('mozbuildsymbols', MozbuildSymbols) + + # Unlike typical Sphinx installs, our documentation is assembled from + # many sources and staged in a common location. This arguably isn't a best + # practice, but it was the easiest to implement at the time. + # + # Here, we invoke our custom code for staging/generating all our + # documentation. + from moztreedocs import SphinxManager + + topsrcdir = app.config._raw_config['topsrcdir'] + manager = SphinxManager(topsrcdir, + os.path.join(topsrcdir, 'tools', 'docs'), + app.outdir) + manager.generate_docs(app) + + app.srcdir = os.path.join(app.outdir, '_staging') + + # We need to adjust sys.path in order for Python API docs to get generated + # properly. We leverage the in-tree virtualenv for this. + from mozbuild.virtualenv import VirtualenvManager + + ve = VirtualenvManager(topsrcdir, + os.path.join(topsrcdir, 'dummy-objdir'), + os.path.join(app.outdir, '_venv'), + sys.stderr, + os.path.join(topsrcdir, 'build', 'virtualenv_packages.txt')) + ve.ensure() + ve.activate() diff --git a/python/mozbuild/mozbuild/test/__init__.py b/python/mozbuild/mozbuild/test/__init__.py new file mode 100644 index 000000000..e69de29bb diff --git a/python/mozbuild/mozbuild/test/action/data/invalid/region.properties b/python/mozbuild/mozbuild/test/action/data/invalid/region.properties new file mode 100644 index 000000000..d4d8109b6 --- /dev/null +++ b/python/mozbuild/mozbuild/test/action/data/invalid/region.properties @@ -0,0 +1,12 @@ +# A region.properties file with invalid unicode byte sequences. The +# sequences were cribbed from Markus Kuhn's "UTF-8 decoder capability +# and stress test", available at +# http://www.cl.cam.ac.uk/~mgk25/ucs/examples/UTF-8-test.txt + +# 3.5 Impossible bytes | +# | +# The following two bytes cannot appear in a correct UTF-8 string | +# | +# 3.5.1 fe = "þ" | +# 3.5.2 ff = "ÿ" | +# 3.5.3 fe fe ff ff = "þþÿÿ" | diff --git a/python/mozbuild/mozbuild/test/action/data/package_fennec_apk/assets/asset.txt b/python/mozbuild/mozbuild/test/action/data/package_fennec_apk/assets/asset.txt new file mode 100644 index 000000000..b01830602 --- /dev/null +++ b/python/mozbuild/mozbuild/test/action/data/package_fennec_apk/assets/asset.txt @@ -0,0 +1 @@ +assets/asset.txt diff --git a/python/mozbuild/mozbuild/test/action/data/package_fennec_apk/classes.dex b/python/mozbuild/mozbuild/test/action/data/package_fennec_apk/classes.dex new file mode 100644 index 000000000..dfc99f9c2 --- /dev/null +++ b/python/mozbuild/mozbuild/test/action/data/package_fennec_apk/classes.dex @@ -0,0 +1 @@ +classes.dex \ No newline at end of file diff --git a/python/mozbuild/mozbuild/test/action/data/package_fennec_apk/input1.ap_ b/python/mozbuild/mozbuild/test/action/data/package_fennec_apk/input1.ap_ new file mode 100644 index 000000000..915be683b Binary files /dev/null and b/python/mozbuild/mozbuild/test/action/data/package_fennec_apk/input1.ap_ differ diff --git a/python/mozbuild/mozbuild/test/action/data/package_fennec_apk/input1/res/res.txt b/python/mozbuild/mozbuild/test/action/data/package_fennec_apk/input1/res/res.txt new file mode 100644 index 000000000..01d2fb0a1 --- /dev/null +++ b/python/mozbuild/mozbuild/test/action/data/package_fennec_apk/input1/res/res.txt @@ -0,0 +1 @@ +input1/res/res.txt diff --git a/python/mozbuild/mozbuild/test/action/data/package_fennec_apk/input1/resources.arsc b/python/mozbuild/mozbuild/test/action/data/package_fennec_apk/input1/resources.arsc new file mode 100644 index 000000000..6274a181a --- /dev/null +++ b/python/mozbuild/mozbuild/test/action/data/package_fennec_apk/input1/resources.arsc @@ -0,0 +1 @@ +input1/resources.arsc \ No newline at end of file diff --git a/python/mozbuild/mozbuild/test/action/data/package_fennec_apk/input2.apk b/python/mozbuild/mozbuild/test/action/data/package_fennec_apk/input2.apk new file mode 100644 index 000000000..3003f5ae9 Binary files /dev/null and b/python/mozbuild/mozbuild/test/action/data/package_fennec_apk/input2.apk differ diff --git a/python/mozbuild/mozbuild/test/action/data/package_fennec_apk/input2/assets/asset.txt b/python/mozbuild/mozbuild/test/action/data/package_fennec_apk/input2/assets/asset.txt new file mode 100644 index 000000000..31a0e5129 --- /dev/null +++ b/python/mozbuild/mozbuild/test/action/data/package_fennec_apk/input2/assets/asset.txt @@ -0,0 +1 @@ +input2/assets/asset.txt diff --git a/python/mozbuild/mozbuild/test/action/data/package_fennec_apk/input2/assets/omni.ja b/python/mozbuild/mozbuild/test/action/data/package_fennec_apk/input2/assets/omni.ja new file mode 100644 index 000000000..36deb6725 --- /dev/null +++ b/python/mozbuild/mozbuild/test/action/data/package_fennec_apk/input2/assets/omni.ja @@ -0,0 +1 @@ +input2/assets/omni.ja \ No newline at end of file diff --git a/python/mozbuild/mozbuild/test/action/data/package_fennec_apk/input2/classes.dex b/python/mozbuild/mozbuild/test/action/data/package_fennec_apk/input2/classes.dex new file mode 100644 index 000000000..99779eb45 --- /dev/null +++ b/python/mozbuild/mozbuild/test/action/data/package_fennec_apk/input2/classes.dex @@ -0,0 +1 @@ +input2/classes.dex \ No newline at end of file diff --git a/python/mozbuild/mozbuild/test/action/data/package_fennec_apk/input2/lib/lib.txt b/python/mozbuild/mozbuild/test/action/data/package_fennec_apk/input2/lib/lib.txt new file mode 100644 index 000000000..7a2594a02 --- /dev/null +++ b/python/mozbuild/mozbuild/test/action/data/package_fennec_apk/input2/lib/lib.txt @@ -0,0 +1 @@ +input2/lib/lib.txt diff --git a/python/mozbuild/mozbuild/test/action/data/package_fennec_apk/input2/res/res.txt b/python/mozbuild/mozbuild/test/action/data/package_fennec_apk/input2/res/res.txt new file mode 100644 index 000000000..2a52ab524 --- /dev/null +++ b/python/mozbuild/mozbuild/test/action/data/package_fennec_apk/input2/res/res.txt @@ -0,0 +1 @@ +input2/res/res.txt diff --git a/python/mozbuild/mozbuild/test/action/data/package_fennec_apk/input2/resources.arsc b/python/mozbuild/mozbuild/test/action/data/package_fennec_apk/input2/resources.arsc new file mode 100644 index 000000000..64f4b77ad --- /dev/null +++ b/python/mozbuild/mozbuild/test/action/data/package_fennec_apk/input2/resources.arsc @@ -0,0 +1 @@ +input/resources.arsc \ No newline at end of file diff --git a/python/mozbuild/mozbuild/test/action/data/package_fennec_apk/input2/root_file.txt b/python/mozbuild/mozbuild/test/action/data/package_fennec_apk/input2/root_file.txt new file mode 100644 index 000000000..9f2f53518 --- /dev/null +++ b/python/mozbuild/mozbuild/test/action/data/package_fennec_apk/input2/root_file.txt @@ -0,0 +1 @@ +input2/root_file.txt diff --git a/python/mozbuild/mozbuild/test/action/data/package_fennec_apk/lib/lib.txt b/python/mozbuild/mozbuild/test/action/data/package_fennec_apk/lib/lib.txt new file mode 100644 index 000000000..acbcebb3d --- /dev/null +++ b/python/mozbuild/mozbuild/test/action/data/package_fennec_apk/lib/lib.txt @@ -0,0 +1 @@ +lib/lib.txt diff --git a/python/mozbuild/mozbuild/test/action/data/package_fennec_apk/omni.ja b/python/mozbuild/mozbuild/test/action/data/package_fennec_apk/omni.ja new file mode 100644 index 000000000..48c422a3a --- /dev/null +++ b/python/mozbuild/mozbuild/test/action/data/package_fennec_apk/omni.ja @@ -0,0 +1 @@ +omni.ja \ No newline at end of file diff --git a/python/mozbuild/mozbuild/test/action/data/package_fennec_apk/root_file.txt b/python/mozbuild/mozbuild/test/action/data/package_fennec_apk/root_file.txt new file mode 100644 index 000000000..89b006da4 --- /dev/null +++ b/python/mozbuild/mozbuild/test/action/data/package_fennec_apk/root_file.txt @@ -0,0 +1 @@ +root_file.txt diff --git a/python/mozbuild/mozbuild/test/action/data/valid-zh-CN/region.properties b/python/mozbuild/mozbuild/test/action/data/valid-zh-CN/region.properties new file mode 100644 index 000000000..d4d7fcfee --- /dev/null +++ b/python/mozbuild/mozbuild/test/action/data/valid-zh-CN/region.properties @@ -0,0 +1,37 @@ +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, You can obtain one at http://mozilla.org/MPL/2.0/. + +# Default search engine +browser.search.defaultenginename=百度 + +# Search engine order (order displayed in the search bar dropdown)s +browser.search.order.1=百度 +browser.search.order.2=Google + +# This is the default set of web based feed handlers shown in the reader +# selection UI +browser.contentHandlers.types.0.title=Bloglines +browser.contentHandlers.types.0.uri=http://www.bloglines.com/login?r=/sub/%s + +# increment this number when anything gets changed in the list below. This will +# cause Firefox to re-read these prefs and inject any new handlers into the +# profile database. Note that "new" is defined as "has a different URL"; this +# means that it's not possible to update the name of existing handler, so +# don't make any spelling errors here. +gecko.handlerService.defaultHandlersVersion=3 + +# The default set of protocol handlers for webcal: +gecko.handlerService.schemes.webcal.0.name=30 Boxes +gecko.handlerService.schemes.webcal.0.uriTemplate=https://30boxes.com/external/widget?refer=ff&url=%s + +# The default set of protocol handlers for mailto: +gecko.handlerService.schemes.mailto.0.name=Yahoo! 邮件 +gecko.handlerService.schemes.mailto.0.uriTemplate=https://compose.mail.yahoo.com/?To=%s +gecko.handlerService.schemes.mailto.1.name=Gmail +gecko.handlerService.schemes.mailto.1.uriTemplate=https://mail.google.com/mail/?extsrc=mailto&url=%s + +# This is the default set of web based feed handlers shown in the reader +# selection UI +browser.contentHandlers.types.0.title=My Yahoo! +browser.contentHandlers.types.0.uri=http://www.bloglines.com/login?r=/sub/%s diff --git a/python/mozbuild/mozbuild/test/action/test_buildlist.py b/python/mozbuild/mozbuild/test/action/test_buildlist.py new file mode 100644 index 000000000..9c2631812 --- /dev/null +++ b/python/mozbuild/mozbuild/test/action/test_buildlist.py @@ -0,0 +1,89 @@ +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, You can obtain one at http://mozilla.org/MPL/2.0/. + +import unittest + +import os, sys, os.path, time +from tempfile import mkdtemp +from shutil import rmtree +import mozunit + +from mozbuild.action.buildlist import addEntriesToListFile + + +class TestBuildList(unittest.TestCase): + """ + Unit tests for buildlist.py + """ + def setUp(self): + self.tmpdir = mkdtemp() + + def tearDown(self): + rmtree(self.tmpdir) + + # utility methods for tests + def touch(self, file, dir=None): + if dir is None: + dir = self.tmpdir + f = os.path.join(dir, file) + open(f, 'w').close() + return f + + def assertFileContains(self, filename, l): + """Assert that the lines in the file |filename| are equal + to the contents of the list |l|, in order.""" + l = l[:] + f = open(filename, 'r') + lines = [line.rstrip() for line in f.readlines()] + f.close() + for line in lines: + self.assert_(len(l) > 0, + "ran out of expected lines! (expected '{0}', got '{1}')" + .format(l, lines)) + self.assertEqual(line, l.pop(0)) + self.assert_(len(l) == 0, + "not enough lines in file! (expected '{0}'," + " got '{1}'".format(l, lines)) + + def test_basic(self): + "Test that addEntriesToListFile works when file doesn't exist." + testfile = os.path.join(self.tmpdir, "test.list") + l = ["a", "b", "c"] + addEntriesToListFile(testfile, l) + self.assertFileContains(testfile, l) + # ensure that attempting to add the same entries again doesn't change it + addEntriesToListFile(testfile, l) + self.assertFileContains(testfile, l) + + def test_append(self): + "Test adding new entries." + testfile = os.path.join(self.tmpdir, "test.list") + l = ["a", "b", "c"] + addEntriesToListFile(testfile, l) + self.assertFileContains(testfile, l) + l2 = ["x","y","z"] + addEntriesToListFile(testfile, l2) + l.extend(l2) + self.assertFileContains(testfile, l) + + def test_append_some(self): + "Test adding new entries mixed with existing entries." + testfile = os.path.join(self.tmpdir, "test.list") + l = ["a", "b", "c"] + addEntriesToListFile(testfile, l) + self.assertFileContains(testfile, l) + addEntriesToListFile(testfile, ["a", "x", "c", "z"]) + self.assertFileContains(testfile, ["a", "b", "c", "x", "z"]) + + def test_add_multiple(self): + """Test that attempting to add the same entry multiple times results in + only one entry being added.""" + testfile = os.path.join(self.tmpdir, "test.list") + addEntriesToListFile(testfile, ["a","b","a","a","b"]) + self.assertFileContains(testfile, ["a","b"]) + addEntriesToListFile(testfile, ["c","a","c","b","c"]) + self.assertFileContains(testfile, ["a","b","c"]) + +if __name__ == '__main__': + mozunit.main() diff --git a/python/mozbuild/mozbuild/test/action/test_generate_browsersearch.py b/python/mozbuild/mozbuild/test/action/test_generate_browsersearch.py new file mode 100644 index 000000000..4c7f5635e --- /dev/null +++ b/python/mozbuild/mozbuild/test/action/test_generate_browsersearch.py @@ -0,0 +1,55 @@ +# -*- coding: utf-8 -*- + +# Any copyright is dedicated to the Public Domain. +# http://creativecommons.org/publicdomain/zero/1.0/ + +from __future__ import unicode_literals + +import json +import os +import unittest + +import mozunit + +import mozbuild.action.generate_browsersearch as generate_browsersearch + +from mozfile.mozfile import ( + NamedTemporaryFile, + TemporaryDirectory, +) + +import mozpack.path as mozpath + + +test_data_path = mozpath.abspath(mozpath.dirname(__file__)) +test_data_path = mozpath.join(test_data_path, 'data') + + +class TestGenerateBrowserSearch(unittest.TestCase): + """ + Unit tests for generate_browsersearch.py. + """ + + def _test_one(self, name): + with TemporaryDirectory() as tmpdir: + with NamedTemporaryFile(mode='r+') as temp: + srcdir = os.path.join(test_data_path, name) + + generate_browsersearch.main([ + '--silent', + '--srcdir', srcdir, + temp.name]) + return json.load(temp) + + def test_valid_unicode(self): + o = self._test_one('valid-zh-CN') + self.assertEquals(o['default'], '百度') + self.assertEquals(o['engines'], ['百度', 'Google']) + + def test_invalid_unicode(self): + with self.assertRaises(UnicodeDecodeError): + self._test_one('invalid') + + +if __name__ == '__main__': + mozunit.main() diff --git a/python/mozbuild/mozbuild/test/action/test_package_fennec_apk.py b/python/mozbuild/mozbuild/test/action/test_package_fennec_apk.py new file mode 100644 index 000000000..5b7760836 --- /dev/null +++ b/python/mozbuild/mozbuild/test/action/test_package_fennec_apk.py @@ -0,0 +1,70 @@ +# -*- coding: utf-8 -*- + +# Any copyright is dedicated to the Public Domain. +# http://creativecommons.org/publicdomain/zero/1.0/ + +from __future__ import unicode_literals + +import os +import unittest + +import mozunit + +from mozbuild.action.package_fennec_apk import ( + package_fennec_apk as package, +) +from mozpack.mozjar import JarReader +import mozpack.path as mozpath + + +test_data_path = mozpath.abspath(mozpath.dirname(__file__)) +test_data_path = mozpath.join(test_data_path, 'data', 'package_fennec_apk') + + +def data(name): + return os.path.join(test_data_path, name) + + +class TestPackageFennecAPK(unittest.TestCase): + """ + Unit tests for package_fennec_apk.py. + """ + + def test_arguments(self): + # Language repacks take updated resources from an ap_ and pack them + # into an apk. Make sure the second input overrides the first. + jarrer = package(inputs=[], + omni_ja=data('omni.ja'), + classes_dex=data('classes.dex'), + assets_dirs=[data('assets')], + lib_dirs=[data('lib')], + root_files=[data('root_file.txt')]) + + # omni.ja ends up in assets/omni.ja. + self.assertEquals(jarrer['assets/omni.ja'].open().read().strip(), 'omni.ja') + + # Everything else is in place. + for name in ('classes.dex', + 'assets/asset.txt', + 'lib/lib.txt', + 'root_file.txt'): + self.assertEquals(jarrer[name].open().read().strip(), name) + + def test_inputs(self): + # Language repacks take updated resources from an ap_ and pack them + # into an apk. In this case, the first input is the original package, + # the second input the update ap_. Make sure the second input + # overrides the first. + jarrer = package(inputs=[data('input2.apk'), data('input1.ap_')]) + + files1 = JarReader(data('input1.ap_')).entries.keys() + files2 = JarReader(data('input2.apk')).entries.keys() + for name in files2: + self.assertTrue(name in files1 or + jarrer[name].open().read().startswith('input2/')) + for name in files1: + self.assertTrue(jarrer[name].open().read().startswith('input1/')) + + +if __name__ == '__main__': + mozunit.main() diff --git a/python/mozbuild/mozbuild/test/backend/__init__.py b/python/mozbuild/mozbuild/test/backend/__init__.py new file mode 100644 index 000000000..e69de29bb diff --git a/python/mozbuild/mozbuild/test/backend/common.py b/python/mozbuild/mozbuild/test/backend/common.py new file mode 100644 index 000000000..85ccb1037 --- /dev/null +++ b/python/mozbuild/mozbuild/test/backend/common.py @@ -0,0 +1,156 @@ +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, You can obtain one at http://mozilla.org/MPL/2.0/. + +from __future__ import unicode_literals + +import os +import unittest + +from collections import defaultdict +from shutil import rmtree +from tempfile import mkdtemp + +from mach.logging import LoggingManager + +from mozbuild.backend.configenvironment import ConfigEnvironment +from mozbuild.frontend.emitter import TreeMetadataEmitter +from mozbuild.frontend.reader import BuildReader + +import mozpack.path as mozpath + + +log_manager = LoggingManager() +log_manager.add_terminal_logging() + + +test_data_path = mozpath.abspath(mozpath.dirname(__file__)) +test_data_path = mozpath.join(test_data_path, 'data') + + +CONFIGS = defaultdict(lambda: { + 'defines': {}, + 'non_global_defines': [], + 'substs': {'OS_TARGET': 'WINNT'}, +}, { + 'android_eclipse': { + 'defines': { + 'MOZ_ANDROID_MIN_SDK_VERSION': '15', + }, + 'non_global_defines': [], + 'substs': { + 'ANDROID_TARGET_SDK': '16', + 'MOZ_WIDGET_TOOLKIT': 'android', + }, + }, + 'binary-components': { + 'defines': {}, + 'non_global_defines': [], + 'substs': { + 'LIB_PREFIX': 'lib', + 'LIB_SUFFIX': 'a', + 'COMPILE_ENVIRONMENT': '1', + }, + }, + 'sources': { + 'defines': {}, + 'non_global_defines': [], + 'substs': { + 'LIB_PREFIX': 'lib', + 'LIB_SUFFIX': 'a', + }, + }, + 'stub0': { + 'defines': { + 'MOZ_TRUE_1': '1', + 'MOZ_TRUE_2': '1', + }, + 'non_global_defines': [ + 'MOZ_NONGLOBAL_1', + 'MOZ_NONGLOBAL_2', + ], + 'substs': { + 'MOZ_FOO': 'foo', + 'MOZ_BAR': 'bar', + }, + }, + 'substitute_config_files': { + 'defines': {}, + 'non_global_defines': [], + 'substs': { + 'MOZ_FOO': 'foo', + 'MOZ_BAR': 'bar', + }, + }, + 'test_config': { + 'defines': { + 'foo': 'baz qux', + 'baz': 1, + }, + 'non_global_defines': [], + 'substs': { + 'foo': 'bar baz', + }, + }, + 'visual-studio': { + 'defines': {}, + 'non_global_defines': [], + 'substs': { + 'MOZ_APP_NAME': 'my_app', + }, + }, +}) + + +class BackendTester(unittest.TestCase): + def setUp(self): + self._old_env = dict(os.environ) + os.environ.pop('MOZ_OBJDIR', None) + + def tearDown(self): + os.environ.clear() + os.environ.update(self._old_env) + + def _get_environment(self, name): + """Obtain a new instance of a ConfigEnvironment for a known profile. + + A new temporary object directory is created for the environment. The + environment is cleaned up automatically when the test finishes. + """ + config = CONFIGS[name] + + objdir = mkdtemp() + self.addCleanup(rmtree, objdir) + + srcdir = mozpath.join(test_data_path, name) + config['substs']['top_srcdir'] = srcdir + return ConfigEnvironment(srcdir, objdir, **config) + + def _emit(self, name, env=None): + env = env or self._get_environment(name) + reader = BuildReader(env) + emitter = TreeMetadataEmitter(env) + + return env, emitter.emit(reader.read_topsrcdir()) + + def _consume(self, name, cls, env=None): + env, objs = self._emit(name, env=env) + backend = cls(env) + backend.consume(objs) + + return env + + def _tree_paths(self, topdir, filename): + for dirpath, dirnames, filenames in os.walk(topdir): + for f in filenames: + if f == filename: + yield mozpath.relpath(mozpath.join(dirpath, f), topdir) + + def _mozbuild_paths(self, env): + return self._tree_paths(env.topsrcdir, 'moz.build') + + def _makefile_in_paths(self, env): + return self._tree_paths(env.topsrcdir, 'Makefile.in') + + +__all__ = ['BackendTester'] diff --git a/python/mozbuild/mozbuild/test/backend/data/android_eclipse/library1/resources/values/strings.xml b/python/mozbuild/mozbuild/test/backend/data/android_eclipse/library1/resources/values/strings.xml new file mode 100644 index 000000000..a7337c554 --- /dev/null +++ b/python/mozbuild/mozbuild/test/backend/data/android_eclipse/library1/resources/values/strings.xml @@ -0,0 +1 @@ +library1 diff --git a/python/mozbuild/mozbuild/test/backend/data/android_eclipse/main1/AndroidManifest.xml b/python/mozbuild/mozbuild/test/backend/data/android_eclipse/main1/AndroidManifest.xml new file mode 100644 index 000000000..7a906454d --- /dev/null +++ b/python/mozbuild/mozbuild/test/backend/data/android_eclipse/main1/AndroidManifest.xml @@ -0,0 +1 @@ + diff --git a/python/mozbuild/mozbuild/test/backend/data/android_eclipse/main2/AndroidManifest.xml b/python/mozbuild/mozbuild/test/backend/data/android_eclipse/main2/AndroidManifest.xml new file mode 100644 index 000000000..7a906454d --- /dev/null +++ b/python/mozbuild/mozbuild/test/backend/data/android_eclipse/main2/AndroidManifest.xml @@ -0,0 +1 @@ + diff --git a/python/mozbuild/mozbuild/test/backend/data/android_eclipse/main2/assets/dummy.txt b/python/mozbuild/mozbuild/test/backend/data/android_eclipse/main2/assets/dummy.txt new file mode 100644 index 000000000..c32a95993 --- /dev/null +++ b/python/mozbuild/mozbuild/test/backend/data/android_eclipse/main2/assets/dummy.txt @@ -0,0 +1 @@ +# Placeholder. \ No newline at end of file diff --git a/python/mozbuild/mozbuild/test/backend/data/android_eclipse/main2/extra.jar b/python/mozbuild/mozbuild/test/backend/data/android_eclipse/main2/extra.jar new file mode 100644 index 000000000..c32a95993 --- /dev/null +++ b/python/mozbuild/mozbuild/test/backend/data/android_eclipse/main2/extra.jar @@ -0,0 +1 @@ +# Placeholder. \ No newline at end of file diff --git a/python/mozbuild/mozbuild/test/backend/data/android_eclipse/main2/res/values/strings.xml b/python/mozbuild/mozbuild/test/backend/data/android_eclipse/main2/res/values/strings.xml new file mode 100644 index 000000000..0b28bf41e --- /dev/null +++ b/python/mozbuild/mozbuild/test/backend/data/android_eclipse/main2/res/values/strings.xml @@ -0,0 +1 @@ +main1 diff --git a/python/mozbuild/mozbuild/test/backend/data/android_eclipse/main3/AndroidManifest.xml b/python/mozbuild/mozbuild/test/backend/data/android_eclipse/main3/AndroidManifest.xml new file mode 100644 index 000000000..7a906454d --- /dev/null +++ b/python/mozbuild/mozbuild/test/backend/data/android_eclipse/main3/AndroidManifest.xml @@ -0,0 +1 @@ + diff --git a/python/mozbuild/mozbuild/test/backend/data/android_eclipse/main3/a/A.java b/python/mozbuild/mozbuild/test/backend/data/android_eclipse/main3/a/A.java new file mode 100644 index 000000000..0ab867d3d --- /dev/null +++ b/python/mozbuild/mozbuild/test/backend/data/android_eclipse/main3/a/A.java @@ -0,0 +1 @@ +package a.a; diff --git a/python/mozbuild/mozbuild/test/backend/data/android_eclipse/main3/b/B.java b/python/mozbuild/mozbuild/test/backend/data/android_eclipse/main3/b/B.java new file mode 100644 index 000000000..66eb44c15 --- /dev/null +++ b/python/mozbuild/mozbuild/test/backend/data/android_eclipse/main3/b/B.java @@ -0,0 +1 @@ +package b; diff --git a/python/mozbuild/mozbuild/test/backend/data/android_eclipse/main3/c/C.java b/python/mozbuild/mozbuild/test/backend/data/android_eclipse/main3/c/C.java new file mode 100644 index 000000000..ca474ff33 --- /dev/null +++ b/python/mozbuild/mozbuild/test/backend/data/android_eclipse/main3/c/C.java @@ -0,0 +1 @@ +package d.e; diff --git a/python/mozbuild/mozbuild/test/backend/data/android_eclipse/main4 b/python/mozbuild/mozbuild/test/backend/data/android_eclipse/main4 new file mode 100644 index 000000000..7a906454d --- /dev/null +++ b/python/mozbuild/mozbuild/test/backend/data/android_eclipse/main4 @@ -0,0 +1 @@ + diff --git a/python/mozbuild/mozbuild/test/backend/data/android_eclipse/moz.build b/python/mozbuild/mozbuild/test/backend/data/android_eclipse/moz.build new file mode 100644 index 000000000..327284c88 --- /dev/null +++ b/python/mozbuild/mozbuild/test/backend/data/android_eclipse/moz.build @@ -0,0 +1,37 @@ +# -*- Mode: python; indent-tabs-mode: nil; tab-width: 40 -*- +# Any copyright is dedicated to the Public Domain. +# http://creativecommons.org/publicdomain/zero/1.0/ + + +p = add_android_eclipse_library_project('library1') +p.package_name = 'org.mozilla.test.library1' +p.res = 'library1/resources' + +p = add_android_eclipse_library_project('library2') +p.package_name = 'org.mozilla.test.library2' + +p = add_android_eclipse_project('main1', 'main1/AndroidManifest.xml') +p.package_name = 'org.mozilla.test.main1' +p.recursive_make_targets += ['target1', 'target2'] + +p = add_android_eclipse_project('main2', 'main2/AndroidManifest.xml') +p.package_name = 'org.mozilla.test.main2' +p.res = 'main2/res' +p.assets = 'main2/assets' +p.extra_jars = ['main2/extra.jar'] + +p = add_android_eclipse_project('main3', 'main3/AndroidManifest.xml') +p.package_name = 'org.mozilla.test.main3' +cpe = p.add_classpathentry('a', 'main3/a', dstdir='a/a') +cpe = p.add_classpathentry('b', 'main3/b', dstdir='b') +cpe.exclude_patterns += ['b/Excludes.java', 'b/Excludes2.java'] +cpe = p.add_classpathentry('c', 'main3/c', dstdir='d/e') +cpe.ignore_warnings = True + +p = add_android_eclipse_project('main4', 'main3/AndroidManifest.xml') +p.package_name = 'org.mozilla.test.main3' +p.referenced_projects += ['library1'] +p.included_projects += ['library2'] +p.recursive_make_targets += ['target3', 'target4'] + +DIRS += ['subdir'] diff --git a/python/mozbuild/mozbuild/test/backend/data/android_eclipse/subdir/moz.build b/python/mozbuild/mozbuild/test/backend/data/android_eclipse/subdir/moz.build new file mode 100644 index 000000000..c75aec456 --- /dev/null +++ b/python/mozbuild/mozbuild/test/backend/data/android_eclipse/subdir/moz.build @@ -0,0 +1,13 @@ +# -*- Mode: python; indent-tabs-mode: nil; tab-width: 40 -*- +# Any copyright is dedicated to the Public Domain. +# http://creativecommons.org/publicdomain/zero/1.0/ + +DEFINES['FOO'] = 'FOO' + +p = add_android_eclipse_library_project('sublibrary') +p.package_name = 'org.mozilla.test.sublibrary' +p.is_library = True + +p = add_android_eclipse_project('submain', 'submain/AndroidManifest.xml') +p.package_name = 'org.mozilla.test.submain' +p.recursive_make_targets += ['subtarget1', 'subtarget2'] diff --git a/python/mozbuild/mozbuild/test/backend/data/android_eclipse/subdir/submain/AndroidManifest.xml b/python/mozbuild/mozbuild/test/backend/data/android_eclipse/subdir/submain/AndroidManifest.xml new file mode 100644 index 000000000..7a906454d --- /dev/null +++ b/python/mozbuild/mozbuild/test/backend/data/android_eclipse/subdir/submain/AndroidManifest.xml @@ -0,0 +1 @@ + diff --git a/python/mozbuild/mozbuild/test/backend/data/binary-components/bar/moz.build b/python/mozbuild/mozbuild/test/backend/data/binary-components/bar/moz.build new file mode 100644 index 000000000..2946e42aa --- /dev/null +++ b/python/mozbuild/mozbuild/test/backend/data/binary-components/bar/moz.build @@ -0,0 +1,2 @@ +Component('bar') +NO_COMPONENTS_MANIFEST = True diff --git a/python/mozbuild/mozbuild/test/backend/data/binary-components/foo/moz.build b/python/mozbuild/mozbuild/test/backend/data/binary-components/foo/moz.build new file mode 100644 index 000000000..8611a74be --- /dev/null +++ b/python/mozbuild/mozbuild/test/backend/data/binary-components/foo/moz.build @@ -0,0 +1 @@ +Component('foo') diff --git a/python/mozbuild/mozbuild/test/backend/data/binary-components/moz.build b/python/mozbuild/mozbuild/test/backend/data/binary-components/moz.build new file mode 100644 index 000000000..1776d0514 --- /dev/null +++ b/python/mozbuild/mozbuild/test/backend/data/binary-components/moz.build @@ -0,0 +1,10 @@ +@template +def Component(name): + LIBRARY_NAME = name + FORCE_SHARED_LIB = True + IS_COMPONENT = True + +DIRS += [ + 'foo', + 'bar', +] diff --git a/python/mozbuild/mozbuild/test/backend/data/branding-files/bar.ico b/python/mozbuild/mozbuild/test/backend/data/branding-files/bar.ico new file mode 100644 index 000000000..e69de29bb diff --git a/python/mozbuild/mozbuild/test/backend/data/branding-files/foo.ico b/python/mozbuild/mozbuild/test/backend/data/branding-files/foo.ico new file mode 100644 index 000000000..e69de29bb diff --git a/python/mozbuild/mozbuild/test/backend/data/branding-files/moz.build b/python/mozbuild/mozbuild/test/backend/data/branding-files/moz.build new file mode 100644 index 000000000..083f0f82d --- /dev/null +++ b/python/mozbuild/mozbuild/test/backend/data/branding-files/moz.build @@ -0,0 +1,12 @@ +# Any copyright is dedicated to the Public Domain. +# http://creativecommons.org/publicdomain/zero/1.0/ + +BRANDING_FILES += [ + 'bar.ico', + 'sub/quux.png', +] + +BRANDING_FILES.icons += [ + 'foo.ico', +] + diff --git a/python/mozbuild/mozbuild/test/backend/data/branding-files/sub/quux.png b/python/mozbuild/mozbuild/test/backend/data/branding-files/sub/quux.png new file mode 100644 index 000000000..e69de29bb diff --git a/python/mozbuild/mozbuild/test/backend/data/build/app/moz.build b/python/mozbuild/mozbuild/test/backend/data/build/app/moz.build new file mode 100644 index 000000000..8d6218ea9 --- /dev/null +++ b/python/mozbuild/mozbuild/test/backend/data/build/app/moz.build @@ -0,0 +1,54 @@ +DIST_SUBDIR = 'app' + +EXTRA_JS_MODULES += [ + '../foo.jsm', +] + +EXTRA_JS_MODULES.child += [ + '../bar.jsm', +] + +EXTRA_PP_JS_MODULES += [ + '../baz.jsm', +] + +EXTRA_PP_JS_MODULES.child2 += [ + '../qux.jsm', +] + +FINAL_TARGET_FILES += [ + '../foo.ini', +] + +FINAL_TARGET_FILES.child += [ + '../bar.ini', +] + +FINAL_TARGET_PP_FILES += [ + '../baz.ini', + '../foo.css', +] + +FINAL_TARGET_PP_FILES.child2 += [ + '../qux.ini', +] + +EXTRA_COMPONENTS += [ + '../components.manifest', + '../foo.js', +] + +EXTRA_PP_COMPONENTS += [ + '../bar.js', +] + +JS_PREFERENCE_FILES += [ + '../prefs.js', +] + +JAR_MANIFESTS += [ + '../jar.mn', +] + +DEFINES['FOO'] = 'bar' +DEFINES['BAR'] = True diff --git a/python/mozbuild/mozbuild/test/backend/data/build/bar.ini b/python/mozbuild/mozbuild/test/backend/data/build/bar.ini new file mode 100644 index 000000000..91dcbe153 --- /dev/null +++ b/python/mozbuild/mozbuild/test/backend/data/build/bar.ini @@ -0,0 +1 @@ +bar.ini diff --git a/python/mozbuild/mozbuild/test/backend/data/build/bar.js b/python/mozbuild/mozbuild/test/backend/data/build/bar.js new file mode 100644 index 000000000..1a608e8a5 --- /dev/null +++ b/python/mozbuild/mozbuild/test/backend/data/build/bar.js @@ -0,0 +1,2 @@ +#filter substitution +bar.js: FOO is @FOO@ diff --git a/python/mozbuild/mozbuild/test/backend/data/build/bar.jsm b/python/mozbuild/mozbuild/test/backend/data/build/bar.jsm new file mode 100644 index 000000000..05db2e2f6 --- /dev/null +++ b/python/mozbuild/mozbuild/test/backend/data/build/bar.jsm @@ -0,0 +1 @@ +bar.jsm diff --git a/python/mozbuild/mozbuild/test/backend/data/build/baz.ini b/python/mozbuild/mozbuild/test/backend/data/build/baz.ini new file mode 100644 index 000000000..975a1e437 --- /dev/null +++ b/python/mozbuild/mozbuild/test/backend/data/build/baz.ini @@ -0,0 +1,2 @@ +#filter substitution +baz.ini: FOO is @FOO@ diff --git a/python/mozbuild/mozbuild/test/backend/data/build/baz.jsm b/python/mozbuild/mozbuild/test/backend/data/build/baz.jsm new file mode 100644 index 000000000..f39ed0208 --- /dev/null +++ b/python/mozbuild/mozbuild/test/backend/data/build/baz.jsm @@ -0,0 +1,2 @@ +#filter substitution +baz.jsm: FOO is @FOO@ diff --git a/python/mozbuild/mozbuild/test/backend/data/build/components.manifest b/python/mozbuild/mozbuild/test/backend/data/build/components.manifest new file mode 100644 index 000000000..b5bb87254 --- /dev/null +++ b/python/mozbuild/mozbuild/test/backend/data/build/components.manifest @@ -0,0 +1,2 @@ +component {foo} foo.js +component {bar} bar.js diff --git a/python/mozbuild/mozbuild/test/backend/data/build/foo.css b/python/mozbuild/mozbuild/test/backend/data/build/foo.css new file mode 100644 index 000000000..1803d6c57 --- /dev/null +++ b/python/mozbuild/mozbuild/test/backend/data/build/foo.css @@ -0,0 +1,2 @@ +%filter substitution +foo.css: FOO is @FOO@ diff --git a/python/mozbuild/mozbuild/test/backend/data/build/foo.ini b/python/mozbuild/mozbuild/test/backend/data/build/foo.ini new file mode 100644 index 000000000..c93c9d765 --- /dev/null +++ b/python/mozbuild/mozbuild/test/backend/data/build/foo.ini @@ -0,0 +1 @@ +foo.ini diff --git a/python/mozbuild/mozbuild/test/backend/data/build/foo.js b/python/mozbuild/mozbuild/test/backend/data/build/foo.js new file mode 100644 index 000000000..4fa71e2d2 --- /dev/null +++ b/python/mozbuild/mozbuild/test/backend/data/build/foo.js @@ -0,0 +1 @@ +foo.js diff --git a/python/mozbuild/mozbuild/test/backend/data/build/foo.jsm b/python/mozbuild/mozbuild/test/backend/data/build/foo.jsm new file mode 100644 index 000000000..d58fd61c1 --- /dev/null +++ b/python/mozbuild/mozbuild/test/backend/data/build/foo.jsm @@ -0,0 +1 @@ +foo.jsm diff --git a/python/mozbuild/mozbuild/test/backend/data/build/jar.mn b/python/mozbuild/mozbuild/test/backend/data/build/jar.mn new file mode 100644 index 000000000..393055c4e --- /dev/null +++ b/python/mozbuild/mozbuild/test/backend/data/build/jar.mn @@ -0,0 +1,11 @@ +foo.jar: +% content bar %child/ +% content foo % + foo.js +* foo.css + bar.js (subdir/bar.js) + qux.js (subdir/bar.js) +* child/hoge.js (bar.js) +* child/baz.jsm + +% override chrome://foo/bar.svg#hello chrome://bar/bar.svg#hello diff --git a/python/mozbuild/mozbuild/test/backend/data/build/moz.build b/python/mozbuild/mozbuild/test/backend/data/build/moz.build new file mode 100644 index 000000000..b0b0cabd1 --- /dev/null +++ b/python/mozbuild/mozbuild/test/backend/data/build/moz.build @@ -0,0 +1,68 @@ +CONFIGURE_SUBST_FILES += [ + '/config/autoconf.mk', + '/config/emptyvars.mk', +] + +EXTRA_JS_MODULES += [ + 'foo.jsm', +] + +EXTRA_JS_MODULES.child += [ + 'bar.jsm', +] + +EXTRA_PP_JS_MODULES += [ + 'baz.jsm', +] + +EXTRA_PP_JS_MODULES.child2 += [ + 'qux.jsm', +] + +FINAL_TARGET_FILES += [ + 'foo.ini', +] + +FINAL_TARGET_FILES.child += [ + 'bar.ini', +] + +FINAL_TARGET_PP_FILES += [ + 'baz.ini', +] + +FINAL_TARGET_PP_FILES.child2 += [ + 'foo.css', + 'qux.ini', +] + +EXTRA_COMPONENTS += [ + 'components.manifest', + 'foo.js', +] + +EXTRA_PP_COMPONENTS += [ + 'bar.js', +] + +JS_PREFERENCE_FILES += [ + 'prefs.js', +] + +RESOURCE_FILES += [ + 'resource', +] + +RESOURCE_FILES.child += [ + 'resource2', +] + +DEFINES['FOO'] = 'foo' + +JAR_MANIFESTS += [ + 'jar.mn', +] + +DIRS += [ + 'app', +] diff --git a/python/mozbuild/mozbuild/test/backend/data/build/prefs.js b/python/mozbuild/mozbuild/test/backend/data/build/prefs.js new file mode 100644 index 000000000..a030da9fd --- /dev/null +++ b/python/mozbuild/mozbuild/test/backend/data/build/prefs.js @@ -0,0 +1 @@ +prefs.js diff --git a/python/mozbuild/mozbuild/test/backend/data/build/qux.ini b/python/mozbuild/mozbuild/test/backend/data/build/qux.ini new file mode 100644 index 000000000..3ce157eb6 --- /dev/null +++ b/python/mozbuild/mozbuild/test/backend/data/build/qux.ini @@ -0,0 +1,5 @@ +#ifdef BAR +qux.ini: BAR is defined +#else +qux.ini: BAR is not defined +#endif diff --git a/python/mozbuild/mozbuild/test/backend/data/build/qux.jsm b/python/mozbuild/mozbuild/test/backend/data/build/qux.jsm new file mode 100644 index 000000000..9c5fe28d5 --- /dev/null +++ b/python/mozbuild/mozbuild/test/backend/data/build/qux.jsm @@ -0,0 +1,5 @@ +#ifdef BAR +qux.jsm: BAR is defined +#else +qux.jsm: BAR is not defined +#endif diff --git a/python/mozbuild/mozbuild/test/backend/data/build/resource b/python/mozbuild/mozbuild/test/backend/data/build/resource new file mode 100644 index 000000000..91e75c679 --- /dev/null +++ b/python/mozbuild/mozbuild/test/backend/data/build/resource @@ -0,0 +1 @@ +resource diff --git a/python/mozbuild/mozbuild/test/backend/data/build/resource2 b/python/mozbuild/mozbuild/test/backend/data/build/resource2 new file mode 100644 index 000000000..b7c270096 --- /dev/null +++ b/python/mozbuild/mozbuild/test/backend/data/build/resource2 @@ -0,0 +1 @@ +resource2 diff --git a/python/mozbuild/mozbuild/test/backend/data/build/subdir/bar.js b/python/mozbuild/mozbuild/test/backend/data/build/subdir/bar.js new file mode 100644 index 000000000..80c887a84 --- /dev/null +++ b/python/mozbuild/mozbuild/test/backend/data/build/subdir/bar.js @@ -0,0 +1 @@ +bar.js diff --git a/python/mozbuild/mozbuild/test/backend/data/defines/moz.build b/python/mozbuild/mozbuild/test/backend/data/defines/moz.build new file mode 100644 index 000000000..be4b31143 --- /dev/null +++ b/python/mozbuild/mozbuild/test/backend/data/defines/moz.build @@ -0,0 +1,14 @@ +# Any copyright is dedicated to the Public Domain. +# http://creativecommons.org/publicdomain/zero/1.0/ + +value = 'xyz' +DEFINES = { + 'FOO': True, +} + +DEFINES['BAZ'] = '"ab\'cd"' +DEFINES.update({ + 'BAR': 7, + 'VALUE': value, + 'QUX': False, +}) diff --git a/python/mozbuild/mozbuild/test/backend/data/dist-files/install.rdf b/python/mozbuild/mozbuild/test/backend/data/dist-files/install.rdf new file mode 100644 index 000000000..e69de29bb diff --git a/python/mozbuild/mozbuild/test/backend/data/dist-files/main.js b/python/mozbuild/mozbuild/test/backend/data/dist-files/main.js new file mode 100644 index 000000000..e69de29bb diff --git a/python/mozbuild/mozbuild/test/backend/data/dist-files/moz.build b/python/mozbuild/mozbuild/test/backend/data/dist-files/moz.build new file mode 100644 index 000000000..cbd2c942b --- /dev/null +++ b/python/mozbuild/mozbuild/test/backend/data/dist-files/moz.build @@ -0,0 +1,8 @@ +# -*- Mode: python; indent-tabs-mode: nil; tab-width: 40 -*- +# Any copyright is dedicated to the Public Domain. +# http://creativecommons.org/publicdomain/zero/1.0/ + +FINAL_TARGET_PP_FILES += [ + 'install.rdf', + 'main.js', +] diff --git a/python/mozbuild/mozbuild/test/backend/data/exports-generated/dom1.h b/python/mozbuild/mozbuild/test/backend/data/exports-generated/dom1.h new file mode 100644 index 000000000..e69de29bb diff --git a/python/mozbuild/mozbuild/test/backend/data/exports-generated/foo.h b/python/mozbuild/mozbuild/test/backend/data/exports-generated/foo.h new file mode 100644 index 000000000..e69de29bb diff --git a/python/mozbuild/mozbuild/test/backend/data/exports-generated/gfx.h b/python/mozbuild/mozbuild/test/backend/data/exports-generated/gfx.h new file mode 100644 index 000000000..e69de29bb diff --git a/python/mozbuild/mozbuild/test/backend/data/exports-generated/moz.build b/python/mozbuild/mozbuild/test/backend/data/exports-generated/moz.build new file mode 100644 index 000000000..b604ef1a0 --- /dev/null +++ b/python/mozbuild/mozbuild/test/backend/data/exports-generated/moz.build @@ -0,0 +1,12 @@ +# Any copyright is dedicated to the Public Domain. +# http://creativecommons.org/publicdomain/zero/1.0/ + +EXPORTS += ['!bar.h', 'foo.h'] +EXPORTS.mozilla += ['!mozilla2.h', 'mozilla1.h'] +EXPORTS.mozilla.dom += ['!dom2.h', '!dom3.h', 'dom1.h'] +EXPORTS.gfx += ['gfx.h'] + +GENERATED_FILES += ['bar.h'] +GENERATED_FILES += ['mozilla2.h'] +GENERATED_FILES += ['dom2.h'] +GENERATED_FILES += ['dom3.h'] diff --git a/python/mozbuild/mozbuild/test/backend/data/exports-generated/mozilla1.h b/python/mozbuild/mozbuild/test/backend/data/exports-generated/mozilla1.h new file mode 100644 index 000000000..e69de29bb diff --git a/python/mozbuild/mozbuild/test/backend/data/exports/dom1.h b/python/mozbuild/mozbuild/test/backend/data/exports/dom1.h new file mode 100644 index 000000000..e69de29bb diff --git a/python/mozbuild/mozbuild/test/backend/data/exports/dom2.h b/python/mozbuild/mozbuild/test/backend/data/exports/dom2.h new file mode 100644 index 000000000..e69de29bb diff --git a/python/mozbuild/mozbuild/test/backend/data/exports/foo.h b/python/mozbuild/mozbuild/test/backend/data/exports/foo.h new file mode 100644 index 000000000..e69de29bb diff --git a/python/mozbuild/mozbuild/test/backend/data/exports/gfx.h b/python/mozbuild/mozbuild/test/backend/data/exports/gfx.h new file mode 100644 index 000000000..e69de29bb diff --git a/python/mozbuild/mozbuild/test/backend/data/exports/moz.build b/python/mozbuild/mozbuild/test/backend/data/exports/moz.build new file mode 100644 index 000000000..725fa1fd4 --- /dev/null +++ b/python/mozbuild/mozbuild/test/backend/data/exports/moz.build @@ -0,0 +1,8 @@ +# Any copyright is dedicated to the Public Domain. +# http://creativecommons.org/publicdomain/zero/1.0/ + +EXPORTS += ['foo.h'] +EXPORTS.mozilla += ['mozilla1.h', 'mozilla2.h'] +EXPORTS.mozilla.dom += ['dom1.h', 'dom2.h'] +EXPORTS.mozilla.gfx += ['gfx.h'] +EXPORTS.nspr.private += ['pprio.h'] diff --git a/python/mozbuild/mozbuild/test/backend/data/exports/mozilla1.h b/python/mozbuild/mozbuild/test/backend/data/exports/mozilla1.h new file mode 100644 index 000000000..e69de29bb diff --git a/python/mozbuild/mozbuild/test/backend/data/exports/mozilla2.h b/python/mozbuild/mozbuild/test/backend/data/exports/mozilla2.h new file mode 100644 index 000000000..e69de29bb diff --git a/python/mozbuild/mozbuild/test/backend/data/exports/pprio.h b/python/mozbuild/mozbuild/test/backend/data/exports/pprio.h new file mode 100644 index 000000000..e69de29bb diff --git a/python/mozbuild/mozbuild/test/backend/data/final_target/both/moz.build b/python/mozbuild/mozbuild/test/backend/data/final_target/both/moz.build new file mode 100644 index 000000000..c926e3788 --- /dev/null +++ b/python/mozbuild/mozbuild/test/backend/data/final_target/both/moz.build @@ -0,0 +1,6 @@ +# -*- Mode: python; indent-tabs-mode: nil; tab-width: 40 -*- +# Any copyright is dedicated to the Public Domain. +# http://creativecommons.org/publicdomain/zero/1.0/ + +XPI_NAME = 'mycrazyxpi' +DIST_SUBDIR = 'asubdir' diff --git a/python/mozbuild/mozbuild/test/backend/data/final_target/dist-subdir/moz.build b/python/mozbuild/mozbuild/test/backend/data/final_target/dist-subdir/moz.build new file mode 100644 index 000000000..8dcf066a4 --- /dev/null +++ b/python/mozbuild/mozbuild/test/backend/data/final_target/dist-subdir/moz.build @@ -0,0 +1,5 @@ +# -*- Mode: python; indent-tabs-mode: nil; tab-width: 40 -*- +# Any copyright is dedicated to the Public Domain. +# http://creativecommons.org/publicdomain/zero/1.0/ + +DIST_SUBDIR = 'asubdir' diff --git a/python/mozbuild/mozbuild/test/backend/data/final_target/final-target/moz.build b/python/mozbuild/mozbuild/test/backend/data/final_target/final-target/moz.build new file mode 100644 index 000000000..1d746eea5 --- /dev/null +++ b/python/mozbuild/mozbuild/test/backend/data/final_target/final-target/moz.build @@ -0,0 +1,5 @@ +# -*- Mode: python; indent-tabs-mode: nil; tab-width: 40 -*- +# Any copyright is dedicated to the Public Domain. +# http://creativecommons.org/publicdomain/zero/1.0/ + +FINAL_TARGET = 'random-final-target' diff --git a/python/mozbuild/mozbuild/test/backend/data/final_target/moz.build b/python/mozbuild/mozbuild/test/backend/data/final_target/moz.build new file mode 100644 index 000000000..280299475 --- /dev/null +++ b/python/mozbuild/mozbuild/test/backend/data/final_target/moz.build @@ -0,0 +1,5 @@ +# -*- Mode: python; indent-tabs-mode: nil; tab-width: 40 -*- +# Any copyright is dedicated to the Public Domain. +# http://creativecommons.org/publicdomain/zero/1.0/ + +DIRS += ['xpi-name', 'dist-subdir', 'both', 'final-target'] diff --git a/python/mozbuild/mozbuild/test/backend/data/final_target/xpi-name/moz.build b/python/mozbuild/mozbuild/test/backend/data/final_target/xpi-name/moz.build new file mode 100644 index 000000000..54bc30fec --- /dev/null +++ b/python/mozbuild/mozbuild/test/backend/data/final_target/xpi-name/moz.build @@ -0,0 +1,5 @@ +# -*- Mode: python; indent-tabs-mode: nil; tab-width: 40 -*- +# Any copyright is dedicated to the Public Domain. +# http://creativecommons.org/publicdomain/zero/1.0/ + +XPI_NAME = 'mycrazyxpi' diff --git a/python/mozbuild/mozbuild/test/backend/data/generated-files/foo-data b/python/mozbuild/mozbuild/test/backend/data/generated-files/foo-data new file mode 100644 index 000000000..e69de29bb diff --git a/python/mozbuild/mozbuild/test/backend/data/generated-files/generate-bar.py b/python/mozbuild/mozbuild/test/backend/data/generated-files/generate-bar.py new file mode 100644 index 000000000..e69de29bb diff --git a/python/mozbuild/mozbuild/test/backend/data/generated-files/generate-foo.py b/python/mozbuild/mozbuild/test/backend/data/generated-files/generate-foo.py new file mode 100644 index 000000000..e69de29bb diff --git a/python/mozbuild/mozbuild/test/backend/data/generated-files/moz.build b/python/mozbuild/mozbuild/test/backend/data/generated-files/moz.build new file mode 100644 index 000000000..1fa389f51 --- /dev/null +++ b/python/mozbuild/mozbuild/test/backend/data/generated-files/moz.build @@ -0,0 +1,12 @@ +# -*- Mode: python; indent-tabs-mode: nil; tab-width: 40 -*- +# Any copyright is dedicated to the Public Domain. +# http://creativecommons.org/publicdomain/zero/1.0/ + +GENERATED_FILES += [ 'bar.c', 'foo.c', 'quux.c' ] + +bar = GENERATED_FILES['bar.c'] +bar.script = 'generate-bar.py:baz' + +foo = GENERATED_FILES['foo.c'] +foo.script = 'generate-foo.py' +foo.inputs = ['foo-data'] diff --git a/python/mozbuild/mozbuild/test/backend/data/generated_includes/moz.build b/python/mozbuild/mozbuild/test/backend/data/generated_includes/moz.build new file mode 100644 index 000000000..14deaf8cf --- /dev/null +++ b/python/mozbuild/mozbuild/test/backend/data/generated_includes/moz.build @@ -0,0 +1,5 @@ +# -*- Mode: python; indent-tabs-mode: nil; tab-width: 40 -*- +# Any copyright is dedicated to the Public Domain. +# http://creativecommons.org/publicdomain/zero/1.0/ + +LOCAL_INCLUDES += ['!/bar/baz', '!foo'] diff --git a/python/mozbuild/mozbuild/test/backend/data/host-defines/moz.build b/python/mozbuild/mozbuild/test/backend/data/host-defines/moz.build new file mode 100644 index 000000000..30f8c160f --- /dev/null +++ b/python/mozbuild/mozbuild/test/backend/data/host-defines/moz.build @@ -0,0 +1,14 @@ +# Any copyright is dedicated to the Public Domain. +# http://creativecommons.org/publicdomain/zero/1.0/ + +value = 'xyz' +HOST_DEFINES = { + 'FOO': True, +} + +HOST_DEFINES['BAZ'] = '"ab\'cd"' +HOST_DEFINES.update({ + 'BAR': 7, + 'VALUE': value, + 'QUX': False, +}) diff --git a/python/mozbuild/mozbuild/test/backend/data/install_substitute_config_files/moz.build b/python/mozbuild/mozbuild/test/backend/data/install_substitute_config_files/moz.build new file mode 100644 index 000000000..dbadef914 --- /dev/null +++ b/python/mozbuild/mozbuild/test/backend/data/install_substitute_config_files/moz.build @@ -0,0 +1,6 @@ +# -*- Mode: python; indent-tabs-mode: nil; tab-width: 40 -*- +# Any copyright is dedicated to the Public Domain. +# http://creativecommons.org/publicdomain/zero/1.0/ + +# We want to test recursion into the subdir, so do the real work in 'sub' +DIRS += ['sub'] diff --git a/python/mozbuild/mozbuild/test/backend/data/install_substitute_config_files/sub/foo.h.in b/python/mozbuild/mozbuild/test/backend/data/install_substitute_config_files/sub/foo.h.in new file mode 100644 index 000000000..da287dfca --- /dev/null +++ b/python/mozbuild/mozbuild/test/backend/data/install_substitute_config_files/sub/foo.h.in @@ -0,0 +1 @@ +#define MOZ_FOO @MOZ_FOO@ diff --git a/python/mozbuild/mozbuild/test/backend/data/install_substitute_config_files/sub/moz.build b/python/mozbuild/mozbuild/test/backend/data/install_substitute_config_files/sub/moz.build new file mode 100644 index 000000000..c2ef44079 --- /dev/null +++ b/python/mozbuild/mozbuild/test/backend/data/install_substitute_config_files/sub/moz.build @@ -0,0 +1,7 @@ +# -*- Mode: python; indent-tabs-mode: nil; tab-width: 40 -*- +# Any copyright is dedicated to the Public Domain. +# http://creativecommons.org/publicdomain/zero/1.0/ + +CONFIGURE_SUBST_FILES = ['foo.h'] + +EXPORTS.out += ['!foo.h'] diff --git a/python/mozbuild/mozbuild/test/backend/data/ipdl_sources/bar/moz.build b/python/mozbuild/mozbuild/test/backend/data/ipdl_sources/bar/moz.build new file mode 100644 index 000000000..f189212fd --- /dev/null +++ b/python/mozbuild/mozbuild/test/backend/data/ipdl_sources/bar/moz.build @@ -0,0 +1,10 @@ +# -*- Mode: python; indent-tabs-mode: nil; tab-width: 40 -*- +# vim: set filetype=python: +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, You can obtain one at http://mozilla.org/MPL/2.0/. + +IPDL_SOURCES += [ + 'bar.ipdl', + 'bar2.ipdlh', +] diff --git a/python/mozbuild/mozbuild/test/backend/data/ipdl_sources/foo/moz.build b/python/mozbuild/mozbuild/test/backend/data/ipdl_sources/foo/moz.build new file mode 100644 index 000000000..4e1554559 --- /dev/null +++ b/python/mozbuild/mozbuild/test/backend/data/ipdl_sources/foo/moz.build @@ -0,0 +1,10 @@ +# -*- Mode: python; indent-tabs-mode: nil; tab-width: 40 -*- +# vim: set filetype=python: +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, You can obtain one at http://mozilla.org/MPL/2.0/. + +IPDL_SOURCES += [ + 'foo.ipdl', + 'foo2.ipdlh', +] diff --git a/python/mozbuild/mozbuild/test/backend/data/ipdl_sources/moz.build b/python/mozbuild/mozbuild/test/backend/data/ipdl_sources/moz.build new file mode 100644 index 000000000..03cf5e236 --- /dev/null +++ b/python/mozbuild/mozbuild/test/backend/data/ipdl_sources/moz.build @@ -0,0 +1,10 @@ +# -*- Mode: python; indent-tabs-mode: nil; tab-width: 40 -*- +# vim: set filetype=python: +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, You can obtain one at http://mozilla.org/MPL/2.0/. + +DIRS += [ + 'bar', + 'foo', +] diff --git a/python/mozbuild/mozbuild/test/backend/data/jar-manifests/moz.build b/python/mozbuild/mozbuild/test/backend/data/jar-manifests/moz.build new file mode 100644 index 000000000..7daa419f1 --- /dev/null +++ b/python/mozbuild/mozbuild/test/backend/data/jar-manifests/moz.build @@ -0,0 +1,8 @@ +# -*- Mode: python; indent-tabs-mode: nil; tab-width: 40 -*- +# vim: set filetype=python: +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, You can obtain one at http://mozilla.org/MPL/2.0/. + +JAR_MANIFESTS += ['jar.mn'] + diff --git a/python/mozbuild/mozbuild/test/backend/data/local_includes/bar/baz/dummy_file_for_nonempty_directory b/python/mozbuild/mozbuild/test/backend/data/local_includes/bar/baz/dummy_file_for_nonempty_directory new file mode 100644 index 000000000..e69de29bb diff --git a/python/mozbuild/mozbuild/test/backend/data/local_includes/foo/dummy_file_for_nonempty_directory b/python/mozbuild/mozbuild/test/backend/data/local_includes/foo/dummy_file_for_nonempty_directory new file mode 100644 index 000000000..e69de29bb diff --git a/python/mozbuild/mozbuild/test/backend/data/local_includes/moz.build b/python/mozbuild/mozbuild/test/backend/data/local_includes/moz.build new file mode 100644 index 000000000..565c2bee6 --- /dev/null +++ b/python/mozbuild/mozbuild/test/backend/data/local_includes/moz.build @@ -0,0 +1,5 @@ +# -*- Mode: python; indent-tabs-mode: nil; tab-width: 40 -*- +# Any copyright is dedicated to the Public Domain. +# http://creativecommons.org/publicdomain/zero/1.0/ + +LOCAL_INCLUDES += ['/bar/baz', 'foo'] diff --git a/python/mozbuild/mozbuild/test/backend/data/resources/bar.res.in b/python/mozbuild/mozbuild/test/backend/data/resources/bar.res.in new file mode 100644 index 000000000..e69de29bb diff --git a/python/mozbuild/mozbuild/test/backend/data/resources/cursor.cur b/python/mozbuild/mozbuild/test/backend/data/resources/cursor.cur new file mode 100644 index 000000000..e69de29bb diff --git a/python/mozbuild/mozbuild/test/backend/data/resources/desktop1.ttf b/python/mozbuild/mozbuild/test/backend/data/resources/desktop1.ttf new file mode 100644 index 000000000..e69de29bb diff --git a/python/mozbuild/mozbuild/test/backend/data/resources/desktop2.ttf b/python/mozbuild/mozbuild/test/backend/data/resources/desktop2.ttf new file mode 100644 index 000000000..e69de29bb diff --git a/python/mozbuild/mozbuild/test/backend/data/resources/extra.manifest b/python/mozbuild/mozbuild/test/backend/data/resources/extra.manifest new file mode 100644 index 000000000..e69de29bb diff --git a/python/mozbuild/mozbuild/test/backend/data/resources/font1.ttf b/python/mozbuild/mozbuild/test/backend/data/resources/font1.ttf new file mode 100644 index 000000000..e69de29bb diff --git a/python/mozbuild/mozbuild/test/backend/data/resources/font2.ttf b/python/mozbuild/mozbuild/test/backend/data/resources/font2.ttf new file mode 100644 index 000000000..e69de29bb diff --git a/python/mozbuild/mozbuild/test/backend/data/resources/foo.res b/python/mozbuild/mozbuild/test/backend/data/resources/foo.res new file mode 100644 index 000000000..e69de29bb diff --git a/python/mozbuild/mozbuild/test/backend/data/resources/mobile.ttf b/python/mozbuild/mozbuild/test/backend/data/resources/mobile.ttf new file mode 100644 index 000000000..e69de29bb diff --git a/python/mozbuild/mozbuild/test/backend/data/resources/moz.build b/python/mozbuild/mozbuild/test/backend/data/resources/moz.build new file mode 100644 index 000000000..a5771c808 --- /dev/null +++ b/python/mozbuild/mozbuild/test/backend/data/resources/moz.build @@ -0,0 +1,9 @@ +# Any copyright is dedicated to the Public Domain. +# http://creativecommons.org/publicdomain/zero/1.0/ + +RESOURCE_FILES += ['bar.res.in', 'foo.res'] +RESOURCE_FILES.cursors += ['cursor.cur'] +RESOURCE_FILES.fonts += ['font1.ttf', 'font2.ttf'] +RESOURCE_FILES.fonts.desktop += ['desktop1.ttf', 'desktop2.ttf'] +RESOURCE_FILES.fonts.mobile += ['mobile.ttf'] +RESOURCE_FILES.tests += ['extra.manifest', 'test.manifest'] diff --git a/python/mozbuild/mozbuild/test/backend/data/resources/test.manifest b/python/mozbuild/mozbuild/test/backend/data/resources/test.manifest new file mode 100644 index 000000000..e69de29bb diff --git a/python/mozbuild/mozbuild/test/backend/data/sdk-files/bar.ico b/python/mozbuild/mozbuild/test/backend/data/sdk-files/bar.ico new file mode 100644 index 000000000..e69de29bb diff --git a/python/mozbuild/mozbuild/test/backend/data/sdk-files/foo.ico b/python/mozbuild/mozbuild/test/backend/data/sdk-files/foo.ico new file mode 100644 index 000000000..e69de29bb diff --git a/python/mozbuild/mozbuild/test/backend/data/sdk-files/moz.build b/python/mozbuild/mozbuild/test/backend/data/sdk-files/moz.build new file mode 100644 index 000000000..342987741 --- /dev/null +++ b/python/mozbuild/mozbuild/test/backend/data/sdk-files/moz.build @@ -0,0 +1,11 @@ +# Any copyright is dedicated to the Public Domain. +# http://creativecommons.org/publicdomain/zero/1.0/ + +SDK_FILES += [ + 'bar.ico', + 'sub/quux.png', +] + +SDK_FILES.icons += [ + 'foo.ico', +] diff --git a/python/mozbuild/mozbuild/test/backend/data/sdk-files/sub/quux.png b/python/mozbuild/mozbuild/test/backend/data/sdk-files/sub/quux.png new file mode 100644 index 000000000..e69de29bb diff --git a/python/mozbuild/mozbuild/test/backend/data/sources/bar.c b/python/mozbuild/mozbuild/test/backend/data/sources/bar.c new file mode 100644 index 000000000..e69de29bb diff --git a/python/mozbuild/mozbuild/test/backend/data/sources/bar.cpp b/python/mozbuild/mozbuild/test/backend/data/sources/bar.cpp new file mode 100644 index 000000000..e69de29bb diff --git a/python/mozbuild/mozbuild/test/backend/data/sources/bar.mm b/python/mozbuild/mozbuild/test/backend/data/sources/bar.mm new file mode 100644 index 000000000..e69de29bb diff --git a/python/mozbuild/mozbuild/test/backend/data/sources/bar.s b/python/mozbuild/mozbuild/test/backend/data/sources/bar.s new file mode 100644 index 000000000..e69de29bb diff --git a/python/mozbuild/mozbuild/test/backend/data/sources/baz.S b/python/mozbuild/mozbuild/test/backend/data/sources/baz.S new file mode 100644 index 000000000..e69de29bb diff --git a/python/mozbuild/mozbuild/test/backend/data/sources/foo.S b/python/mozbuild/mozbuild/test/backend/data/sources/foo.S new file mode 100644 index 000000000..e69de29bb diff --git a/python/mozbuild/mozbuild/test/backend/data/sources/foo.asm b/python/mozbuild/mozbuild/test/backend/data/sources/foo.asm new file mode 100644 index 000000000..e69de29bb diff --git a/python/mozbuild/mozbuild/test/backend/data/sources/foo.c b/python/mozbuild/mozbuild/test/backend/data/sources/foo.c new file mode 100644 index 000000000..e69de29bb diff --git a/python/mozbuild/mozbuild/test/backend/data/sources/foo.cpp b/python/mozbuild/mozbuild/test/backend/data/sources/foo.cpp new file mode 100644 index 000000000..e69de29bb diff --git a/python/mozbuild/mozbuild/test/backend/data/sources/foo.mm b/python/mozbuild/mozbuild/test/backend/data/sources/foo.mm new file mode 100644 index 000000000..e69de29bb diff --git a/python/mozbuild/mozbuild/test/backend/data/sources/moz.build b/python/mozbuild/mozbuild/test/backend/data/sources/moz.build new file mode 100644 index 000000000..d31acae3d --- /dev/null +++ b/python/mozbuild/mozbuild/test/backend/data/sources/moz.build @@ -0,0 +1,21 @@ +# -*- Mode: python; indent-tabs-mode: nil; tab-width: 40 -*- +# Any copyright is dedicated to the Public Domain. +# http://creativecommons.org/publicdomain/zero/1.0/ + +@template +def Library(name): + '''Template for libraries.''' + LIBRARY_NAME = name + +Library('dummy') + +SOURCES += ['bar.s', 'foo.asm'] + +HOST_SOURCES += ['bar.cpp', 'foo.cpp'] +HOST_SOURCES += ['bar.c', 'foo.c'] + +SOURCES += ['bar.c', 'foo.c'] + +SOURCES += ['bar.mm', 'foo.mm'] + +SOURCES += ['baz.S', 'foo.S'] diff --git a/python/mozbuild/mozbuild/test/backend/data/stub0/Makefile.in b/python/mozbuild/mozbuild/test/backend/data/stub0/Makefile.in new file mode 100644 index 000000000..02ff0a3f9 --- /dev/null +++ b/python/mozbuild/mozbuild/test/backend/data/stub0/Makefile.in @@ -0,0 +1,4 @@ +# Any copyright is dedicated to the Public Domain. +# http://creativecommons.org/publicdomain/zero/1.0/ + +FOO := foo diff --git a/python/mozbuild/mozbuild/test/backend/data/stub0/dir1/Makefile.in b/python/mozbuild/mozbuild/test/backend/data/stub0/dir1/Makefile.in new file mode 100644 index 000000000..17c147d97 --- /dev/null +++ b/python/mozbuild/mozbuild/test/backend/data/stub0/dir1/Makefile.in @@ -0,0 +1,7 @@ +# Any copyright is dedicated to the Public Domain. +# http://creativecommons.org/publicdomain/zero/1.0/ + +include $(DEPTH)/config/autoconf.mk + +include $(topsrcdir)/config/rules.mk + diff --git a/python/mozbuild/mozbuild/test/backend/data/stub0/dir1/moz.build b/python/mozbuild/mozbuild/test/backend/data/stub0/dir1/moz.build new file mode 100644 index 000000000..041381548 --- /dev/null +++ b/python/mozbuild/mozbuild/test/backend/data/stub0/dir1/moz.build @@ -0,0 +1,5 @@ +# -*- Mode: python; indent-tabs-mode: nil; tab-width: 40 -*- +# Any copyright is dedicated to the Public Domain. +# http://creativecommons.org/publicdomain/zero/1.0/ + + diff --git a/python/mozbuild/mozbuild/test/backend/data/stub0/dir2/moz.build b/python/mozbuild/mozbuild/test/backend/data/stub0/dir2/moz.build new file mode 100644 index 000000000..32a37fe46 --- /dev/null +++ b/python/mozbuild/mozbuild/test/backend/data/stub0/dir2/moz.build @@ -0,0 +1,4 @@ +# -*- Mode: python; indent-tabs-mode: nil; tab-width: 40 -*- +# Any copyright is dedicated to the Public Domain. +# http://creativecommons.org/publicdomain/zero/1.0/ + diff --git a/python/mozbuild/mozbuild/test/backend/data/stub0/dir3/Makefile.in b/python/mozbuild/mozbuild/test/backend/data/stub0/dir3/Makefile.in new file mode 100644 index 000000000..17c147d97 --- /dev/null +++ b/python/mozbuild/mozbuild/test/backend/data/stub0/dir3/Makefile.in @@ -0,0 +1,7 @@ +# Any copyright is dedicated to the Public Domain. +# http://creativecommons.org/publicdomain/zero/1.0/ + +include $(DEPTH)/config/autoconf.mk + +include $(topsrcdir)/config/rules.mk + diff --git a/python/mozbuild/mozbuild/test/backend/data/stub0/dir3/moz.build b/python/mozbuild/mozbuild/test/backend/data/stub0/dir3/moz.build new file mode 100644 index 000000000..32a37fe46 --- /dev/null +++ b/python/mozbuild/mozbuild/test/backend/data/stub0/dir3/moz.build @@ -0,0 +1,4 @@ +# -*- Mode: python; indent-tabs-mode: nil; tab-width: 40 -*- +# Any copyright is dedicated to the Public Domain. +# http://creativecommons.org/publicdomain/zero/1.0/ + diff --git a/python/mozbuild/mozbuild/test/backend/data/stub0/moz.build b/python/mozbuild/mozbuild/test/backend/data/stub0/moz.build new file mode 100644 index 000000000..0d92bb7c3 --- /dev/null +++ b/python/mozbuild/mozbuild/test/backend/data/stub0/moz.build @@ -0,0 +1,7 @@ +# -*- Mode: python; indent-tabs-mode: nil; tab-width: 40 -*- +# Any copyright is dedicated to the Public Domain. +# http://creativecommons.org/publicdomain/zero/1.0/ + +DIRS += ['dir1'] +DIRS += ['dir2'] +TEST_DIRS += ['dir3'] diff --git a/python/mozbuild/mozbuild/test/backend/data/substitute_config_files/Makefile.in b/python/mozbuild/mozbuild/test/backend/data/substitute_config_files/Makefile.in new file mode 100644 index 000000000..e69de29bb diff --git a/python/mozbuild/mozbuild/test/backend/data/substitute_config_files/foo.in b/python/mozbuild/mozbuild/test/backend/data/substitute_config_files/foo.in new file mode 100644 index 000000000..5331f1f05 --- /dev/null +++ b/python/mozbuild/mozbuild/test/backend/data/substitute_config_files/foo.in @@ -0,0 +1 @@ +TEST = @MOZ_FOO@ diff --git a/python/mozbuild/mozbuild/test/backend/data/substitute_config_files/moz.build b/python/mozbuild/mozbuild/test/backend/data/substitute_config_files/moz.build new file mode 100644 index 000000000..01545c250 --- /dev/null +++ b/python/mozbuild/mozbuild/test/backend/data/substitute_config_files/moz.build @@ -0,0 +1,5 @@ +# -*- Mode: python; indent-tabs-mode: nil; tab-width: 40 -*- +# Any copyright is dedicated to the Public Domain. +# http://creativecommons.org/publicdomain/zero/1.0/ + +CONFIGURE_SUBST_FILES = ['foo'] diff --git a/python/mozbuild/mozbuild/test/backend/data/test-manifest-shared-support/child/another-file.sjs b/python/mozbuild/mozbuild/test/backend/data/test-manifest-shared-support/child/another-file.sjs new file mode 100644 index 000000000..e69de29bb diff --git a/python/mozbuild/mozbuild/test/backend/data/test-manifest-shared-support/child/browser.ini b/python/mozbuild/mozbuild/test/backend/data/test-manifest-shared-support/child/browser.ini new file mode 100644 index 000000000..4f1335d6b --- /dev/null +++ b/python/mozbuild/mozbuild/test/backend/data/test-manifest-shared-support/child/browser.ini @@ -0,0 +1,6 @@ +[DEFAULT] +support-files = + another-file.sjs + data/** + +[test_sub.js] \ No newline at end of file diff --git a/python/mozbuild/mozbuild/test/backend/data/test-manifest-shared-support/child/data/one.txt b/python/mozbuild/mozbuild/test/backend/data/test-manifest-shared-support/child/data/one.txt new file mode 100644 index 000000000..e69de29bb diff --git a/python/mozbuild/mozbuild/test/backend/data/test-manifest-shared-support/child/data/two.txt b/python/mozbuild/mozbuild/test/backend/data/test-manifest-shared-support/child/data/two.txt new file mode 100644 index 000000000..e69de29bb diff --git a/python/mozbuild/mozbuild/test/backend/data/test-manifest-shared-support/child/test_sub.js b/python/mozbuild/mozbuild/test/backend/data/test-manifest-shared-support/child/test_sub.js new file mode 100644 index 000000000..e69de29bb diff --git a/python/mozbuild/mozbuild/test/backend/data/test-manifest-shared-support/mochitest.ini b/python/mozbuild/mozbuild/test/backend/data/test-manifest-shared-support/mochitest.ini new file mode 100644 index 000000000..a9860f3de --- /dev/null +++ b/python/mozbuild/mozbuild/test/backend/data/test-manifest-shared-support/mochitest.ini @@ -0,0 +1,8 @@ +[DEFAULT] +support-files = + support-file.txt + !/child/test_sub.js + !/child/another-file.sjs + !/child/data/** + +[test_foo.js] diff --git a/python/mozbuild/mozbuild/test/backend/data/test-manifest-shared-support/moz.build b/python/mozbuild/mozbuild/test/backend/data/test-manifest-shared-support/moz.build new file mode 100644 index 000000000..1c1d064ea --- /dev/null +++ b/python/mozbuild/mozbuild/test/backend/data/test-manifest-shared-support/moz.build @@ -0,0 +1,5 @@ +# Any copyright is dedicated to the Public Domain. +# http://creativecommons.org/publicdomain/zero/1.0/ + +MOCHITEST_MANIFESTS += ['mochitest.ini'] +BROWSER_CHROME_MANIFESTS += ['child/browser.ini'] diff --git a/python/mozbuild/mozbuild/test/backend/data/test-manifest-shared-support/support-file.txt b/python/mozbuild/mozbuild/test/backend/data/test-manifest-shared-support/support-file.txt new file mode 100644 index 000000000..e69de29bb diff --git a/python/mozbuild/mozbuild/test/backend/data/test-manifest-shared-support/test_foo.js b/python/mozbuild/mozbuild/test/backend/data/test-manifest-shared-support/test_foo.js new file mode 100644 index 000000000..e69de29bb diff --git a/python/mozbuild/mozbuild/test/backend/data/test-manifests-duplicate-support-files/mochitest1.ini b/python/mozbuild/mozbuild/test/backend/data/test-manifests-duplicate-support-files/mochitest1.ini new file mode 100644 index 000000000..1f9816a89 --- /dev/null +++ b/python/mozbuild/mozbuild/test/backend/data/test-manifests-duplicate-support-files/mochitest1.ini @@ -0,0 +1,4 @@ +[DEFAULT] +support-files = support-file.txt + +[test_foo.js] diff --git a/python/mozbuild/mozbuild/test/backend/data/test-manifests-duplicate-support-files/mochitest2.ini b/python/mozbuild/mozbuild/test/backend/data/test-manifests-duplicate-support-files/mochitest2.ini new file mode 100644 index 000000000..e2a2fc96a --- /dev/null +++ b/python/mozbuild/mozbuild/test/backend/data/test-manifests-duplicate-support-files/mochitest2.ini @@ -0,0 +1,4 @@ +[DEFAULT] +support-files = support-file.txt + +[test_bar.js] diff --git a/python/mozbuild/mozbuild/test/backend/data/test-manifests-duplicate-support-files/moz.build b/python/mozbuild/mozbuild/test/backend/data/test-manifests-duplicate-support-files/moz.build new file mode 100644 index 000000000..d10500f8d --- /dev/null +++ b/python/mozbuild/mozbuild/test/backend/data/test-manifests-duplicate-support-files/moz.build @@ -0,0 +1,7 @@ +# Any copyright is dedicated to the Public Domain. +# http://creativecommons.org/publicdomain/zero/1.0/ + +MOCHITEST_MANIFESTS += [ + 'mochitest1.ini', + 'mochitest2.ini', +] diff --git a/python/mozbuild/mozbuild/test/backend/data/test-manifests-duplicate-support-files/test_bar.js b/python/mozbuild/mozbuild/test/backend/data/test-manifests-duplicate-support-files/test_bar.js new file mode 100644 index 000000000..e69de29bb diff --git a/python/mozbuild/mozbuild/test/backend/data/test-manifests-duplicate-support-files/test_foo.js b/python/mozbuild/mozbuild/test/backend/data/test-manifests-duplicate-support-files/test_foo.js new file mode 100644 index 000000000..e69de29bb diff --git a/python/mozbuild/mozbuild/test/backend/data/test-manifests-package-tests/instrumentation.ini b/python/mozbuild/mozbuild/test/backend/data/test-manifests-package-tests/instrumentation.ini new file mode 100644 index 000000000..03d4f794e --- /dev/null +++ b/python/mozbuild/mozbuild/test/backend/data/test-manifests-package-tests/instrumentation.ini @@ -0,0 +1 @@ +[not_packaged.java] diff --git a/python/mozbuild/mozbuild/test/backend/data/test-manifests-package-tests/mochitest.ini b/python/mozbuild/mozbuild/test/backend/data/test-manifests-package-tests/mochitest.ini new file mode 100644 index 000000000..009b2b223 --- /dev/null +++ b/python/mozbuild/mozbuild/test/backend/data/test-manifests-package-tests/mochitest.ini @@ -0,0 +1 @@ +[mochitest.js] diff --git a/python/mozbuild/mozbuild/test/backend/data/test-manifests-package-tests/mochitest.js b/python/mozbuild/mozbuild/test/backend/data/test-manifests-package-tests/mochitest.js new file mode 100644 index 000000000..e69de29bb diff --git a/python/mozbuild/mozbuild/test/backend/data/test-manifests-package-tests/moz.build b/python/mozbuild/mozbuild/test/backend/data/test-manifests-package-tests/moz.build new file mode 100644 index 000000000..82dba29dc --- /dev/null +++ b/python/mozbuild/mozbuild/test/backend/data/test-manifests-package-tests/moz.build @@ -0,0 +1,10 @@ +# Any copyright is dedicated to the Public Domain. +# http://creativecommons.org/publicdomain/zero/1.0/ + +MOCHITEST_MANIFESTS += [ + 'mochitest.ini', +] + +ANDROID_INSTRUMENTATION_MANIFESTS += [ + 'instrumentation.ini', +] diff --git a/python/mozbuild/mozbuild/test/backend/data/test-manifests-package-tests/not_packaged.java b/python/mozbuild/mozbuild/test/backend/data/test-manifests-package-tests/not_packaged.java new file mode 100644 index 000000000..e69de29bb diff --git a/python/mozbuild/mozbuild/test/backend/data/test-manifests-written/dir1/test_bar.js b/python/mozbuild/mozbuild/test/backend/data/test-manifests-written/dir1/test_bar.js new file mode 100644 index 000000000..e69de29bb diff --git a/python/mozbuild/mozbuild/test/backend/data/test-manifests-written/dir1/xpcshell.ini b/python/mozbuild/mozbuild/test/backend/data/test-manifests-written/dir1/xpcshell.ini new file mode 100644 index 000000000..0cddad8ba --- /dev/null +++ b/python/mozbuild/mozbuild/test/backend/data/test-manifests-written/dir1/xpcshell.ini @@ -0,0 +1,3 @@ +[DEFAULT] + +[test_bar.js] diff --git a/python/mozbuild/mozbuild/test/backend/data/test-manifests-written/mochitest.ini b/python/mozbuild/mozbuild/test/backend/data/test-manifests-written/mochitest.ini new file mode 100644 index 000000000..81869e1fa --- /dev/null +++ b/python/mozbuild/mozbuild/test/backend/data/test-manifests-written/mochitest.ini @@ -0,0 +1,3 @@ +[DEFAULT] + +[mochitest.js] diff --git a/python/mozbuild/mozbuild/test/backend/data/test-manifests-written/mochitest.js b/python/mozbuild/mozbuild/test/backend/data/test-manifests-written/mochitest.js new file mode 100644 index 000000000..e69de29bb diff --git a/python/mozbuild/mozbuild/test/backend/data/test-manifests-written/moz.build b/python/mozbuild/mozbuild/test/backend/data/test-manifests-written/moz.build new file mode 100644 index 000000000..d004cdd0f --- /dev/null +++ b/python/mozbuild/mozbuild/test/backend/data/test-manifests-written/moz.build @@ -0,0 +1,9 @@ +# Any copyright is dedicated to the Public Domain. +# http://creativecommons.org/publicdomain/zero/1.0/ + +XPCSHELL_TESTS_MANIFESTS += [ + 'dir1/xpcshell.ini', + 'xpcshell.ini', +] + +MOCHITEST_MANIFESTS += ['mochitest.ini'] diff --git a/python/mozbuild/mozbuild/test/backend/data/test-manifests-written/xpcshell.ini b/python/mozbuild/mozbuild/test/backend/data/test-manifests-written/xpcshell.ini new file mode 100644 index 000000000..f6a5351e9 --- /dev/null +++ b/python/mozbuild/mozbuild/test/backend/data/test-manifests-written/xpcshell.ini @@ -0,0 +1,4 @@ +[DEFAULT] +support-files = support/** + +[xpcshell.js] diff --git a/python/mozbuild/mozbuild/test/backend/data/test-manifests-written/xpcshell.js b/python/mozbuild/mozbuild/test/backend/data/test-manifests-written/xpcshell.js new file mode 100644 index 000000000..e69de29bb diff --git a/python/mozbuild/mozbuild/test/backend/data/test_config/file.in b/python/mozbuild/mozbuild/test/backend/data/test_config/file.in new file mode 100644 index 000000000..07aa30deb --- /dev/null +++ b/python/mozbuild/mozbuild/test/backend/data/test_config/file.in @@ -0,0 +1,3 @@ +#ifdef foo +@foo@ +@bar@ diff --git a/python/mozbuild/mozbuild/test/backend/data/test_config/moz.build b/python/mozbuild/mozbuild/test/backend/data/test_config/moz.build new file mode 100644 index 000000000..f0c357aaf --- /dev/null +++ b/python/mozbuild/mozbuild/test/backend/data/test_config/moz.build @@ -0,0 +1,3 @@ +CONFIGURE_SUBST_FILES = [ + 'file', +] diff --git a/python/mozbuild/mozbuild/test/backend/data/variable_passthru/Makefile.in b/python/mozbuild/mozbuild/test/backend/data/variable_passthru/Makefile.in new file mode 100644 index 000000000..e69de29bb diff --git a/python/mozbuild/mozbuild/test/backend/data/variable_passthru/moz.build b/python/mozbuild/mozbuild/test/backend/data/variable_passthru/moz.build new file mode 100644 index 000000000..36a2603b1 --- /dev/null +++ b/python/mozbuild/mozbuild/test/backend/data/variable_passthru/moz.build @@ -0,0 +1,23 @@ +# -*- Mode: python; indent-tabs-mode: nil; tab-width: 40 -*- +# Any copyright is dedicated to the Public Domain. +# http://creativecommons.org/publicdomain/zero/1.0/ + +NO_VISIBILITY_FLAGS = True + +DELAYLOAD_DLLS = ['foo.dll', 'bar.dll'] + +RCFILE = 'foo.rc' +RESFILE = 'bar.res' +RCINCLUDE = 'bar.rc' +DEFFILE = 'baz.def' + +CFLAGS += ['-fno-exceptions', '-w'] +CXXFLAGS += ['-fcxx-exceptions', '-option with spaces'] +LDFLAGS += ['-ld flag with spaces', '-x'] +HOST_CFLAGS += ['-funroll-loops', '-wall'] +HOST_CXXFLAGS += ['-funroll-loops-harder', '-wall-day-everyday'] +WIN32_EXE_LDFLAGS += ['-subsystem:console'] + +DISABLE_STL_WRAPPING = True + +ALLOW_COMPILER_WARNINGS = True diff --git a/python/mozbuild/mozbuild/test/backend/data/variable_passthru/test1.c b/python/mozbuild/mozbuild/test/backend/data/variable_passthru/test1.c new file mode 100644 index 000000000..e69de29bb diff --git a/python/mozbuild/mozbuild/test/backend/data/variable_passthru/test1.cpp b/python/mozbuild/mozbuild/test/backend/data/variable_passthru/test1.cpp new file mode 100644 index 000000000..e69de29bb diff --git a/python/mozbuild/mozbuild/test/backend/data/variable_passthru/test1.mm b/python/mozbuild/mozbuild/test/backend/data/variable_passthru/test1.mm new file mode 100644 index 000000000..e69de29bb diff --git a/python/mozbuild/mozbuild/test/backend/data/variable_passthru/test2.c b/python/mozbuild/mozbuild/test/backend/data/variable_passthru/test2.c new file mode 100644 index 000000000..e69de29bb diff --git a/python/mozbuild/mozbuild/test/backend/data/variable_passthru/test2.cpp b/python/mozbuild/mozbuild/test/backend/data/variable_passthru/test2.cpp new file mode 100644 index 000000000..e69de29bb diff --git a/python/mozbuild/mozbuild/test/backend/data/variable_passthru/test2.mm b/python/mozbuild/mozbuild/test/backend/data/variable_passthru/test2.mm new file mode 100644 index 000000000..e69de29bb diff --git a/python/mozbuild/mozbuild/test/backend/data/visual-studio/dir1/bar.cpp b/python/mozbuild/mozbuild/test/backend/data/visual-studio/dir1/bar.cpp new file mode 100644 index 000000000..e69de29bb diff --git a/python/mozbuild/mozbuild/test/backend/data/visual-studio/dir1/foo.cpp b/python/mozbuild/mozbuild/test/backend/data/visual-studio/dir1/foo.cpp new file mode 100644 index 000000000..e69de29bb diff --git a/python/mozbuild/mozbuild/test/backend/data/visual-studio/dir1/moz.build b/python/mozbuild/mozbuild/test/backend/data/visual-studio/dir1/moz.build new file mode 100644 index 000000000..b77e67ade --- /dev/null +++ b/python/mozbuild/mozbuild/test/backend/data/visual-studio/dir1/moz.build @@ -0,0 +1,9 @@ +# -*- Mode: python; indent-tabs-mode: nil; tab-width: 40 -*- +# Any copyright is dedicated to the Public Domain. +# http://creativecommons.org/publicdomain/zero/1.0/ + +FINAL_LIBRARY = 'test' +SOURCES += ['bar.cpp', 'foo.cpp'] +LOCAL_INCLUDES += ['/includeA/foo'] +DEFINES['DEFINEFOO'] = True +DEFINES['DEFINEBAR'] = 'bar' diff --git a/python/mozbuild/mozbuild/test/backend/data/visual-studio/moz.build b/python/mozbuild/mozbuild/test/backend/data/visual-studio/moz.build new file mode 100644 index 000000000..d339b48c4 --- /dev/null +++ b/python/mozbuild/mozbuild/test/backend/data/visual-studio/moz.build @@ -0,0 +1,7 @@ +# -*- Mode: python; indent-tabs-mode: nil; tab-width: 40 -*- +# Any copyright is dedicated to the Public Domain. +# http://creativecommons.org/publicdomain/zero/1.0/ + +DIRS += ['dir1'] + +Library('test') diff --git a/python/mozbuild/mozbuild/test/backend/data/xpidl/config/makefiles/xpidl/Makefile.in b/python/mozbuild/mozbuild/test/backend/data/xpidl/config/makefiles/xpidl/Makefile.in new file mode 100644 index 000000000..e69de29bb diff --git a/python/mozbuild/mozbuild/test/backend/data/xpidl/moz.build b/python/mozbuild/mozbuild/test/backend/data/xpidl/moz.build new file mode 100644 index 000000000..d49efde26 --- /dev/null +++ b/python/mozbuild/mozbuild/test/backend/data/xpidl/moz.build @@ -0,0 +1,6 @@ +# -*- Mode: python; indent-tabs-mode: nil; tab-width: 40 -*- +# Any copyright is dedicated to the Public Domain. +# http://creativecommons.org/publicdomain/zero/1.0/ + +XPIDL_MODULE = 'my_module' +XPIDL_SOURCES = ['bar.idl', 'foo.idl'] diff --git a/python/mozbuild/mozbuild/test/backend/test_android_eclipse.py b/python/mozbuild/mozbuild/test/backend/test_android_eclipse.py new file mode 100644 index 000000000..c4e9221c9 --- /dev/null +++ b/python/mozbuild/mozbuild/test/backend/test_android_eclipse.py @@ -0,0 +1,153 @@ +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, You can obtain one at http://mozilla.org/MPL/2.0/. + +from __future__ import unicode_literals + +import json +import os +import unittest + +from mozbuild.backend.android_eclipse import AndroidEclipseBackend +from mozbuild.frontend.emitter import TreeMetadataEmitter +from mozbuild.frontend.reader import BuildReader +from mozbuild.test.backend.common import BackendTester +from mozpack.manifests import InstallManifest +from mozunit import main + +import mozpack.path as mozpath + +class TestAndroidEclipseBackend(BackendTester): + def __init__(self, *args, **kwargs): + BackendTester.__init__(self, *args, **kwargs) + self.env = None + + def assertExists(self, *args): + p = mozpath.join(self.env.topobjdir, 'android_eclipse', *args) + self.assertTrue(os.path.exists(p), "Path %s exists" % p) + + def assertNotExists(self, *args): + p = mozpath.join(self.env.topobjdir, 'android_eclipse', *args) + self.assertFalse(os.path.exists(p), "Path %s does not exist" % p) + + def test_library_project_files(self): + """Ensure we generate reasonable files for library projects.""" + self.env = self._consume('android_eclipse', AndroidEclipseBackend) + for f in ['.classpath', + '.project', + '.settings', + 'AndroidManifest.xml', + 'project.properties']: + self.assertExists('library1', f) + + def test_main_project_files(self): + """Ensure we generate reasonable files for main (non-library) projects.""" + self.env = self._consume('android_eclipse', AndroidEclipseBackend) + for f in ['.classpath', + '.project', + '.settings', + 'gen', + 'lint.xml', + 'project.properties']: + self.assertExists('main1', f) + + def test_library_manifest(self): + """Ensure we generate manifest for library projects.""" + self.env = self._consume('android_eclipse', AndroidEclipseBackend) + self.assertExists('library1', 'AndroidManifest.xml') + + def test_classpathentries(self): + """Ensure we produce reasonable classpathentries.""" + self.env = self._consume('android_eclipse', AndroidEclipseBackend) + self.assertExists('main3', '.classpath') + # This is brittle but simple. + with open(mozpath.join(self.env.topobjdir, 'android_eclipse', 'main3', '.classpath'), 'rt') as fh: + lines = fh.readlines() + lines = [line.strip() for line in lines] + self.assertIn('', lines) + self.assertIn('', lines) + self.assertIn('', lines) + + def test_library_project_setting(self): + """Ensure we declare a library project correctly.""" + self.env = self._consume('android_eclipse', AndroidEclipseBackend) + + self.assertExists('library1', 'project.properties') + with open(mozpath.join(self.env.topobjdir, 'android_eclipse', 'library1', 'project.properties'), 'rt') as fh: + lines = fh.readlines() + lines = [line.strip() for line in lines] + self.assertIn('android.library=true', lines) + + self.assertExists('main1', 'project.properties') + with open(mozpath.join(self.env.topobjdir, 'android_eclipse', 'main1', 'project.properties'), 'rt') as fh: + lines = fh.readlines() + lines = [line.strip() for line in lines] + self.assertNotIn('android.library=true', lines) + + def test_referenced_projects(self): + """Ensure we reference another project correctly.""" + self.env = self._consume('android_eclipse', AndroidEclipseBackend) + self.assertExists('main4', '.classpath') + # This is brittle but simple. + with open(mozpath.join(self.env.topobjdir, 'android_eclipse', 'main4', '.classpath'), 'rt') as fh: + lines = fh.readlines() + lines = [line.strip() for line in lines] + self.assertIn('', lines) + + def test_extra_jars(self): + """Ensure we add class path entries to extra jars iff asked to.""" + self.env = self._consume('android_eclipse', AndroidEclipseBackend) + self.assertExists('main2', '.classpath') + # This is brittle but simple. + with open(mozpath.join(self.env.topobjdir, 'android_eclipse', 'main2', '.classpath'), 'rt') as fh: + lines = fh.readlines() + lines = [line.strip() for line in lines] + self.assertIn('' % self.env.topsrcdir, lines) + + def test_included_projects(self): + """Ensure we include another project correctly.""" + self.env = self._consume('android_eclipse', AndroidEclipseBackend) + self.assertExists('main4', 'project.properties') + # This is brittle but simple. + with open(mozpath.join(self.env.topobjdir, 'android_eclipse', 'main4', 'project.properties'), 'rt') as fh: + lines = fh.readlines() + lines = [line.strip() for line in lines] + self.assertIn('android.library.reference.1=library2', lines) + + def assertInManifest(self, project_name, *args): + manifest_path = mozpath.join(self.env.topobjdir, 'android_eclipse', '%s.manifest' % project_name) + manifest = InstallManifest(manifest_path) + for arg in args: + self.assertIn(arg, manifest, '%s in manifest for project %s' % (arg, project_name)) + + def assertNotInManifest(self, project_name, *args): + manifest_path = mozpath.join(self.env.topobjdir, 'android_eclipse', '%s.manifest' % project_name) + manifest = InstallManifest(manifest_path) + for arg in args: + self.assertNotIn(arg, manifest, '%s not in manifest for project %s' % (arg, project_name)) + + def test_manifest_main_manifest(self): + """Ensure we symlink manifest if asked to for main projects.""" + self.env = self._consume('android_eclipse', AndroidEclipseBackend) + self.assertInManifest('main1', 'AndroidManifest.xml') + + def test_manifest_res(self): + """Ensure we symlink res/ iff asked to.""" + self.env = self._consume('android_eclipse', AndroidEclipseBackend) + self.assertInManifest('library1', 'res') + self.assertNotInManifest('library2', 'res') + + def test_manifest_classpathentries(self): + """Ensure we symlink classpathentries correctly.""" + self.env = self._consume('android_eclipse', AndroidEclipseBackend) + self.assertInManifest('main3', 'a/a', 'b', 'd/e') + + def test_manifest_assets(self): + """Ensure we symlink assets/ iff asked to.""" + self.env = self._consume('android_eclipse', AndroidEclipseBackend) + self.assertNotInManifest('main1', 'assets') + self.assertInManifest('main2', 'assets') + + +if __name__ == '__main__': + main() diff --git a/python/mozbuild/mozbuild/test/backend/test_build.py b/python/mozbuild/mozbuild/test/backend/test_build.py new file mode 100644 index 000000000..d3f5fb6a9 --- /dev/null +++ b/python/mozbuild/mozbuild/test/backend/test_build.py @@ -0,0 +1,233 @@ +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, You can obtain one at http://mozilla.org/MPL/2.0/. + +from __future__ import unicode_literals, print_function + +import buildconfig +import os +import shutil +import sys +import unittest +import mozpack.path as mozpath +from contextlib import contextmanager +from mozunit import main +from mozbuild.backend import get_backend_class +from mozbuild.backend.configenvironment import ConfigEnvironment +from mozbuild.backend.recursivemake import RecursiveMakeBackend +from mozbuild.backend.fastermake import FasterMakeBackend +from mozbuild.base import MozbuildObject +from mozbuild.frontend.emitter import TreeMetadataEmitter +from mozbuild.frontend.reader import BuildReader +from mozbuild.util import ensureParentDir +from mozpack.files import FileFinder +from tempfile import mkdtemp + + +BASE_SUBSTS = [ + ('PYTHON', mozpath.normsep(sys.executable)), +] + + +class TestBuild(unittest.TestCase): + def setUp(self): + self._old_env = dict(os.environ) + os.environ.pop('MOZCONFIG', None) + os.environ.pop('MOZ_OBJDIR', None) + + def tearDown(self): + os.environ.clear() + os.environ.update(self._old_env) + + @contextmanager + def do_test_backend(self, *backends, **kwargs): + topobjdir = mkdtemp() + try: + config = ConfigEnvironment(buildconfig.topsrcdir, topobjdir, + **kwargs) + reader = BuildReader(config) + emitter = TreeMetadataEmitter(config) + moz_build = mozpath.join(config.topsrcdir, 'test.mozbuild') + definitions = list(emitter.emit( + reader.read_mozbuild(moz_build, config))) + for backend in backends: + backend(config).consume(definitions) + + yield config + except: + raise + finally: + if not os.environ.get('MOZ_NO_CLEANUP'): + shutil.rmtree(topobjdir) + + @contextmanager + def line_handler(self): + lines = [] + + def handle_make_line(line): + lines.append(line) + + try: + yield handle_make_line + except: + print('\n'.join(lines)) + raise + + if os.environ.get('MOZ_VERBOSE_MAKE'): + print('\n'.join(lines)) + + def test_recursive_make(self): + substs = list(BASE_SUBSTS) + with self.do_test_backend(RecursiveMakeBackend, + substs=substs) as config: + build = MozbuildObject(config.topsrcdir, None, None, + config.topobjdir) + overrides = [ + 'install_manifest_depends=', + 'MOZ_JAR_MAKER_FILE_FORMAT=flat', + 'TEST_MOZBUILD=1', + ] + with self.line_handler() as handle_make_line: + build._run_make(directory=config.topobjdir, target=overrides, + silent=False, line_handler=handle_make_line) + + self.validate(config) + + def test_faster_recursive_make(self): + substs = list(BASE_SUBSTS) + [ + ('BUILD_BACKENDS', 'FasterMake+RecursiveMake'), + ] + with self.do_test_backend(get_backend_class( + 'FasterMake+RecursiveMake'), substs=substs) as config: + buildid = mozpath.join(config.topobjdir, 'config', 'buildid') + ensureParentDir(buildid) + with open(buildid, 'w') as fh: + fh.write('20100101012345\n') + + build = MozbuildObject(config.topsrcdir, None, None, + config.topobjdir) + overrides = [ + 'install_manifest_depends=', + 'MOZ_JAR_MAKER_FILE_FORMAT=flat', + 'TEST_MOZBUILD=1', + ] + with self.line_handler() as handle_make_line: + build._run_make(directory=config.topobjdir, target=overrides, + silent=False, line_handler=handle_make_line) + + self.validate(config) + + def test_faster_make(self): + substs = list(BASE_SUBSTS) + [ + ('MOZ_BUILD_APP', 'dummy_app'), + ('MOZ_WIDGET_TOOLKIT', 'dummy_widget'), + ] + with self.do_test_backend(RecursiveMakeBackend, FasterMakeBackend, + substs=substs) as config: + buildid = mozpath.join(config.topobjdir, 'config', 'buildid') + ensureParentDir(buildid) + with open(buildid, 'w') as fh: + fh.write('20100101012345\n') + + build = MozbuildObject(config.topsrcdir, None, None, + config.topobjdir) + overrides = [ + 'TEST_MOZBUILD=1', + ] + with self.line_handler() as handle_make_line: + build._run_make(directory=mozpath.join(config.topobjdir, + 'faster'), + target=overrides, silent=False, + line_handler=handle_make_line) + + self.validate(config) + + def validate(self, config): + self.maxDiff = None + test_path = os.path.join(os.path.dirname(os.path.abspath(__file__)), + 'data', 'build') + os.sep + + # We want unicode instances out of the files, because having plain str + # makes assertEqual diff output in case of error extra verbose because + # of the difference in type. + result = { + p: f.open().read().decode('utf-8') + for p, f in FileFinder(mozpath.join(config.topobjdir, 'dist')) + } + self.assertTrue(len(result)) + self.assertEqual(result, { + 'bin/baz.ini': 'baz.ini: FOO is foo\n', + 'bin/child/bar.ini': 'bar.ini\n', + 'bin/child2/foo.css': 'foo.css: FOO is foo\n', + 'bin/child2/qux.ini': 'qux.ini: BAR is not defined\n', + 'bin/chrome.manifest': + 'manifest chrome/foo.manifest\n' + 'manifest components/components.manifest\n', + 'bin/chrome/foo.manifest': + 'content bar foo/child/\n' + 'content foo foo/\n' + 'override chrome://foo/bar.svg#hello ' + 'chrome://bar/bar.svg#hello\n', + 'bin/chrome/foo/bar.js': 'bar.js\n', + 'bin/chrome/foo/child/baz.jsm': + '//@line 2 "%sbaz.jsm"\nbaz.jsm: FOO is foo\n' % (test_path), + 'bin/chrome/foo/child/hoge.js': + '//@line 2 "%sbar.js"\nbar.js: FOO is foo\n' % (test_path), + 'bin/chrome/foo/foo.css': 'foo.css: FOO is foo\n', + 'bin/chrome/foo/foo.js': 'foo.js\n', + 'bin/chrome/foo/qux.js': 'bar.js\n', + 'bin/components/bar.js': + '//@line 2 "%sbar.js"\nbar.js: FOO is foo\n' % (test_path), + 'bin/components/components.manifest': + 'component {foo} foo.js\ncomponent {bar} bar.js\n', + 'bin/components/foo.js': 'foo.js\n', + 'bin/defaults/pref/prefs.js': 'prefs.js\n', + 'bin/foo.ini': 'foo.ini\n', + 'bin/modules/baz.jsm': + '//@line 2 "%sbaz.jsm"\nbaz.jsm: FOO is foo\n' % (test_path), + 'bin/modules/child/bar.jsm': 'bar.jsm\n', + 'bin/modules/child2/qux.jsm': + '//@line 4 "%squx.jsm"\nqux.jsm: BAR is not defined\n' + % (test_path), + 'bin/modules/foo.jsm': 'foo.jsm\n', + 'bin/res/resource': 'resource\n', + 'bin/res/child/resource2': 'resource2\n', + + 'bin/app/baz.ini': 'baz.ini: FOO is bar\n', + 'bin/app/child/bar.ini': 'bar.ini\n', + 'bin/app/child2/qux.ini': 'qux.ini: BAR is defined\n', + 'bin/app/chrome.manifest': + 'manifest chrome/foo.manifest\n' + 'manifest components/components.manifest\n', + 'bin/app/chrome/foo.manifest': + 'content bar foo/child/\n' + 'content foo foo/\n' + 'override chrome://foo/bar.svg#hello ' + 'chrome://bar/bar.svg#hello\n', + 'bin/app/chrome/foo/bar.js': 'bar.js\n', + 'bin/app/chrome/foo/child/baz.jsm': + '//@line 2 "%sbaz.jsm"\nbaz.jsm: FOO is bar\n' % (test_path), + 'bin/app/chrome/foo/child/hoge.js': + '//@line 2 "%sbar.js"\nbar.js: FOO is bar\n' % (test_path), + 'bin/app/chrome/foo/foo.css': 'foo.css: FOO is bar\n', + 'bin/app/chrome/foo/foo.js': 'foo.js\n', + 'bin/app/chrome/foo/qux.js': 'bar.js\n', + 'bin/app/components/bar.js': + '//@line 2 "%sbar.js"\nbar.js: FOO is bar\n' % (test_path), + 'bin/app/components/components.manifest': + 'component {foo} foo.js\ncomponent {bar} bar.js\n', + 'bin/app/components/foo.js': 'foo.js\n', + 'bin/app/defaults/preferences/prefs.js': 'prefs.js\n', + 'bin/app/foo.css': 'foo.css: FOO is bar\n', + 'bin/app/foo.ini': 'foo.ini\n', + 'bin/app/modules/baz.jsm': + '//@line 2 "%sbaz.jsm"\nbaz.jsm: FOO is bar\n' % (test_path), + 'bin/app/modules/child/bar.jsm': 'bar.jsm\n', + 'bin/app/modules/child2/qux.jsm': + '//@line 2 "%squx.jsm"\nqux.jsm: BAR is defined\n' + % (test_path), + 'bin/app/modules/foo.jsm': 'foo.jsm\n', + }) + +if __name__ == '__main__': + main() diff --git a/python/mozbuild/mozbuild/test/backend/test_configenvironment.py b/python/mozbuild/mozbuild/test/backend/test_configenvironment.py new file mode 100644 index 000000000..95593e186 --- /dev/null +++ b/python/mozbuild/mozbuild/test/backend/test_configenvironment.py @@ -0,0 +1,63 @@ +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, You can obtain one at http://mozilla.org/MPL/2.0/. + +import os, posixpath +from StringIO import StringIO +import unittest +from mozunit import main, MockedOpen + +import mozbuild.backend.configenvironment as ConfigStatus + +from mozbuild.util import ReadOnlyDict + +import mozpack.path as mozpath + + +class ConfigEnvironment(ConfigStatus.ConfigEnvironment): + def __init__(self, *args, **kwargs): + ConfigStatus.ConfigEnvironment.__init__(self, *args, **kwargs) + # Be helpful to unit tests + if not 'top_srcdir' in self.substs: + if os.path.isabs(self.topsrcdir): + top_srcdir = self.topsrcdir.replace(os.sep, '/') + else: + top_srcdir = mozpath.relpath(self.topsrcdir, self.topobjdir).replace(os.sep, '/') + + d = dict(self.substs) + d['top_srcdir'] = top_srcdir + self.substs = ReadOnlyDict(d) + + d = dict(self.substs_unicode) + d[u'top_srcdir'] = top_srcdir.decode('utf-8') + self.substs_unicode = ReadOnlyDict(d) + + +class TestEnvironment(unittest.TestCase): + def test_auto_substs(self): + '''Test the automatically set values of ACDEFINES, ALLSUBSTS + and ALLEMPTYSUBSTS. + ''' + env = ConfigEnvironment('.', '.', + defines = { 'foo': 'bar', 'baz': 'qux 42', + 'abc': "d'e'f", 'extra': 'foobar' }, + non_global_defines = ['extra', 'ignore'], + substs = { 'FOO': 'bar', 'FOOBAR': '', 'ABC': 'def', + 'bar': 'baz qux', 'zzz': '"abc def"', + 'qux': '' }) + # non_global_defines should be filtered out in ACDEFINES. + # Original order of the defines need to be respected in ACDEFINES + self.assertEqual(env.substs['ACDEFINES'], """-Dabc='d'\\''e'\\''f' -Dbaz='qux 42' -Dfoo=bar""") + # Likewise for ALLSUBSTS, which also must contain ACDEFINES + self.assertEqual(env.substs['ALLSUBSTS'], '''ABC = def +ACDEFINES = -Dabc='d'\\''e'\\''f' -Dbaz='qux 42' -Dfoo=bar +FOO = bar +bar = baz qux +zzz = "abc def"''') + # ALLEMPTYSUBSTS contains all substs with no value. + self.assertEqual(env.substs['ALLEMPTYSUBSTS'], '''FOOBAR = +qux =''') + + +if __name__ == "__main__": + main() diff --git a/python/mozbuild/mozbuild/test/backend/test_recursivemake.py b/python/mozbuild/mozbuild/test/backend/test_recursivemake.py new file mode 100644 index 000000000..87f50f497 --- /dev/null +++ b/python/mozbuild/mozbuild/test/backend/test_recursivemake.py @@ -0,0 +1,942 @@ +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, You can obtain one at http://mozilla.org/MPL/2.0/. + +from __future__ import unicode_literals + +import cPickle as pickle +import json +import os +import unittest + +from mozpack.manifests import ( + InstallManifest, +) +from mozunit import main + +from mozbuild.backend.recursivemake import ( + RecursiveMakeBackend, + RecursiveMakeTraversal, +) +from mozbuild.frontend.emitter import TreeMetadataEmitter +from mozbuild.frontend.reader import BuildReader + +from mozbuild.test.backend.common import BackendTester + +import mozpack.path as mozpath + + +class TestRecursiveMakeTraversal(unittest.TestCase): + def test_traversal(self): + traversal = RecursiveMakeTraversal() + traversal.add('', dirs=['A', 'B', 'C']) + traversal.add('', dirs=['D']) + traversal.add('A') + traversal.add('B', dirs=['E', 'F']) + traversal.add('C', dirs=['G', 'H']) + traversal.add('D', dirs=['I', 'K']) + traversal.add('D', dirs=['J', 'L']) + traversal.add('E') + traversal.add('F') + traversal.add('G') + traversal.add('H') + traversal.add('I', dirs=['M', 'N']) + traversal.add('J', dirs=['O', 'P']) + traversal.add('K', dirs=['Q', 'R']) + traversal.add('L', dirs=['S']) + traversal.add('M') + traversal.add('N', dirs=['T']) + traversal.add('O') + traversal.add('P', dirs=['U']) + traversal.add('Q') + traversal.add('R', dirs=['V']) + traversal.add('S', dirs=['W']) + traversal.add('T') + traversal.add('U') + traversal.add('V') + traversal.add('W', dirs=['X']) + traversal.add('X') + + parallels = set(('G', 'H', 'I', 'J', 'O', 'P', 'Q', 'R', 'U')) + def filter(current, subdirs): + return (current, [d for d in subdirs.dirs if d in parallels], + [d for d in subdirs.dirs if d not in parallels]) + + start, deps = traversal.compute_dependencies(filter) + self.assertEqual(start, ('X',)) + self.maxDiff = None + self.assertEqual(deps, { + 'A': ('',), + 'B': ('A',), + 'C': ('F',), + 'D': ('G', 'H'), + 'E': ('B',), + 'F': ('E',), + 'G': ('C',), + 'H': ('C',), + 'I': ('D',), + 'J': ('D',), + 'K': ('T', 'O', 'U'), + 'L': ('Q', 'V'), + 'M': ('I',), + 'N': ('M',), + 'O': ('J',), + 'P': ('J',), + 'Q': ('K',), + 'R': ('K',), + 'S': ('L',), + 'T': ('N',), + 'U': ('P',), + 'V': ('R',), + 'W': ('S',), + 'X': ('W',), + }) + + self.assertEqual(list(traversal.traverse('', filter)), + ['', 'A', 'B', 'E', 'F', 'C', 'G', 'H', 'D', 'I', + 'M', 'N', 'T', 'J', 'O', 'P', 'U', 'K', 'Q', 'R', + 'V', 'L', 'S', 'W', 'X']) + + self.assertEqual(list(traversal.traverse('C', filter)), + ['C', 'G', 'H']) + + def test_traversal_2(self): + traversal = RecursiveMakeTraversal() + traversal.add('', dirs=['A', 'B', 'C']) + traversal.add('A') + traversal.add('B', dirs=['D', 'E', 'F']) + traversal.add('C', dirs=['G', 'H', 'I']) + traversal.add('D') + traversal.add('E') + traversal.add('F') + traversal.add('G') + traversal.add('H') + traversal.add('I') + + start, deps = traversal.compute_dependencies() + self.assertEqual(start, ('I',)) + self.assertEqual(deps, { + 'A': ('',), + 'B': ('A',), + 'C': ('F',), + 'D': ('B',), + 'E': ('D',), + 'F': ('E',), + 'G': ('C',), + 'H': ('G',), + 'I': ('H',), + }) + + def test_traversal_filter(self): + traversal = RecursiveMakeTraversal() + traversal.add('', dirs=['A', 'B', 'C']) + traversal.add('A') + traversal.add('B', dirs=['D', 'E', 'F']) + traversal.add('C', dirs=['G', 'H', 'I']) + traversal.add('D') + traversal.add('E') + traversal.add('F') + traversal.add('G') + traversal.add('H') + traversal.add('I') + + def filter(current, subdirs): + if current == 'B': + current = None + return current, [], subdirs.dirs + + start, deps = traversal.compute_dependencies(filter) + self.assertEqual(start, ('I',)) + self.assertEqual(deps, { + 'A': ('',), + 'C': ('F',), + 'D': ('A',), + 'E': ('D',), + 'F': ('E',), + 'G': ('C',), + 'H': ('G',), + 'I': ('H',), + }) + +class TestRecursiveMakeBackend(BackendTester): + def test_basic(self): + """Ensure the RecursiveMakeBackend works without error.""" + env = self._consume('stub0', RecursiveMakeBackend) + self.assertTrue(os.path.exists(mozpath.join(env.topobjdir, + 'backend.RecursiveMakeBackend'))) + self.assertTrue(os.path.exists(mozpath.join(env.topobjdir, + 'backend.RecursiveMakeBackend.in'))) + + def test_output_files(self): + """Ensure proper files are generated.""" + env = self._consume('stub0', RecursiveMakeBackend) + + expected = ['', 'dir1', 'dir2'] + + for d in expected: + out_makefile = mozpath.join(env.topobjdir, d, 'Makefile') + out_backend = mozpath.join(env.topobjdir, d, 'backend.mk') + + self.assertTrue(os.path.exists(out_makefile)) + self.assertTrue(os.path.exists(out_backend)) + + def test_makefile_conversion(self): + """Ensure Makefile.in is converted properly.""" + env = self._consume('stub0', RecursiveMakeBackend) + + p = mozpath.join(env.topobjdir, 'Makefile') + + lines = [l.strip() for l in open(p, 'rt').readlines()[1:] if not l.startswith('#')] + self.assertEqual(lines, [ + 'DEPTH := .', + 'topobjdir := %s' % env.topobjdir, + 'topsrcdir := %s' % env.topsrcdir, + 'srcdir := %s' % env.topsrcdir, + 'VPATH := %s' % env.topsrcdir, + 'relativesrcdir := .', + 'include $(DEPTH)/config/autoconf.mk', + '', + 'FOO := foo', + '', + 'include $(topsrcdir)/config/recurse.mk', + ]) + + def test_missing_makefile_in(self): + """Ensure missing Makefile.in results in Makefile creation.""" + env = self._consume('stub0', RecursiveMakeBackend) + + p = mozpath.join(env.topobjdir, 'dir2', 'Makefile') + self.assertTrue(os.path.exists(p)) + + lines = [l.strip() for l in open(p, 'rt').readlines()] + self.assertEqual(len(lines), 10) + + self.assertTrue(lines[0].startswith('# THIS FILE WAS AUTOMATICALLY')) + + def test_backend_mk(self): + """Ensure backend.mk file is written out properly.""" + env = self._consume('stub0', RecursiveMakeBackend) + + p = mozpath.join(env.topobjdir, 'backend.mk') + + lines = [l.strip() for l in open(p, 'rt').readlines()[2:]] + self.assertEqual(lines, [ + 'DIRS := dir1 dir2', + ]) + + # Make env.substs writable to add ENABLE_TESTS + env.substs = dict(env.substs) + env.substs['ENABLE_TESTS'] = '1' + self._consume('stub0', RecursiveMakeBackend, env=env) + p = mozpath.join(env.topobjdir, 'backend.mk') + + lines = [l.strip() for l in open(p, 'rt').readlines()[2:]] + self.assertEqual(lines, [ + 'DIRS := dir1 dir2 dir3', + ]) + + def test_mtime_no_change(self): + """Ensure mtime is not updated if file content does not change.""" + + env = self._consume('stub0', RecursiveMakeBackend) + + makefile_path = mozpath.join(env.topobjdir, 'Makefile') + backend_path = mozpath.join(env.topobjdir, 'backend.mk') + makefile_mtime = os.path.getmtime(makefile_path) + backend_mtime = os.path.getmtime(backend_path) + + reader = BuildReader(env) + emitter = TreeMetadataEmitter(env) + backend = RecursiveMakeBackend(env) + backend.consume(emitter.emit(reader.read_topsrcdir())) + + self.assertEqual(os.path.getmtime(makefile_path), makefile_mtime) + self.assertEqual(os.path.getmtime(backend_path), backend_mtime) + + def test_substitute_config_files(self): + """Ensure substituted config files are produced.""" + env = self._consume('substitute_config_files', RecursiveMakeBackend) + + p = mozpath.join(env.topobjdir, 'foo') + self.assertTrue(os.path.exists(p)) + lines = [l.strip() for l in open(p, 'rt').readlines()] + self.assertEqual(lines, [ + 'TEST = foo', + ]) + + def test_install_substitute_config_files(self): + """Ensure we recurse into the dirs that install substituted config files.""" + env = self._consume('install_substitute_config_files', RecursiveMakeBackend) + + root_deps_path = mozpath.join(env.topobjdir, 'root-deps.mk') + lines = [l.strip() for l in open(root_deps_path, 'rt').readlines()] + + # Make sure we actually recurse into the sub directory during export to + # install the subst file. + self.assertTrue(any(l == 'recurse_export: sub/export' for l in lines)) + + def test_variable_passthru(self): + """Ensure variable passthru is written out correctly.""" + env = self._consume('variable_passthru', RecursiveMakeBackend) + + backend_path = mozpath.join(env.topobjdir, 'backend.mk') + lines = [l.strip() for l in open(backend_path, 'rt').readlines()[2:]] + + expected = { + 'ALLOW_COMPILER_WARNINGS': [ + 'ALLOW_COMPILER_WARNINGS := 1', + ], + 'DISABLE_STL_WRAPPING': [ + 'DISABLE_STL_WRAPPING := 1', + ], + 'VISIBILITY_FLAGS': [ + 'VISIBILITY_FLAGS :=', + ], + 'RCFILE': [ + 'RCFILE := foo.rc', + ], + 'RESFILE': [ + 'RESFILE := bar.res', + ], + 'RCINCLUDE': [ + 'RCINCLUDE := bar.rc', + ], + 'DEFFILE': [ + 'DEFFILE := baz.def', + ], + 'MOZBUILD_CFLAGS': [ + 'MOZBUILD_CFLAGS += -fno-exceptions', + 'MOZBUILD_CFLAGS += -w', + ], + 'MOZBUILD_CXXFLAGS': [ + 'MOZBUILD_CXXFLAGS += -fcxx-exceptions', + "MOZBUILD_CXXFLAGS += '-option with spaces'", + ], + 'MOZBUILD_LDFLAGS': [ + "MOZBUILD_LDFLAGS += '-ld flag with spaces'", + 'MOZBUILD_LDFLAGS += -x', + 'MOZBUILD_LDFLAGS += -DELAYLOAD:foo.dll', + 'MOZBUILD_LDFLAGS += -DELAYLOAD:bar.dll', + ], + 'MOZBUILD_HOST_CFLAGS': [ + 'MOZBUILD_HOST_CFLAGS += -funroll-loops', + 'MOZBUILD_HOST_CFLAGS += -wall', + ], + 'MOZBUILD_HOST_CXXFLAGS': [ + 'MOZBUILD_HOST_CXXFLAGS += -funroll-loops-harder', + 'MOZBUILD_HOST_CXXFLAGS += -wall-day-everyday', + ], + 'WIN32_EXE_LDFLAGS': [ + 'WIN32_EXE_LDFLAGS += -subsystem:console', + ], + } + + for var, val in expected.items(): + # print("test_variable_passthru[%s]" % (var)) + found = [str for str in lines if str.startswith(var)] + self.assertEqual(found, val) + + def test_sources(self): + """Ensure SOURCES and HOST_SOURCES are handled properly.""" + env = self._consume('sources', RecursiveMakeBackend) + + backend_path = mozpath.join(env.topobjdir, 'backend.mk') + lines = [l.strip() for l in open(backend_path, 'rt').readlines()[2:]] + + expected = { + 'ASFILES': [ + 'ASFILES += bar.s', + 'ASFILES += foo.asm', + ], + 'CMMSRCS': [ + 'CMMSRCS += bar.mm', + 'CMMSRCS += foo.mm', + ], + 'CSRCS': [ + 'CSRCS += bar.c', + 'CSRCS += foo.c', + ], + 'HOST_CPPSRCS': [ + 'HOST_CPPSRCS += bar.cpp', + 'HOST_CPPSRCS += foo.cpp', + ], + 'HOST_CSRCS': [ + 'HOST_CSRCS += bar.c', + 'HOST_CSRCS += foo.c', + ], + 'SSRCS': [ + 'SSRCS += baz.S', + 'SSRCS += foo.S', + ], + } + + for var, val in expected.items(): + found = [str for str in lines if str.startswith(var)] + self.assertEqual(found, val) + + def test_exports(self): + """Ensure EXPORTS is handled properly.""" + env = self._consume('exports', RecursiveMakeBackend) + + # EXPORTS files should appear in the dist_include install manifest. + m = InstallManifest(path=mozpath.join(env.topobjdir, + '_build_manifests', 'install', 'dist_include')) + self.assertEqual(len(m), 7) + self.assertIn('foo.h', m) + self.assertIn('mozilla/mozilla1.h', m) + self.assertIn('mozilla/dom/dom2.h', m) + + def test_generated_files(self): + """Ensure GENERATED_FILES is handled properly.""" + env = self._consume('generated-files', RecursiveMakeBackend) + + backend_path = mozpath.join(env.topobjdir, 'backend.mk') + lines = [l.strip() for l in open(backend_path, 'rt').readlines()[2:]] + + expected = [ + 'export:: bar.c', + 'GARBAGE += bar.c', + 'EXTRA_MDDEPEND_FILES += bar.c.pp', + 'bar.c: %s/generate-bar.py' % env.topsrcdir, + '$(REPORT_BUILD)', + '$(call py_action,file_generate,%s/generate-bar.py baz bar.c $(MDDEPDIR)/bar.c.pp)' % env.topsrcdir, + '', + 'export:: foo.c', + 'GARBAGE += foo.c', + 'EXTRA_MDDEPEND_FILES += foo.c.pp', + 'foo.c: %s/generate-foo.py $(srcdir)/foo-data' % (env.topsrcdir), + '$(REPORT_BUILD)', + '$(call py_action,file_generate,%s/generate-foo.py main foo.c $(MDDEPDIR)/foo.c.pp $(srcdir)/foo-data)' % (env.topsrcdir), + '', + 'export:: quux.c', + 'GARBAGE += quux.c', + 'EXTRA_MDDEPEND_FILES += quux.c.pp', + ] + + self.maxDiff = None + self.assertEqual(lines, expected) + + def test_exports_generated(self): + """Ensure EXPORTS that are listed in GENERATED_FILES + are handled properly.""" + env = self._consume('exports-generated', RecursiveMakeBackend) + + # EXPORTS files should appear in the dist_include install manifest. + m = InstallManifest(path=mozpath.join(env.topobjdir, + '_build_manifests', 'install', 'dist_include')) + self.assertEqual(len(m), 8) + self.assertIn('foo.h', m) + self.assertIn('mozilla/mozilla1.h', m) + self.assertIn('mozilla/dom/dom1.h', m) + self.assertIn('gfx/gfx.h', m) + self.assertIn('bar.h', m) + self.assertIn('mozilla/mozilla2.h', m) + self.assertIn('mozilla/dom/dom2.h', m) + self.assertIn('mozilla/dom/dom3.h', m) + # EXPORTS files that are also GENERATED_FILES should be handled as + # INSTALL_TARGETS. + backend_path = mozpath.join(env.topobjdir, 'backend.mk') + lines = [l.strip() for l in open(backend_path, 'rt').readlines()[2:]] + expected = [ + 'export:: bar.h', + 'GARBAGE += bar.h', + 'EXTRA_MDDEPEND_FILES += bar.h.pp', + 'export:: mozilla2.h', + 'GARBAGE += mozilla2.h', + 'EXTRA_MDDEPEND_FILES += mozilla2.h.pp', + 'export:: dom2.h', + 'GARBAGE += dom2.h', + 'EXTRA_MDDEPEND_FILES += dom2.h.pp', + 'export:: dom3.h', + 'GARBAGE += dom3.h', + 'EXTRA_MDDEPEND_FILES += dom3.h.pp', + 'dist_include_FILES += bar.h', + 'dist_include_DEST := $(DEPTH)/dist/include/', + 'dist_include_TARGET := export', + 'INSTALL_TARGETS += dist_include', + 'dist_include_mozilla_FILES += mozilla2.h', + 'dist_include_mozilla_DEST := $(DEPTH)/dist/include/mozilla', + 'dist_include_mozilla_TARGET := export', + 'INSTALL_TARGETS += dist_include_mozilla', + 'dist_include_mozilla_dom_FILES += dom2.h', + 'dist_include_mozilla_dom_FILES += dom3.h', + 'dist_include_mozilla_dom_DEST := $(DEPTH)/dist/include/mozilla/dom', + 'dist_include_mozilla_dom_TARGET := export', + 'INSTALL_TARGETS += dist_include_mozilla_dom', + ] + self.maxDiff = None + self.assertEqual(lines, expected) + + def test_resources(self): + """Ensure RESOURCE_FILES is handled properly.""" + env = self._consume('resources', RecursiveMakeBackend) + + # RESOURCE_FILES should appear in the dist_bin install manifest. + m = InstallManifest(path=os.path.join(env.topobjdir, + '_build_manifests', 'install', 'dist_bin')) + self.assertEqual(len(m), 10) + self.assertIn('res/foo.res', m) + self.assertIn('res/fonts/font1.ttf', m) + self.assertIn('res/fonts/desktop/desktop2.ttf', m) + + self.assertIn('res/bar.res.in', m) + self.assertIn('res/tests/test.manifest', m) + self.assertIn('res/tests/extra.manifest', m) + + def test_branding_files(self): + """Ensure BRANDING_FILES is handled properly.""" + env = self._consume('branding-files', RecursiveMakeBackend) + + #BRANDING_FILES should appear in the dist_branding install manifest. + m = InstallManifest(path=os.path.join(env.topobjdir, + '_build_manifests', 'install', 'dist_branding')) + self.assertEqual(len(m), 3) + self.assertIn('bar.ico', m) + self.assertIn('quux.png', m) + self.assertIn('icons/foo.ico', m) + + def test_sdk_files(self): + """Ensure SDK_FILES is handled properly.""" + env = self._consume('sdk-files', RecursiveMakeBackend) + + #SDK_FILES should appear in the dist_sdk install manifest. + m = InstallManifest(path=os.path.join(env.topobjdir, + '_build_manifests', 'install', 'dist_sdk')) + self.assertEqual(len(m), 3) + self.assertIn('bar.ico', m) + self.assertIn('quux.png', m) + self.assertIn('icons/foo.ico', m) + + def test_test_manifests_files_written(self): + """Ensure test manifests get turned into files.""" + env = self._consume('test-manifests-written', RecursiveMakeBackend) + + tests_dir = mozpath.join(env.topobjdir, '_tests') + m_master = mozpath.join(tests_dir, 'testing', 'mochitest', 'tests', 'mochitest.ini') + x_master = mozpath.join(tests_dir, 'xpcshell', 'xpcshell.ini') + self.assertTrue(os.path.exists(m_master)) + self.assertTrue(os.path.exists(x_master)) + + lines = [l.strip() for l in open(x_master, 'rt').readlines()] + self.assertEqual(lines, [ + '; THIS FILE WAS AUTOMATICALLY GENERATED. DO NOT MODIFY BY HAND.', + '', + '[include:dir1/xpcshell.ini]', + '[include:xpcshell.ini]', + ]) + + all_tests_path = mozpath.join(env.topobjdir, 'all-tests.pkl') + self.assertTrue(os.path.exists(all_tests_path)) + + with open(all_tests_path, 'rb') as fh: + o = pickle.load(fh) + + self.assertIn('xpcshell.js', o) + self.assertIn('dir1/test_bar.js', o) + + self.assertEqual(len(o['xpcshell.js']), 1) + + def test_test_manifest_pattern_matches_recorded(self): + """Pattern matches in test manifests' support-files should be recorded.""" + env = self._consume('test-manifests-written', RecursiveMakeBackend) + m = InstallManifest(path=mozpath.join(env.topobjdir, + '_build_manifests', 'install', '_test_files')) + + # This is not the most robust test in the world, but it gets the job + # done. + entries = [e for e in m._dests.keys() if '**' in e] + self.assertEqual(len(entries), 1) + self.assertIn('support/**', entries[0]) + + def test_test_manifest_deffered_installs_written(self): + """Shared support files are written to their own data file by the backend.""" + env = self._consume('test-manifest-shared-support', RecursiveMakeBackend) + all_tests_path = mozpath.join(env.topobjdir, 'all-tests.pkl') + self.assertTrue(os.path.exists(all_tests_path)) + test_installs_path = mozpath.join(env.topobjdir, 'test-installs.pkl') + + with open(test_installs_path, 'r') as fh: + test_installs = pickle.load(fh) + + self.assertEqual(set(test_installs.keys()), + set(['child/test_sub.js', + 'child/data/**', + 'child/another-file.sjs'])) + for key in test_installs.keys(): + self.assertIn(key, test_installs) + + test_files_manifest = mozpath.join(env.topobjdir, + '_build_manifests', + 'install', + '_test_files') + + # First, read the generated for ini manifest contents. + m = InstallManifest(path=test_files_manifest) + + # Then, synthesize one from the test-installs.pkl file. This should + # allow us to re-create a subset of the above. + synthesized_manifest = InstallManifest() + for item, installs in test_installs.items(): + for install_info in installs: + if len(install_info) == 3: + synthesized_manifest.add_pattern_symlink(*install_info) + if len(install_info) == 2: + synthesized_manifest.add_symlink(*install_info) + + self.assertEqual(len(synthesized_manifest), 3) + for item, info in synthesized_manifest._dests.items(): + self.assertIn(item, m) + self.assertEqual(info, m._dests[item]) + + def test_xpidl_generation(self): + """Ensure xpidl files and directories are written out.""" + env = self._consume('xpidl', RecursiveMakeBackend) + + # Install manifests should contain entries. + install_dir = mozpath.join(env.topobjdir, '_build_manifests', + 'install') + self.assertTrue(os.path.isfile(mozpath.join(install_dir, 'dist_idl'))) + self.assertTrue(os.path.isfile(mozpath.join(install_dir, 'xpidl'))) + + m = InstallManifest(path=mozpath.join(install_dir, 'dist_idl')) + self.assertEqual(len(m), 2) + self.assertIn('bar.idl', m) + self.assertIn('foo.idl', m) + + m = InstallManifest(path=mozpath.join(install_dir, 'xpidl')) + self.assertIn('.deps/my_module.pp', m) + + m = InstallManifest(path=os.path.join(install_dir, 'dist_bin')) + self.assertIn('components/my_module.xpt', m) + self.assertIn('components/interfaces.manifest', m) + + m = InstallManifest(path=mozpath.join(install_dir, 'dist_include')) + self.assertIn('foo.h', m) + + p = mozpath.join(env.topobjdir, 'config/makefiles/xpidl') + self.assertTrue(os.path.isdir(p)) + + self.assertTrue(os.path.isfile(mozpath.join(p, 'Makefile'))) + + def test_old_install_manifest_deleted(self): + # Simulate an install manifest from a previous backend version. Ensure + # it is deleted. + env = self._get_environment('stub0') + purge_dir = mozpath.join(env.topobjdir, '_build_manifests', 'install') + manifest_path = mozpath.join(purge_dir, 'old_manifest') + os.makedirs(purge_dir) + m = InstallManifest() + m.write(path=manifest_path) + with open(mozpath.join( + env.topobjdir, 'backend.RecursiveMakeBackend'), 'w') as f: + f.write('%s\n' % manifest_path) + + self.assertTrue(os.path.exists(manifest_path)) + self._consume('stub0', RecursiveMakeBackend, env) + self.assertFalse(os.path.exists(manifest_path)) + + def test_install_manifests_written(self): + env, objs = self._emit('stub0') + backend = RecursiveMakeBackend(env) + + m = InstallManifest() + backend._install_manifests['testing'] = m + m.add_symlink(__file__, 'self') + backend.consume(objs) + + man_dir = mozpath.join(env.topobjdir, '_build_manifests', 'install') + self.assertTrue(os.path.isdir(man_dir)) + + expected = ['testing'] + for e in expected: + full = mozpath.join(man_dir, e) + self.assertTrue(os.path.exists(full)) + + m2 = InstallManifest(path=full) + self.assertEqual(m, m2) + + def test_ipdl_sources(self): + """Test that IPDL_SOURCES are written to ipdlsrcs.mk correctly.""" + env = self._consume('ipdl_sources', RecursiveMakeBackend) + + manifest_path = mozpath.join(env.topobjdir, + 'ipc', 'ipdl', 'ipdlsrcs.mk') + lines = [l.strip() for l in open(manifest_path, 'rt').readlines()] + + # Handle Windows paths correctly + topsrcdir = env.topsrcdir.replace(os.sep, '/') + + expected = [ + "ALL_IPDLSRCS := %s/bar/bar.ipdl %s/bar/bar2.ipdlh %s/foo/foo.ipdl %s/foo/foo2.ipdlh" % tuple([topsrcdir] * 4), + "CPPSRCS := UnifiedProtocols0.cpp", + "IPDLDIRS := %s/bar %s/foo" % (topsrcdir, topsrcdir), + ] + + found = [str for str in lines if str.startswith(('ALL_IPDLSRCS', + 'CPPSRCS', + 'IPDLDIRS'))] + self.assertEqual(found, expected) + + def test_defines(self): + """Test that DEFINES are written to backend.mk correctly.""" + env = self._consume('defines', RecursiveMakeBackend) + + backend_path = mozpath.join(env.topobjdir, 'backend.mk') + lines = [l.strip() for l in open(backend_path, 'rt').readlines()[2:]] + + var = 'DEFINES' + defines = [val for val in lines if val.startswith(var)] + + expected = ['DEFINES += -DFOO \'-DBAZ="ab\'\\\'\'cd"\' -UQUX -DBAR=7 -DVALUE=xyz'] + self.assertEqual(defines, expected) + + def test_host_defines(self): + """Test that HOST_DEFINES are written to backend.mk correctly.""" + env = self._consume('host-defines', RecursiveMakeBackend) + + backend_path = mozpath.join(env.topobjdir, 'backend.mk') + lines = [l.strip() for l in open(backend_path, 'rt').readlines()[2:]] + + var = 'HOST_DEFINES' + defines = [val for val in lines if val.startswith(var)] + + expected = ['HOST_DEFINES += -DFOO \'-DBAZ="ab\'\\\'\'cd"\' -UQUX -DBAR=7 -DVALUE=xyz'] + self.assertEqual(defines, expected) + + def test_local_includes(self): + """Test that LOCAL_INCLUDES are written to backend.mk correctly.""" + env = self._consume('local_includes', RecursiveMakeBackend) + + backend_path = mozpath.join(env.topobjdir, 'backend.mk') + lines = [l.strip() for l in open(backend_path, 'rt').readlines()[2:]] + + expected = [ + 'LOCAL_INCLUDES += -I$(srcdir)/bar/baz', + 'LOCAL_INCLUDES += -I$(srcdir)/foo', + ] + + found = [str for str in lines if str.startswith('LOCAL_INCLUDES')] + self.assertEqual(found, expected) + + def test_generated_includes(self): + """Test that GENERATED_INCLUDES are written to backend.mk correctly.""" + env = self._consume('generated_includes', RecursiveMakeBackend) + + backend_path = mozpath.join(env.topobjdir, 'backend.mk') + lines = [l.strip() for l in open(backend_path, 'rt').readlines()[2:]] + + topobjdir = env.topobjdir.replace('\\', '/') + + expected = [ + 'LOCAL_INCLUDES += -I$(CURDIR)/bar/baz', + 'LOCAL_INCLUDES += -I$(CURDIR)/foo', + ] + + found = [str for str in lines if str.startswith('LOCAL_INCLUDES')] + self.assertEqual(found, expected) + + def test_final_target(self): + """Test that FINAL_TARGET is written to backend.mk correctly.""" + env = self._consume('final_target', RecursiveMakeBackend) + + final_target_rule = "FINAL_TARGET = $(if $(XPI_NAME),$(DIST)/xpi-stage/$(XPI_NAME),$(DIST)/bin)$(DIST_SUBDIR:%=/%)" + expected = dict() + expected[env.topobjdir] = [] + expected[mozpath.join(env.topobjdir, 'both')] = [ + 'XPI_NAME = mycrazyxpi', + 'DIST_SUBDIR = asubdir', + final_target_rule + ] + expected[mozpath.join(env.topobjdir, 'dist-subdir')] = [ + 'DIST_SUBDIR = asubdir', + final_target_rule + ] + expected[mozpath.join(env.topobjdir, 'xpi-name')] = [ + 'XPI_NAME = mycrazyxpi', + final_target_rule + ] + expected[mozpath.join(env.topobjdir, 'final-target')] = [ + 'FINAL_TARGET = $(DEPTH)/random-final-target' + ] + for key, expected_rules in expected.iteritems(): + backend_path = mozpath.join(key, 'backend.mk') + lines = [l.strip() for l in open(backend_path, 'rt').readlines()[2:]] + found = [str for str in lines if + str.startswith('FINAL_TARGET') or str.startswith('XPI_NAME') or + str.startswith('DIST_SUBDIR')] + self.assertEqual(found, expected_rules) + + def test_final_target_pp_files(self): + """Test that FINAL_TARGET_PP_FILES is written to backend.mk correctly.""" + env = self._consume('dist-files', RecursiveMakeBackend) + + backend_path = mozpath.join(env.topobjdir, 'backend.mk') + lines = [l.strip() for l in open(backend_path, 'rt').readlines()[2:]] + + expected = [ + 'DIST_FILES_0 += $(srcdir)/install.rdf', + 'DIST_FILES_0 += $(srcdir)/main.js', + 'DIST_FILES_0_PATH := $(DEPTH)/dist/bin/', + 'DIST_FILES_0_TARGET := misc', + 'PP_TARGETS += DIST_FILES_0', + ] + + found = [str for str in lines if 'DIST_FILES' in str] + self.assertEqual(found, expected) + + def test_config(self): + """Test that CONFIGURE_SUBST_FILES are properly handled.""" + env = self._consume('test_config', RecursiveMakeBackend) + + self.assertEqual( + open(os.path.join(env.topobjdir, 'file'), 'r').readlines(), [ + '#ifdef foo\n', + 'bar baz\n', + '@bar@\n', + ]) + + def test_jar_manifests(self): + env = self._consume('jar-manifests', RecursiveMakeBackend) + + with open(os.path.join(env.topobjdir, 'backend.mk'), 'rb') as fh: + lines = fh.readlines() + + lines = [line.rstrip() for line in lines] + + self.assertIn('JAR_MANIFEST := %s/jar.mn' % env.topsrcdir, lines) + + def test_test_manifests_duplicate_support_files(self): + """Ensure duplicate support-files in test manifests work.""" + env = self._consume('test-manifests-duplicate-support-files', + RecursiveMakeBackend) + + p = os.path.join(env.topobjdir, '_build_manifests', 'install', '_test_files') + m = InstallManifest(p) + self.assertIn('testing/mochitest/tests/support-file.txt', m) + + def test_android_eclipse(self): + env = self._consume('android_eclipse', RecursiveMakeBackend) + + with open(mozpath.join(env.topobjdir, 'backend.mk'), 'rb') as fh: + lines = fh.readlines() + + lines = [line.rstrip() for line in lines] + + # Dependencies first. + self.assertIn('ANDROID_ECLIPSE_PROJECT_main1: target1 target2', lines) + self.assertIn('ANDROID_ECLIPSE_PROJECT_main4: target3 target4', lines) + + command_template = '\t$(call py_action,process_install_manifest,' + \ + '--no-remove --no-remove-all-directory-symlinks ' + \ + '--no-remove-empty-directories %s %s.manifest)' + # Commands second. + for project_name in ['main1', 'main2', 'library1', 'library2']: + stem = '%s/android_eclipse/%s' % (env.topobjdir, project_name) + self.assertIn(command_template % (stem, stem), lines) + + # Projects declared in subdirectories. + with open(mozpath.join(env.topobjdir, 'subdir', 'backend.mk'), 'rb') as fh: + lines = fh.readlines() + + lines = [line.rstrip() for line in lines] + + self.assertIn('ANDROID_ECLIPSE_PROJECT_submain: subtarget1 subtarget2', lines) + + for project_name in ['submain', 'sublibrary']: + # Destination and install manifest are relative to topobjdir. + stem = '%s/android_eclipse/%s' % (env.topobjdir, project_name) + self.assertIn(command_template % (stem, stem), lines) + + def test_install_manifests_package_tests(self): + """Ensure test suites honor package_tests=False.""" + env = self._consume('test-manifests-package-tests', RecursiveMakeBackend) + + all_tests_path = mozpath.join(env.topobjdir, 'all-tests.pkl') + self.assertTrue(os.path.exists(all_tests_path)) + + with open(all_tests_path, 'rb') as fh: + o = pickle.load(fh) + self.assertIn('mochitest.js', o) + self.assertIn('not_packaged.java', o) + + man_dir = mozpath.join(env.topobjdir, '_build_manifests', 'install') + self.assertTrue(os.path.isdir(man_dir)) + + full = mozpath.join(man_dir, '_test_files') + self.assertTrue(os.path.exists(full)) + + m = InstallManifest(path=full) + + # Only mochitest.js should be in the install manifest. + self.assertTrue('testing/mochitest/tests/mochitest.js' in m) + + # The path is odd here because we do not normalize at test manifest + # processing time. This is a fragile test because there's currently no + # way to iterate the manifest. + self.assertFalse('instrumentation/./not_packaged.java' in m) + + def test_binary_components(self): + """Ensure binary components are correctly handled.""" + env = self._consume('binary-components', RecursiveMakeBackend) + + with open(mozpath.join(env.topobjdir, 'foo', 'backend.mk')) as fh: + lines = fh.readlines()[2:] + + self.assertEqual(lines, [ + 'misc::\n', + '\t$(call py_action,buildlist,$(DEPTH)/dist/bin/chrome.manifest ' + + "'manifest components/components.manifest')\n", + '\t$(call py_action,buildlist,' + + '$(DEPTH)/dist/bin/components/components.manifest ' + + "'binary-component foo')\n", + 'LIBRARY_NAME := foo\n', + 'FORCE_SHARED_LIB := 1\n', + 'IMPORT_LIBRARY := foo\n', + 'SHARED_LIBRARY := foo\n', + 'IS_COMPONENT := 1\n', + 'DSO_SONAME := foo\n', + 'LIB_IS_C_ONLY := 1\n', + ]) + + with open(mozpath.join(env.topobjdir, 'bar', 'backend.mk')) as fh: + lines = fh.readlines()[2:] + + self.assertEqual(lines, [ + 'LIBRARY_NAME := bar\n', + 'FORCE_SHARED_LIB := 1\n', + 'IMPORT_LIBRARY := bar\n', + 'SHARED_LIBRARY := bar\n', + 'IS_COMPONENT := 1\n', + 'DSO_SONAME := bar\n', + 'LIB_IS_C_ONLY := 1\n', + ]) + + self.assertTrue(os.path.exists(mozpath.join(env.topobjdir, 'binaries.json'))) + with open(mozpath.join(env.topobjdir, 'binaries.json'), 'rb') as fh: + binaries = json.load(fh) + + self.assertEqual(binaries, { + 'programs': [], + 'shared_libraries': [ + { + 'basename': 'foo', + 'import_name': 'foo', + 'install_target': 'dist/bin', + 'lib_name': 'foo', + 'relobjdir': 'foo', + 'soname': 'foo', + }, + { + 'basename': 'bar', + 'import_name': 'bar', + 'install_target': 'dist/bin', + 'lib_name': 'bar', + 'relobjdir': 'bar', + 'soname': 'bar', + } + ], + }) + + +if __name__ == '__main__': + main() diff --git a/python/mozbuild/mozbuild/test/backend/test_visualstudio.py b/python/mozbuild/mozbuild/test/backend/test_visualstudio.py new file mode 100644 index 000000000..bfc95e552 --- /dev/null +++ b/python/mozbuild/mozbuild/test/backend/test_visualstudio.py @@ -0,0 +1,64 @@ +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, You can obtain one at http://mozilla.org/MPL/2.0/. + +from __future__ import unicode_literals + +from xml.dom.minidom import parse +import os +import unittest + +from mozbuild.backend.visualstudio import VisualStudioBackend +from mozbuild.test.backend.common import BackendTester + +from mozunit import main + + +class TestVisualStudioBackend(BackendTester): + @unittest.skip('Failing inconsistently in automation.') + def test_basic(self): + """Ensure we can consume our stub project.""" + + env = self._consume('visual-studio', VisualStudioBackend) + + msvc = os.path.join(env.topobjdir, 'msvc') + self.assertTrue(os.path.isdir(msvc)) + + self.assertTrue(os.path.isfile(os.path.join(msvc, 'mozilla.sln'))) + self.assertTrue(os.path.isfile(os.path.join(msvc, 'mozilla.props'))) + self.assertTrue(os.path.isfile(os.path.join(msvc, 'mach.bat'))) + self.assertTrue(os.path.isfile(os.path.join(msvc, 'binary_my_app.vcxproj'))) + self.assertTrue(os.path.isfile(os.path.join(msvc, 'target_full.vcxproj'))) + self.assertTrue(os.path.isfile(os.path.join(msvc, 'library_dir1.vcxproj'))) + self.assertTrue(os.path.isfile(os.path.join(msvc, 'library_dir1.vcxproj.user'))) + + d = parse(os.path.join(msvc, 'library_dir1.vcxproj')) + self.assertEqual(d.documentElement.tagName, 'Project') + els = d.getElementsByTagName('ClCompile') + self.assertEqual(len(els), 2) + + # mozilla-config.h should be explicitly listed as an include. + els = d.getElementsByTagName('NMakeForcedIncludes') + self.assertEqual(len(els), 1) + self.assertEqual(els[0].firstChild.nodeValue, + '$(TopObjDir)\\dist\\include\\mozilla-config.h') + + # LOCAL_INCLUDES get added to the include search path. + els = d.getElementsByTagName('NMakeIncludeSearchPath') + self.assertEqual(len(els), 1) + includes = els[0].firstChild.nodeValue.split(';') + self.assertIn(os.path.normpath('$(TopSrcDir)/includeA/foo'), includes) + self.assertIn(os.path.normpath('$(TopSrcDir)/dir1'), includes) + self.assertIn(os.path.normpath('$(TopObjDir)/dir1'), includes) + self.assertIn(os.path.normpath('$(TopObjDir)\\dist\\include'), includes) + + # DEFINES get added to the project. + els = d.getElementsByTagName('NMakePreprocessorDefinitions') + self.assertEqual(len(els), 1) + defines = els[0].firstChild.nodeValue.split(';') + self.assertIn('DEFINEFOO', defines) + self.assertIn('DEFINEBAR=bar', defines) + + +if __name__ == '__main__': + main() diff --git a/python/mozbuild/mozbuild/test/common.py b/python/mozbuild/mozbuild/test/common.py new file mode 100644 index 000000000..76a39b313 --- /dev/null +++ b/python/mozbuild/mozbuild/test/common.py @@ -0,0 +1,50 @@ +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, You can obtain one at http://mozilla.org/MPL/2.0/. + +from __future__ import unicode_literals + +from mach.logging import LoggingManager + +from mozbuild.util import ReadOnlyDict + +import mozpack.path as mozpath + + +# By including this module, tests get structured logging. +log_manager = LoggingManager() +log_manager.add_terminal_logging() + +# mozconfig is not a reusable type (it's actually a module) so, we +# have to mock it. +class MockConfig(object): + def __init__(self, + topsrcdir='/path/to/topsrcdir', + extra_substs={}, + error_is_fatal=True, + ): + self.topsrcdir = mozpath.abspath(topsrcdir) + self.topobjdir = mozpath.abspath('/path/to/topobjdir') + + self.substs = ReadOnlyDict({ + 'MOZ_FOO': 'foo', + 'MOZ_BAR': 'bar', + 'MOZ_TRUE': '1', + 'MOZ_FALSE': '', + 'DLL_PREFIX': 'lib', + 'DLL_SUFFIX': '.so' + }, **extra_substs) + + self.substs_unicode = ReadOnlyDict({k.decode('utf-8'): v.decode('utf-8', + 'replace') for k, v in self.substs.items()}) + + self.defines = self.substs + + self.external_source_dir = None + self.lib_prefix = 'lib' + self.lib_suffix = '.a' + self.import_prefix = 'lib' + self.import_suffix = '.so' + self.dll_prefix = 'lib' + self.dll_suffix = '.so' + self.error_is_fatal = error_is_fatal diff --git a/python/mozbuild/mozbuild/test/compilation/__init__.py b/python/mozbuild/mozbuild/test/compilation/__init__.py new file mode 100644 index 000000000..e69de29bb diff --git a/python/mozbuild/mozbuild/test/compilation/test_warnings.py b/python/mozbuild/mozbuild/test/compilation/test_warnings.py new file mode 100644 index 000000000..cd2406dfc --- /dev/null +++ b/python/mozbuild/mozbuild/test/compilation/test_warnings.py @@ -0,0 +1,241 @@ +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this file, +# You can obtain one at http://mozilla.org/MPL/2.0/. + +import os +import unittest + +from mozfile.mozfile import NamedTemporaryFile + +from mozbuild.compilation.warnings import CompilerWarning +from mozbuild.compilation.warnings import WarningsCollector +from mozbuild.compilation.warnings import WarningsDatabase + +from mozunit import main + +CLANG_TESTS = [ + ('foobar.cpp:123:10: warning: you messed up [-Wfoo]', + 'foobar.cpp', 123, 10, 'you messed up', '-Wfoo'), + ("c_locale_dummy.c:457:1: warning: (near initialization for " + "'full_wmonthname[0]') [-Wpointer-sign]", + 'c_locale_dummy.c', 457, 1, + "(near initialization for 'full_wmonthname[0]')", '-Wpointer-sign') +] + +MSVC_TESTS = [ + ("C:/mozilla-central/test/foo.cpp(793) : warning C4244: 'return' : " + "conversion from 'double' to 'uint32_t', possible loss of data", + 'C:/mozilla-central/test/foo.cpp', 793, 'C4244', + "'return' : conversion from 'double' to 'uint32_t', possible loss of " + 'data') +] + +CURRENT_LINE = 1 + +def get_warning(): + global CURRENT_LINE + + w = CompilerWarning() + w['filename'] = '/foo/bar/baz.cpp' + w['line'] = CURRENT_LINE + w['column'] = 12 + w['message'] = 'This is irrelevant' + + CURRENT_LINE += 1 + + return w + +class TestCompilerWarning(unittest.TestCase): + def test_equivalence(self): + w1 = CompilerWarning() + w2 = CompilerWarning() + + s = set() + + # Empty warnings should be equal. + self.assertEqual(w1, w2) + + s.add(w1) + s.add(w2) + + self.assertEqual(len(s), 1) + + w1['filename'] = '/foo.c' + w2['filename'] = '/bar.c' + + self.assertNotEqual(w1, w2) + + s = set() + s.add(w1) + s.add(w2) + + self.assertEqual(len(s), 2) + + w1['filename'] = '/foo.c' + w1['line'] = 5 + w2['line'] = 5 + + w2['filename'] = '/foo.c' + w1['column'] = 3 + w2['column'] = 3 + + self.assertEqual(w1, w2) + + def test_comparison(self): + w1 = CompilerWarning() + w2 = CompilerWarning() + + w1['filename'] = '/aaa.c' + w1['line'] = 5 + w1['column'] = 5 + + w2['filename'] = '/bbb.c' + w2['line'] = 5 + w2['column'] = 5 + + self.assertLess(w1, w2) + self.assertGreater(w2, w1) + self.assertGreaterEqual(w2, w1) + + w2['filename'] = '/aaa.c' + w2['line'] = 4 + w2['column'] = 6 + + self.assertLess(w2, w1) + self.assertGreater(w1, w2) + self.assertGreaterEqual(w1, w2) + + w2['filename'] = '/aaa.c' + w2['line'] = 5 + w2['column'] = 10 + + self.assertLess(w1, w2) + self.assertGreater(w2, w1) + self.assertGreaterEqual(w2, w1) + + w2['filename'] = '/aaa.c' + w2['line'] = 5 + w2['column'] = 5 + + self.assertLessEqual(w1, w2) + self.assertLessEqual(w2, w1) + self.assertGreaterEqual(w2, w1) + self.assertGreaterEqual(w1, w2) + +class TestWarningsParsing(unittest.TestCase): + def test_clang_parsing(self): + for source, filename, line, column, message, flag in CLANG_TESTS: + collector = WarningsCollector(resolve_files=False) + warning = collector.process_line(source) + + self.assertIsNotNone(warning) + + self.assertEqual(warning['filename'], filename) + self.assertEqual(warning['line'], line) + self.assertEqual(warning['column'], column) + self.assertEqual(warning['message'], message) + self.assertEqual(warning['flag'], flag) + + def test_msvc_parsing(self): + for source, filename, line, flag, message in MSVC_TESTS: + collector = WarningsCollector(resolve_files=False) + warning = collector.process_line(source) + + self.assertIsNotNone(warning) + + self.assertEqual(warning['filename'], os.path.normpath(filename)) + self.assertEqual(warning['line'], line) + self.assertEqual(warning['flag'], flag) + self.assertEqual(warning['message'], message) + +class TestWarningsDatabase(unittest.TestCase): + def test_basic(self): + db = WarningsDatabase() + + self.assertEqual(len(db), 0) + + for i in range(10): + db.insert(get_warning(), compute_hash=False) + + self.assertEqual(len(db), 10) + + warnings = list(db) + self.assertEqual(len(warnings), 10) + + def test_hashing(self): + """Ensure that hashing files on insert works.""" + db = WarningsDatabase() + + temp = NamedTemporaryFile(mode='wt') + temp.write('x' * 100) + temp.flush() + + w = CompilerWarning() + w['filename'] = temp.name + w['line'] = 1 + w['column'] = 4 + w['message'] = 'foo bar' + + # Should not throw. + db.insert(w) + + w['filename'] = 'DOES_NOT_EXIST' + + with self.assertRaises(Exception): + db.insert(w) + + def test_pruning(self): + """Ensure old warnings are removed from database appropriately.""" + db = WarningsDatabase() + + source_files = [] + for i in range(1, 21): + temp = NamedTemporaryFile(mode='wt') + temp.write('x' * (100 * i)) + temp.flush() + + # Keep reference so it doesn't get GC'd and deleted. + source_files.append(temp) + + w = CompilerWarning() + w['filename'] = temp.name + w['line'] = 1 + w['column'] = i * 10 + w['message'] = 'irrelevant' + + db.insert(w) + + self.assertEqual(len(db), 20) + + # If we change a source file, inserting a new warning should nuke the + # old one. + source_files[0].write('extra') + source_files[0].flush() + + w = CompilerWarning() + w['filename'] = source_files[0].name + w['line'] = 1 + w['column'] = 50 + w['message'] = 'replaced' + + db.insert(w) + + self.assertEqual(len(db), 20) + + warnings = list(db.warnings_for_file(source_files[0].name)) + self.assertEqual(len(warnings), 1) + self.assertEqual(warnings[0]['column'], w['column']) + + # If we delete the source file, calling prune should cause the warnings + # to go away. + old_filename = source_files[0].name + del source_files[0] + + self.assertFalse(os.path.exists(old_filename)) + + db.prune() + self.assertEqual(len(db), 19) + + +if __name__ == '__main__': + main() diff --git a/python/mozbuild/mozbuild/test/configure/common.py b/python/mozbuild/mozbuild/test/configure/common.py new file mode 100644 index 000000000..089d61a0d --- /dev/null +++ b/python/mozbuild/mozbuild/test/configure/common.py @@ -0,0 +1,279 @@ +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, You can obtain one at http://mozilla.org/MPL/2.0/. + +from __future__ import absolute_import, print_function, unicode_literals + +import copy +import errno +import os +import subprocess +import sys +import tempfile +import unittest + +from mozbuild.configure import ConfigureSandbox +from mozbuild.util import ReadOnlyNamespace +from mozpack import path as mozpath + +from StringIO import StringIO +from which import WhichError + +from buildconfig import ( + topobjdir, + topsrcdir, +) + + +def fake_short_path(path): + if sys.platform.startswith('win'): + return '/'.join(p.split(' ', 1)[0] + '~1' if ' 'in p else p + for p in mozpath.split(path)) + return path + +def ensure_exe_extension(path): + if sys.platform.startswith('win'): + return path + '.exe' + return path + + +class ConfigureTestVFS(object): + def __init__(self, paths): + self._paths = set(mozpath.abspath(p) for p in paths) + + def exists(self, path): + path = mozpath.abspath(path) + if path in self._paths: + return True + if mozpath.basedir(path, [topsrcdir, topobjdir]): + return os.path.exists(path) + return False + + def isfile(self, path): + path = mozpath.abspath(path) + if path in self._paths: + return True + if mozpath.basedir(path, [topsrcdir, topobjdir]): + return os.path.isfile(path) + return False + + +class ConfigureTestSandbox(ConfigureSandbox): + '''Wrapper around the ConfigureSandbox for testing purposes. + + Its arguments are the same as ConfigureSandbox, except for the additional + `paths` argument, which is a dict where the keys are file paths and the + values are either None or a function that will be called when the sandbox + calls an implemented function from subprocess with the key as command. + When the command is CONFIG_SHELL, the function for the path of the script + that follows will be called. + + The API for those functions is: + retcode, stdout, stderr = func(stdin, args) + + This class is only meant to implement the minimal things to make + moz.configure testing possible. As such, it takes shortcuts. + ''' + def __init__(self, paths, config, environ, *args, **kwargs): + self._search_path = environ.get('PATH', '').split(os.pathsep) + + self._subprocess_paths = { + mozpath.abspath(k): v for k, v in paths.iteritems() if v + } + + paths = paths.keys() + + environ = dict(environ) + if 'CONFIG_SHELL' not in environ: + environ['CONFIG_SHELL'] = mozpath.abspath('/bin/sh') + self._subprocess_paths[environ['CONFIG_SHELL']] = self.shell + paths.append(environ['CONFIG_SHELL']) + self._environ = copy.copy(environ) + + vfs = ConfigureTestVFS(paths) + + os_path = { + k: getattr(vfs, k) for k in dir(vfs) if not k.startswith('_') + } + + os_path.update(self.OS.path.__dict__) + + self.imported_os = ReadOnlyNamespace(path=ReadOnlyNamespace(**os_path)) + + super(ConfigureTestSandbox, self).__init__(config, environ, *args, + **kwargs) + + def _get_one_import(self, what): + if what == 'which.which': + return self.which + + if what == 'which': + return ReadOnlyNamespace( + which=self.which, + WhichError=WhichError, + ) + + if what == 'subprocess.Popen': + return self.Popen + + if what == 'subprocess': + return ReadOnlyNamespace( + CalledProcessError=subprocess.CalledProcessError, + check_output=self.check_output, + PIPE=subprocess.PIPE, + STDOUT=subprocess.STDOUT, + Popen=self.Popen, + ) + + if what == 'os.environ': + return self._environ + + if what == 'ctypes.wintypes': + return ReadOnlyNamespace( + LPCWSTR=0, + LPWSTR=1, + DWORD=2, + ) + + if what == 'ctypes': + class CTypesFunc(object): + def __init__(self, func): + self._func = func + + def __call__(self, *args, **kwargs): + return self._func(*args, **kwargs) + + + return ReadOnlyNamespace( + create_unicode_buffer=self.create_unicode_buffer, + windll=ReadOnlyNamespace( + kernel32=ReadOnlyNamespace( + GetShortPathNameW=CTypesFunc(self.GetShortPathNameW), + ) + ), + ) + + if what == '_winreg': + def OpenKey(*args, **kwargs): + raise WindowsError() + + return ReadOnlyNamespace( + HKEY_LOCAL_MACHINE=0, + OpenKey=OpenKey, + ) + + return super(ConfigureTestSandbox, self)._get_one_import(what) + + def create_unicode_buffer(self, *args, **kwargs): + class Buffer(object): + def __init__(self): + self.value = '' + + return Buffer() + + def GetShortPathNameW(self, path_in, path_out, length): + path_out.value = fake_short_path(path_in) + return length + + def which(self, command, path=None): + for parent in (path or self._search_path): + c = mozpath.abspath(mozpath.join(parent, command)) + for candidate in (c, ensure_exe_extension(c)): + if self.imported_os.path.exists(candidate): + return candidate + raise WhichError() + + def Popen(self, args, stdin=None, stdout=None, stderr=None, **kargs): + try: + program = self.which(args[0]) + except WhichError: + raise OSError(errno.ENOENT, 'File not found') + + func = self._subprocess_paths.get(program) + retcode, stdout, stderr = func(stdin, args[1:]) + + class Process(object): + def communicate(self, stdin=None): + return stdout, stderr + + def wait(self): + return retcode + + return Process() + + def check_output(self, args, **kwargs): + proc = self.Popen(args, **kwargs) + stdout, stderr = proc.communicate() + retcode = proc.wait() + if retcode: + raise subprocess.CalledProcessError(retcode, args, stdout) + return stdout + + def shell(self, stdin, args): + script = mozpath.abspath(args[0]) + if script in self._subprocess_paths: + return self._subprocess_paths[script](stdin, args[1:]) + return 127, '', 'File not found' + + +class BaseConfigureTest(unittest.TestCase): + HOST = 'x86_64-pc-linux-gnu' + + def setUp(self): + self._cwd = os.getcwd() + os.chdir(topobjdir) + + def tearDown(self): + os.chdir(self._cwd) + + def config_guess(self, stdin, args): + return 0, self.HOST, '' + + def config_sub(self, stdin, args): + return 0, args[0], '' + + def get_sandbox(self, paths, config, args=[], environ={}, mozconfig='', + out=None, logger=None): + kwargs = {} + if logger: + kwargs['logger'] = logger + else: + if not out: + out = StringIO() + kwargs['stdout'] = out + kwargs['stderr'] = out + + if hasattr(self, 'TARGET'): + target = ['--target=%s' % self.TARGET] + else: + target = [] + + if mozconfig: + fh, mozconfig_path = tempfile.mkstemp() + os.write(fh, mozconfig) + os.close(fh) + else: + mozconfig_path = os.path.join(os.path.dirname(__file__), 'data', + 'empty_mozconfig') + + try: + environ = dict( + environ, + OLD_CONFIGURE=os.path.join(topsrcdir, 'old-configure'), + MOZCONFIG=mozconfig_path) + + paths = dict(paths) + autoconf_dir = mozpath.join(topsrcdir, 'build', 'autoconf') + paths[mozpath.join(autoconf_dir, + 'config.guess')] = self.config_guess + paths[mozpath.join(autoconf_dir, 'config.sub')] = self.config_sub + + sandbox = ConfigureTestSandbox(paths, config, environ, + ['configure'] + target + args, + **kwargs) + sandbox.include_file(os.path.join(topsrcdir, 'moz.configure')) + + return sandbox + finally: + if mozconfig: + os.remove(mozconfig_path) diff --git a/python/mozbuild/mozbuild/test/configure/data/decorators.configure b/python/mozbuild/mozbuild/test/configure/data/decorators.configure new file mode 100644 index 000000000..e5e41c68a --- /dev/null +++ b/python/mozbuild/mozbuild/test/configure/data/decorators.configure @@ -0,0 +1,44 @@ +# -*- Mode: python; indent-tabs-mode: nil; tab-width: 40 -*- +# vim: set filetype=python: +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, You can obtain one at http://mozilla.org/MPL/2.0/. + +@template +def simple_decorator(func): + return func + +@template +def wrapper_decorator(func): + def wrapper(*args, **kwargs): + return func(*args, **kwargs) + return wrapper + +@template +def function_decorator(*args, **kwargs): + # We could return wrapper_decorator from above here, but then we wouldn't + # know if this works as expected because wrapper_decorator itself was + # modified or because the right thing happened here. + def wrapper_decorator(func): + def wrapper(*args, **kwargs): + return func(*args, **kwargs) + return wrapper + return wrapper_decorator + +@depends('--help') +@simple_decorator +def foo(help): + global FOO + FOO = 1 + +@depends('--help') +@wrapper_decorator +def bar(help): + global BAR + BAR = 1 + +@depends('--help') +@function_decorator('a', 'b', 'c') +def qux(help): + global QUX + QUX = 1 diff --git a/python/mozbuild/mozbuild/test/configure/data/empty_mozconfig b/python/mozbuild/mozbuild/test/configure/data/empty_mozconfig new file mode 100644 index 000000000..e69de29bb diff --git a/python/mozbuild/mozbuild/test/configure/data/extra.configure b/python/mozbuild/mozbuild/test/configure/data/extra.configure new file mode 100644 index 000000000..43fbf7c5d --- /dev/null +++ b/python/mozbuild/mozbuild/test/configure/data/extra.configure @@ -0,0 +1,13 @@ +# -*- Mode: python; indent-tabs-mode: nil; tab-width: 40 -*- +# vim: set filetype=python: +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, You can obtain one at http://mozilla.org/MPL/2.0/. + +option('--extra', help='Extra') + +@depends('--extra') +def extra(extra): + return extra + +set_config('EXTRA', extra) diff --git a/python/mozbuild/mozbuild/test/configure/data/imply_option/imm.configure b/python/mozbuild/mozbuild/test/configure/data/imply_option/imm.configure new file mode 100644 index 000000000..ad05e383c --- /dev/null +++ b/python/mozbuild/mozbuild/test/configure/data/imply_option/imm.configure @@ -0,0 +1,32 @@ +# -*- Mode: python; indent-tabs-mode: nil; tab-width: 40 -*- +# vim: set filetype=python: +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, You can obtain one at http://mozilla.org/MPL/2.0/. + +imply_option('--enable-foo', True) + +option('--enable-foo', help='enable foo') + +@depends('--enable-foo', '--help') +def foo(value, help): + if value: + return True + +imply_option('--enable-bar', ('foo', 'bar')) + +option('--enable-bar', nargs='*', help='enable bar') + +@depends('--enable-bar') +def bar(value): + if value: + return value + +imply_option('--enable-baz', 'BAZ') + +option('--enable-baz', nargs=1, help='enable baz') + +@depends('--enable-baz') +def bar(value): + if value: + return value diff --git a/python/mozbuild/mozbuild/test/configure/data/imply_option/infer.configure b/python/mozbuild/mozbuild/test/configure/data/imply_option/infer.configure new file mode 100644 index 000000000..2ad1506ef --- /dev/null +++ b/python/mozbuild/mozbuild/test/configure/data/imply_option/infer.configure @@ -0,0 +1,24 @@ +# -*- Mode: python; indent-tabs-mode: nil; tab-width: 40 -*- +# vim: set filetype=python: +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, You can obtain one at http://mozilla.org/MPL/2.0/. + +option('--enable-foo', help='enable foo') + +@depends('--enable-foo', '--help') +def foo(value, help): + if value: + return True + +imply_option('--enable-bar', foo) + + +option('--enable-bar', help='enable bar') + +@depends('--enable-bar') +def bar(value): + if value: + return value + +set_config('BAR', bar) diff --git a/python/mozbuild/mozbuild/test/configure/data/imply_option/infer_ko.configure b/python/mozbuild/mozbuild/test/configure/data/imply_option/infer_ko.configure new file mode 100644 index 000000000..72b88d7b5 --- /dev/null +++ b/python/mozbuild/mozbuild/test/configure/data/imply_option/infer_ko.configure @@ -0,0 +1,31 @@ +# -*- Mode: python; indent-tabs-mode: nil; tab-width: 40 -*- +# vim: set filetype=python: +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, You can obtain one at http://mozilla.org/MPL/2.0/. + +option('--enable-hoge', help='enable hoge') + +@depends('--enable-hoge') +def hoge(value): + return value + + +option('--enable-foo', help='enable foo') + +@depends('--enable-foo', hoge) +def foo(value, hoge): + if value: + return True + +imply_option('--enable-bar', foo) + + +option('--enable-bar', help='enable bar') + +@depends('--enable-bar') +def bar(value): + if value: + return value + +set_config('BAR', bar) diff --git a/python/mozbuild/mozbuild/test/configure/data/imply_option/negative.configure b/python/mozbuild/mozbuild/test/configure/data/imply_option/negative.configure new file mode 100644 index 000000000..ca8e9df3a --- /dev/null +++ b/python/mozbuild/mozbuild/test/configure/data/imply_option/negative.configure @@ -0,0 +1,34 @@ +# -*- Mode: python; indent-tabs-mode: nil; tab-width: 40 -*- +# vim: set filetype=python: +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, You can obtain one at http://mozilla.org/MPL/2.0/. + +option('--enable-foo', help='enable foo') + +@depends('--enable-foo') +def foo(value): + if value: + return False + +imply_option('--enable-bar', foo) + + +option('--disable-hoge', help='enable hoge') + +@depends('--disable-hoge') +def hoge(value): + if not value: + return False + +imply_option('--enable-bar', hoge) + + +option('--enable-bar', default=True, help='enable bar') + +@depends('--enable-bar') +def bar(value): + if not value: + return value + +set_config('BAR', bar) diff --git a/python/mozbuild/mozbuild/test/configure/data/imply_option/simple.configure b/python/mozbuild/mozbuild/test/configure/data/imply_option/simple.configure new file mode 100644 index 000000000..6d905ebbb --- /dev/null +++ b/python/mozbuild/mozbuild/test/configure/data/imply_option/simple.configure @@ -0,0 +1,24 @@ +# -*- Mode: python; indent-tabs-mode: nil; tab-width: 40 -*- +# vim: set filetype=python: +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, You can obtain one at http://mozilla.org/MPL/2.0/. + +option('--enable-foo', help='enable foo') + +@depends('--enable-foo') +def foo(value): + if value: + return True + +imply_option('--enable-bar', foo) + + +option('--enable-bar', help='enable bar') + +@depends('--enable-bar') +def bar(value): + if value: + return value + +set_config('BAR', bar) diff --git a/python/mozbuild/mozbuild/test/configure/data/imply_option/values.configure b/python/mozbuild/mozbuild/test/configure/data/imply_option/values.configure new file mode 100644 index 000000000..6af4b1eda --- /dev/null +++ b/python/mozbuild/mozbuild/test/configure/data/imply_option/values.configure @@ -0,0 +1,24 @@ +# -*- Mode: python; indent-tabs-mode: nil; tab-width: 40 -*- +# vim: set filetype=python: +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, You can obtain one at http://mozilla.org/MPL/2.0/. + +option('--enable-foo', nargs='*', help='enable foo') + +@depends('--enable-foo') +def foo(value): + if value: + return value + +imply_option('--enable-bar', foo) + + +option('--enable-bar', nargs='*', help='enable bar') + +@depends('--enable-bar') +def bar(value): + if value: + return value + +set_config('BAR', bar) diff --git a/python/mozbuild/mozbuild/test/configure/data/included.configure b/python/mozbuild/mozbuild/test/configure/data/included.configure new file mode 100644 index 000000000..5c056764d --- /dev/null +++ b/python/mozbuild/mozbuild/test/configure/data/included.configure @@ -0,0 +1,53 @@ +# -*- Mode: python; indent-tabs-mode: nil; tab-width: 40 -*- +# vim: set filetype=python: +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, You can obtain one at http://mozilla.org/MPL/2.0/. + +# For more complex and repetitive things, we can create templates +@template +def check_compiler_flag(flag): + @depends(is_gcc) + def check(value): + if value: + return [flag] + set_config('CFLAGS', check) + return check + + +check_compiler_flag('-Werror=foobar') + +# Normal functions can be used in @depends functions. +def fortytwo(): + return 42 + +def twentyone(): + yield 21 + +@depends(is_gcc) +def check(value): + if value: + return fortytwo() + +set_config('TEMPLATE_VALUE', check) + +@depends(is_gcc) +def check(value): + if value: + for val in twentyone(): + return val + +set_config('TEMPLATE_VALUE_2', check) + +# Normal functions can use @imports too to import modules. +@imports('sys') +def platform(): + return sys.platform + +option('--enable-imports-in-template', help='Imports in template') +@depends('--enable-imports-in-template') +def check(value): + if value: + return platform() + +set_config('PLATFORM', check) diff --git a/python/mozbuild/mozbuild/test/configure/data/moz.configure b/python/mozbuild/mozbuild/test/configure/data/moz.configure new file mode 100644 index 000000000..32c4b8535 --- /dev/null +++ b/python/mozbuild/mozbuild/test/configure/data/moz.configure @@ -0,0 +1,174 @@ +# -*- Mode: python; indent-tabs-mode: nil; tab-width: 40 -*- +# vim: set filetype=python: +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, You can obtain one at http://mozilla.org/MPL/2.0/. + +option('--enable-simple', help='Enable simple') + +# Setting MOZ_WITH_ENV in the environment has the same effect as passing +# --enable-with-env. +option('--enable-with-env', env='MOZ_WITH_ENV', help='Enable with env') + +# Optional values +option('--enable-values', nargs='*', help='Enable values') + +# Everything supported in the Option class is supported in option(). Assume +# the tests of the Option class are extensive about this. + +# Alternatively to --enable/--disable, there also is --with/--without. The +# difference is semantic only. Behavior is the same as --enable/--disable. + +# When the option name starts with --disable/--without, the default is for +# the option to be enabled. +option('--without-thing', help='Build without thing') + +# A --enable/--with option with a default of False is equivalent to a +# --disable/--without option. This can be used to change the defaults +# depending on e.g. the target or the built application. +option('--with-stuff', default=False, help='Build with stuff') + +# Other kinds of arbitrary options are also allowed. This is effectively +# equivalent to --enable/--with, with no possibility of --disable/--without. +option('--option', env='MOZ_OPTION', help='Option') + +# It is also possible to pass options through the environment only. +option(env='CC', nargs=1, help='C Compiler') + +# Call the function when the --enable-simple option is processed, with its +# OptionValue as argument. +@depends('--enable-simple') +def simple(simple): + if simple: + return simple + +set_config('ENABLED_SIMPLE', simple) + +# There can be multiple functions depending on the same option. +@depends('--enable-simple') +def simple(simple): + return simple + +set_config('SIMPLE', simple) + +@depends('--enable-with-env') +def with_env(with_env): + return with_env + +set_config('WITH_ENV', with_env) + +# It doesn't matter if the dependency is on --enable or --disable +@depends('--disable-values') +def with_env2(values): + return values + +set_config('VALUES', with_env2) + +# It is possible to @depends on environment-only options. +@depends('CC') +def is_gcc(cc): + return cc and 'gcc' in cc[0] + +set_config('IS_GCC', is_gcc) + +# It is possible to depend on the result from another function. +@depends(with_env2) +def with_env3(values): + return values + +set_config('VALUES2', with_env3) + +# @depends functions can also return results for use as input to another +# @depends. +@depends(with_env3) +def with_env4(values): + return values + +@depends(with_env4) +def with_env5(values): + return values + +set_config('VALUES3', with_env5) + +# The result from @depends functions can also be used as input to options. +# The result must be returned, not implied. The function must also depend +# on --help. +@depends('--enable-simple', '--help') +def simple(simple, help): + return 'simple' if simple else 'not-simple' + +option('--with-returned-default', default=simple, help='Returned default') + +@depends('--with-returned-default') +def default(value): + return value + +set_config('DEFAULTED', default) + +@depends('--enable-values', '--help') +def choices(values, help): + if len(values): + return { + 'alpha': ('a', 'b', 'c'), + 'numeric': ('0', '1', '2'), + }.get(values[0]) + +option('--returned-choices', choices=choices, help='Choices') + +@depends('--returned-choices') +def returned_choices(values): + return values + +set_config('CHOICES', returned_choices) + +# All options must be referenced by some @depends function. +# It is possible to depend on multiple options/functions +@depends('--without-thing', '--with-stuff', with_env4, '--option') +def remainder(*args): + return args + +set_config('REMAINDER', remainder) + +# It is possible to include other files to extend the configuration script. +include('included.configure') + +# It is also possible for the include file path to come from the result of a +# @depends function. That function needs to depend on '--help' like for option +# defaults and choices. +option('--enable-include', nargs=1, help='Include') +@depends('--enable-include', '--help') +def include_path(path, help): + return path[0] if path else None + +include(include_path) + +# Sandboxed functions can import from modules through the use of the @imports +# decorator. +# The order of the decorators matter: @imports needs to appear after other +# decorators. +option('--with-imports', nargs='?', help='Imports') + +# A limited set of functions from os.path are exposed by default. +@depends('--with-imports') +def with_imports(value): + if len(value): + return hasattr(os.path, 'abspath') + +set_config('HAS_ABSPATH', with_imports) + +# It is still possible to import the full set from os.path. +# It is also possible to cherry-pick builtins. +@depends('--with-imports') +@imports('os.path') +def with_imports(value): + if len(value): + return hasattr(os.path, 'getatime') + +set_config('HAS_GETATIME', with_imports) + +@depends('--with-imports') +def with_imports(value): + if len(value): + return hasattr(os.path, 'getatime') + +set_config('HAS_GETATIME2', with_imports) diff --git a/python/mozbuild/mozbuild/test/configure/data/set_config.configure b/python/mozbuild/mozbuild/test/configure/data/set_config.configure new file mode 100644 index 000000000..cf5743963 --- /dev/null +++ b/python/mozbuild/mozbuild/test/configure/data/set_config.configure @@ -0,0 +1,43 @@ +# -*- Mode: python; indent-tabs-mode: nil; tab-width: 40 -*- +# vim: set filetype=python: +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, You can obtain one at http://mozilla.org/MPL/2.0/. + +option('--set-foo', help='set foo') + +@depends('--set-foo') +def foo(value): + if value: + return True + +set_config('FOO', foo) + + +option('--set-bar', help='set bar') + +@depends('--set-bar') +def bar(value): + return bool(value) + +set_config('BAR', bar) + + +option('--set-value', nargs=1, help='set value') + +@depends('--set-value') +def set_value(value): + if value: + return value[0] + +set_config('VALUE', set_value) + + +option('--set-name', nargs=1, help='set name') + +@depends('--set-name') +def set_name(value): + if value: + return value[0] + +set_config(set_name, True) diff --git a/python/mozbuild/mozbuild/test/configure/data/set_define.configure b/python/mozbuild/mozbuild/test/configure/data/set_define.configure new file mode 100644 index 000000000..422263427 --- /dev/null +++ b/python/mozbuild/mozbuild/test/configure/data/set_define.configure @@ -0,0 +1,43 @@ +# -*- Mode: python; indent-tabs-mode: nil; tab-width: 40 -*- +# vim: set filetype=python: +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, You can obtain one at http://mozilla.org/MPL/2.0/. + +option('--set-foo', help='set foo') + +@depends('--set-foo') +def foo(value): + if value: + return True + +set_define('FOO', foo) + + +option('--set-bar', help='set bar') + +@depends('--set-bar') +def bar(value): + return bool(value) + +set_define('BAR', bar) + + +option('--set-value', nargs=1, help='set value') + +@depends('--set-value') +def set_value(value): + if value: + return value[0] + +set_define('VALUE', set_value) + + +option('--set-name', nargs=1, help='set name') + +@depends('--set-name') +def set_name(value): + if value: + return value[0] + +set_define(set_name, True) diff --git a/python/mozbuild/mozbuild/test/configure/data/subprocess.configure b/python/mozbuild/mozbuild/test/configure/data/subprocess.configure new file mode 100644 index 000000000..de6be9cec --- /dev/null +++ b/python/mozbuild/mozbuild/test/configure/data/subprocess.configure @@ -0,0 +1,23 @@ +# -*- Mode: python; indent-tabs-mode: nil; tab-width: 40 -*- +# vim: set filetype=python: +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, You can obtain one at http://mozilla.org/MPL/2.0/. + +@depends('--help') +@imports('codecs') +@imports(_from='mozbuild.configure.util', _import='getpreferredencoding') +@imports('os') +@imports(_from='__builtin__', _import='open') +def dies_when_logging(_): + test_file = 'test.txt' + quote_char = "'" + if getpreferredencoding().lower() == 'utf-8': + quote_char = '\u00B4'.encode('utf-8') + try: + with open(test_file, 'w+') as fh: + fh.write(quote_char) + out = check_cmd_output('cat', 'test.txt') + log.info(out) + finally: + os.remove(test_file) diff --git a/python/mozbuild/mozbuild/test/configure/lint.py b/python/mozbuild/mozbuild/test/configure/lint.py new file mode 100644 index 000000000..9965a60e9 --- /dev/null +++ b/python/mozbuild/mozbuild/test/configure/lint.py @@ -0,0 +1,65 @@ +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, You can obtain one at http://mozilla.org/MPL/2.0/. + +from __future__ import absolute_import, print_function, unicode_literals + +import os +import unittest +from StringIO import StringIO +from mozunit import main +from buildconfig import ( + topobjdir, + topsrcdir, +) + +from mozbuild.configure.lint import LintSandbox + + +test_path = os.path.abspath(__file__) + + +class LintMeta(type): + def __new__(mcs, name, bases, attrs): + def create_test(project, func): + def test(self): + return func(self, project) + return test + + for project in ( + 'b2g', + 'b2g/dev', + 'b2g/graphene', + 'browser', + 'embedding/ios', + 'extensions', + 'js', + 'mobile/android', + ): + attrs['test_%s' % project.replace('/', '_')] = create_test( + project, attrs['lint']) + + return type.__new__(mcs, name, bases, attrs) + + +class Lint(unittest.TestCase): + __metaclass__ = LintMeta + + def setUp(self): + self._curdir = os.getcwd() + os.chdir(topobjdir) + + def tearDown(self): + os.chdir(self._curdir) + + def lint(self, project): + sandbox = LintSandbox({ + 'OLD_CONFIGURE': os.path.join(topsrcdir, 'old-configure'), + 'MOZCONFIG': os.path.join(os.path.dirname(test_path), 'data', + 'empty_mozconfig'), + }, ['--enable-project=%s' % project]) + sandbox.run(os.path.join(topsrcdir, 'moz.configure')) + + +if __name__ == '__main__': + main() diff --git a/python/mozbuild/mozbuild/test/configure/test_checks_configure.py b/python/mozbuild/mozbuild/test/configure/test_checks_configure.py new file mode 100644 index 000000000..181c7acbd --- /dev/null +++ b/python/mozbuild/mozbuild/test/configure/test_checks_configure.py @@ -0,0 +1,940 @@ +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, You can obtain one at http://mozilla.org/MPL/2.0/. + +from __future__ import absolute_import, print_function, unicode_literals + +from StringIO import StringIO +import os +import sys +import textwrap +import unittest + +from mozunit import ( + main, + MockedOpen, +) + +from mozbuild.configure import ( + ConfigureError, + ConfigureSandbox, +) +from mozbuild.util import exec_ +from mozpack import path as mozpath + +from buildconfig import topsrcdir +from common import ( + ConfigureTestSandbox, + ensure_exe_extension, + fake_short_path, +) + + +class TestChecksConfigure(unittest.TestCase): + def test_checking(self): + out = StringIO() + sandbox = ConfigureSandbox({}, stdout=out, stderr=out) + base_dir = os.path.join(topsrcdir, 'build', 'moz.configure') + sandbox.include_file(os.path.join(base_dir, 'checks.configure')) + + exec_(textwrap.dedent(''' + @checking('for a thing') + def foo(value): + return value + '''), sandbox) + + foo = sandbox['foo'] + + foo(True) + self.assertEqual(out.getvalue(), 'checking for a thing... yes\n') + + out.truncate(0) + foo(False) + self.assertEqual(out.getvalue(), 'checking for a thing... no\n') + + out.truncate(0) + foo(42) + self.assertEqual(out.getvalue(), 'checking for a thing... 42\n') + + out.truncate(0) + foo('foo') + self.assertEqual(out.getvalue(), 'checking for a thing... foo\n') + + out.truncate(0) + data = ['foo', 'bar'] + foo(data) + self.assertEqual(out.getvalue(), 'checking for a thing... %r\n' % data) + + # When the function given to checking does nothing interesting, the + # behavior is not altered + exec_(textwrap.dedent(''' + @checking('for a thing', lambda x: x) + def foo(value): + return value + '''), sandbox) + + foo = sandbox['foo'] + + out.truncate(0) + foo(True) + self.assertEqual(out.getvalue(), 'checking for a thing... yes\n') + + out.truncate(0) + foo(False) + self.assertEqual(out.getvalue(), 'checking for a thing... no\n') + + out.truncate(0) + foo(42) + self.assertEqual(out.getvalue(), 'checking for a thing... 42\n') + + out.truncate(0) + foo('foo') + self.assertEqual(out.getvalue(), 'checking for a thing... foo\n') + + out.truncate(0) + data = ['foo', 'bar'] + foo(data) + self.assertEqual(out.getvalue(), 'checking for a thing... %r\n' % data) + + exec_(textwrap.dedent(''' + def munge(x): + if not x: + return 'not found' + if isinstance(x, (str, bool, int)): + return x + return ' '.join(x) + + @checking('for a thing', munge) + def foo(value): + return value + '''), sandbox) + + foo = sandbox['foo'] + + out.truncate(0) + foo(True) + self.assertEqual(out.getvalue(), 'checking for a thing... yes\n') + + out.truncate(0) + foo(False) + self.assertEqual(out.getvalue(), 'checking for a thing... not found\n') + + out.truncate(0) + foo(42) + self.assertEqual(out.getvalue(), 'checking for a thing... 42\n') + + out.truncate(0) + foo('foo') + self.assertEqual(out.getvalue(), 'checking for a thing... foo\n') + + out.truncate(0) + foo(['foo', 'bar']) + self.assertEqual(out.getvalue(), 'checking for a thing... foo bar\n') + + KNOWN_A = ensure_exe_extension(mozpath.abspath('/usr/bin/known-a')) + KNOWN_B = ensure_exe_extension(mozpath.abspath('/usr/local/bin/known-b')) + KNOWN_C = ensure_exe_extension(mozpath.abspath('/home/user/bin/known c')) + OTHER_A = ensure_exe_extension(mozpath.abspath('/lib/other/known-a')) + + def get_result(self, command='', args=[], environ={}, + prog='/bin/configure', extra_paths=None, + includes=('util.configure', 'checks.configure')): + config = {} + out = StringIO() + paths = { + self.KNOWN_A: None, + self.KNOWN_B: None, + self.KNOWN_C: None, + } + if extra_paths: + paths.update(extra_paths) + environ = dict(environ) + if 'PATH' not in environ: + environ['PATH'] = os.pathsep.join(os.path.dirname(p) for p in paths) + paths[self.OTHER_A] = None + sandbox = ConfigureTestSandbox(paths, config, environ, [prog] + args, + out, out) + base_dir = os.path.join(topsrcdir, 'build', 'moz.configure') + for f in includes: + sandbox.include_file(os.path.join(base_dir, f)) + + status = 0 + try: + exec_(command, sandbox) + sandbox.run() + except SystemExit as e: + status = e.code + + return config, out.getvalue(), status + + def test_check_prog(self): + config, out, status = self.get_result( + 'check_prog("FOO", ("known-a",))') + self.assertEqual(status, 0) + self.assertEqual(config, {'FOO': self.KNOWN_A}) + self.assertEqual(out, 'checking for foo... %s\n' % self.KNOWN_A) + + config, out, status = self.get_result( + 'check_prog("FOO", ("unknown", "known-b", "known c"))') + self.assertEqual(status, 0) + self.assertEqual(config, {'FOO': self.KNOWN_B}) + self.assertEqual(out, 'checking for foo... %s\n' % self.KNOWN_B) + + config, out, status = self.get_result( + 'check_prog("FOO", ("unknown", "unknown-2", "known c"))') + self.assertEqual(status, 0) + self.assertEqual(config, {'FOO': fake_short_path(self.KNOWN_C)}) + self.assertEqual(out, "checking for foo... '%s'\n" + % fake_short_path(self.KNOWN_C)) + + config, out, status = self.get_result( + 'check_prog("FOO", ("unknown",))') + self.assertEqual(status, 1) + self.assertEqual(config, {}) + self.assertEqual(out, textwrap.dedent('''\ + checking for foo... not found + DEBUG: foo: Trying unknown + ERROR: Cannot find foo + ''')) + + config, out, status = self.get_result( + 'check_prog("FOO", ("unknown", "unknown-2", "unknown 3"))') + self.assertEqual(status, 1) + self.assertEqual(config, {}) + self.assertEqual(out, textwrap.dedent('''\ + checking for foo... not found + DEBUG: foo: Trying unknown + DEBUG: foo: Trying unknown-2 + DEBUG: foo: Trying 'unknown 3' + ERROR: Cannot find foo + ''')) + + config, out, status = self.get_result( + 'check_prog("FOO", ("unknown", "unknown-2", "unknown 3"), ' + 'allow_missing=True)') + self.assertEqual(status, 0) + self.assertEqual(config, {'FOO': ':'}) + self.assertEqual(out, 'checking for foo... not found\n') + + @unittest.skipIf(not sys.platform.startswith('win'), 'Windows-only test') + def test_check_prog_exe(self): + config, out, status = self.get_result( + 'check_prog("FOO", ("unknown", "known-b", "known c"))', + ['FOO=known-a.exe']) + self.assertEqual(status, 0) + self.assertEqual(config, {'FOO': self.KNOWN_A}) + self.assertEqual(out, 'checking for foo... %s\n' % self.KNOWN_A) + + config, out, status = self.get_result( + 'check_prog("FOO", ("unknown", "known-b", "known c"))', + ['FOO=%s' % os.path.splitext(self.KNOWN_A)[0]]) + self.assertEqual(status, 0) + self.assertEqual(config, {'FOO': self.KNOWN_A}) + self.assertEqual(out, 'checking for foo... %s\n' % self.KNOWN_A) + + + def test_check_prog_with_args(self): + config, out, status = self.get_result( + 'check_prog("FOO", ("unknown", "known-b", "known c"))', + ['FOO=known-a']) + self.assertEqual(status, 0) + self.assertEqual(config, {'FOO': self.KNOWN_A}) + self.assertEqual(out, 'checking for foo... %s\n' % self.KNOWN_A) + + config, out, status = self.get_result( + 'check_prog("FOO", ("unknown", "known-b", "known c"))', + ['FOO=%s' % self.KNOWN_A]) + self.assertEqual(status, 0) + self.assertEqual(config, {'FOO': self.KNOWN_A}) + self.assertEqual(out, 'checking for foo... %s\n' % self.KNOWN_A) + + path = self.KNOWN_B.replace('known-b', 'known-a') + config, out, status = self.get_result( + 'check_prog("FOO", ("unknown", "known-b", "known c"))', + ['FOO=%s' % path]) + self.assertEqual(status, 1) + self.assertEqual(config, {}) + self.assertEqual(out, textwrap.dedent('''\ + checking for foo... not found + DEBUG: foo: Trying %s + ERROR: Cannot find foo + ''') % path) + + config, out, status = self.get_result( + 'check_prog("FOO", ("unknown",))', + ['FOO=known c']) + self.assertEqual(status, 0) + self.assertEqual(config, {'FOO': fake_short_path(self.KNOWN_C)}) + self.assertEqual(out, "checking for foo... '%s'\n" + % fake_short_path(self.KNOWN_C)) + + config, out, status = self.get_result( + 'check_prog("FOO", ("unknown", "unknown-2", "unknown 3"), ' + 'allow_missing=True)', ['FOO=unknown']) + self.assertEqual(status, 1) + self.assertEqual(config, {}) + self.assertEqual(out, textwrap.dedent('''\ + checking for foo... not found + DEBUG: foo: Trying unknown + ERROR: Cannot find foo + ''')) + + def test_check_prog_what(self): + config, out, status = self.get_result( + 'check_prog("CC", ("known-a",), what="the target C compiler")') + self.assertEqual(status, 0) + self.assertEqual(config, {'CC': self.KNOWN_A}) + self.assertEqual( + out, 'checking for the target C compiler... %s\n' % self.KNOWN_A) + + config, out, status = self.get_result( + 'check_prog("CC", ("unknown", "unknown-2", "unknown 3"),' + ' what="the target C compiler")') + self.assertEqual(status, 1) + self.assertEqual(config, {}) + self.assertEqual(out, textwrap.dedent('''\ + checking for the target C compiler... not found + DEBUG: cc: Trying unknown + DEBUG: cc: Trying unknown-2 + DEBUG: cc: Trying 'unknown 3' + ERROR: Cannot find the target C compiler + ''')) + + def test_check_prog_input(self): + config, out, status = self.get_result(textwrap.dedent(''' + option("--with-ccache", nargs=1, help="ccache") + check_prog("CCACHE", ("known-a",), input="--with-ccache") + '''), ['--with-ccache=known-b']) + self.assertEqual(status, 0) + self.assertEqual(config, {'CCACHE': self.KNOWN_B}) + self.assertEqual(out, 'checking for ccache... %s\n' % self.KNOWN_B) + + script = textwrap.dedent(''' + option(env="CC", nargs=1, help="compiler") + @depends("CC") + def compiler(value): + return value[0].split()[0] if value else None + check_prog("CC", ("known-a",), input=compiler) + ''') + config, out, status = self.get_result(script) + self.assertEqual(status, 0) + self.assertEqual(config, {'CC': self.KNOWN_A}) + self.assertEqual(out, 'checking for cc... %s\n' % self.KNOWN_A) + + config, out, status = self.get_result(script, ['CC=known-b']) + self.assertEqual(status, 0) + self.assertEqual(config, {'CC': self.KNOWN_B}) + self.assertEqual(out, 'checking for cc... %s\n' % self.KNOWN_B) + + config, out, status = self.get_result(script, ['CC=known-b -m32']) + self.assertEqual(status, 0) + self.assertEqual(config, {'CC': self.KNOWN_B}) + self.assertEqual(out, 'checking for cc... %s\n' % self.KNOWN_B) + + def test_check_prog_progs(self): + config, out, status = self.get_result( + 'check_prog("FOO", ())') + self.assertEqual(status, 0) + self.assertEqual(config, {}) + self.assertEqual(out, '') + + config, out, status = self.get_result( + 'check_prog("FOO", ())', ['FOO=known-a']) + self.assertEqual(status, 0) + self.assertEqual(config, {'FOO': self.KNOWN_A}) + self.assertEqual(out, 'checking for foo... %s\n' % self.KNOWN_A) + + script = textwrap.dedent(''' + option(env="TARGET", nargs=1, default="linux", help="target") + @depends("TARGET") + def compiler(value): + if value: + if value[0] == "linux": + return ("gcc", "clang") + if value[0] == "winnt": + return ("cl", "clang-cl") + check_prog("CC", compiler) + ''') + config, out, status = self.get_result(script) + self.assertEqual(status, 1) + self.assertEqual(config, {}) + self.assertEqual(out, textwrap.dedent('''\ + checking for cc... not found + DEBUG: cc: Trying gcc + DEBUG: cc: Trying clang + ERROR: Cannot find cc + ''')) + + config, out, status = self.get_result(script, ['TARGET=linux']) + self.assertEqual(status, 1) + self.assertEqual(config, {}) + self.assertEqual(out, textwrap.dedent('''\ + checking for cc... not found + DEBUG: cc: Trying gcc + DEBUG: cc: Trying clang + ERROR: Cannot find cc + ''')) + + config, out, status = self.get_result(script, ['TARGET=winnt']) + self.assertEqual(status, 1) + self.assertEqual(config, {}) + self.assertEqual(out, textwrap.dedent('''\ + checking for cc... not found + DEBUG: cc: Trying cl + DEBUG: cc: Trying clang-cl + ERROR: Cannot find cc + ''')) + + config, out, status = self.get_result(script, ['TARGET=none']) + self.assertEqual(status, 0) + self.assertEqual(config, {}) + self.assertEqual(out, '') + + config, out, status = self.get_result(script, ['TARGET=winnt', + 'CC=known-a']) + self.assertEqual(status, 0) + self.assertEqual(config, {'CC': self.KNOWN_A}) + self.assertEqual(out, 'checking for cc... %s\n' % self.KNOWN_A) + + config, out, status = self.get_result(script, ['TARGET=none', + 'CC=known-a']) + self.assertEqual(status, 0) + self.assertEqual(config, {'CC': self.KNOWN_A}) + self.assertEqual(out, 'checking for cc... %s\n' % self.KNOWN_A) + + def test_check_prog_configure_error(self): + with self.assertRaises(ConfigureError) as e: + self.get_result('check_prog("FOO", "foo")') + + self.assertEqual(e.exception.message, + 'progs must resolve to a list or tuple!') + + with self.assertRaises(ConfigureError) as e: + self.get_result( + 'foo = depends(when=True)(lambda: ("a", "b"))\n' + 'check_prog("FOO", ("known-a",), input=foo)' + ) + + self.assertEqual(e.exception.message, + 'input must resolve to a tuple or a list with a ' + 'single element, or a string') + + with self.assertRaises(ConfigureError) as e: + self.get_result( + 'foo = depends(when=True)(lambda: {"a": "b"})\n' + 'check_prog("FOO", ("known-a",), input=foo)' + ) + + self.assertEqual(e.exception.message, + 'input must resolve to a tuple or a list with a ' + 'single element, or a string') + + def test_check_prog_with_path(self): + config, out, status = self.get_result('check_prog("A", ("known-a",), paths=["/some/path"])') + self.assertEqual(status, 1) + self.assertEqual(config, {}) + self.assertEqual(out, textwrap.dedent('''\ + checking for a... not found + DEBUG: a: Trying known-a + ERROR: Cannot find a + ''')) + + config, out, status = self.get_result('check_prog("A", ("known-a",), paths=["%s"])' % + os.path.dirname(self.OTHER_A)) + self.assertEqual(status, 0) + self.assertEqual(config, {'A': self.OTHER_A}) + self.assertEqual(out, textwrap.dedent('''\ + checking for a... %s + ''' % self.OTHER_A)) + + dirs = map(mozpath.dirname, (self.OTHER_A, self.KNOWN_A)) + config, out, status = self.get_result(textwrap.dedent('''\ + check_prog("A", ("known-a",), paths=["%s"]) + ''' % os.pathsep.join(dirs))) + self.assertEqual(status, 0) + self.assertEqual(config, {'A': self.OTHER_A}) + self.assertEqual(out, textwrap.dedent('''\ + checking for a... %s + ''' % self.OTHER_A)) + + dirs = map(mozpath.dirname, (self.KNOWN_A, self.KNOWN_B)) + config, out, status = self.get_result(textwrap.dedent('''\ + check_prog("A", ("known-a",), paths=["%s", "%s"]) + ''' % (os.pathsep.join(dirs), self.OTHER_A))) + self.assertEqual(status, 0) + self.assertEqual(config, {'A': self.KNOWN_A}) + self.assertEqual(out, textwrap.dedent('''\ + checking for a... %s + ''' % self.KNOWN_A)) + + config, out, status = self.get_result('check_prog("A", ("known-a",), paths="%s")' % + os.path.dirname(self.OTHER_A)) + + self.assertEqual(status, 1) + self.assertEqual(config, {}) + self.assertEqual(out, textwrap.dedent('''\ + checking for a... + DEBUG: a: Trying known-a + ERROR: Paths provided to find_program must be a list of strings, not %r + ''' % mozpath.dirname(self.OTHER_A))) + + def test_java_tool_checks(self): + includes = ('util.configure', 'checks.configure', 'java.configure') + + def mock_valid_javac(_, args): + if len(args) == 1 and args[0] == '-version': + return 0, '1.7', '' + self.fail("Unexpected arguments to mock_valid_javac: %s" % args) + + # A valid set of tools in a standard location. + java = mozpath.abspath('/usr/bin/java') + javah = mozpath.abspath('/usr/bin/javah') + javac = mozpath.abspath('/usr/bin/javac') + jar = mozpath.abspath('/usr/bin/jar') + jarsigner = mozpath.abspath('/usr/bin/jarsigner') + keytool = mozpath.abspath('/usr/bin/keytool') + + paths = { + java: None, + javah: None, + javac: mock_valid_javac, + jar: None, + jarsigner: None, + keytool: None, + } + + config, out, status = self.get_result(includes=includes, extra_paths=paths) + self.assertEqual(status, 0) + self.assertEqual(config, { + 'JAVA': java, + 'JAVAH': javah, + 'JAVAC': javac, + 'JAR': jar, + 'JARSIGNER': jarsigner, + 'KEYTOOL': keytool, + }) + self.assertEqual(out, textwrap.dedent('''\ + checking for java... %s + checking for javah... %s + checking for jar... %s + checking for jarsigner... %s + checking for keytool... %s + checking for javac... %s + checking for javac version... 1.7 + ''' % (java, javah, jar, jarsigner, keytool, javac))) + + # An alternative valid set of tools referred to by JAVA_HOME. + alt_java = mozpath.abspath('/usr/local/bin/java') + alt_javah = mozpath.abspath('/usr/local/bin/javah') + alt_javac = mozpath.abspath('/usr/local/bin/javac') + alt_jar = mozpath.abspath('/usr/local/bin/jar') + alt_jarsigner = mozpath.abspath('/usr/local/bin/jarsigner') + alt_keytool = mozpath.abspath('/usr/local/bin/keytool') + alt_java_home = mozpath.dirname(mozpath.dirname(alt_java)) + + paths.update({ + alt_java: None, + alt_javah: None, + alt_javac: mock_valid_javac, + alt_jar: None, + alt_jarsigner: None, + alt_keytool: None, + }) + + config, out, status = self.get_result(includes=includes, + extra_paths=paths, + environ={ + 'JAVA_HOME': alt_java_home, + 'PATH': mozpath.dirname(java) + }) + self.assertEqual(status, 0) + self.assertEqual(config, { + 'JAVA': alt_java, + 'JAVAH': alt_javah, + 'JAVAC': alt_javac, + 'JAR': alt_jar, + 'JARSIGNER': alt_jarsigner, + 'KEYTOOL': alt_keytool, + }) + self.assertEqual(out, textwrap.dedent('''\ + checking for java... %s + checking for javah... %s + checking for jar... %s + checking for jarsigner... %s + checking for keytool... %s + checking for javac... %s + checking for javac version... 1.7 + ''' % (alt_java, alt_javah, alt_jar, alt_jarsigner, + alt_keytool, alt_javac))) + + # We can use --with-java-bin-path instead of JAVA_HOME to similar + # effect. + config, out, status = self.get_result( + args=['--with-java-bin-path=%s' % mozpath.dirname(alt_java)], + includes=includes, + extra_paths=paths, + environ={ + 'PATH': mozpath.dirname(java) + }) + self.assertEqual(status, 0) + self.assertEqual(config, { + 'JAVA': alt_java, + 'JAVAH': alt_javah, + 'JAVAC': alt_javac, + 'JAR': alt_jar, + 'JARSIGNER': alt_jarsigner, + 'KEYTOOL': alt_keytool, + }) + self.assertEqual(out, textwrap.dedent('''\ + checking for java... %s + checking for javah... %s + checking for jar... %s + checking for jarsigner... %s + checking for keytool... %s + checking for javac... %s + checking for javac version... 1.7 + ''' % (alt_java, alt_javah, alt_jar, alt_jarsigner, + alt_keytool, alt_javac))) + + # If --with-java-bin-path and JAVA_HOME are both set, + # --with-java-bin-path takes precedence. + config, out, status = self.get_result( + args=['--with-java-bin-path=%s' % mozpath.dirname(alt_java)], + includes=includes, + extra_paths=paths, + environ={ + 'PATH': mozpath.dirname(java), + 'JAVA_HOME': mozpath.dirname(mozpath.dirname(java)), + }) + self.assertEqual(status, 0) + self.assertEqual(config, { + 'JAVA': alt_java, + 'JAVAH': alt_javah, + 'JAVAC': alt_javac, + 'JAR': alt_jar, + 'JARSIGNER': alt_jarsigner, + 'KEYTOOL': alt_keytool, + }) + self.assertEqual(out, textwrap.dedent('''\ + checking for java... %s + checking for javah... %s + checking for jar... %s + checking for jarsigner... %s + checking for keytool... %s + checking for javac... %s + checking for javac version... 1.7 + ''' % (alt_java, alt_javah, alt_jar, alt_jarsigner, + alt_keytool, alt_javac))) + + def mock_old_javac(_, args): + if len(args) == 1 and args[0] == '-version': + return 0, '1.6.9', '' + self.fail("Unexpected arguments to mock_old_javac: %s" % args) + + # An old javac is fatal. + paths[javac] = mock_old_javac + config, out, status = self.get_result(includes=includes, + extra_paths=paths, + environ={ + 'PATH': mozpath.dirname(java) + }) + self.assertEqual(status, 1) + self.assertEqual(config, { + 'JAVA': java, + 'JAVAH': javah, + 'JAVAC': javac, + 'JAR': jar, + 'JARSIGNER': jarsigner, + 'KEYTOOL': keytool, + }) + self.assertEqual(out, textwrap.dedent('''\ + checking for java... %s + checking for javah... %s + checking for jar... %s + checking for jarsigner... %s + checking for keytool... %s + checking for javac... %s + checking for javac version... + ERROR: javac 1.7 or higher is required (found 1.6.9) + ''' % (java, javah, jar, jarsigner, keytool, javac))) + + # Any missing tool is fatal when these checks run. + del paths[jarsigner] + config, out, status = self.get_result(includes=includes, + extra_paths=paths, + environ={ + 'PATH': mozpath.dirname(java) + }) + self.assertEqual(status, 1) + self.assertEqual(config, { + 'JAVA': java, + 'JAVAH': javah, + 'JAR': jar, + 'JARSIGNER': ':', + }) + self.assertEqual(out, textwrap.dedent('''\ + checking for java... %s + checking for javah... %s + checking for jar... %s + checking for jarsigner... not found + ERROR: The program jarsigner was not found. Set $JAVA_HOME to your Java SDK directory or use '--with-java-bin-path={java-bin-dir}' + ''' % (java, javah, jar))) + + def test_pkg_check_modules(self): + mock_pkg_config_version = '0.10.0' + mock_pkg_config_path = mozpath.abspath('/usr/bin/pkg-config') + + def mock_pkg_config(_, args): + if args[0:2] == ['--errors-to-stdout', '--print-errors']: + assert len(args) == 3 + package = args[2] + if package == 'unknown': + return (1, "Package unknown was not found in the pkg-config search path.\n" + "Perhaps you should add the directory containing `unknown.pc'\n" + "to the PKG_CONFIG_PATH environment variable\n" + "No package 'unknown' found", '') + if package == 'valid': + return 0, '', '' + if package == 'new > 1.1': + return 1, "Requested 'new > 1.1' but version of new is 1.1", '' + if args[0] == '--cflags': + assert len(args) == 2 + return 0, '-I/usr/include/%s' % args[1], '' + if args[0] == '--libs': + assert len(args) == 2 + return 0, '-l%s' % args[1], '' + if args[0] == '--version': + return 0, mock_pkg_config_version, '' + self.fail("Unexpected arguments to mock_pkg_config: %s" % args) + + def get_result(cmd, args=[], extra_paths=None): + return self.get_result(textwrap.dedent('''\ + option('--disable-compile-environment', help='compile env') + include('%(topsrcdir)s/build/moz.configure/util.configure') + include('%(topsrcdir)s/build/moz.configure/checks.configure') + include('%(topsrcdir)s/build/moz.configure/pkg.configure') + ''' % {'topsrcdir': topsrcdir}) + cmd, args=args, extra_paths=extra_paths, + includes=()) + + extra_paths = { + mock_pkg_config_path: mock_pkg_config, + } + includes = ('util.configure', 'checks.configure', 'pkg.configure') + + config, output, status = get_result("pkg_check_modules('MOZ_VALID', 'valid')") + self.assertEqual(status, 1) + self.assertEqual(output, textwrap.dedent('''\ + checking for pkg_config... not found + ERROR: *** The pkg-config script could not be found. Make sure it is + *** in your path, or set the PKG_CONFIG environment variable + *** to the full path to pkg-config. + ''')) + + + config, output, status = get_result("pkg_check_modules('MOZ_VALID', 'valid')", + extra_paths=extra_paths) + self.assertEqual(status, 0) + self.assertEqual(output, textwrap.dedent('''\ + checking for pkg_config... %s + checking for pkg-config version... %s + checking for valid... yes + checking MOZ_VALID_CFLAGS... -I/usr/include/valid + checking MOZ_VALID_LIBS... -lvalid + ''' % (mock_pkg_config_path, mock_pkg_config_version))) + self.assertEqual(config, { + 'PKG_CONFIG': mock_pkg_config_path, + 'MOZ_VALID_CFLAGS': ('-I/usr/include/valid',), + 'MOZ_VALID_LIBS': ('-lvalid',), + }) + + config, output, status = get_result("pkg_check_modules('MOZ_UKNOWN', 'unknown')", + extra_paths=extra_paths) + self.assertEqual(status, 1) + self.assertEqual(output, textwrap.dedent('''\ + checking for pkg_config... %s + checking for pkg-config version... %s + checking for unknown... no + ERROR: Package unknown was not found in the pkg-config search path. + ERROR: Perhaps you should add the directory containing `unknown.pc' + ERROR: to the PKG_CONFIG_PATH environment variable + ERROR: No package 'unknown' found + ''' % (mock_pkg_config_path, mock_pkg_config_version))) + self.assertEqual(config, { + 'PKG_CONFIG': mock_pkg_config_path, + }) + + config, output, status = get_result("pkg_check_modules('MOZ_NEW', 'new > 1.1')", + extra_paths=extra_paths) + self.assertEqual(status, 1) + self.assertEqual(output, textwrap.dedent('''\ + checking for pkg_config... %s + checking for pkg-config version... %s + checking for new > 1.1... no + ERROR: Requested 'new > 1.1' but version of new is 1.1 + ''' % (mock_pkg_config_path, mock_pkg_config_version))) + self.assertEqual(config, { + 'PKG_CONFIG': mock_pkg_config_path, + }) + + # allow_missing makes missing packages non-fatal. + cmd = textwrap.dedent('''\ + have_new_module = pkg_check_modules('MOZ_NEW', 'new > 1.1', allow_missing=True) + @depends(have_new_module) + def log_new_module_error(mod): + if mod is not True: + log.info('Module not found.') + ''') + + config, output, status = get_result(cmd, extra_paths=extra_paths) + self.assertEqual(status, 0) + self.assertEqual(output, textwrap.dedent('''\ + checking for pkg_config... %s + checking for pkg-config version... %s + checking for new > 1.1... no + WARNING: Requested 'new > 1.1' but version of new is 1.1 + Module not found. + ''' % (mock_pkg_config_path, mock_pkg_config_version))) + self.assertEqual(config, { + 'PKG_CONFIG': mock_pkg_config_path, + }) + + config, output, status = get_result(cmd, + args=['--disable-compile-environment'], + extra_paths=extra_paths) + self.assertEqual(status, 0) + self.assertEqual(output, 'Module not found.\n') + self.assertEqual(config, {}) + + def mock_old_pkg_config(_, args): + if args[0] == '--version': + return 0, '0.8.10', '' + self.fail("Unexpected arguments to mock_old_pkg_config: %s" % args) + + extra_paths = { + mock_pkg_config_path: mock_old_pkg_config, + } + + config, output, status = get_result("pkg_check_modules('MOZ_VALID', 'valid')", + extra_paths=extra_paths) + self.assertEqual(status, 1) + self.assertEqual(output, textwrap.dedent('''\ + checking for pkg_config... %s + checking for pkg-config version... 0.8.10 + ERROR: *** Your version of pkg-config is too old. You need version 0.9.0 or newer. + ''' % mock_pkg_config_path)) + + def test_simple_keyfile(self): + includes = ('util.configure', 'checks.configure', 'keyfiles.configure') + + config, output, status = self.get_result( + "simple_keyfile('Mozilla API')", includes=includes) + self.assertEqual(status, 0) + self.assertEqual(output, textwrap.dedent('''\ + checking for the Mozilla API key... no + ''')) + self.assertEqual(config, { + 'MOZ_MOZILLA_API_KEY': 'no-mozilla-api-key', + }) + + config, output, status = self.get_result( + "simple_keyfile('Mozilla API')", + args=['--with-mozilla-api-keyfile=/foo/bar/does/not/exist'], + includes=includes) + self.assertEqual(status, 1) + self.assertEqual(output, textwrap.dedent('''\ + checking for the Mozilla API key... no + ERROR: '/foo/bar/does/not/exist': No such file or directory. + ''')) + self.assertEqual(config, {}) + + with MockedOpen({'key': ''}): + config, output, status = self.get_result( + "simple_keyfile('Mozilla API')", + args=['--with-mozilla-api-keyfile=key'], + includes=includes) + self.assertEqual(status, 1) + self.assertEqual(output, textwrap.dedent('''\ + checking for the Mozilla API key... no + ERROR: 'key' is empty. + ''')) + self.assertEqual(config, {}) + + with MockedOpen({'key': 'fake-key\n'}): + config, output, status = self.get_result( + "simple_keyfile('Mozilla API')", + args=['--with-mozilla-api-keyfile=key'], + includes=includes) + self.assertEqual(status, 0) + self.assertEqual(output, textwrap.dedent('''\ + checking for the Mozilla API key... yes + ''')) + self.assertEqual(config, { + 'MOZ_MOZILLA_API_KEY': 'fake-key', + }) + + def test_id_and_secret_keyfile(self): + includes = ('util.configure', 'checks.configure', 'keyfiles.configure') + + config, output, status = self.get_result( + "id_and_secret_keyfile('Bing API')", includes=includes) + self.assertEqual(status, 0) + self.assertEqual(output, textwrap.dedent('''\ + checking for the Bing API key... no + ''')) + self.assertEqual(config, { + 'MOZ_BING_API_CLIENTID': 'no-bing-api-clientid', + 'MOZ_BING_API_KEY': 'no-bing-api-key', + }) + + config, output, status = self.get_result( + "id_and_secret_keyfile('Bing API')", + args=['--with-bing-api-keyfile=/foo/bar/does/not/exist'], + includes=includes) + self.assertEqual(status, 1) + self.assertEqual(output, textwrap.dedent('''\ + checking for the Bing API key... no + ERROR: '/foo/bar/does/not/exist': No such file or directory. + ''')) + self.assertEqual(config, {}) + + with MockedOpen({'key': ''}): + config, output, status = self.get_result( + "id_and_secret_keyfile('Bing API')", + args=['--with-bing-api-keyfile=key'], + includes=includes) + self.assertEqual(status, 1) + self.assertEqual(output, textwrap.dedent('''\ + checking for the Bing API key... no + ERROR: 'key' is empty. + ''')) + self.assertEqual(config, {}) + + with MockedOpen({'key': 'fake-id fake-key\n'}): + config, output, status = self.get_result( + "id_and_secret_keyfile('Bing API')", + args=['--with-bing-api-keyfile=key'], + includes=includes) + self.assertEqual(status, 0) + self.assertEqual(output, textwrap.dedent('''\ + checking for the Bing API key... yes + ''')) + self.assertEqual(config, { + 'MOZ_BING_API_CLIENTID': 'fake-id', + 'MOZ_BING_API_KEY': 'fake-key', + }) + + with MockedOpen({'key': 'fake-key\n'}): + config, output, status = self.get_result( + "id_and_secret_keyfile('Bing API')", + args=['--with-bing-api-keyfile=key'], + includes=includes) + self.assertEqual(status, 1) + self.assertEqual(output, textwrap.dedent('''\ + checking for the Bing API key... no + ERROR: Bing API key file has an invalid format. + ''')) + self.assertEqual(config, {}) + + +if __name__ == '__main__': + main() diff --git a/python/mozbuild/mozbuild/test/configure/test_compile_checks.py b/python/mozbuild/mozbuild/test/configure/test_compile_checks.py new file mode 100644 index 000000000..5913dbe3d --- /dev/null +++ b/python/mozbuild/mozbuild/test/configure/test_compile_checks.py @@ -0,0 +1,403 @@ +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, You can obtain one at http://mozilla.org/MPL/2.0/. + +from __future__ import absolute_import, print_function, unicode_literals + +import os +import textwrap +import unittest +import mozpack.path as mozpath + +from StringIO import StringIO + +from buildconfig import topsrcdir +from common import ConfigureTestSandbox +from mozbuild.util import exec_ +from mozunit import main +from test_toolchain_helpers import FakeCompiler + + +class BaseCompileChecks(unittest.TestCase): + def get_mock_compiler(self, expected_test_content=None, expected_flags=None): + expected_flags = expected_flags or [] + def mock_compiler(stdin, args): + args, test_file = args[:-1], args[-1] + self.assertIn('-c', args) + for flag in expected_flags: + self.assertIn(flag, args) + + if expected_test_content: + with open(test_file) as fh: + test_content = fh.read() + self.assertEqual(test_content, expected_test_content) + + return FakeCompiler()(None, args) + return mock_compiler + + def do_compile_test(self, command, expected_test_content=None, + expected_flags=None): + + paths = { + os.path.abspath('/usr/bin/mockcc'): self.get_mock_compiler( + expected_test_content=expected_test_content, + expected_flags=expected_flags), + } + + base_dir = os.path.join(topsrcdir, 'build', 'moz.configure') + + mock_compiler_defs = textwrap.dedent('''\ + @depends(when=True) + def extra_toolchain_flags(): + return [] + + include('%s/compilers-util.configure') + + @compiler_class + @depends(when=True) + def c_compiler(): + return namespace( + flags=[], + type='gcc', + compiler=os.path.abspath('/usr/bin/mockcc'), + wrapper=[], + language='C', + ) + + @compiler_class + @depends(when=True) + def cxx_compiler(): + return namespace( + flags=[], + type='gcc', + compiler=os.path.abspath('/usr/bin/mockcc'), + wrapper=[], + language='C++', + ) + ''' % mozpath.normsep(base_dir)) + + config = {} + out = StringIO() + sandbox = ConfigureTestSandbox(paths, config, {}, ['/bin/configure'], + out, out) + sandbox.include_file(os.path.join(base_dir, 'util.configure')) + sandbox.include_file(os.path.join(base_dir, 'checks.configure')) + exec_(mock_compiler_defs, sandbox) + sandbox.include_file(os.path.join(base_dir, 'compile-checks.configure')) + + status = 0 + try: + exec_(command, sandbox) + sandbox.run() + except SystemExit as e: + status = e.code + + return config, out.getvalue(), status + + +class TestHeaderChecks(BaseCompileChecks): + def test_try_compile_include(self): + expected_test_content = textwrap.dedent('''\ + #include + #include + int + main(void) + { + + ; + return 0; + } + ''') + + cmd = textwrap.dedent('''\ + try_compile(['foo.h', 'bar.h'], language='C') + ''') + + config, out, status = self.do_compile_test(cmd, expected_test_content) + self.assertEqual(status, 0) + self.assertEqual(config, {}) + + def test_try_compile_flags(self): + expected_flags = ['--extra', '--flags'] + + cmd = textwrap.dedent('''\ + try_compile(language='C++', flags=['--flags', '--extra']) + ''') + + config, out, status = self.do_compile_test(cmd, expected_flags=expected_flags) + self.assertEqual(status, 0) + self.assertEqual(config, {}) + + def test_try_compile_failure(self): + cmd = textwrap.dedent('''\ + have_fn = try_compile(body='somefn();', flags=['-funknown-flag']) + set_config('HAVE_SOMEFN', have_fn) + + have_another = try_compile(body='anotherfn();', language='C') + set_config('HAVE_ANOTHERFN', have_another) + ''') + + config, out, status = self.do_compile_test(cmd) + self.assertEqual(status, 0) + self.assertEqual(config, { + 'HAVE_ANOTHERFN': True, + }) + + def test_try_compile_msg(self): + cmd = textwrap.dedent('''\ + known_flag = try_compile(language='C++', flags=['-fknown-flag'], + check_msg='whether -fknown-flag works') + set_config('HAVE_KNOWN_FLAG', known_flag) + ''') + config, out, status = self.do_compile_test(cmd) + self.assertEqual(status, 0) + self.assertEqual(config, {'HAVE_KNOWN_FLAG': True}) + self.assertEqual(out, textwrap.dedent('''\ + checking whether -fknown-flag works... yes + ''')) + + def test_check_header(self): + expected_test_content = textwrap.dedent('''\ + #include + int + main(void) + { + + ; + return 0; + } + ''') + + cmd = textwrap.dedent('''\ + check_header('foo.h') + ''') + + config, out, status = self.do_compile_test(cmd, + expected_test_content=expected_test_content) + self.assertEqual(status, 0) + self.assertEqual(config, {'DEFINES': {'HAVE_FOO_H': True}}) + self.assertEqual(out, textwrap.dedent('''\ + checking for foo.h... yes + ''')) + + def test_check_header_conditional(self): + cmd = textwrap.dedent('''\ + check_headers('foo.h', 'bar.h', when=never) + ''') + + config, out, status = self.do_compile_test(cmd) + self.assertEqual(status, 0) + self.assertEqual(out, '') + self.assertEqual(config, {'DEFINES':{}}) + + def test_check_header_include(self): + expected_test_content = textwrap.dedent('''\ + #include + #include + #include + int + main(void) + { + + ; + return 0; + } + ''') + + cmd = textwrap.dedent('''\ + have_foo = check_header('foo.h', includes=['std.h', 'bar.h']) + set_config('HAVE_FOO_H', have_foo) + ''') + + config, out, status = self.do_compile_test(cmd, + expected_test_content=expected_test_content) + + self.assertEqual(status, 0) + self.assertEqual(config, { + 'HAVE_FOO_H': True, + 'DEFINES': { + 'HAVE_FOO_H': True, + } + }) + self.assertEqual(out, textwrap.dedent('''\ + checking for foo.h... yes + ''')) + + def test_check_headers_multiple(self): + cmd = textwrap.dedent('''\ + baz_bar, quux_bar = check_headers('baz/foo-bar.h', 'baz-quux/foo-bar.h') + set_config('HAVE_BAZ_BAR', baz_bar) + set_config('HAVE_QUUX_BAR', quux_bar) + ''') + + config, out, status = self.do_compile_test(cmd) + self.assertEqual(status, 0) + self.assertEqual(config, { + 'HAVE_BAZ_BAR': True, + 'HAVE_QUUX_BAR': True, + 'DEFINES': { + 'HAVE_BAZ_FOO_BAR_H': True, + 'HAVE_BAZ_QUUX_FOO_BAR_H': True, + } + }) + self.assertEqual(out, textwrap.dedent('''\ + checking for baz/foo-bar.h... yes + checking for baz-quux/foo-bar.h... yes + ''')) + + def test_check_headers_not_found(self): + + cmd = textwrap.dedent('''\ + baz_bar, quux_bar = check_headers('baz/foo-bar.h', 'baz-quux/foo-bar.h', + flags=['-funknown-flag']) + set_config('HAVE_BAZ_BAR', baz_bar) + set_config('HAVE_QUUX_BAR', quux_bar) + ''') + + config, out, status = self.do_compile_test(cmd) + self.assertEqual(status, 0) + self.assertEqual(config, {'DEFINES': {}}) + self.assertEqual(out, textwrap.dedent('''\ + checking for baz/foo-bar.h... no + checking for baz-quux/foo-bar.h... no + ''')) + + +class TestWarningChecks(BaseCompileChecks): + def get_warnings(self): + return textwrap.dedent('''\ + set_config('_WARNINGS_CFLAGS', warnings_cflags) + set_config('_WARNINGS_CXXFLAGS', warnings_cxxflags) + ''') + + def test_check_and_add_gcc_warning(self): + for flag, expected_flags in ( + ('-Wfoo', ['-Werror', '-Wfoo']), + ('-Wno-foo', ['-Werror', '-Wfoo']), + ('-Werror=foo', ['-Werror=foo']), + ('-Wno-error=foo', ['-Wno-error=foo']), + ): + cmd = textwrap.dedent('''\ + check_and_add_gcc_warning('%s') + ''' % flag) + self.get_warnings() + + config, out, status = self.do_compile_test( + cmd, expected_flags=expected_flags) + self.assertEqual(status, 0) + self.assertEqual(config, { + '_WARNINGS_CFLAGS': [flag], + '_WARNINGS_CXXFLAGS': [flag], + }) + self.assertEqual(out, textwrap.dedent('''\ + checking whether the C compiler supports {flag}... yes + checking whether the C++ compiler supports {flag}... yes + '''.format(flag=flag))) + + def test_check_and_add_gcc_warning_one(self): + cmd = textwrap.dedent('''\ + check_and_add_gcc_warning('-Wfoo', cxx_compiler) + ''') + self.get_warnings() + + config, out, status = self.do_compile_test(cmd) + self.assertEqual(status, 0) + self.assertEqual(config, { + '_WARNINGS_CFLAGS': [], + '_WARNINGS_CXXFLAGS': ['-Wfoo'], + }) + self.assertEqual(out, textwrap.dedent('''\ + checking whether the C++ compiler supports -Wfoo... yes + ''')) + + def test_check_and_add_gcc_warning_when(self): + cmd = textwrap.dedent('''\ + @depends(when=True) + def never(): + return False + check_and_add_gcc_warning('-Wfoo', cxx_compiler, when=never) + ''') + self.get_warnings() + + config, out, status = self.do_compile_test(cmd) + self.assertEqual(status, 0) + self.assertEqual(config, { + '_WARNINGS_CFLAGS': [], + '_WARNINGS_CXXFLAGS': [], + }) + self.assertEqual(out, '') + + cmd = textwrap.dedent('''\ + @depends(when=True) + def always(): + return True + check_and_add_gcc_warning('-Wfoo', cxx_compiler, when=always) + ''') + self.get_warnings() + + config, out, status = self.do_compile_test(cmd) + self.assertEqual(status, 0) + self.assertEqual(config, { + '_WARNINGS_CFLAGS': [], + '_WARNINGS_CXXFLAGS': ['-Wfoo'], + }) + self.assertEqual(out, textwrap.dedent('''\ + checking whether the C++ compiler supports -Wfoo... yes + ''')) + + def test_add_gcc_warning(self): + cmd = textwrap.dedent('''\ + add_gcc_warning('-Wfoo') + ''') + self.get_warnings() + + config, out, status = self.do_compile_test(cmd) + self.assertEqual(status, 0) + self.assertEqual(config, { + '_WARNINGS_CFLAGS': ['-Wfoo'], + '_WARNINGS_CXXFLAGS': ['-Wfoo'], + }) + self.assertEqual(out, '') + + def test_add_gcc_warning_one(self): + cmd = textwrap.dedent('''\ + add_gcc_warning('-Wfoo', c_compiler) + ''') + self.get_warnings() + + config, out, status = self.do_compile_test(cmd) + self.assertEqual(status, 0) + self.assertEqual(config, { + '_WARNINGS_CFLAGS': ['-Wfoo'], + '_WARNINGS_CXXFLAGS': [], + }) + self.assertEqual(out, '') + + def test_add_gcc_warning_when(self): + cmd = textwrap.dedent('''\ + @depends(when=True) + def never(): + return False + add_gcc_warning('-Wfoo', c_compiler, when=never) + ''') + self.get_warnings() + + config, out, status = self.do_compile_test(cmd) + self.assertEqual(status, 0) + self.assertEqual(config, { + '_WARNINGS_CFLAGS': [], + '_WARNINGS_CXXFLAGS': [], + }) + self.assertEqual(out, '') + + cmd = textwrap.dedent('''\ + @depends(when=True) + def always(): + return True + add_gcc_warning('-Wfoo', c_compiler, when=always) + ''') + self.get_warnings() + + config, out, status = self.do_compile_test(cmd) + self.assertEqual(status, 0) + self.assertEqual(config, { + '_WARNINGS_CFLAGS': ['-Wfoo'], + '_WARNINGS_CXXFLAGS': [], + }) + self.assertEqual(out, '') + + +if __name__ == '__main__': + main() diff --git a/python/mozbuild/mozbuild/test/configure/test_configure.py b/python/mozbuild/mozbuild/test/configure/test_configure.py new file mode 100644 index 000000000..df97ba70d --- /dev/null +++ b/python/mozbuild/mozbuild/test/configure/test_configure.py @@ -0,0 +1,1273 @@ +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, You can obtain one at http://mozilla.org/MPL/2.0/. + +from __future__ import absolute_import, print_function, unicode_literals + +from StringIO import StringIO +import os +import sys +import textwrap +import unittest + +from mozunit import ( + main, + MockedOpen, +) + +from mozbuild.configure.options import ( + InvalidOptionError, + NegativeOptionValue, + PositiveOptionValue, +) +from mozbuild.configure import ( + ConfigureError, + ConfigureSandbox, +) +from mozbuild.util import exec_ + +import mozpack.path as mozpath + +test_data_path = mozpath.abspath(mozpath.dirname(__file__)) +test_data_path = mozpath.join(test_data_path, 'data') + + +class TestConfigure(unittest.TestCase): + def get_config(self, options=[], env={}, configure='moz.configure', + prog='/bin/configure'): + config = {} + out = StringIO() + sandbox = ConfigureSandbox(config, env, [prog] + options, out, out) + + sandbox.run(mozpath.join(test_data_path, configure)) + + if '--help' in options: + return out.getvalue(), config + self.assertEquals('', out.getvalue()) + return config + + def moz_configure(self, source): + return MockedOpen({ + os.path.join(test_data_path, + 'moz.configure'): textwrap.dedent(source) + }) + + def test_defaults(self): + config = self.get_config() + self.maxDiff = None + self.assertEquals({ + 'CHOICES': NegativeOptionValue(), + 'DEFAULTED': PositiveOptionValue(('not-simple',)), + 'IS_GCC': NegativeOptionValue(), + 'REMAINDER': (PositiveOptionValue(), NegativeOptionValue(), + NegativeOptionValue(), NegativeOptionValue()), + 'SIMPLE': NegativeOptionValue(), + 'VALUES': NegativeOptionValue(), + 'VALUES2': NegativeOptionValue(), + 'VALUES3': NegativeOptionValue(), + 'WITH_ENV': NegativeOptionValue(), + }, config) + + def test_help(self): + help, config = self.get_config(['--help'], prog='configure') + + self.assertEquals({}, config) + self.maxDiff = None + self.assertEquals( + 'Usage: configure [options]\n' + '\n' + 'Options: [defaults in brackets after descriptions]\n' + ' --help print this message\n' + ' --enable-simple Enable simple\n' + ' --enable-with-env Enable with env\n' + ' --enable-values Enable values\n' + ' --without-thing Build without thing\n' + ' --with-stuff Build with stuff\n' + ' --option Option\n' + ' --with-returned-default Returned default [not-simple]\n' + ' --returned-choices Choices\n' + ' --enable-imports-in-template\n' + ' Imports in template\n' + ' --enable-include Include\n' + ' --with-imports Imports\n' + '\n' + 'Environment variables:\n' + ' CC C Compiler\n', + help + ) + + def test_unknown(self): + with self.assertRaises(InvalidOptionError): + self.get_config(['--unknown']) + + def test_simple(self): + for config in ( + self.get_config(), + self.get_config(['--disable-simple']), + # Last option wins. + self.get_config(['--enable-simple', '--disable-simple']), + ): + self.assertNotIn('ENABLED_SIMPLE', config) + self.assertIn('SIMPLE', config) + self.assertEquals(NegativeOptionValue(), config['SIMPLE']) + + for config in ( + self.get_config(['--enable-simple']), + self.get_config(['--disable-simple', '--enable-simple']), + ): + self.assertIn('ENABLED_SIMPLE', config) + self.assertIn('SIMPLE', config) + self.assertEquals(PositiveOptionValue(), config['SIMPLE']) + self.assertIs(config['SIMPLE'], config['ENABLED_SIMPLE']) + + # --enable-simple doesn't take values. + with self.assertRaises(InvalidOptionError): + self.get_config(['--enable-simple=value']) + + def test_with_env(self): + for config in ( + self.get_config(), + self.get_config(['--disable-with-env']), + self.get_config(['--enable-with-env', '--disable-with-env']), + self.get_config(env={'MOZ_WITH_ENV': ''}), + # Options win over environment + self.get_config(['--disable-with-env'], + env={'MOZ_WITH_ENV': '1'}), + ): + self.assertIn('WITH_ENV', config) + self.assertEquals(NegativeOptionValue(), config['WITH_ENV']) + + for config in ( + self.get_config(['--enable-with-env']), + self.get_config(['--disable-with-env', '--enable-with-env']), + self.get_config(env={'MOZ_WITH_ENV': '1'}), + self.get_config(['--enable-with-env'], + env={'MOZ_WITH_ENV': ''}), + ): + self.assertIn('WITH_ENV', config) + self.assertEquals(PositiveOptionValue(), config['WITH_ENV']) + + with self.assertRaises(InvalidOptionError): + self.get_config(['--enable-with-env=value']) + + with self.assertRaises(InvalidOptionError): + self.get_config(env={'MOZ_WITH_ENV': 'value'}) + + def test_values(self, name='VALUES'): + for config in ( + self.get_config(), + self.get_config(['--disable-values']), + self.get_config(['--enable-values', '--disable-values']), + ): + self.assertIn(name, config) + self.assertEquals(NegativeOptionValue(), config[name]) + + for config in ( + self.get_config(['--enable-values']), + self.get_config(['--disable-values', '--enable-values']), + ): + self.assertIn(name, config) + self.assertEquals(PositiveOptionValue(), config[name]) + + config = self.get_config(['--enable-values=foo']) + self.assertIn(name, config) + self.assertEquals(PositiveOptionValue(('foo',)), config[name]) + + config = self.get_config(['--enable-values=foo,bar']) + self.assertIn(name, config) + self.assertTrue(config[name]) + self.assertEquals(PositiveOptionValue(('foo', 'bar')), config[name]) + + def test_values2(self): + self.test_values('VALUES2') + + def test_values3(self): + self.test_values('VALUES3') + + def test_returned_default(self): + config = self.get_config(['--enable-simple']) + self.assertIn('DEFAULTED', config) + self.assertEquals( + PositiveOptionValue(('simple',)), config['DEFAULTED']) + + config = self.get_config(['--disable-simple']) + self.assertIn('DEFAULTED', config) + self.assertEquals( + PositiveOptionValue(('not-simple',)), config['DEFAULTED']) + + def test_returned_choices(self): + for val in ('a', 'b', 'c'): + config = self.get_config( + ['--enable-values=alpha', '--returned-choices=%s' % val]) + self.assertIn('CHOICES', config) + self.assertEquals(PositiveOptionValue((val,)), config['CHOICES']) + + for val in ('0', '1', '2'): + config = self.get_config( + ['--enable-values=numeric', '--returned-choices=%s' % val]) + self.assertIn('CHOICES', config) + self.assertEquals(PositiveOptionValue((val,)), config['CHOICES']) + + with self.assertRaises(InvalidOptionError): + self.get_config(['--enable-values=numeric', + '--returned-choices=a']) + + with self.assertRaises(InvalidOptionError): + self.get_config(['--enable-values=alpha', '--returned-choices=0']) + + def test_included(self): + config = self.get_config(env={'CC': 'gcc'}) + self.assertIn('IS_GCC', config) + self.assertEquals(config['IS_GCC'], True) + + config = self.get_config( + ['--enable-include=extra.configure', '--extra']) + self.assertIn('EXTRA', config) + self.assertEquals(PositiveOptionValue(), config['EXTRA']) + + with self.assertRaises(InvalidOptionError): + self.get_config(['--extra']) + + def test_template(self): + config = self.get_config(env={'CC': 'gcc'}) + self.assertIn('CFLAGS', config) + self.assertEquals(config['CFLAGS'], ['-Werror=foobar']) + + config = self.get_config(env={'CC': 'clang'}) + self.assertNotIn('CFLAGS', config) + + def test_imports(self): + config = {} + out = StringIO() + sandbox = ConfigureSandbox(config, {}, [], out, out) + + with self.assertRaises(ImportError): + exec_(textwrap.dedent(''' + @template + def foo(): + import sys + foo()'''), + sandbox + ) + + exec_(textwrap.dedent(''' + @template + @imports('sys') + def foo(): + return sys'''), + sandbox + ) + + self.assertIs(sandbox['foo'](), sys) + + exec_(textwrap.dedent(''' + @template + @imports(_from='os', _import='path') + def foo(): + return path'''), + sandbox + ) + + self.assertIs(sandbox['foo'](), os.path) + + exec_(textwrap.dedent(''' + @template + @imports(_from='os', _import='path', _as='os_path') + def foo(): + return os_path'''), + sandbox + ) + + self.assertIs(sandbox['foo'](), os.path) + + exec_(textwrap.dedent(''' + @template + @imports('__builtin__') + def foo(): + return __builtin__'''), + sandbox + ) + + import __builtin__ + self.assertIs(sandbox['foo'](), __builtin__) + + exec_(textwrap.dedent(''' + @template + @imports(_from='__builtin__', _import='open') + def foo(): + return open('%s')''' % os.devnull), + sandbox + ) + + f = sandbox['foo']() + self.assertEquals(f.name, os.devnull) + f.close() + + # This unlocks the sandbox + exec_(textwrap.dedent(''' + @template + @imports(_import='__builtin__', _as='__builtins__') + def foo(): + import sys + return sys'''), + sandbox + ) + + self.assertIs(sandbox['foo'](), sys) + + exec_(textwrap.dedent(''' + @template + @imports('__sandbox__') + def foo(): + return __sandbox__'''), + sandbox + ) + + self.assertIs(sandbox['foo'](), sandbox) + + exec_(textwrap.dedent(''' + @template + @imports(_import='__sandbox__', _as='s') + def foo(): + return s'''), + sandbox + ) + + self.assertIs(sandbox['foo'](), sandbox) + + # Nothing leaked from the function being executed + self.assertEquals(sandbox.keys(), ['__builtins__', 'foo']) + self.assertEquals(sandbox['__builtins__'], ConfigureSandbox.BUILTINS) + + exec_(textwrap.dedent(''' + @template + @imports('sys') + def foo(): + @depends(when=True) + def bar(): + return sys + return bar + bar = foo()'''), + sandbox + ) + + with self.assertRaises(NameError) as e: + sandbox._depends[sandbox['bar']].result + + self.assertEquals(e.exception.message, + "global name 'sys' is not defined") + + def test_apply_imports(self): + imports = [] + + class CountApplyImportsSandbox(ConfigureSandbox): + def _apply_imports(self, *args, **kwargs): + imports.append((args, kwargs)) + super(CountApplyImportsSandbox, self)._apply_imports( + *args, **kwargs) + + config = {} + out = StringIO() + sandbox = CountApplyImportsSandbox(config, {}, [], out, out) + + exec_(textwrap.dedent(''' + @template + @imports('sys') + def foo(): + return sys + foo() + foo()'''), + sandbox + ) + + self.assertEquals(len(imports), 1) + + def test_os_path(self): + config = self.get_config(['--with-imports=%s' % __file__]) + self.assertIn('HAS_ABSPATH', config) + self.assertEquals(config['HAS_ABSPATH'], True) + self.assertIn('HAS_GETATIME', config) + self.assertEquals(config['HAS_GETATIME'], True) + self.assertIn('HAS_GETATIME2', config) + self.assertEquals(config['HAS_GETATIME2'], False) + + def test_template_call(self): + config = self.get_config(env={'CC': 'gcc'}) + self.assertIn('TEMPLATE_VALUE', config) + self.assertEquals(config['TEMPLATE_VALUE'], 42) + self.assertIn('TEMPLATE_VALUE_2', config) + self.assertEquals(config['TEMPLATE_VALUE_2'], 21) + + def test_template_imports(self): + config = self.get_config(['--enable-imports-in-template']) + self.assertIn('PLATFORM', config) + self.assertEquals(config['PLATFORM'], sys.platform) + + def test_decorators(self): + config = {} + out = StringIO() + sandbox = ConfigureSandbox(config, {}, [], out, out) + + sandbox.include_file(mozpath.join(test_data_path, 'decorators.configure')) + + self.assertNotIn('FOO', sandbox) + self.assertNotIn('BAR', sandbox) + self.assertNotIn('QUX', sandbox) + + def test_set_config(self): + def get_config(*args): + return self.get_config(*args, configure='set_config.configure') + + help, config = get_config(['--help']) + self.assertEquals(config, {}) + + config = get_config(['--set-foo']) + self.assertIn('FOO', config) + self.assertEquals(config['FOO'], True) + + config = get_config(['--set-bar']) + self.assertNotIn('FOO', config) + self.assertIn('BAR', config) + self.assertEquals(config['BAR'], True) + + config = get_config(['--set-value=qux']) + self.assertIn('VALUE', config) + self.assertEquals(config['VALUE'], 'qux') + + config = get_config(['--set-name=hoge']) + self.assertIn('hoge', config) + self.assertEquals(config['hoge'], True) + + config = get_config([]) + self.assertEquals(config, {'BAR': False}) + + with self.assertRaises(ConfigureError): + # Both --set-foo and --set-name=FOO are going to try to + # set_config('FOO'...) + get_config(['--set-foo', '--set-name=FOO']) + + def test_set_config_when(self): + with self.moz_configure(''' + option('--with-qux', help='qux') + set_config('FOO', 'foo', when=True) + set_config('BAR', 'bar', when=False) + set_config('QUX', 'qux', when='--with-qux') + '''): + config = self.get_config() + self.assertEquals(config, { + 'FOO': 'foo', + }) + config = self.get_config(['--with-qux']) + self.assertEquals(config, { + 'FOO': 'foo', + 'QUX': 'qux', + }) + + def test_set_define(self): + def get_config(*args): + return self.get_config(*args, configure='set_define.configure') + + help, config = get_config(['--help']) + self.assertEquals(config, {'DEFINES': {}}) + + config = get_config(['--set-foo']) + self.assertIn('FOO', config['DEFINES']) + self.assertEquals(config['DEFINES']['FOO'], True) + + config = get_config(['--set-bar']) + self.assertNotIn('FOO', config['DEFINES']) + self.assertIn('BAR', config['DEFINES']) + self.assertEquals(config['DEFINES']['BAR'], True) + + config = get_config(['--set-value=qux']) + self.assertIn('VALUE', config['DEFINES']) + self.assertEquals(config['DEFINES']['VALUE'], 'qux') + + config = get_config(['--set-name=hoge']) + self.assertIn('hoge', config['DEFINES']) + self.assertEquals(config['DEFINES']['hoge'], True) + + config = get_config([]) + self.assertEquals(config['DEFINES'], {'BAR': False}) + + with self.assertRaises(ConfigureError): + # Both --set-foo and --set-name=FOO are going to try to + # set_define('FOO'...) + get_config(['--set-foo', '--set-name=FOO']) + + def test_set_define_when(self): + with self.moz_configure(''' + option('--with-qux', help='qux') + set_define('FOO', 'foo', when=True) + set_define('BAR', 'bar', when=False) + set_define('QUX', 'qux', when='--with-qux') + '''): + config = self.get_config() + self.assertEquals(config['DEFINES'], { + 'FOO': 'foo', + }) + config = self.get_config(['--with-qux']) + self.assertEquals(config['DEFINES'], { + 'FOO': 'foo', + 'QUX': 'qux', + }) + + def test_imply_option_simple(self): + def get_config(*args): + return self.get_config( + *args, configure='imply_option/simple.configure') + + help, config = get_config(['--help']) + self.assertEquals(config, {}) + + config = get_config([]) + self.assertEquals(config, {}) + + config = get_config(['--enable-foo']) + self.assertIn('BAR', config) + self.assertEquals(config['BAR'], PositiveOptionValue()) + + with self.assertRaises(InvalidOptionError) as e: + get_config(['--enable-foo', '--disable-bar']) + + self.assertEquals( + e.exception.message, + "'--enable-bar' implied by '--enable-foo' conflicts with " + "'--disable-bar' from the command-line") + + def test_imply_option_negative(self): + def get_config(*args): + return self.get_config( + *args, configure='imply_option/negative.configure') + + help, config = get_config(['--help']) + self.assertEquals(config, {}) + + config = get_config([]) + self.assertEquals(config, {}) + + config = get_config(['--enable-foo']) + self.assertIn('BAR', config) + self.assertEquals(config['BAR'], NegativeOptionValue()) + + with self.assertRaises(InvalidOptionError) as e: + get_config(['--enable-foo', '--enable-bar']) + + self.assertEquals( + e.exception.message, + "'--disable-bar' implied by '--enable-foo' conflicts with " + "'--enable-bar' from the command-line") + + config = get_config(['--disable-hoge']) + self.assertIn('BAR', config) + self.assertEquals(config['BAR'], NegativeOptionValue()) + + with self.assertRaises(InvalidOptionError) as e: + get_config(['--disable-hoge', '--enable-bar']) + + self.assertEquals( + e.exception.message, + "'--disable-bar' implied by '--disable-hoge' conflicts with " + "'--enable-bar' from the command-line") + + def test_imply_option_values(self): + def get_config(*args): + return self.get_config( + *args, configure='imply_option/values.configure') + + help, config = get_config(['--help']) + self.assertEquals(config, {}) + + config = get_config([]) + self.assertEquals(config, {}) + + config = get_config(['--enable-foo=a']) + self.assertIn('BAR', config) + self.assertEquals(config['BAR'], PositiveOptionValue(('a',))) + + config = get_config(['--enable-foo=a,b']) + self.assertIn('BAR', config) + self.assertEquals(config['BAR'], PositiveOptionValue(('a','b'))) + + with self.assertRaises(InvalidOptionError) as e: + get_config(['--enable-foo=a,b', '--disable-bar']) + + self.assertEquals( + e.exception.message, + "'--enable-bar=a,b' implied by '--enable-foo' conflicts with " + "'--disable-bar' from the command-line") + + def test_imply_option_infer(self): + def get_config(*args): + return self.get_config( + *args, configure='imply_option/infer.configure') + + help, config = get_config(['--help']) + self.assertEquals(config, {}) + + config = get_config([]) + self.assertEquals(config, {}) + + with self.assertRaises(InvalidOptionError) as e: + get_config(['--enable-foo', '--disable-bar']) + + self.assertEquals( + e.exception.message, + "'--enable-bar' implied by '--enable-foo' conflicts with " + "'--disable-bar' from the command-line") + + with self.assertRaises(ConfigureError) as e: + self.get_config([], configure='imply_option/infer_ko.configure') + + self.assertEquals( + e.exception.message, + "Cannot infer what implies '--enable-bar'. Please add a `reason` " + "to the `imply_option` call.") + + def test_imply_option_immediate_value(self): + def get_config(*args): + return self.get_config( + *args, configure='imply_option/imm.configure') + + help, config = get_config(['--help']) + self.assertEquals(config, {}) + + config = get_config([]) + self.assertEquals(config, {}) + + config_path = mozpath.abspath( + mozpath.join(test_data_path, 'imply_option', 'imm.configure')) + + with self.assertRaisesRegexp(InvalidOptionError, + "--enable-foo' implied by 'imply_option at %s:7' conflicts with " + "'--disable-foo' from the command-line" % config_path): + get_config(['--disable-foo']) + + with self.assertRaisesRegexp(InvalidOptionError, + "--enable-bar=foo,bar' implied by 'imply_option at %s:16' conflicts" + " with '--enable-bar=a,b,c' from the command-line" % config_path): + get_config(['--enable-bar=a,b,c']) + + with self.assertRaisesRegexp(InvalidOptionError, + "--enable-baz=BAZ' implied by 'imply_option at %s:25' conflicts" + " with '--enable-baz=QUUX' from the command-line" % config_path): + get_config(['--enable-baz=QUUX']) + + def test_imply_option_failures(self): + with self.assertRaises(ConfigureError) as e: + with self.moz_configure(''' + imply_option('--with-foo', ('a',), 'bar') + '''): + self.get_config() + + self.assertEquals(e.exception.message, + "`--with-foo`, emitted from `%s` line 2, is unknown." + % mozpath.join(test_data_path, 'moz.configure')) + + with self.assertRaises(TypeError) as e: + with self.moz_configure(''' + imply_option('--with-foo', 42, 'bar') + + option('--with-foo', help='foo') + @depends('--with-foo') + def foo(value): + return value + '''): + self.get_config() + + self.assertEquals(e.exception.message, + "Unexpected type: 'int'") + + def test_imply_option_when(self): + with self.moz_configure(''' + option('--with-foo', help='foo') + imply_option('--with-qux', True, when='--with-foo') + option('--with-qux', help='qux') + set_config('QUX', depends('--with-qux')(lambda x: x)) + '''): + config = self.get_config() + self.assertEquals(config, { + 'QUX': NegativeOptionValue(), + }) + + config = self.get_config(['--with-foo']) + self.assertEquals(config, { + 'QUX': PositiveOptionValue(), + }) + + def test_option_failures(self): + with self.assertRaises(ConfigureError) as e: + with self.moz_configure('option("--with-foo", help="foo")'): + self.get_config() + + self.assertEquals( + e.exception.message, + 'Option `--with-foo` is not handled ; reference it with a @depends' + ) + + with self.assertRaises(ConfigureError) as e: + with self.moz_configure(''' + option("--with-foo", help="foo") + option("--with-foo", help="foo") + '''): + self.get_config() + + self.assertEquals( + e.exception.message, + 'Option `--with-foo` already defined' + ) + + with self.assertRaises(ConfigureError) as e: + with self.moz_configure(''' + option(env="MOZ_FOO", help="foo") + option(env="MOZ_FOO", help="foo") + '''): + self.get_config() + + self.assertEquals( + e.exception.message, + 'Option `MOZ_FOO` already defined' + ) + + with self.assertRaises(ConfigureError) as e: + with self.moz_configure(''' + option('--with-foo', env="MOZ_FOO", help="foo") + option(env="MOZ_FOO", help="foo") + '''): + self.get_config() + + self.assertEquals( + e.exception.message, + 'Option `MOZ_FOO` already defined' + ) + + with self.assertRaises(ConfigureError) as e: + with self.moz_configure(''' + option(env="MOZ_FOO", help="foo") + option('--with-foo', env="MOZ_FOO", help="foo") + '''): + self.get_config() + + self.assertEquals( + e.exception.message, + 'Option `MOZ_FOO` already defined' + ) + + with self.assertRaises(ConfigureError) as e: + with self.moz_configure(''' + option('--with-foo', env="MOZ_FOO", help="foo") + option('--with-foo', help="foo") + '''): + self.get_config() + + self.assertEquals( + e.exception.message, + 'Option `--with-foo` already defined' + ) + + def test_option_when(self): + with self.moz_configure(''' + option('--with-foo', help='foo', when=True) + option('--with-bar', help='bar', when=False) + option('--with-qux', env="QUX", help='qux', when='--with-foo') + + set_config('FOO', depends('--with-foo', when=True)(lambda x: x)) + set_config('BAR', depends('--with-bar', when=False)(lambda x: x)) + set_config('QUX', depends('--with-qux', when='--with-foo')(lambda x: x)) + '''): + config = self.get_config() + self.assertEquals(config, { + 'FOO': NegativeOptionValue(), + }) + + config = self.get_config(['--with-foo']) + self.assertEquals(config, { + 'FOO': PositiveOptionValue(), + 'QUX': NegativeOptionValue(), + }) + + config = self.get_config(['--with-foo', '--with-qux']) + self.assertEquals(config, { + 'FOO': PositiveOptionValue(), + 'QUX': PositiveOptionValue(), + }) + + with self.assertRaises(InvalidOptionError) as e: + self.get_config(['--with-bar']) + + self.assertEquals( + e.exception.message, + '--with-bar is not available in this configuration' + ) + + with self.assertRaises(InvalidOptionError) as e: + self.get_config(['--with-qux']) + + self.assertEquals( + e.exception.message, + '--with-qux is not available in this configuration' + ) + + with self.assertRaises(InvalidOptionError) as e: + self.get_config(['QUX=1']) + + self.assertEquals( + e.exception.message, + 'QUX is not available in this configuration' + ) + + config = self.get_config(env={'QUX': '1'}) + self.assertEquals(config, { + 'FOO': NegativeOptionValue(), + }) + + help, config = self.get_config(['--help']) + self.assertEquals(help, textwrap.dedent('''\ + Usage: configure [options] + + Options: [defaults in brackets after descriptions] + --help print this message + --with-foo foo + + Environment variables: + ''')) + + help, config = self.get_config(['--help', '--with-foo']) + self.assertEquals(help, textwrap.dedent('''\ + Usage: configure [options] + + Options: [defaults in brackets after descriptions] + --help print this message + --with-foo foo + --with-qux qux + + Environment variables: + ''')) + + with self.moz_configure(''' + option('--with-foo', help='foo', when=True) + set_config('FOO', depends('--with-foo')(lambda x: x)) + '''): + with self.assertRaises(ConfigureError) as e: + self.get_config() + + self.assertEquals(e.exception.message, + '@depends function needs the same `when` as ' + 'options it depends on') + + with self.moz_configure(''' + @depends(when=True) + def always(): + return True + @depends(when=True) + def always2(): + return True + option('--with-foo', help='foo', when=always) + set_config('FOO', depends('--with-foo', when=always2)(lambda x: x)) + '''): + with self.assertRaises(ConfigureError) as e: + self.get_config() + + self.assertEquals(e.exception.message, + '@depends function needs the same `when` as ' + 'options it depends on') + + def test_include_failures(self): + with self.assertRaises(ConfigureError) as e: + with self.moz_configure('include("../foo.configure")'): + self.get_config() + + self.assertEquals( + e.exception.message, + 'Cannot include `%s` because it is not in a subdirectory of `%s`' + % (mozpath.normpath(mozpath.join(test_data_path, '..', + 'foo.configure')), + mozpath.normsep(test_data_path)) + ) + + with self.assertRaises(ConfigureError) as e: + with self.moz_configure(''' + include('extra.configure') + include('extra.configure') + '''): + self.get_config() + + self.assertEquals( + e.exception.message, + 'Cannot include `%s` because it was included already.' + % mozpath.normpath(mozpath.join(test_data_path, + 'extra.configure')) + ) + + with self.assertRaises(TypeError) as e: + with self.moz_configure(''' + include(42) + '''): + self.get_config() + + self.assertEquals(e.exception.message, "Unexpected type: 'int'") + + def test_include_when(self): + with MockedOpen({ + os.path.join(test_data_path, 'moz.configure'): textwrap.dedent(''' + option('--with-foo', help='foo') + + include('always.configure', when=True) + include('never.configure', when=False) + include('foo.configure', when='--with-foo') + + set_config('FOO', foo) + set_config('BAR', bar) + set_config('QUX', qux) + '''), + os.path.join(test_data_path, 'always.configure'): textwrap.dedent(''' + option('--with-bar', help='bar') + @depends('--with-bar') + def bar(x): + if x: + return 'bar' + '''), + os.path.join(test_data_path, 'never.configure'): textwrap.dedent(''' + option('--with-qux', help='qux') + @depends('--with-qux') + def qux(x): + if x: + return 'qux' + '''), + os.path.join(test_data_path, 'foo.configure'): textwrap.dedent(''' + option('--with-foo-really', help='really foo') + @depends('--with-foo-really') + def foo(x): + if x: + return 'foo' + + include('foo2.configure', when='--with-foo-really') + '''), + os.path.join(test_data_path, 'foo2.configure'): textwrap.dedent(''' + set_config('FOO2', True) + '''), + }): + config = self.get_config() + self.assertEquals(config, {}) + + config = self.get_config(['--with-foo']) + self.assertEquals(config, {}) + + config = self.get_config(['--with-bar']) + self.assertEquals(config, { + 'BAR': 'bar', + }) + + with self.assertRaises(InvalidOptionError) as e: + self.get_config(['--with-qux']) + + self.assertEquals( + e.exception.message, + '--with-qux is not available in this configuration' + ) + + config = self.get_config(['--with-foo', '--with-foo-really']) + self.assertEquals(config, { + 'FOO': 'foo', + 'FOO2': True, + }) + + def test_sandbox_failures(self): + with self.assertRaises(KeyError) as e: + with self.moz_configure(''' + include = 42 + '''): + self.get_config() + + self.assertEquals(e.exception.message, 'Cannot reassign builtins') + + with self.assertRaises(KeyError) as e: + with self.moz_configure(''' + foo = 42 + '''): + self.get_config() + + self.assertEquals(e.exception.message, + 'Cannot assign `foo` because it is neither a ' + '@depends nor a @template') + + def test_depends_failures(self): + with self.assertRaises(ConfigureError) as e: + with self.moz_configure(''' + @depends() + def foo(): + return + '''): + self.get_config() + + self.assertEquals(e.exception.message, + "@depends needs at least one argument") + + with self.assertRaises(ConfigureError) as e: + with self.moz_configure(''' + @depends('--with-foo') + def foo(value): + return value + '''): + self.get_config() + + self.assertEquals(e.exception.message, + "'--with-foo' is not a known option. Maybe it's " + "declared too late?") + + with self.assertRaises(ConfigureError) as e: + with self.moz_configure(''' + @depends('--with-foo=42') + def foo(value): + return value + '''): + self.get_config() + + self.assertEquals(e.exception.message, + "Option must not contain an '='") + + with self.assertRaises(TypeError) as e: + with self.moz_configure(''' + @depends(42) + def foo(value): + return value + '''): + self.get_config() + + self.assertEquals(e.exception.message, + "Cannot use object of type 'int' as argument " + "to @depends") + + with self.assertRaises(ConfigureError) as e: + with self.moz_configure(''' + @depends('--help') + def foo(value): + yield + '''): + self.get_config() + + self.assertEquals(e.exception.message, + "Cannot decorate generator functions with @depends") + + with self.assertRaises(TypeError) as e: + with self.moz_configure(''' + depends('--help')(42) + '''): + self.get_config() + + self.assertEquals(e.exception.message, + "Unexpected type: 'int'") + + with self.assertRaises(ConfigureError) as e: + with self.moz_configure(''' + option('--foo', help='foo') + @depends('--foo') + def foo(value): + return value + + foo() + '''): + self.get_config() + + self.assertEquals(e.exception.message, + "The `foo` function may not be called") + + with self.assertRaises(TypeError) as e: + with self.moz_configure(''' + @depends('--help', foo=42) + def foo(_): + return + '''): + self.get_config() + + self.assertEquals(e.exception.message, + "depends_impl() got an unexpected keyword argument 'foo'") + + def test_depends_when(self): + with self.moz_configure(''' + @depends(when=True) + def foo(): + return 'foo' + + set_config('FOO', foo) + + @depends(when=False) + def bar(): + return 'bar' + + set_config('BAR', bar) + + option('--with-qux', help='qux') + @depends(when='--with-qux') + def qux(): + return 'qux' + + set_config('QUX', qux) + '''): + config = self.get_config() + self.assertEquals(config, { + 'FOO': 'foo', + }) + + config = self.get_config(['--with-qux']) + self.assertEquals(config, { + 'FOO': 'foo', + 'QUX': 'qux', + }) + + def test_imports_failures(self): + with self.assertRaises(ConfigureError) as e: + with self.moz_configure(''' + @imports('os') + @template + def foo(value): + return value + '''): + self.get_config() + + self.assertEquals(e.exception.message, + '@imports must appear after @template') + + with self.assertRaises(ConfigureError) as e: + with self.moz_configure(''' + option('--foo', help='foo') + @imports('os') + @depends('--foo') + def foo(value): + return value + '''): + self.get_config() + + self.assertEquals(e.exception.message, + '@imports must appear after @depends') + + for import_ in ( + "42", + "_from=42, _import='os'", + "_from='os', _import='path', _as=42", + ): + with self.assertRaises(TypeError) as e: + with self.moz_configure(''' + @imports(%s) + @template + def foo(value): + return value + ''' % import_): + self.get_config() + + self.assertEquals(e.exception.message, "Unexpected type: 'int'") + + with self.assertRaises(TypeError) as e: + with self.moz_configure(''' + @imports('os', 42) + @template + def foo(value): + return value + '''): + self.get_config() + + self.assertEquals(e.exception.message, "Unexpected type: 'int'") + + with self.assertRaises(ValueError) as e: + with self.moz_configure(''' + @imports('os*') + def foo(value): + return value + '''): + self.get_config() + + self.assertEquals(e.exception.message, + "Invalid argument to @imports: 'os*'") + + def test_only_when(self): + moz_configure = ''' + option('--enable-when', help='when') + @depends('--enable-when', '--help') + def when(value, _): + return bool(value) + + with only_when(when): + option('--foo', nargs='*', help='foo') + @depends('--foo') + def foo(value): + return value + + set_config('FOO', foo) + set_define('FOO', foo) + + # It is possible to depend on a function defined in a only_when + # block. It then resolves to `None`. + set_config('BAR', depends(foo)(lambda x: x)) + set_define('BAR', depends(foo)(lambda x: x)) + ''' + + with self.moz_configure(moz_configure): + config = self.get_config() + self.assertEqual(config, { + 'DEFINES': {}, + }) + + config = self.get_config(['--enable-when']) + self.assertEqual(config, { + 'BAR': NegativeOptionValue(), + 'FOO': NegativeOptionValue(), + 'DEFINES': { + 'BAR': NegativeOptionValue(), + 'FOO': NegativeOptionValue(), + }, + }) + + config = self.get_config(['--enable-when', '--foo=bar']) + self.assertEqual(config, { + 'BAR': PositiveOptionValue(['bar']), + 'FOO': PositiveOptionValue(['bar']), + 'DEFINES': { + 'BAR': PositiveOptionValue(['bar']), + 'FOO': PositiveOptionValue(['bar']), + }, + }) + + # The --foo option doesn't exist when --enable-when is not given. + with self.assertRaises(InvalidOptionError) as e: + self.get_config(['--foo']) + + self.assertEquals(e.exception.message, + '--foo is not available in this configuration') + + # Cannot depend on an option defined in a only_when block, because we + # don't know what OptionValue would make sense. + with self.moz_configure(moz_configure + ''' + set_config('QUX', depends('--foo')(lambda x: x)) + '''): + with self.assertRaises(ConfigureError) as e: + self.get_config() + + self.assertEquals(e.exception.message, + '@depends function needs the same `when` as ' + 'options it depends on') + + with self.moz_configure(moz_configure + ''' + set_config('QUX', depends('--foo', when=when)(lambda x: x)) + '''): + self.get_config(['--enable-when']) + + # Using imply_option for an option defined in a only_when block fails + # similarly if the imply_option happens outside the block. + with self.moz_configure(''' + imply_option('--foo', True) + ''' + moz_configure): + with self.assertRaises(InvalidOptionError) as e: + self.get_config() + + self.assertEquals(e.exception.message, + '--foo is not available in this configuration') + + # And similarly doesn't fail when the condition is true. + with self.moz_configure(''' + imply_option('--foo', True) + ''' + moz_configure): + self.get_config(['--enable-when']) + + +if __name__ == '__main__': + main() diff --git a/python/mozbuild/mozbuild/test/configure/test_lint.py b/python/mozbuild/mozbuild/test/configure/test_lint.py new file mode 100644 index 000000000..6ac2bb356 --- /dev/null +++ b/python/mozbuild/mozbuild/test/configure/test_lint.py @@ -0,0 +1,132 @@ +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, You can obtain one at http://mozilla.org/MPL/2.0/. + +from __future__ import absolute_import, print_function, unicode_literals + +from StringIO import StringIO +import os +import textwrap +import unittest + +from mozunit import ( + main, + MockedOpen, +) + +from mozbuild.configure import ConfigureError +from mozbuild.configure.lint import LintSandbox + +import mozpack.path as mozpath + +test_data_path = mozpath.abspath(mozpath.dirname(__file__)) +test_data_path = mozpath.join(test_data_path, 'data') + + +class TestLint(unittest.TestCase): + def lint_test(self, options=[], env={}): + sandbox = LintSandbox(env, ['configure'] + options) + + sandbox.run(mozpath.join(test_data_path, 'moz.configure')) + + def moz_configure(self, source): + return MockedOpen({ + os.path.join(test_data_path, + 'moz.configure'): textwrap.dedent(source) + }) + + def test_depends_failures(self): + with self.moz_configure(''' + option('--foo', help='foo') + @depends('--foo') + def foo(value): + return value + + @depends('--help', foo) + def bar(help, foo): + return + '''): + self.lint_test() + + with self.assertRaises(ConfigureError) as e: + with self.moz_configure(''' + option('--foo', help='foo') + @depends('--foo') + @imports('os') + def foo(value): + return value + + @depends('--help', foo) + def bar(help, foo): + return + '''): + self.lint_test() + + self.assertEquals(e.exception.message, + "`bar` depends on '--help' and `foo`. " + "`foo` must depend on '--help'") + + with self.assertRaises(ConfigureError) as e: + with self.moz_configure(''' + @template + def tmpl(): + qux = 42 + + option('--foo', help='foo') + @depends('--foo') + def foo(value): + qux + return value + + @depends('--help', foo) + def bar(help, foo): + return + tmpl() + '''): + self.lint_test() + + self.assertEquals(e.exception.message, + "`bar` depends on '--help' and `foo`. " + "`foo` must depend on '--help'") + + with self.moz_configure(''' + option('--foo', help='foo') + @depends('--foo') + def foo(value): + return value + + include(foo) + '''): + self.lint_test() + + with self.assertRaises(ConfigureError) as e: + with self.moz_configure(''' + option('--foo', help='foo') + @depends('--foo') + @imports('os') + def foo(value): + return value + + include(foo) + '''): + self.lint_test() + + self.assertEquals(e.exception.message, + "Missing @depends for `foo`: '--help'") + + # There is a default restricted `os` module when there is no explicit + # @imports, and it's fine to use it without a dependency on --help. + with self.moz_configure(''' + option('--foo', help='foo') + @depends('--foo') + def foo(value): + os + return value + + include(foo) + '''): + self.lint_test() + + +if __name__ == '__main__': + main() diff --git a/python/mozbuild/mozbuild/test/configure/test_moz_configure.py b/python/mozbuild/mozbuild/test/configure/test_moz_configure.py new file mode 100644 index 000000000..7c318adef --- /dev/null +++ b/python/mozbuild/mozbuild/test/configure/test_moz_configure.py @@ -0,0 +1,93 @@ +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, You can obtain one at http://mozilla.org/MPL/2.0/. + +from __future__ import absolute_import, print_function, unicode_literals + +from mozunit import main +from mozpack import path as mozpath + +from common import BaseConfigureTest + + +class TestMozConfigure(BaseConfigureTest): + def test_moz_configure_options(self): + def get_value_for(args=[], environ={}, mozconfig=''): + sandbox = self.get_sandbox({}, {}, args, environ, mozconfig) + + # Add a fake old-configure option + sandbox.option_impl('--with-foo', nargs='*', + help='Help missing for old configure options') + + result = sandbox._value_for(sandbox['all_configure_options']) + shell = mozpath.abspath('/bin/sh') + return result.replace('CONFIG_SHELL=%s ' % shell, '') + + self.assertEquals('--enable-application=browser', + get_value_for(['--enable-application=browser'])) + + self.assertEquals('--enable-application=browser ' + 'MOZ_PROFILING=1', + get_value_for(['--enable-application=browser', + 'MOZ_PROFILING=1'])) + + value = get_value_for( + environ={'MOZ_PROFILING': '1'}, + mozconfig='ac_add_options --enable-project=js') + + self.assertEquals('--enable-project=js MOZ_PROFILING=1', + value) + + # --disable-js-shell is the default, so it's filtered out. + self.assertEquals('--enable-application=browser', + get_value_for(['--enable-application=browser', + '--disable-js-shell'])) + + # Normally, --without-foo would be filtered out because that's the + # default, but since it is a (fake) old-configure option, it always + # appears. + self.assertEquals('--enable-application=browser --without-foo', + get_value_for(['--enable-application=browser', + '--without-foo'])) + self.assertEquals('--enable-application=browser --with-foo', + get_value_for(['--enable-application=browser', + '--with-foo'])) + + self.assertEquals("--enable-application=browser '--with-foo=foo bar'", + get_value_for(['--enable-application=browser', + '--with-foo=foo bar'])) + + def test_nsis_version(self): + this = self + + class FakeNSIS(object): + def __init__(self, version): + self.version = version + + def __call__(self, stdin, args): + this.assertEquals(args, ('-version',)) + return 0, self.version, '' + + def check_nsis_version(version): + sandbox = self.get_sandbox( + {'/usr/bin/makensis': FakeNSIS(version)}, {}, [], + {'PATH': '/usr/bin', 'MAKENSISU': '/usr/bin/makensis'}) + return sandbox._value_for(sandbox['nsis_version']) + + with self.assertRaises(SystemExit) as e: + check_nsis_version('v2.5') + + with self.assertRaises(SystemExit) as e: + check_nsis_version('v3.0a2') + + self.assertEquals(check_nsis_version('v3.0b1'), '3.0b1') + self.assertEquals(check_nsis_version('v3.0b2'), '3.0b2') + self.assertEquals(check_nsis_version('v3.0rc1'), '3.0rc1') + self.assertEquals(check_nsis_version('v3.0'), '3.0') + self.assertEquals(check_nsis_version('v3.0-2'), '3.0') + self.assertEquals(check_nsis_version('v3.0.1'), '3.0') + self.assertEquals(check_nsis_version('v3.1'), '3.1') + + +if __name__ == '__main__': + main() diff --git a/python/mozbuild/mozbuild/test/configure/test_options.py b/python/mozbuild/mozbuild/test/configure/test_options.py new file mode 100644 index 000000000..e504f9e05 --- /dev/null +++ b/python/mozbuild/mozbuild/test/configure/test_options.py @@ -0,0 +1,852 @@ +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, You can obtain one at http://mozilla.org/MPL/2.0/. + +from __future__ import absolute_import, print_function, unicode_literals + +import unittest + +from mozunit import main + +from mozbuild.configure.options import ( + CommandLineHelper, + ConflictingOptionError, + InvalidOptionError, + NegativeOptionValue, + Option, + PositiveOptionValue, +) + + +class Option(Option): + def __init__(self, *args, **kwargs): + kwargs['help'] = 'Dummy help' + super(Option, self).__init__(*args, **kwargs) + + +class TestOption(unittest.TestCase): + def test_option(self): + option = Option('--option') + self.assertEquals(option.prefix, '') + self.assertEquals(option.name, 'option') + self.assertEquals(option.env, None) + self.assertFalse(option.default) + + option = Option('--enable-option') + self.assertEquals(option.prefix, 'enable') + self.assertEquals(option.name, 'option') + self.assertEquals(option.env, None) + self.assertFalse(option.default) + + option = Option('--disable-option') + self.assertEquals(option.prefix, 'disable') + self.assertEquals(option.name, 'option') + self.assertEquals(option.env, None) + self.assertTrue(option.default) + + option = Option('--with-option') + self.assertEquals(option.prefix, 'with') + self.assertEquals(option.name, 'option') + self.assertEquals(option.env, None) + self.assertFalse(option.default) + + option = Option('--without-option') + self.assertEquals(option.prefix, 'without') + self.assertEquals(option.name, 'option') + self.assertEquals(option.env, None) + self.assertTrue(option.default) + + option = Option('--without-option-foo', env='MOZ_OPTION') + self.assertEquals(option.env, 'MOZ_OPTION') + + option = Option(env='MOZ_OPTION') + self.assertEquals(option.prefix, '') + self.assertEquals(option.name, None) + self.assertEquals(option.env, 'MOZ_OPTION') + self.assertFalse(option.default) + + with self.assertRaises(InvalidOptionError) as e: + Option('--option', nargs=0, default=('a',)) + self.assertEquals(e.exception.message, + "The given `default` doesn't satisfy `nargs`") + + with self.assertRaises(InvalidOptionError) as e: + Option('--option', nargs=1, default=()) + self.assertEquals( + e.exception.message, + 'default must be a bool, a string or a tuple of strings') + + with self.assertRaises(InvalidOptionError) as e: + Option('--option', nargs=1, default=True) + self.assertEquals(e.exception.message, + "The given `default` doesn't satisfy `nargs`") + + with self.assertRaises(InvalidOptionError) as e: + Option('--option', nargs=1, default=('a', 'b')) + self.assertEquals(e.exception.message, + "The given `default` doesn't satisfy `nargs`") + + with self.assertRaises(InvalidOptionError) as e: + Option('--option', nargs=2, default=()) + self.assertEquals( + e.exception.message, + 'default must be a bool, a string or a tuple of strings') + + with self.assertRaises(InvalidOptionError) as e: + Option('--option', nargs=2, default=True) + self.assertEquals(e.exception.message, + "The given `default` doesn't satisfy `nargs`") + + with self.assertRaises(InvalidOptionError) as e: + Option('--option', nargs=2, default=('a',)) + self.assertEquals(e.exception.message, + "The given `default` doesn't satisfy `nargs`") + + with self.assertRaises(InvalidOptionError) as e: + Option('--option', nargs='?', default=('a', 'b')) + self.assertEquals(e.exception.message, + "The given `default` doesn't satisfy `nargs`") + + with self.assertRaises(InvalidOptionError) as e: + Option('--option', nargs='+', default=()) + self.assertEquals( + e.exception.message, + 'default must be a bool, a string or a tuple of strings') + + with self.assertRaises(InvalidOptionError) as e: + Option('--option', nargs='+', default=True) + self.assertEquals(e.exception.message, + "The given `default` doesn't satisfy `nargs`") + + # --disable options with a nargs value that requires at least one + # argument need to be given a default. + with self.assertRaises(InvalidOptionError) as e: + Option('--disable-option', nargs=1) + self.assertEquals(e.exception.message, + "The given `default` doesn't satisfy `nargs`") + + with self.assertRaises(InvalidOptionError) as e: + Option('--disable-option', nargs='+') + self.assertEquals(e.exception.message, + "The given `default` doesn't satisfy `nargs`") + + # Test nargs inference from default value + option = Option('--with-foo', default=True) + self.assertEquals(option.nargs, 0) + + option = Option('--with-foo', default=False) + self.assertEquals(option.nargs, 0) + + option = Option('--with-foo', default='a') + self.assertEquals(option.nargs, '?') + + option = Option('--with-foo', default=('a',)) + self.assertEquals(option.nargs, '?') + + option = Option('--with-foo', default=('a', 'b')) + self.assertEquals(option.nargs, '*') + + option = Option(env='FOO', default=True) + self.assertEquals(option.nargs, 0) + + option = Option(env='FOO', default=False) + self.assertEquals(option.nargs, 0) + + option = Option(env='FOO', default='a') + self.assertEquals(option.nargs, '?') + + option = Option(env='FOO', default=('a',)) + self.assertEquals(option.nargs, '?') + + option = Option(env='FOO', default=('a', 'b')) + self.assertEquals(option.nargs, '*') + + def test_option_option(self): + for option in ( + '--option', + '--enable-option', + '--disable-option', + '--with-option', + '--without-option', + ): + self.assertEquals(Option(option).option, option) + self.assertEquals(Option(option, env='FOO').option, option) + + opt = Option(option, default=False) + self.assertEquals(opt.option, + option.replace('-disable-', '-enable-') + .replace('-without-', '-with-')) + + opt = Option(option, default=True) + self.assertEquals(opt.option, + option.replace('-enable-', '-disable-') + .replace('-with-', '-without-')) + + self.assertEquals(Option(env='FOO').option, 'FOO') + + def test_option_choices(self): + with self.assertRaises(InvalidOptionError) as e: + Option('--option', nargs=3, choices=('a', 'b')) + self.assertEquals(e.exception.message, + 'Not enough `choices` for `nargs`') + + with self.assertRaises(InvalidOptionError) as e: + Option('--without-option', nargs=1, choices=('a', 'b')) + self.assertEquals(e.exception.message, + 'A `default` must be given along with `choices`') + + with self.assertRaises(InvalidOptionError) as e: + Option('--without-option', nargs='+', choices=('a', 'b')) + self.assertEquals(e.exception.message, + 'A `default` must be given along with `choices`') + + with self.assertRaises(InvalidOptionError) as e: + Option('--without-option', default='c', choices=('a', 'b')) + self.assertEquals(e.exception.message, + "The `default` value must be one of 'a', 'b'") + + with self.assertRaises(InvalidOptionError) as e: + Option('--without-option', default=('a', 'c',), choices=('a', 'b')) + self.assertEquals(e.exception.message, + "The `default` value must be one of 'a', 'b'") + + with self.assertRaises(InvalidOptionError) as e: + Option('--without-option', default=('c',), choices=('a', 'b')) + self.assertEquals(e.exception.message, + "The `default` value must be one of 'a', 'b'") + + option = Option('--with-option', nargs='+', choices=('a', 'b')) + with self.assertRaises(InvalidOptionError) as e: + option.get_value('--with-option=c') + self.assertEquals(e.exception.message, "'c' is not one of 'a', 'b'") + + value = option.get_value('--with-option=b,a') + self.assertTrue(value) + self.assertEquals(PositiveOptionValue(('b', 'a')), value) + + option = Option('--without-option', nargs='*', default='a', + choices=('a', 'b')) + with self.assertRaises(InvalidOptionError) as e: + option.get_value('--with-option=c') + self.assertEquals(e.exception.message, "'c' is not one of 'a', 'b'") + + value = option.get_value('--with-option=b,a') + self.assertTrue(value) + self.assertEquals(PositiveOptionValue(('b', 'a')), value) + + # Test nargs inference from choices + option = Option('--with-option', choices=('a', 'b')) + self.assertEqual(option.nargs, 1) + + # Test "relative" values + option = Option('--with-option', nargs='*', default=('b', 'c'), + choices=('a', 'b', 'c', 'd')) + + value = option.get_value('--with-option=+d') + self.assertEquals(PositiveOptionValue(('b', 'c', 'd')), value) + + value = option.get_value('--with-option=-b') + self.assertEquals(PositiveOptionValue(('c',)), value) + + value = option.get_value('--with-option=-b,+d') + self.assertEquals(PositiveOptionValue(('c','d')), value) + + # Adding something that is in the default is fine + value = option.get_value('--with-option=+b') + self.assertEquals(PositiveOptionValue(('b', 'c')), value) + + # Removing something that is not in the default is fine, as long as it + # is one of the choices + value = option.get_value('--with-option=-a') + self.assertEquals(PositiveOptionValue(('b', 'c')), value) + + with self.assertRaises(InvalidOptionError) as e: + option.get_value('--with-option=-e') + self.assertEquals(e.exception.message, + "'e' is not one of 'a', 'b', 'c', 'd'") + + # Other "not a choice" errors. + with self.assertRaises(InvalidOptionError) as e: + option.get_value('--with-option=+e') + self.assertEquals(e.exception.message, + "'e' is not one of 'a', 'b', 'c', 'd'") + + with self.assertRaises(InvalidOptionError) as e: + option.get_value('--with-option=e') + self.assertEquals(e.exception.message, + "'e' is not one of 'a', 'b', 'c', 'd'") + + def test_option_value_format(self): + val = PositiveOptionValue() + self.assertEquals('--with-value', val.format('--with-value')) + self.assertEquals('--with-value', val.format('--without-value')) + self.assertEquals('--enable-value', val.format('--enable-value')) + self.assertEquals('--enable-value', val.format('--disable-value')) + self.assertEquals('--value', val.format('--value')) + self.assertEquals('VALUE=1', val.format('VALUE')) + + val = PositiveOptionValue(('a',)) + self.assertEquals('--with-value=a', val.format('--with-value')) + self.assertEquals('--with-value=a', val.format('--without-value')) + self.assertEquals('--enable-value=a', val.format('--enable-value')) + self.assertEquals('--enable-value=a', val.format('--disable-value')) + self.assertEquals('--value=a', val.format('--value')) + self.assertEquals('VALUE=a', val.format('VALUE')) + + val = PositiveOptionValue(('a', 'b')) + self.assertEquals('--with-value=a,b', val.format('--with-value')) + self.assertEquals('--with-value=a,b', val.format('--without-value')) + self.assertEquals('--enable-value=a,b', val.format('--enable-value')) + self.assertEquals('--enable-value=a,b', val.format('--disable-value')) + self.assertEquals('--value=a,b', val.format('--value')) + self.assertEquals('VALUE=a,b', val.format('VALUE')) + + val = NegativeOptionValue() + self.assertEquals('--without-value', val.format('--with-value')) + self.assertEquals('--without-value', val.format('--without-value')) + self.assertEquals('--disable-value', val.format('--enable-value')) + self.assertEquals('--disable-value', val.format('--disable-value')) + self.assertEquals('', val.format('--value')) + self.assertEquals('VALUE=', val.format('VALUE')) + + def test_option_value(self, name='option', nargs=0, default=None): + disabled = name.startswith(('disable-', 'without-')) + if disabled: + negOptionValue = PositiveOptionValue + posOptionValue = NegativeOptionValue + else: + posOptionValue = PositiveOptionValue + negOptionValue = NegativeOptionValue + defaultValue = (PositiveOptionValue(default) + if default else negOptionValue()) + + option = Option('--%s' % name, nargs=nargs, default=default) + + if nargs in (0, '?', '*') or disabled: + value = option.get_value('--%s' % name, 'option') + self.assertEquals(value, posOptionValue()) + self.assertEquals(value.origin, 'option') + else: + with self.assertRaises(InvalidOptionError) as e: + option.get_value('--%s' % name) + if nargs == 1: + self.assertEquals(e.exception.message, + '--%s takes 1 value' % name) + elif nargs == '+': + self.assertEquals(e.exception.message, + '--%s takes 1 or more values' % name) + else: + self.assertEquals(e.exception.message, + '--%s takes 2 values' % name) + + value = option.get_value('') + self.assertEquals(value, defaultValue) + self.assertEquals(value.origin, 'default') + + value = option.get_value(None) + self.assertEquals(value, defaultValue) + self.assertEquals(value.origin, 'default') + + with self.assertRaises(AssertionError): + value = option.get_value('MOZ_OPTION=', 'environment') + + with self.assertRaises(AssertionError): + value = option.get_value('MOZ_OPTION=1', 'environment') + + with self.assertRaises(AssertionError): + value = option.get_value('--foo') + + if nargs in (1, '?', '*', '+') and not disabled: + value = option.get_value('--%s=' % name, 'option') + self.assertEquals(value, PositiveOptionValue(('',))) + self.assertEquals(value.origin, 'option') + else: + with self.assertRaises(InvalidOptionError) as e: + option.get_value('--%s=' % name) + if disabled: + self.assertEquals(e.exception.message, + 'Cannot pass a value to --%s' % name) + else: + self.assertEquals(e.exception.message, + '--%s takes %d values' % (name, nargs)) + + if nargs in (1, '?', '*', '+') and not disabled: + value = option.get_value('--%s=foo' % name, 'option') + self.assertEquals(value, PositiveOptionValue(('foo',))) + self.assertEquals(value.origin, 'option') + else: + with self.assertRaises(InvalidOptionError) as e: + option.get_value('--%s=foo' % name) + if disabled: + self.assertEquals(e.exception.message, + 'Cannot pass a value to --%s' % name) + else: + self.assertEquals(e.exception.message, + '--%s takes %d values' % (name, nargs)) + + if nargs in (2, '*', '+') and not disabled: + value = option.get_value('--%s=foo,bar' % name, 'option') + self.assertEquals(value, PositiveOptionValue(('foo', 'bar'))) + self.assertEquals(value.origin, 'option') + else: + with self.assertRaises(InvalidOptionError) as e: + option.get_value('--%s=foo,bar' % name, 'option') + if disabled: + self.assertEquals(e.exception.message, + 'Cannot pass a value to --%s' % name) + elif nargs == '?': + self.assertEquals(e.exception.message, + '--%s takes 0 or 1 values' % name) + else: + self.assertEquals(e.exception.message, + '--%s takes %d value%s' + % (name, nargs, 's' if nargs != 1 else '')) + + option = Option('--%s' % name, env='MOZ_OPTION', nargs=nargs, + default=default) + if nargs in (0, '?', '*') or disabled: + value = option.get_value('--%s' % name, 'option') + self.assertEquals(value, posOptionValue()) + self.assertEquals(value.origin, 'option') + else: + with self.assertRaises(InvalidOptionError) as e: + option.get_value('--%s' % name) + if disabled: + self.assertEquals(e.exception.message, + 'Cannot pass a value to --%s' % name) + elif nargs == '+': + self.assertEquals(e.exception.message, + '--%s takes 1 or more values' % name) + else: + self.assertEquals(e.exception.message, + '--%s takes %d value%s' + % (name, nargs, 's' if nargs != 1 else '')) + + value = option.get_value('') + self.assertEquals(value, defaultValue) + self.assertEquals(value.origin, 'default') + + value = option.get_value(None) + self.assertEquals(value, defaultValue) + self.assertEquals(value.origin, 'default') + + value = option.get_value('MOZ_OPTION=', 'environment') + self.assertEquals(value, NegativeOptionValue()) + self.assertEquals(value.origin, 'environment') + + if nargs in (0, '?', '*'): + value = option.get_value('MOZ_OPTION=1', 'environment') + self.assertEquals(value, PositiveOptionValue()) + self.assertEquals(value.origin, 'environment') + elif nargs in (1, '+'): + value = option.get_value('MOZ_OPTION=1', 'environment') + self.assertEquals(value, PositiveOptionValue(('1',))) + self.assertEquals(value.origin, 'environment') + else: + with self.assertRaises(InvalidOptionError) as e: + option.get_value('MOZ_OPTION=1', 'environment') + self.assertEquals(e.exception.message, 'MOZ_OPTION takes 2 values') + + if nargs in (1, '?', '*', '+') and not disabled: + value = option.get_value('--%s=' % name, 'option') + self.assertEquals(value, PositiveOptionValue(('',))) + self.assertEquals(value.origin, 'option') + else: + with self.assertRaises(InvalidOptionError) as e: + option.get_value('--%s=' % name, 'option') + if disabled: + self.assertEquals(e.exception.message, + 'Cannot pass a value to --%s' % name) + else: + self.assertEquals(e.exception.message, + '--%s takes %d values' % (name, nargs)) + + with self.assertRaises(AssertionError): + value = option.get_value('--foo', 'option') + + if nargs in (1, '?', '*', '+'): + value = option.get_value('MOZ_OPTION=foo', 'environment') + self.assertEquals(value, PositiveOptionValue(('foo',))) + self.assertEquals(value.origin, 'environment') + else: + with self.assertRaises(InvalidOptionError) as e: + option.get_value('MOZ_OPTION=foo', 'environment') + self.assertEquals(e.exception.message, + 'MOZ_OPTION takes %d values' % nargs) + + if nargs in (2, '*', '+'): + value = option.get_value('MOZ_OPTION=foo,bar', 'environment') + self.assertEquals(value, PositiveOptionValue(('foo', 'bar'))) + self.assertEquals(value.origin, 'environment') + else: + with self.assertRaises(InvalidOptionError) as e: + option.get_value('MOZ_OPTION=foo,bar', 'environment') + if nargs == '?': + self.assertEquals(e.exception.message, + 'MOZ_OPTION takes 0 or 1 values') + else: + self.assertEquals(e.exception.message, + 'MOZ_OPTION takes %d value%s' + % (nargs, 's' if nargs != 1 else '')) + + if disabled: + return option + + env_option = Option(env='MOZ_OPTION', nargs=nargs, default=default) + with self.assertRaises(AssertionError): + env_option.get_value('--%s' % name) + + value = env_option.get_value('') + self.assertEquals(value, defaultValue) + self.assertEquals(value.origin, 'default') + + value = env_option.get_value('MOZ_OPTION=', 'environment') + self.assertEquals(value, negOptionValue()) + self.assertEquals(value.origin, 'environment') + + if nargs in (0, '?', '*'): + value = env_option.get_value('MOZ_OPTION=1', 'environment') + self.assertEquals(value, posOptionValue()) + self.assertTrue(value) + self.assertEquals(value.origin, 'environment') + elif nargs in (1, '+'): + value = env_option.get_value('MOZ_OPTION=1', 'environment') + self.assertEquals(value, PositiveOptionValue(('1',))) + self.assertEquals(value.origin, 'environment') + else: + with self.assertRaises(InvalidOptionError) as e: + env_option.get_value('MOZ_OPTION=1', 'environment') + self.assertEquals(e.exception.message, 'MOZ_OPTION takes 2 values') + + with self.assertRaises(AssertionError) as e: + env_option.get_value('--%s' % name) + + with self.assertRaises(AssertionError) as e: + env_option.get_value('--foo') + + if nargs in (1, '?', '*', '+'): + value = env_option.get_value('MOZ_OPTION=foo', 'environment') + self.assertEquals(value, PositiveOptionValue(('foo',))) + self.assertEquals(value.origin, 'environment') + else: + with self.assertRaises(InvalidOptionError) as e: + env_option.get_value('MOZ_OPTION=foo', 'environment') + self.assertEquals(e.exception.message, + 'MOZ_OPTION takes %d values' % nargs) + + if nargs in (2, '*', '+'): + value = env_option.get_value('MOZ_OPTION=foo,bar', 'environment') + self.assertEquals(value, PositiveOptionValue(('foo', 'bar'))) + self.assertEquals(value.origin, 'environment') + else: + with self.assertRaises(InvalidOptionError) as e: + env_option.get_value('MOZ_OPTION=foo,bar', 'environment') + if nargs == '?': + self.assertEquals(e.exception.message, + 'MOZ_OPTION takes 0 or 1 values') + else: + self.assertEquals(e.exception.message, + 'MOZ_OPTION takes %d value%s' + % (nargs, 's' if nargs != 1 else '')) + + return option + + def test_option_value_enable(self, enable='enable', disable='disable', + nargs=0, default=None): + option = self.test_option_value('%s-option' % enable, nargs=nargs, + default=default) + + value = option.get_value('--%s-option' % disable, 'option') + self.assertEquals(value, NegativeOptionValue()) + self.assertEquals(value.origin, 'option') + + option = self.test_option_value('%s-option' % disable, nargs=nargs, + default=default) + + if nargs in (0, '?', '*'): + value = option.get_value('--%s-option' % enable, 'option') + self.assertEquals(value, PositiveOptionValue()) + self.assertEquals(value.origin, 'option') + else: + with self.assertRaises(InvalidOptionError) as e: + option.get_value('--%s-option' % enable, 'option') + if nargs == 1: + self.assertEquals(e.exception.message, + '--%s-option takes 1 value' % enable) + elif nargs == '+': + self.assertEquals(e.exception.message, + '--%s-option takes 1 or more values' + % enable) + else: + self.assertEquals(e.exception.message, + '--%s-option takes 2 values' % enable) + + def test_option_value_with(self): + self.test_option_value_enable('with', 'without') + + def test_option_value_invalid_nargs(self): + with self.assertRaises(InvalidOptionError) as e: + Option('--option', nargs='foo') + self.assertEquals(e.exception.message, + "nargs must be a positive integer, '?', '*' or '+'") + + with self.assertRaises(InvalidOptionError) as e: + Option('--option', nargs=-2) + self.assertEquals(e.exception.message, + "nargs must be a positive integer, '?', '*' or '+'") + + def test_option_value_nargs_1(self): + self.test_option_value(nargs=1) + self.test_option_value(nargs=1, default=('a',)) + self.test_option_value_enable(nargs=1, default=('a',)) + + # A default is required + with self.assertRaises(InvalidOptionError) as e: + Option('--disable-option', nargs=1) + self.assertEquals(e.exception.message, + "The given `default` doesn't satisfy `nargs`") + + def test_option_value_nargs_2(self): + self.test_option_value(nargs=2) + self.test_option_value(nargs=2, default=('a', 'b')) + self.test_option_value_enable(nargs=2, default=('a', 'b')) + + # A default is required + with self.assertRaises(InvalidOptionError) as e: + Option('--disable-option', nargs=2) + self.assertEquals(e.exception.message, + "The given `default` doesn't satisfy `nargs`") + + def test_option_value_nargs_0_or_1(self): + self.test_option_value(nargs='?') + self.test_option_value(nargs='?', default=('a',)) + self.test_option_value_enable(nargs='?') + self.test_option_value_enable(nargs='?', default=('a',)) + + def test_option_value_nargs_0_or_more(self): + self.test_option_value(nargs='*') + self.test_option_value(nargs='*', default=('a',)) + self.test_option_value(nargs='*', default=('a', 'b')) + self.test_option_value_enable(nargs='*') + self.test_option_value_enable(nargs='*', default=('a',)) + self.test_option_value_enable(nargs='*', default=('a', 'b')) + + def test_option_value_nargs_1_or_more(self): + self.test_option_value(nargs='+') + self.test_option_value(nargs='+', default=('a',)) + self.test_option_value(nargs='+', default=('a', 'b')) + self.test_option_value_enable(nargs='+', default=('a',)) + self.test_option_value_enable(nargs='+', default=('a', 'b')) + + # A default is required + with self.assertRaises(InvalidOptionError) as e: + Option('--disable-option', nargs='+') + self.assertEquals(e.exception.message, + "The given `default` doesn't satisfy `nargs`") + + +class TestCommandLineHelper(unittest.TestCase): + def test_basic(self): + helper = CommandLineHelper({}, ['cmd', '--foo', '--bar']) + + self.assertEquals(['--foo', '--bar'], list(helper)) + + helper.add('--enable-qux') + + self.assertEquals(['--foo', '--bar', '--enable-qux'], list(helper)) + + value, option = helper.handle(Option('--bar')) + self.assertEquals(['--foo', '--enable-qux'], list(helper)) + self.assertEquals(PositiveOptionValue(), value) + self.assertEquals('--bar', option) + + value, option = helper.handle(Option('--baz')) + self.assertEquals(['--foo', '--enable-qux'], list(helper)) + self.assertEquals(NegativeOptionValue(), value) + self.assertEquals(None, option) + + def test_precedence(self): + foo = Option('--with-foo', nargs='*') + helper = CommandLineHelper({}, ['cmd', '--with-foo=a,b']) + value, option = helper.handle(foo) + self.assertEquals(PositiveOptionValue(('a', 'b')), value) + self.assertEquals('command-line', value.origin) + self.assertEquals('--with-foo=a,b', option) + + helper = CommandLineHelper({}, ['cmd', '--with-foo=a,b', + '--without-foo']) + value, option = helper.handle(foo) + self.assertEquals(NegativeOptionValue(), value) + self.assertEquals('command-line', value.origin) + self.assertEquals('--without-foo', option) + + helper = CommandLineHelper({}, ['cmd', '--without-foo', + '--with-foo=a,b']) + value, option = helper.handle(foo) + self.assertEquals(PositiveOptionValue(('a', 'b')), value) + self.assertEquals('command-line', value.origin) + self.assertEquals('--with-foo=a,b', option) + + foo = Option('--with-foo', env='FOO', nargs='*') + helper = CommandLineHelper({'FOO': ''}, ['cmd', '--with-foo=a,b']) + value, option = helper.handle(foo) + self.assertEquals(PositiveOptionValue(('a', 'b')), value) + self.assertEquals('command-line', value.origin) + self.assertEquals('--with-foo=a,b', option) + + helper = CommandLineHelper({'FOO': 'a,b'}, ['cmd', '--without-foo']) + value, option = helper.handle(foo) + self.assertEquals(NegativeOptionValue(), value) + self.assertEquals('command-line', value.origin) + self.assertEquals('--without-foo', option) + + helper = CommandLineHelper({'FOO': ''}, ['cmd', '--with-bar=a,b']) + value, option = helper.handle(foo) + self.assertEquals(NegativeOptionValue(), value) + self.assertEquals('environment', value.origin) + self.assertEquals('FOO=', option) + + helper = CommandLineHelper({'FOO': 'a,b'}, ['cmd', '--without-bar']) + value, option = helper.handle(foo) + self.assertEquals(PositiveOptionValue(('a', 'b')), value) + self.assertEquals('environment', value.origin) + self.assertEquals('FOO=a,b', option) + + helper = CommandLineHelper({}, ['cmd', '--with-foo=a,b', 'FOO=']) + value, option = helper.handle(foo) + self.assertEquals(NegativeOptionValue(), value) + self.assertEquals('command-line', value.origin) + self.assertEquals('FOO=', option) + + helper = CommandLineHelper({}, ['cmd', '--without-foo', 'FOO=a,b']) + value, option = helper.handle(foo) + self.assertEquals(PositiveOptionValue(('a', 'b')), value) + self.assertEquals('command-line', value.origin) + self.assertEquals('FOO=a,b', option) + + helper = CommandLineHelper({}, ['cmd', 'FOO=', '--with-foo=a,b']) + value, option = helper.handle(foo) + self.assertEquals(PositiveOptionValue(('a', 'b')), value) + self.assertEquals('command-line', value.origin) + self.assertEquals('--with-foo=a,b', option) + + helper = CommandLineHelper({}, ['cmd', 'FOO=a,b', '--without-foo']) + value, option = helper.handle(foo) + self.assertEquals(NegativeOptionValue(), value) + self.assertEquals('command-line', value.origin) + self.assertEquals('--without-foo', option) + + def test_extra_args(self): + foo = Option('--with-foo', env='FOO', nargs='*') + helper = CommandLineHelper({}, ['cmd']) + helper.add('FOO=a,b,c', 'other-origin') + value, option = helper.handle(foo) + self.assertEquals(PositiveOptionValue(('a', 'b', 'c')), value) + self.assertEquals('other-origin', value.origin) + self.assertEquals('FOO=a,b,c', option) + + helper = CommandLineHelper({}, ['cmd']) + helper.add('FOO=a,b,c', 'other-origin') + helper.add('--with-foo=a,b,c', 'other-origin') + value, option = helper.handle(foo) + self.assertEquals(PositiveOptionValue(('a', 'b', 'c')), value) + self.assertEquals('other-origin', value.origin) + self.assertEquals('--with-foo=a,b,c', option) + + # Adding conflicting options is not allowed. + helper = CommandLineHelper({}, ['cmd']) + helper.add('FOO=a,b,c', 'other-origin') + with self.assertRaises(ConflictingOptionError) as cm: + helper.add('FOO=', 'other-origin') + self.assertEqual('FOO=', cm.exception.arg) + self.assertEqual('other-origin', cm.exception.origin) + self.assertEqual('FOO=a,b,c', cm.exception.old_arg) + self.assertEqual('other-origin', cm.exception.old_origin) + with self.assertRaises(ConflictingOptionError) as cm: + helper.add('FOO=a,b', 'other-origin') + self.assertEqual('FOO=a,b', cm.exception.arg) + self.assertEqual('other-origin', cm.exception.origin) + self.assertEqual('FOO=a,b,c', cm.exception.old_arg) + self.assertEqual('other-origin', cm.exception.old_origin) + # But adding the same is allowed. + helper.add('FOO=a,b,c', 'other-origin') + value, option = helper.handle(foo) + self.assertEquals(PositiveOptionValue(('a', 'b', 'c')), value) + self.assertEquals('other-origin', value.origin) + self.assertEquals('FOO=a,b,c', option) + + # The same rule as above applies when using the option form vs. the + # variable form. But we can't detect it when .add is called. + helper = CommandLineHelper({}, ['cmd']) + helper.add('FOO=a,b,c', 'other-origin') + helper.add('--without-foo', 'other-origin') + with self.assertRaises(ConflictingOptionError) as cm: + helper.handle(foo) + self.assertEqual('--without-foo', cm.exception.arg) + self.assertEqual('other-origin', cm.exception.origin) + self.assertEqual('FOO=a,b,c', cm.exception.old_arg) + self.assertEqual('other-origin', cm.exception.old_origin) + helper = CommandLineHelper({}, ['cmd']) + helper.add('FOO=a,b,c', 'other-origin') + helper.add('--with-foo=a,b', 'other-origin') + with self.assertRaises(ConflictingOptionError) as cm: + helper.handle(foo) + self.assertEqual('--with-foo=a,b', cm.exception.arg) + self.assertEqual('other-origin', cm.exception.origin) + self.assertEqual('FOO=a,b,c', cm.exception.old_arg) + self.assertEqual('other-origin', cm.exception.old_origin) + helper = CommandLineHelper({}, ['cmd']) + helper.add('FOO=a,b,c', 'other-origin') + helper.add('--with-foo=a,b,c', 'other-origin') + value, option = helper.handle(foo) + self.assertEquals(PositiveOptionValue(('a', 'b', 'c')), value) + self.assertEquals('other-origin', value.origin) + self.assertEquals('--with-foo=a,b,c', option) + + # Conflicts are also not allowed against what is in the + # environment/on the command line. + helper = CommandLineHelper({}, ['cmd', '--with-foo=a,b']) + helper.add('FOO=a,b,c', 'other-origin') + with self.assertRaises(ConflictingOptionError) as cm: + helper.handle(foo) + self.assertEqual('FOO=a,b,c', cm.exception.arg) + self.assertEqual('other-origin', cm.exception.origin) + self.assertEqual('--with-foo=a,b', cm.exception.old_arg) + self.assertEqual('command-line', cm.exception.old_origin) + + helper = CommandLineHelper({}, ['cmd', '--with-foo=a,b']) + helper.add('--without-foo', 'other-origin') + with self.assertRaises(ConflictingOptionError) as cm: + helper.handle(foo) + self.assertEqual('--without-foo', cm.exception.arg) + self.assertEqual('other-origin', cm.exception.origin) + self.assertEqual('--with-foo=a,b', cm.exception.old_arg) + self.assertEqual('command-line', cm.exception.old_origin) + + def test_possible_origins(self): + with self.assertRaises(InvalidOptionError): + Option('--foo', possible_origins='command-line') + + helper = CommandLineHelper({'BAZ': '1'}, ['cmd', '--foo', '--bar']) + foo = Option('--foo', + possible_origins=('command-line',)) + value, option = helper.handle(foo) + self.assertEquals(PositiveOptionValue(), value) + self.assertEquals('command-line', value.origin) + self.assertEquals('--foo', option) + + bar = Option('--bar', + possible_origins=('mozconfig',)) + with self.assertRaisesRegexp(InvalidOptionError, + "--bar can not be set by command-line. Values are accepted from: mozconfig"): + helper.handle(bar) + + baz = Option(env='BAZ', + possible_origins=('implied',)) + with self.assertRaisesRegexp(InvalidOptionError, + "BAZ=1 can not be set by environment. Values are accepted from: implied"): + helper.handle(baz) + + +if __name__ == '__main__': + main() diff --git a/python/mozbuild/mozbuild/test/configure/test_toolchain_configure.py b/python/mozbuild/mozbuild/test/configure/test_toolchain_configure.py new file mode 100644 index 000000000..2ef93792b --- /dev/null +++ b/python/mozbuild/mozbuild/test/configure/test_toolchain_configure.py @@ -0,0 +1,1271 @@ +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, You can obtain one at http://mozilla.org/MPL/2.0/. + +from __future__ import absolute_import, print_function, unicode_literals + +import logging +import os + +from StringIO import StringIO + +from mozunit import main + +from common import BaseConfigureTest +from mozbuild.configure.util import Version +from mozbuild.util import memoize +from mozpack import path as mozpath +from test_toolchain_helpers import ( + FakeCompiler, + CompilerResult, +) + + +DEFAULT_C99 = { + '__STDC_VERSION__': '199901L', +} + +DEFAULT_C11 = { + '__STDC_VERSION__': '201112L', +} + +DEFAULT_CXX_97 = { + '__cplusplus': '199711L', +} + +DEFAULT_CXX_11 = { + '__cplusplus': '201103L', +} + +DEFAULT_CXX_14 = { + '__cplusplus': '201402L', +} + +SUPPORTS_GNU99 = { + '-std=gnu99': DEFAULT_C99, +} + +SUPPORTS_GNUXX11 = { + '-std=gnu++11': DEFAULT_CXX_11, +} + +SUPPORTS_CXX14 = { + '-std=c++14': DEFAULT_CXX_14, +} + + +@memoize +def GCC_BASE(version): + version = Version(version) + return FakeCompiler({ + '__GNUC__': version.major, + '__GNUC_MINOR__': version.minor, + '__GNUC_PATCHLEVEL__': version.patch, + '__STDC__': 1, + '__ORDER_LITTLE_ENDIAN__': 1234, + '__ORDER_BIG_ENDIAN__': 4321, + }) + + +@memoize +def GCC(version): + return GCC_BASE(version) + SUPPORTS_GNU99 + + +@memoize +def GXX(version): + return GCC_BASE(version) + DEFAULT_CXX_97 + SUPPORTS_GNUXX11 + + +GCC_4_7 = GCC('4.7.3') +GXX_4_7 = GXX('4.7.3') +GCC_4_9 = GCC('4.9.3') +GXX_4_9 = GXX('4.9.3') +GCC_5 = GCC('5.2.1') + DEFAULT_C11 +GXX_5 = GXX('5.2.1') + +GCC_PLATFORM_LITTLE_ENDIAN = { + '__BYTE_ORDER__': 1234, +} + +GCC_PLATFORM_BIG_ENDIAN = { + '__BYTE_ORDER__': 4321, +} + +GCC_PLATFORM_X86 = FakeCompiler(GCC_PLATFORM_LITTLE_ENDIAN) + { + None: { + '__i386__': 1, + }, + '-m64': { + '__i386__': False, + '__x86_64__': 1, + }, +} + +GCC_PLATFORM_X86_64 = FakeCompiler(GCC_PLATFORM_LITTLE_ENDIAN) + { + None: { + '__x86_64__': 1, + }, + '-m32': { + '__x86_64__': False, + '__i386__': 1, + }, +} + +GCC_PLATFORM_ARM = FakeCompiler(GCC_PLATFORM_LITTLE_ENDIAN) + { + '__arm__': 1, +} + +GCC_PLATFORM_LINUX = { + '__linux__': 1, +} + +GCC_PLATFORM_DARWIN = { + '__APPLE__': 1, +} + +GCC_PLATFORM_WIN = { + '_WIN32': 1, + 'WINNT': 1, +} + +GCC_PLATFORM_X86_LINUX = FakeCompiler(GCC_PLATFORM_X86, GCC_PLATFORM_LINUX) +GCC_PLATFORM_X86_64_LINUX = FakeCompiler(GCC_PLATFORM_X86_64, + GCC_PLATFORM_LINUX) +GCC_PLATFORM_ARM_LINUX = FakeCompiler(GCC_PLATFORM_ARM, GCC_PLATFORM_LINUX) +GCC_PLATFORM_X86_OSX = FakeCompiler(GCC_PLATFORM_X86, GCC_PLATFORM_DARWIN) +GCC_PLATFORM_X86_64_OSX = FakeCompiler(GCC_PLATFORM_X86_64, + GCC_PLATFORM_DARWIN) +GCC_PLATFORM_X86_WIN = FakeCompiler(GCC_PLATFORM_X86, GCC_PLATFORM_WIN) +GCC_PLATFORM_X86_64_WIN = FakeCompiler(GCC_PLATFORM_X86_64, GCC_PLATFORM_WIN) + + +@memoize +def CLANG_BASE(version): + version = Version(version) + return FakeCompiler({ + '__clang__': 1, + '__clang_major__': version.major, + '__clang_minor__': version.minor, + '__clang_patchlevel__': version.patch, + }) + + +@memoize +def CLANG(version): + return GCC_BASE('4.2.1') + CLANG_BASE(version) + SUPPORTS_GNU99 + + +@memoize +def CLANGXX(version): + return (GCC_BASE('4.2.1') + CLANG_BASE(version) + DEFAULT_CXX_97 + + SUPPORTS_GNUXX11) + + +CLANG_3_3 = CLANG('3.3.0') + DEFAULT_C99 +CLANGXX_3_3 = CLANGXX('3.3.0') +CLANG_3_6 = CLANG('3.6.2') + DEFAULT_C11 +CLANGXX_3_6 = CLANGXX('3.6.2') + { + '-std=gnu++11': { + '__has_feature(cxx_alignof)': '1', + }, +} + + +def CLANG_PLATFORM(gcc_platform): + base = { + '--target=x86_64-linux-gnu': GCC_PLATFORM_X86_64_LINUX[None], + '--target=x86_64-darwin11.2.0': GCC_PLATFORM_X86_64_OSX[None], + '--target=i686-linux-gnu': GCC_PLATFORM_X86_LINUX[None], + '--target=i686-darwin11.2.0': GCC_PLATFORM_X86_OSX[None], + '--target=arm-linux-gnu': GCC_PLATFORM_ARM_LINUX[None], + } + undo_gcc_platform = { + k: {symbol: False for symbol in gcc_platform[None]} + for k in base + } + return FakeCompiler(gcc_platform, undo_gcc_platform, base) + + +CLANG_PLATFORM_X86_LINUX = CLANG_PLATFORM(GCC_PLATFORM_X86_LINUX) +CLANG_PLATFORM_X86_64_LINUX = CLANG_PLATFORM(GCC_PLATFORM_X86_64_LINUX) +CLANG_PLATFORM_X86_OSX = CLANG_PLATFORM(GCC_PLATFORM_X86_OSX) +CLANG_PLATFORM_X86_64_OSX = CLANG_PLATFORM(GCC_PLATFORM_X86_64_OSX) +CLANG_PLATFORM_X86_WIN = CLANG_PLATFORM(GCC_PLATFORM_X86_WIN) +CLANG_PLATFORM_X86_64_WIN = CLANG_PLATFORM(GCC_PLATFORM_X86_64_WIN) + + +@memoize +def VS(version): + version = Version(version) + return FakeCompiler({ + None: { + '_MSC_VER': '%02d%02d' % (version.major, version.minor), + '_MSC_FULL_VER': '%02d%02d%05d' % (version.major, version.minor, + version.patch), + }, + '*.cpp': DEFAULT_CXX_97, + }) + + +VS_2013u2 = VS('18.00.30501') +VS_2013u3 = VS('18.00.30723') +VS_2015 = VS('19.00.23026') +VS_2015u1 = VS('19.00.23506') +VS_2015u2 = VS('19.00.23918') +VS_2015u3 = VS('19.00.24213') + +VS_PLATFORM_X86 = { + '_M_IX86': 600, + '_WIN32': 1, +} + +VS_PLATFORM_X86_64 = { + '_M_X64': 100, + '_WIN32': 1, + '_WIN64': 1, +} + +# Note: In reality, the -std=gnu* options are only supported when preceded by +# -Xclang. +CLANG_CL_3_9 = (CLANG_BASE('3.9.0') + VS('18.00.00000') + DEFAULT_C11 + + SUPPORTS_GNU99 + SUPPORTS_GNUXX11 + SUPPORTS_CXX14) + { + '*.cpp': { + '__STDC_VERSION__': False, + '__cplusplus': '201103L', + }, + '-fms-compatibility-version=19.00.24213': VS('19.00.24213')[None], +} + +CLANG_CL_PLATFORM_X86 = FakeCompiler(VS_PLATFORM_X86, GCC_PLATFORM_X86[None]) +CLANG_CL_PLATFORM_X86_64 = FakeCompiler(VS_PLATFORM_X86_64, GCC_PLATFORM_X86_64[None]) + + +class BaseToolchainTest(BaseConfigureTest): + def setUp(self): + super(BaseToolchainTest, self).setUp() + self.out = StringIO() + self.logger = logging.getLogger('BaseToolchainTest') + self.logger.setLevel(logging.ERROR) + self.handler = logging.StreamHandler(self.out) + self.logger.addHandler(self.handler) + + def tearDown(self): + self.logger.removeHandler(self.handler) + del self.handler + del self.out + super(BaseToolchainTest, self).tearDown() + + def do_toolchain_test(self, paths, results, args=[], environ={}): + '''Helper to test the toolchain checks from toolchain.configure. + + - `paths` is a dict associating compiler paths to FakeCompiler + definitions from above. + - `results` is a dict associating result variable names from + toolchain.configure (c_compiler, cxx_compiler, host_c_compiler, + host_cxx_compiler) with a result. + The result can either be an error string, or a CompilerResult + corresponding to the object returned by toolchain.configure checks. + When the results for host_c_compiler are identical to c_compiler, + they can be omitted. Likewise for host_cxx_compiler vs. + cxx_compiler. + ''' + environ = dict(environ) + if 'PATH' not in environ: + environ['PATH'] = os.pathsep.join( + mozpath.abspath(p) for p in ('/bin', '/usr/bin')) + + sandbox = self.get_sandbox(paths, {}, args, environ, + logger=self.logger) + + for var in ('c_compiler', 'cxx_compiler', 'host_c_compiler', + 'host_cxx_compiler'): + if var in results: + result = results[var] + elif var.startswith('host_'): + result = results.get(var[5:], {}) + else: + result = {} + try: + self.out.truncate(0) + compiler = sandbox._value_for(sandbox[var]) + # Add var on both ends to make it clear which of the + # variables is failing the test when that happens. + self.assertEquals((var, compiler), (var, result)) + except SystemExit: + self.assertEquals((var, result), + (var, self.out.getvalue().strip())) + return + + +class LinuxToolchainTest(BaseToolchainTest): + PATHS = { + '/usr/bin/gcc': GCC_4_9 + GCC_PLATFORM_X86_64_LINUX, + '/usr/bin/g++': GXX_4_9 + GCC_PLATFORM_X86_64_LINUX, + '/usr/bin/gcc-4.7': GCC_4_7 + GCC_PLATFORM_X86_64_LINUX, + '/usr/bin/g++-4.7': GXX_4_7 + GCC_PLATFORM_X86_64_LINUX, + '/usr/bin/gcc-5': GCC_5 + GCC_PLATFORM_X86_64_LINUX, + '/usr/bin/g++-5': GXX_5 + GCC_PLATFORM_X86_64_LINUX, + '/usr/bin/clang': CLANG_3_6 + CLANG_PLATFORM_X86_64_LINUX, + '/usr/bin/clang++': CLANGXX_3_6 + CLANG_PLATFORM_X86_64_LINUX, + '/usr/bin/clang-3.6': CLANG_3_6 + CLANG_PLATFORM_X86_64_LINUX, + '/usr/bin/clang++-3.6': CLANGXX_3_6 + CLANG_PLATFORM_X86_64_LINUX, + '/usr/bin/clang-3.3': CLANG_3_3 + CLANG_PLATFORM_X86_64_LINUX, + '/usr/bin/clang++-3.3': CLANGXX_3_3 + CLANG_PLATFORM_X86_64_LINUX, + } + GCC_4_7_RESULT = ('Only GCC 4.8 or newer is supported ' + '(found version 4.7.3).') + GXX_4_7_RESULT = GCC_4_7_RESULT + GCC_4_9_RESULT = CompilerResult( + flags=['-std=gnu99'], + version='4.9.3', + type='gcc', + compiler='/usr/bin/gcc', + language='C', + ) + GXX_4_9_RESULT = CompilerResult( + flags=['-std=gnu++11'], + version='4.9.3', + type='gcc', + compiler='/usr/bin/g++', + language='C++', + ) + GCC_5_RESULT = CompilerResult( + flags=['-std=gnu99'], + version='5.2.1', + type='gcc', + compiler='/usr/bin/gcc-5', + language='C', + ) + GXX_5_RESULT = CompilerResult( + flags=['-std=gnu++11'], + version='5.2.1', + type='gcc', + compiler='/usr/bin/g++-5', + language='C++', + ) + CLANG_3_3_RESULT = CompilerResult( + flags=[], + version='3.3.0', + type='clang', + compiler='/usr/bin/clang-3.3', + language='C', + ) + CLANGXX_3_3_RESULT = 'Only clang/llvm 3.6 or newer is supported.' + CLANG_3_6_RESULT = CompilerResult( + flags=['-std=gnu99'], + version='3.6.2', + type='clang', + compiler='/usr/bin/clang', + language='C', + ) + CLANGXX_3_6_RESULT = CompilerResult( + flags=['-std=gnu++11'], + version='3.6.2', + type='clang', + compiler='/usr/bin/clang++', + language='C++', + ) + + def test_gcc(self): + # We'll try gcc and clang, and find gcc first. + self.do_toolchain_test(self.PATHS, { + 'c_compiler': self.GCC_4_9_RESULT, + 'cxx_compiler': self.GXX_4_9_RESULT, + }) + + def test_unsupported_gcc(self): + self.do_toolchain_test(self.PATHS, { + 'c_compiler': self.GCC_4_7_RESULT, + }, environ={ + 'CC': 'gcc-4.7', + 'CXX': 'g++-4.7', + }) + + # Maybe this should be reporting the mismatched version instead. + self.do_toolchain_test(self.PATHS, { + 'c_compiler': self.GCC_4_9_RESULT, + 'cxx_compiler': self.GXX_4_7_RESULT, + }, environ={ + 'CXX': 'g++-4.7', + }) + + def test_overridden_gcc(self): + self.do_toolchain_test(self.PATHS, { + 'c_compiler': self.GCC_5_RESULT, + 'cxx_compiler': self.GXX_5_RESULT, + }, environ={ + 'CC': 'gcc-5', + 'CXX': 'g++-5', + }) + + def test_guess_cxx(self): + # When CXX is not set, we guess it from CC. + self.do_toolchain_test(self.PATHS, { + 'c_compiler': self.GCC_5_RESULT, + 'cxx_compiler': self.GXX_5_RESULT, + }, environ={ + 'CC': 'gcc-5', + }) + + def test_mismatched_gcc(self): + self.do_toolchain_test(self.PATHS, { + 'c_compiler': self.GCC_4_9_RESULT, + 'cxx_compiler': ( + 'The target C compiler is version 4.9.3, while the target ' + 'C++ compiler is version 5.2.1. Need to use the same compiler ' + 'version.'), + }, environ={ + 'CXX': 'g++-5', + }) + + self.do_toolchain_test(self.PATHS, { + 'c_compiler': self.GCC_4_9_RESULT, + 'cxx_compiler': self.GXX_4_9_RESULT, + 'host_c_compiler': self.GCC_4_9_RESULT, + 'host_cxx_compiler': ( + 'The host C compiler is version 4.9.3, while the host ' + 'C++ compiler is version 5.2.1. Need to use the same compiler ' + 'version.'), + }, environ={ + 'HOST_CXX': 'g++-5', + }) + + def test_mismatched_compiler(self): + self.do_toolchain_test(self.PATHS, { + 'c_compiler': self.GCC_4_9_RESULT, + 'cxx_compiler': ( + 'The target C compiler is gcc, while the target C++ compiler ' + 'is clang. Need to use the same compiler suite.'), + }, environ={ + 'CXX': 'clang++', + }) + + self.do_toolchain_test(self.PATHS, { + 'c_compiler': self.GCC_4_9_RESULT, + 'cxx_compiler': self.GXX_4_9_RESULT, + 'host_c_compiler': self.GCC_4_9_RESULT, + 'host_cxx_compiler': ( + 'The host C compiler is gcc, while the host C++ compiler ' + 'is clang. Need to use the same compiler suite.'), + }, environ={ + 'HOST_CXX': 'clang++', + }) + + self.do_toolchain_test(self.PATHS, { + 'c_compiler': '`%s` is not a C compiler.' + % mozpath.abspath('/usr/bin/g++'), + }, environ={ + 'CC': 'g++', + }) + + self.do_toolchain_test(self.PATHS, { + 'c_compiler': self.GCC_4_9_RESULT, + 'cxx_compiler': '`%s` is not a C++ compiler.' + % mozpath.abspath('/usr/bin/gcc'), + }, environ={ + 'CXX': 'gcc', + }) + + def test_clang(self): + # We'll try gcc and clang, but since there is no gcc (gcc-x.y doesn't + # count), find clang. + paths = { + k: v for k, v in self.PATHS.iteritems() + if os.path.basename(k) not in ('gcc', 'g++') + } + self.do_toolchain_test(paths, { + 'c_compiler': self.CLANG_3_6_RESULT, + 'cxx_compiler': self.CLANGXX_3_6_RESULT, + }) + + def test_guess_cxx_clang(self): + # When CXX is not set, we guess it from CC. + self.do_toolchain_test(self.PATHS, { + 'c_compiler': self.CLANG_3_6_RESULT + { + 'compiler': '/usr/bin/clang-3.6', + }, + 'cxx_compiler': self.CLANGXX_3_6_RESULT + { + 'compiler': '/usr/bin/clang++-3.6', + }, + }, environ={ + 'CC': 'clang-3.6', + }) + + def test_unsupported_clang(self): + # clang 3.3 C compiler is perfectly fine, but we need more for C++. + self.do_toolchain_test(self.PATHS, { + 'c_compiler': self.CLANG_3_3_RESULT, + 'cxx_compiler': self.CLANGXX_3_3_RESULT, + }, environ={ + 'CC': 'clang-3.3', + 'CXX': 'clang++-3.3', + }) + + def test_no_supported_compiler(self): + # Even if there are gcc-x.y or clang-x.y compilers available, we + # don't try them. This could be considered something to improve. + paths = { + k: v for k, v in self.PATHS.iteritems() + if os.path.basename(k) not in ('gcc', 'g++', 'clang', 'clang++') + } + self.do_toolchain_test(paths, { + 'c_compiler': 'Cannot find the target C compiler', + }) + + def test_absolute_path(self): + paths = dict(self.PATHS) + paths.update({ + '/opt/clang/bin/clang': paths['/usr/bin/clang'], + '/opt/clang/bin/clang++': paths['/usr/bin/clang++'], + }) + result = { + 'c_compiler': self.CLANG_3_6_RESULT + { + 'compiler': '/opt/clang/bin/clang', + }, + 'cxx_compiler': self.CLANGXX_3_6_RESULT + { + 'compiler': '/opt/clang/bin/clang++' + }, + } + self.do_toolchain_test(paths, result, environ={ + 'CC': '/opt/clang/bin/clang', + 'CXX': '/opt/clang/bin/clang++', + }) + # With CXX guess too. + self.do_toolchain_test(paths, result, environ={ + 'CC': '/opt/clang/bin/clang', + }) + + def test_atypical_name(self): + paths = dict(self.PATHS) + paths.update({ + '/usr/bin/afl-clang-fast': paths['/usr/bin/clang'], + '/usr/bin/afl-clang-fast++': paths['/usr/bin/clang++'], + }) + self.do_toolchain_test(paths, { + 'c_compiler': self.CLANG_3_6_RESULT + { + 'compiler': '/usr/bin/afl-clang-fast', + }, + 'cxx_compiler': self.CLANGXX_3_6_RESULT + { + 'compiler': '/usr/bin/afl-clang-fast++', + }, + }, environ={ + 'CC': 'afl-clang-fast', + 'CXX': 'afl-clang-fast++', + }) + + def test_mixed_compilers(self): + self.do_toolchain_test(self.PATHS, { + 'c_compiler': self.CLANG_3_6_RESULT, + 'cxx_compiler': self.CLANGXX_3_6_RESULT, + 'host_c_compiler': self.GCC_4_9_RESULT, + 'host_cxx_compiler': self.GXX_4_9_RESULT, + }, environ={ + 'CC': 'clang', + 'HOST_CC': 'gcc', + }) + + self.do_toolchain_test(self.PATHS, { + 'c_compiler': self.CLANG_3_6_RESULT, + 'cxx_compiler': self.CLANGXX_3_6_RESULT, + 'host_c_compiler': self.GCC_4_9_RESULT, + 'host_cxx_compiler': self.GXX_4_9_RESULT, + }, environ={ + 'CC': 'clang', + 'CXX': 'clang++', + 'HOST_CC': 'gcc', + }) + + +class LinuxSimpleCrossToolchainTest(BaseToolchainTest): + TARGET = 'i686-pc-linux-gnu' + PATHS = LinuxToolchainTest.PATHS + GCC_4_9_RESULT = LinuxToolchainTest.GCC_4_9_RESULT + GXX_4_9_RESULT = LinuxToolchainTest.GXX_4_9_RESULT + CLANG_3_6_RESULT = LinuxToolchainTest.CLANG_3_6_RESULT + CLANGXX_3_6_RESULT = LinuxToolchainTest.CLANGXX_3_6_RESULT + + def test_cross_gcc(self): + self.do_toolchain_test(self.PATHS, { + 'c_compiler': self.GCC_4_9_RESULT + { + 'flags': ['-m32'] + }, + 'cxx_compiler': self.GXX_4_9_RESULT + { + 'flags': ['-m32'] + }, + 'host_c_compiler': self.GCC_4_9_RESULT, + 'host_cxx_compiler': self.GXX_4_9_RESULT, + }) + + def test_cross_clang(self): + self.do_toolchain_test(self.PATHS, { + 'c_compiler': self.CLANG_3_6_RESULT + { + 'flags': ['--target=i686-linux-gnu'], + }, + 'cxx_compiler': self.CLANGXX_3_6_RESULT + { + 'flags': ['--target=i686-linux-gnu'], + }, + 'host_c_compiler': self.CLANG_3_6_RESULT, + 'host_cxx_compiler': self.CLANGXX_3_6_RESULT, + }, environ={ + 'CC': 'clang', + }) + + +class LinuxX86_64CrossToolchainTest(BaseToolchainTest): + HOST = 'i686-pc-linux-gnu' + TARGET = 'x86_64-pc-linux-gnu' + PATHS = { + '/usr/bin/gcc': GCC_4_9 + GCC_PLATFORM_X86_LINUX, + '/usr/bin/g++': GXX_4_9 + GCC_PLATFORM_X86_LINUX, + '/usr/bin/clang': CLANG_3_6 + CLANG_PLATFORM_X86_LINUX, + '/usr/bin/clang++': CLANGXX_3_6 + CLANG_PLATFORM_X86_LINUX, + } + GCC_4_9_RESULT = LinuxToolchainTest.GCC_4_9_RESULT + GXX_4_9_RESULT = LinuxToolchainTest.GXX_4_9_RESULT + CLANG_3_6_RESULT = LinuxToolchainTest.CLANG_3_6_RESULT + CLANGXX_3_6_RESULT = LinuxToolchainTest.CLANGXX_3_6_RESULT + + def test_cross_gcc(self): + self.do_toolchain_test(self.PATHS, { + 'c_compiler': self.GCC_4_9_RESULT + { + 'flags': ['-m64'] + }, + 'cxx_compiler': self.GXX_4_9_RESULT + { + 'flags': ['-m64'] + }, + 'host_c_compiler': self.GCC_4_9_RESULT, + 'host_cxx_compiler': self.GXX_4_9_RESULT, + }) + + def test_cross_clang(self): + self.do_toolchain_test(self.PATHS, { + 'c_compiler': self.CLANG_3_6_RESULT + { + 'flags': ['--target=x86_64-linux-gnu'], + }, + 'cxx_compiler': self.CLANGXX_3_6_RESULT + { + 'flags': ['--target=x86_64-linux-gnu'], + }, + 'host_c_compiler': self.CLANG_3_6_RESULT, + 'host_cxx_compiler': self.CLANGXX_3_6_RESULT, + }, environ={ + 'CC': 'clang', + }) + + +class OSXToolchainTest(BaseToolchainTest): + HOST = 'x86_64-apple-darwin11.2.0' + PATHS = { + '/usr/bin/gcc': GCC_4_9 + GCC_PLATFORM_X86_64_OSX, + '/usr/bin/g++': GXX_4_9 + GCC_PLATFORM_X86_64_OSX, + '/usr/bin/gcc-4.7': GCC_4_7 + GCC_PLATFORM_X86_64_OSX, + '/usr/bin/g++-4.7': GXX_4_7 + GCC_PLATFORM_X86_64_OSX, + '/usr/bin/gcc-5': GCC_5 + GCC_PLATFORM_X86_64_OSX, + '/usr/bin/g++-5': GXX_5 + GCC_PLATFORM_X86_64_OSX, + '/usr/bin/clang': CLANG_3_6 + CLANG_PLATFORM_X86_64_OSX, + '/usr/bin/clang++': CLANGXX_3_6 + CLANG_PLATFORM_X86_64_OSX, + '/usr/bin/clang-3.6': CLANG_3_6 + CLANG_PLATFORM_X86_64_OSX, + '/usr/bin/clang++-3.6': CLANGXX_3_6 + CLANG_PLATFORM_X86_64_OSX, + '/usr/bin/clang-3.3': CLANG_3_3 + CLANG_PLATFORM_X86_64_OSX, + '/usr/bin/clang++-3.3': CLANGXX_3_3 + CLANG_PLATFORM_X86_64_OSX, + } + CLANG_3_3_RESULT = LinuxToolchainTest.CLANG_3_3_RESULT + CLANGXX_3_3_RESULT = LinuxToolchainTest.CLANGXX_3_3_RESULT + CLANG_3_6_RESULT = LinuxToolchainTest.CLANG_3_6_RESULT + CLANGXX_3_6_RESULT = LinuxToolchainTest.CLANGXX_3_6_RESULT + GCC_4_7_RESULT = LinuxToolchainTest.GCC_4_7_RESULT + GCC_5_RESULT = LinuxToolchainTest.GCC_5_RESULT + GXX_5_RESULT = LinuxToolchainTest.GXX_5_RESULT + + def test_clang(self): + # We only try clang because gcc is known not to work. + self.do_toolchain_test(self.PATHS, { + 'c_compiler': self.CLANG_3_6_RESULT, + 'cxx_compiler': self.CLANGXX_3_6_RESULT, + }) + + def test_not_gcc(self): + # We won't pick GCC if it's the only thing available. + paths = { + k: v for k, v in self.PATHS.iteritems() + if os.path.basename(k) not in ('clang', 'clang++') + } + self.do_toolchain_test(paths, { + 'c_compiler': 'Cannot find the target C compiler', + }) + + def test_unsupported_clang(self): + # clang 3.3 C compiler is perfectly fine, but we need more for C++. + self.do_toolchain_test(self.PATHS, { + 'c_compiler': self.CLANG_3_3_RESULT, + 'cxx_compiler': self.CLANGXX_3_3_RESULT, + }, environ={ + 'CC': 'clang-3.3', + 'CXX': 'clang++-3.3', + }) + + def test_forced_gcc(self): + # GCC can still be forced if the user really wants it. + self.do_toolchain_test(self.PATHS, { + 'c_compiler': self.GCC_5_RESULT, + 'cxx_compiler': self.GXX_5_RESULT, + }, environ={ + 'CC': 'gcc-5', + 'CXX': 'g++-5', + }) + + def test_forced_unsupported_gcc(self): + self.do_toolchain_test(self.PATHS, { + 'c_compiler': self.GCC_4_7_RESULT, + }, environ={ + 'CC': 'gcc-4.7', + 'CXX': 'g++-4.7', + }) + + +class WindowsToolchainTest(BaseToolchainTest): + HOST = 'i686-pc-mingw32' + + # For the purpose of this test, it doesn't matter that the paths are not + # real Windows paths. + PATHS = { + '/opt/VS_2013u2/bin/cl': VS_2013u2 + VS_PLATFORM_X86, + '/opt/VS_2013u3/bin/cl': VS_2013u3 + VS_PLATFORM_X86, + '/opt/VS_2015/bin/cl': VS_2015 + VS_PLATFORM_X86, + '/opt/VS_2015u1/bin/cl': VS_2015u1 + VS_PLATFORM_X86, + '/opt/VS_2015u2/bin/cl': VS_2015u2 + VS_PLATFORM_X86, + '/usr/bin/cl': VS_2015u3 + VS_PLATFORM_X86, + '/usr/bin/clang-cl': CLANG_CL_3_9 + CLANG_CL_PLATFORM_X86, + '/usr/bin/gcc': GCC_4_9 + GCC_PLATFORM_X86_WIN, + '/usr/bin/g++': GXX_4_9 + GCC_PLATFORM_X86_WIN, + '/usr/bin/gcc-4.7': GCC_4_7 + GCC_PLATFORM_X86_WIN, + '/usr/bin/g++-4.7': GXX_4_7 + GCC_PLATFORM_X86_WIN, + '/usr/bin/gcc-5': GCC_5 + GCC_PLATFORM_X86_WIN, + '/usr/bin/g++-5': GXX_5 + GCC_PLATFORM_X86_WIN, + '/usr/bin/clang': CLANG_3_6 + CLANG_PLATFORM_X86_WIN, + '/usr/bin/clang++': CLANGXX_3_6 + CLANG_PLATFORM_X86_WIN, + '/usr/bin/clang-3.6': CLANG_3_6 + CLANG_PLATFORM_X86_WIN, + '/usr/bin/clang++-3.6': CLANGXX_3_6 + CLANG_PLATFORM_X86_WIN, + '/usr/bin/clang-3.3': CLANG_3_3 + CLANG_PLATFORM_X86_WIN, + '/usr/bin/clang++-3.3': CLANGXX_3_3 + CLANG_PLATFORM_X86_WIN, + } + + VS_2013u2_RESULT = ( + 'This version (18.00.30501) of the MSVC compiler is not supported.\n' + 'You must install Visual C++ 2015 Update 3 or newer in order to build.\n' + 'See https://developer.mozilla.org/en/Windows_Build_Prerequisites') + VS_2013u3_RESULT = ( + 'This version (18.00.30723) of the MSVC compiler is not supported.\n' + 'You must install Visual C++ 2015 Update 3 or newer in order to build.\n' + 'See https://developer.mozilla.org/en/Windows_Build_Prerequisites') + VS_2015_RESULT = ( + 'This version (19.00.23026) of the MSVC compiler is not supported.\n' + 'You must install Visual C++ 2015 Update 3 or newer in order to build.\n' + 'See https://developer.mozilla.org/en/Windows_Build_Prerequisites') + VS_2015u1_RESULT = ( + 'This version (19.00.23506) of the MSVC compiler is not supported.\n' + 'You must install Visual C++ 2015 Update 3 or newer in order to build.\n' + 'See https://developer.mozilla.org/en/Windows_Build_Prerequisites') + VS_2015u2_RESULT = ( + 'This version (19.00.23918) of the MSVC compiler is not supported.\n' + 'You must install Visual C++ 2015 Update 3 or newer in order to build.\n' + 'See https://developer.mozilla.org/en/Windows_Build_Prerequisites') + VS_2015u3_RESULT = CompilerResult( + flags=[], + version='19.00.24213', + type='msvc', + compiler='/usr/bin/cl', + language='C', + ) + VSXX_2015u3_RESULT = CompilerResult( + flags=[], + version='19.00.24213', + type='msvc', + compiler='/usr/bin/cl', + language='C++', + ) + CLANG_CL_3_9_RESULT = CompilerResult( + flags=['-Xclang', '-std=gnu99', + '-fms-compatibility-version=19.00.24213', '-fallback'], + version='19.00.24213', + type='clang-cl', + compiler='/usr/bin/clang-cl', + language='C', + ) + CLANGXX_CL_3_9_RESULT = CompilerResult( + flags=['-Xclang', '-std=c++14', + '-fms-compatibility-version=19.00.24213', '-fallback'], + version='19.00.24213', + type='clang-cl', + compiler='/usr/bin/clang-cl', + language='C++', + ) + CLANG_3_3_RESULT = LinuxToolchainTest.CLANG_3_3_RESULT + CLANGXX_3_3_RESULT = LinuxToolchainTest.CLANGXX_3_3_RESULT + CLANG_3_6_RESULT = LinuxToolchainTest.CLANG_3_6_RESULT + CLANGXX_3_6_RESULT = LinuxToolchainTest.CLANGXX_3_6_RESULT + GCC_4_7_RESULT = LinuxToolchainTest.GCC_4_7_RESULT + GCC_4_9_RESULT = LinuxToolchainTest.GCC_4_9_RESULT + GXX_4_9_RESULT = LinuxToolchainTest.GXX_4_9_RESULT + GCC_5_RESULT = LinuxToolchainTest.GCC_5_RESULT + GXX_5_RESULT = LinuxToolchainTest.GXX_5_RESULT + + # VS2015u3 or greater is required. + def test_msvc(self): + self.do_toolchain_test(self.PATHS, { + 'c_compiler': self.VS_2015u3_RESULT, + 'cxx_compiler': self.VSXX_2015u3_RESULT, + }) + + def test_unsupported_msvc(self): + self.do_toolchain_test(self.PATHS, { + 'c_compiler': self.VS_2015u2_RESULT, + }, environ={ + 'CC': '/opt/VS_2015u2/bin/cl', + }) + + self.do_toolchain_test(self.PATHS, { + 'c_compiler': self.VS_2015u1_RESULT, + }, environ={ + 'CC': '/opt/VS_2015u1/bin/cl', + }) + + self.do_toolchain_test(self.PATHS, { + 'c_compiler': self.VS_2015_RESULT, + }, environ={ + 'CC': '/opt/VS_2015/bin/cl', + }) + + self.do_toolchain_test(self.PATHS, { + 'c_compiler': self.VS_2013u3_RESULT, + }, environ={ + 'CC': '/opt/VS_2013u3/bin/cl', + }) + + self.do_toolchain_test(self.PATHS, { + 'c_compiler': self.VS_2013u2_RESULT, + }, environ={ + 'CC': '/opt/VS_2013u2/bin/cl', + }) + + def test_clang_cl(self): + # We'll pick clang-cl if msvc can't be found. + paths = { + k: v for k, v in self.PATHS.iteritems() + if os.path.basename(k) != 'cl' + } + self.do_toolchain_test(paths, { + 'c_compiler': self.CLANG_CL_3_9_RESULT, + 'cxx_compiler': self.CLANGXX_CL_3_9_RESULT, + }) + + def test_gcc(self): + # We'll pick GCC if msvc and clang-cl can't be found. + paths = { + k: v for k, v in self.PATHS.iteritems() + if os.path.basename(k) not in ('cl', 'clang-cl') + } + self.do_toolchain_test(paths, { + 'c_compiler': self.GCC_4_9_RESULT, + 'cxx_compiler': self.GXX_4_9_RESULT, + }) + + def test_overridden_unsupported_gcc(self): + self.do_toolchain_test(self.PATHS, { + 'c_compiler': self.GCC_4_7_RESULT, + }, environ={ + 'CC': 'gcc-4.7', + 'CXX': 'g++-4.7', + }) + + def test_clang(self): + # We'll pick clang if nothing else is found. + paths = { + k: v for k, v in self.PATHS.iteritems() + if os.path.basename(k) not in ('cl', 'clang-cl', 'gcc') + } + self.do_toolchain_test(paths, { + 'c_compiler': self.CLANG_3_6_RESULT, + 'cxx_compiler': self.CLANGXX_3_6_RESULT, + }) + + def test_overridden_unsupported_clang(self): + # clang 3.3 C compiler is perfectly fine, but we need more for C++. + self.do_toolchain_test(self.PATHS, { + 'c_compiler': self.CLANG_3_3_RESULT, + 'cxx_compiler': self.CLANGXX_3_3_RESULT, + }, environ={ + 'CC': 'clang-3.3', + 'CXX': 'clang++-3.3', + }) + + def test_cannot_cross(self): + paths = { + '/usr/bin/cl': VS_2015u3 + VS_PLATFORM_X86_64, + } + self.do_toolchain_test(paths, { + 'c_compiler': ('Target C compiler target CPU (x86_64) ' + 'does not match --target CPU (i686)'), + }) + + +class Windows64ToolchainTest(WindowsToolchainTest): + HOST = 'x86_64-pc-mingw32' + + # For the purpose of this test, it doesn't matter that the paths are not + # real Windows paths. + PATHS = { + '/opt/VS_2013u2/bin/cl': VS_2013u2 + VS_PLATFORM_X86_64, + '/opt/VS_2013u3/bin/cl': VS_2013u3 + VS_PLATFORM_X86_64, + '/opt/VS_2015/bin/cl': VS_2015 + VS_PLATFORM_X86_64, + '/opt/VS_2015u1/bin/cl': VS_2015u1 + VS_PLATFORM_X86_64, + '/opt/VS_2015u2/bin/cl': VS_2015u2 + VS_PLATFORM_X86_64, + '/usr/bin/cl': VS_2015u3 + VS_PLATFORM_X86_64, + '/usr/bin/clang-cl': CLANG_CL_3_9 + CLANG_CL_PLATFORM_X86_64, + '/usr/bin/gcc': GCC_4_9 + GCC_PLATFORM_X86_64_WIN, + '/usr/bin/g++': GXX_4_9 + GCC_PLATFORM_X86_64_WIN, + '/usr/bin/gcc-4.7': GCC_4_7 + GCC_PLATFORM_X86_64_WIN, + '/usr/bin/g++-4.7': GXX_4_7 + GCC_PLATFORM_X86_64_WIN, + '/usr/bin/gcc-5': GCC_5 + GCC_PLATFORM_X86_64_WIN, + '/usr/bin/g++-5': GXX_5 + GCC_PLATFORM_X86_64_WIN, + '/usr/bin/clang': CLANG_3_6 + CLANG_PLATFORM_X86_64_WIN, + '/usr/bin/clang++': CLANGXX_3_6 + CLANG_PLATFORM_X86_64_WIN, + '/usr/bin/clang-3.6': CLANG_3_6 + CLANG_PLATFORM_X86_64_WIN, + '/usr/bin/clang++-3.6': CLANGXX_3_6 + CLANG_PLATFORM_X86_64_WIN, + '/usr/bin/clang-3.3': CLANG_3_3 + CLANG_PLATFORM_X86_64_WIN, + '/usr/bin/clang++-3.3': CLANGXX_3_3 + CLANG_PLATFORM_X86_64_WIN, + } + + def test_cannot_cross(self): + paths = { + '/usr/bin/cl': VS_2015u3 + VS_PLATFORM_X86, + } + self.do_toolchain_test(paths, { + 'c_compiler': ('Target C compiler target CPU (x86) ' + 'does not match --target CPU (x86_64)'), + }) + + +class LinuxCrossCompileToolchainTest(BaseToolchainTest): + TARGET = 'arm-unknown-linux-gnu' + PATHS = { + '/usr/bin/arm-linux-gnu-gcc': GCC_4_9 + GCC_PLATFORM_ARM_LINUX, + '/usr/bin/arm-linux-gnu-g++': GXX_4_9 + GCC_PLATFORM_ARM_LINUX, + '/usr/bin/arm-linux-gnu-gcc-4.7': GCC_4_7 + GCC_PLATFORM_ARM_LINUX, + '/usr/bin/arm-linux-gnu-g++-4.7': GXX_4_7 + GCC_PLATFORM_ARM_LINUX, + '/usr/bin/arm-linux-gnu-gcc-5': GCC_5 + GCC_PLATFORM_ARM_LINUX, + '/usr/bin/arm-linux-gnu-g++-5': GXX_5 + GCC_PLATFORM_ARM_LINUX, + } + PATHS.update(LinuxToolchainTest.PATHS) + ARM_GCC_4_7_RESULT = LinuxToolchainTest.GXX_4_7_RESULT + ARM_GCC_5_RESULT = LinuxToolchainTest.GCC_5_RESULT + { + 'compiler': '/usr/bin/arm-linux-gnu-gcc-5', + } + ARM_GXX_5_RESULT = LinuxToolchainTest.GXX_5_RESULT + { + 'compiler': '/usr/bin/arm-linux-gnu-g++-5', + } + CLANG_3_6_RESULT = LinuxToolchainTest.CLANG_3_6_RESULT + CLANGXX_3_6_RESULT = LinuxToolchainTest.CLANGXX_3_6_RESULT + GCC_4_9_RESULT = LinuxToolchainTest.GCC_4_9_RESULT + GXX_4_9_RESULT = LinuxToolchainTest.GXX_4_9_RESULT + + little_endian = FakeCompiler(GCC_PLATFORM_LINUX, + GCC_PLATFORM_LITTLE_ENDIAN) + big_endian = FakeCompiler(GCC_PLATFORM_LINUX, GCC_PLATFORM_BIG_ENDIAN) + + PLATFORMS = { + 'i686-pc-linux-gnu': GCC_PLATFORM_X86_LINUX, + 'x86_64-pc-linux-gnu': GCC_PLATFORM_X86_64_LINUX, + 'arm-unknown-linux-gnu': GCC_PLATFORM_ARM_LINUX, + 'aarch64-unknown-linux-gnu': little_endian + { + '__aarch64__': 1, + }, + 'ia64-unknown-linux-gnu': little_endian + { + '__ia64__': 1, + }, + 's390x-unknown-linux-gnu': big_endian + { + '__s390x__': 1, + '__s390__': 1, + }, + 's390-unknown-linux-gnu': big_endian + { + '__s390__': 1, + }, + 'powerpc64-unknown-linux-gnu': big_endian + { + None: { + '__powerpc64__': 1, + '__powerpc__': 1, + }, + '-m32': { + '__powerpc64__': False, + }, + }, + 'powerpc-unknown-linux-gnu': big_endian + { + None: { + '__powerpc__': 1, + }, + '-m64': { + '__powerpc64__': 1, + }, + }, + 'alpha-unknown-linux-gnu': little_endian + { + '__alpha__': 1, + }, + 'hppa-unknown-linux-gnu': big_endian + { + '__hppa__': 1, + }, + 'sparc64-unknown-linux-gnu': big_endian + { + None: { + '__arch64__': 1, + '__sparc__': 1, + }, + '-m32': { + '__arch64__': False, + }, + }, + 'sparc-unknown-linux-gnu': big_endian + { + None: { + '__sparc__': 1, + }, + '-m64': { + '__arch64__': 1, + }, + }, + 'mips64-unknown-linux-gnuabi64': big_endian + { + '__mips64': 1, + '__mips__': 1, + }, + 'mips-unknown-linux-gnu': big_endian + { + '__mips__': 1, + }, + } + + PLATFORMS['powerpc64le-unknown-linux-gnu'] = \ + PLATFORMS['powerpc64-unknown-linux-gnu'] + GCC_PLATFORM_LITTLE_ENDIAN + PLATFORMS['mips64el-unknown-linux-gnuabi64'] = \ + PLATFORMS['mips64-unknown-linux-gnuabi64'] + GCC_PLATFORM_LITTLE_ENDIAN + PLATFORMS['mipsel-unknown-linux-gnu'] = \ + PLATFORMS['mips-unknown-linux-gnu'] + GCC_PLATFORM_LITTLE_ENDIAN + + def do_test_cross_gcc_32_64(self, host, target): + self.HOST = host + self.TARGET = target + paths = { + '/usr/bin/gcc': GCC_4_9 + self.PLATFORMS[host], + '/usr/bin/g++': GXX_4_9 + self.PLATFORMS[host], + } + cross_flags = { + 'flags': ['-m64' if '64' in target else '-m32'] + } + self.do_toolchain_test(paths, { + 'c_compiler': self.GCC_4_9_RESULT + cross_flags, + 'cxx_compiler': self.GXX_4_9_RESULT + cross_flags, + 'host_c_compiler': self.GCC_4_9_RESULT, + 'host_cxx_compiler': self.GXX_4_9_RESULT, + }) + self.HOST = LinuxCrossCompileToolchainTest.HOST + self.TARGET = LinuxCrossCompileToolchainTest.TARGET + + def test_cross_x86_x64(self): + self.do_test_cross_gcc_32_64( + 'i686-pc-linux-gnu', 'x86_64-pc-linux-gnu') + self.do_test_cross_gcc_32_64( + 'x86_64-pc-linux-gnu', 'i686-pc-linux-gnu') + + def test_cross_sparc_sparc64(self): + self.do_test_cross_gcc_32_64( + 'sparc-unknown-linux-gnu', 'sparc64-unknown-linux-gnu') + self.do_test_cross_gcc_32_64( + 'sparc64-unknown-linux-gnu', 'sparc-unknown-linux-gnu') + + def test_cross_ppc_ppc64(self): + self.do_test_cross_gcc_32_64( + 'powerpc-unknown-linux-gnu', 'powerpc64-unknown-linux-gnu') + self.do_test_cross_gcc_32_64( + 'powerpc64-unknown-linux-gnu', 'powerpc-unknown-linux-gnu') + + def do_test_cross_gcc(self, host, target): + self.HOST = host + self.TARGET = target + host_cpu = host.split('-')[0] + cpu, manufacturer, os = target.split('-', 2) + toolchain_prefix = '/usr/bin/%s-%s' % (cpu, os) + paths = { + '/usr/bin/gcc': GCC_4_9 + self.PLATFORMS[host], + '/usr/bin/g++': GXX_4_9 + self.PLATFORMS[host], + } + self.do_toolchain_test(paths, { + 'c_compiler': ('Target C compiler target CPU (%s) ' + 'does not match --target CPU (%s)' + % (host_cpu, cpu)), + }) + + paths.update({ + '%s-gcc' % toolchain_prefix: GCC_4_9 + self.PLATFORMS[target], + '%s-g++' % toolchain_prefix: GXX_4_9 + self.PLATFORMS[target], + }) + self.do_toolchain_test(paths, { + 'c_compiler': self.GCC_4_9_RESULT + { + 'compiler': '%s-gcc' % toolchain_prefix, + }, + 'cxx_compiler': self.GXX_4_9_RESULT + { + 'compiler': '%s-g++' % toolchain_prefix, + }, + 'host_c_compiler': self.GCC_4_9_RESULT, + 'host_cxx_compiler': self.GXX_4_9_RESULT, + }) + self.HOST = LinuxCrossCompileToolchainTest.HOST + self.TARGET = LinuxCrossCompileToolchainTest.TARGET + + def test_cross_gcc_misc(self): + for target in self.PLATFORMS: + if not target.endswith('-pc-linux-gnu'): + self.do_test_cross_gcc('x86_64-pc-linux-gnu', target) + + def test_cannot_cross(self): + self.TARGET = 'mipsel-unknown-linux-gnu' + + paths = { + '/usr/bin/gcc': GCC_4_9 + self.PLATFORMS['mips-unknown-linux-gnu'], + '/usr/bin/g++': GXX_4_9 + self.PLATFORMS['mips-unknown-linux-gnu'], + } + self.do_toolchain_test(paths, { + 'c_compiler': ('Target C compiler target endianness (big) ' + 'does not match --target endianness (little)'), + }) + self.TARGET = LinuxCrossCompileToolchainTest.TARGET + + def test_overridden_cross_gcc(self): + self.do_toolchain_test(self.PATHS, { + 'c_compiler': self.ARM_GCC_5_RESULT, + 'cxx_compiler': self.ARM_GXX_5_RESULT, + 'host_c_compiler': self.GCC_4_9_RESULT, + 'host_cxx_compiler': self.GXX_4_9_RESULT, + }, environ={ + 'CC': 'arm-linux-gnu-gcc-5', + 'CXX': 'arm-linux-gnu-g++-5', + }) + + def test_overridden_unsupported_cross_gcc(self): + self.do_toolchain_test(self.PATHS, { + 'c_compiler': self.ARM_GCC_4_7_RESULT, + }, environ={ + 'CC': 'arm-linux-gnu-gcc-4.7', + 'CXX': 'arm-linux-gnu-g++-4.7', + }) + + def test_guess_cross_cxx(self): + # When CXX is not set, we guess it from CC. + self.do_toolchain_test(self.PATHS, { + 'c_compiler': self.ARM_GCC_5_RESULT, + 'cxx_compiler': self.ARM_GXX_5_RESULT, + 'host_c_compiler': self.GCC_4_9_RESULT, + 'host_cxx_compiler': self.GXX_4_9_RESULT, + }, environ={ + 'CC': 'arm-linux-gnu-gcc-5', + }) + + self.do_toolchain_test(self.PATHS, { + 'c_compiler': self.ARM_GCC_5_RESULT, + 'cxx_compiler': self.ARM_GXX_5_RESULT, + 'host_c_compiler': self.CLANG_3_6_RESULT, + 'host_cxx_compiler': self.CLANGXX_3_6_RESULT, + }, environ={ + 'CC': 'arm-linux-gnu-gcc-5', + 'HOST_CC': 'clang', + }) + + self.do_toolchain_test(self.PATHS, { + 'c_compiler': self.ARM_GCC_5_RESULT, + 'cxx_compiler': self.ARM_GXX_5_RESULT, + 'host_c_compiler': self.CLANG_3_6_RESULT, + 'host_cxx_compiler': self.CLANGXX_3_6_RESULT, + }, environ={ + 'CC': 'arm-linux-gnu-gcc-5', + 'CXX': 'arm-linux-gnu-g++-5', + 'HOST_CC': 'clang', + }) + + def test_cross_clang(self): + cross_clang_result = self.CLANG_3_6_RESULT + { + 'flags': ['--target=arm-linux-gnu'], + } + cross_clangxx_result = self.CLANGXX_3_6_RESULT + { + 'flags': ['--target=arm-linux-gnu'], + } + self.do_toolchain_test(self.PATHS, { + 'c_compiler': cross_clang_result, + 'cxx_compiler': cross_clangxx_result, + 'host_c_compiler': self.CLANG_3_6_RESULT, + 'host_cxx_compiler': self.CLANGXX_3_6_RESULT, + }, environ={ + 'CC': 'clang', + 'HOST_CC': 'clang', + }) + + self.do_toolchain_test(self.PATHS, { + 'c_compiler': cross_clang_result, + 'cxx_compiler': cross_clangxx_result, + 'host_c_compiler': self.CLANG_3_6_RESULT, + 'host_cxx_compiler': self.CLANGXX_3_6_RESULT, + }, environ={ + 'CC': 'clang', + }) + + def test_cross_atypical_clang(self): + paths = dict(self.PATHS) + paths.update({ + '/usr/bin/afl-clang-fast': paths['/usr/bin/clang'], + '/usr/bin/afl-clang-fast++': paths['/usr/bin/clang++'], + }) + afl_clang_result = self.CLANG_3_6_RESULT + { + 'compiler': '/usr/bin/afl-clang-fast', + } + afl_clangxx_result = self.CLANGXX_3_6_RESULT + { + 'compiler': '/usr/bin/afl-clang-fast++', + } + self.do_toolchain_test(paths, { + 'c_compiler': afl_clang_result + { + 'flags': ['--target=arm-linux-gnu'], + }, + 'cxx_compiler': afl_clangxx_result + { + 'flags': ['--target=arm-linux-gnu'], + }, + 'host_c_compiler': afl_clang_result, + 'host_cxx_compiler': afl_clangxx_result, + }, environ={ + 'CC': 'afl-clang-fast', + 'CXX': 'afl-clang-fast++', + }) + + +class OSXCrossToolchainTest(BaseToolchainTest): + TARGET = 'i686-apple-darwin11.2.0' + PATHS = LinuxToolchainTest.PATHS + CLANG_3_6_RESULT = LinuxToolchainTest.CLANG_3_6_RESULT + CLANGXX_3_6_RESULT = LinuxToolchainTest.CLANGXX_3_6_RESULT + + def test_osx_cross(self): + self.do_toolchain_test(self.PATHS, { + 'c_compiler': self.CLANG_3_6_RESULT + { + 'flags': ['--target=i686-darwin11.2.0'], + }, + 'cxx_compiler': self.CLANGXX_3_6_RESULT + { + 'flags': ['--target=i686-darwin11.2.0'], + }, + 'host_c_compiler': self.CLANG_3_6_RESULT, + 'host_cxx_compiler': self.CLANGXX_3_6_RESULT, + }, environ={ + 'CC': 'clang', + }) + + def test_cannot_osx_cross(self): + self.do_toolchain_test(self.PATHS, { + 'c_compiler': 'Target C compiler target kernel (Linux) does not ' + 'match --target kernel (Darwin)', + }, environ={ + 'CC': 'gcc', + }) + + +if __name__ == '__main__': + main() diff --git a/python/mozbuild/mozbuild/test/configure/test_toolchain_helpers.py b/python/mozbuild/mozbuild/test/configure/test_toolchain_helpers.py new file mode 100644 index 000000000..8ec33a8b7 --- /dev/null +++ b/python/mozbuild/mozbuild/test/configure/test_toolchain_helpers.py @@ -0,0 +1,437 @@ +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, You can obtain one at http://mozilla.org/MPL/2.0/. + +from __future__ import absolute_import, print_function, unicode_literals + +import copy +import re +import types +import unittest + +from fnmatch import fnmatch +from StringIO import StringIO +from textwrap import dedent + +from mozunit import ( + main, + MockedOpen, +) + +from mozbuild.preprocessor import Preprocessor +from mozbuild.util import ReadOnlyNamespace +from mozpack import path as mozpath + + +class CompilerPreprocessor(Preprocessor): + # The C preprocessor only expands macros when they are not in C strings. + # For now, we don't look very hard for C strings because they don't matter + # that much for our unit tests, but we at least avoid expanding in the + # simple "FOO" case. + VARSUBST = re.compile('(?\w+)(?!")', re.U) + NON_WHITESPACE = re.compile('\S') + HAS_FEATURE = re.compile('(__has_feature)\(([^\)]*)\)') + + def __init__(self, *args, **kwargs): + Preprocessor.__init__(self, *args, **kwargs) + self.do_filter('c_substitution') + self.setMarker('#\s*') + + def do_if(self, expression, **kwargs): + # The C preprocessor handles numbers following C rules, which is a + # different handling than what our Preprocessor does out of the box. + # Hack around it enough that the configure tests work properly. + context = self.context + def normalize_numbers(value): + if isinstance(value, types.StringTypes): + if value[-1:] == 'L' and value[:-1].isdigit(): + value = int(value[:-1]) + return value + # Our Preprocessor doesn't handle macros with parameters, so we hack + # around that for __has_feature()-like things. + def normalize_has_feature(expr): + return self.HAS_FEATURE.sub(r'\1\2', expr) + self.context = self.Context( + (normalize_has_feature(k), normalize_numbers(v)) + for k, v in context.iteritems() + ) + try: + return Preprocessor.do_if(self, normalize_has_feature(expression), + **kwargs) + finally: + self.context = context + + class Context(dict): + def __missing__(self, key): + return None + + def filter_c_substitution(self, line): + def repl(matchobj): + varname = matchobj.group('VAR') + if varname in self.context: + result = str(self.context[varname]) + # The C preprocessor inserts whitespaces around expanded + # symbols. + start, end = matchobj.span('VAR') + if self.NON_WHITESPACE.match(line[start-1:start]): + result = ' ' + result + if self.NON_WHITESPACE.match(line[end:end+1]): + result = result + ' ' + return result + return matchobj.group(0) + return self.VARSUBST.sub(repl, line) + + +class TestCompilerPreprocessor(unittest.TestCase): + def test_expansion(self): + pp = CompilerPreprocessor({ + 'A': 1, + 'B': '2', + 'C': 'c', + 'D': 'd' + }) + pp.out = StringIO() + input = StringIO('A.B.C "D"') + input.name = 'foo' + pp.do_include(input) + + self.assertEquals(pp.out.getvalue(), '1 . 2 . c "D"') + + def test_condition(self): + pp = CompilerPreprocessor({ + 'A': 1, + 'B': '2', + 'C': '0L', + }) + pp.out = StringIO() + input = StringIO(dedent('''\ + #ifdef A + IFDEF_A + #endif + #if A + IF_A + #endif + # if B + IF_B + # else + IF_NOT_B + # endif + #if !C + IF_NOT_C + #else + IF_C + #endif + ''')) + input.name = 'foo' + pp.do_include(input) + + self.assertEquals('IFDEF_A\nIF_A\nIF_B\nIF_NOT_C\n', pp.out.getvalue()) + + +class FakeCompiler(dict): + '''Defines a fake compiler for use in toolchain tests below. + + The definitions given when creating an instance can have one of two + forms: + - a dict giving preprocessor symbols and their respective value, e.g. + { '__GNUC__': 4, '__STDC__': 1 } + - a dict associating flags to preprocessor symbols. An entry for `None` + is required in this case. Those are the baseline preprocessor symbols. + Additional entries describe additional flags to set or existing flags + to unset (with a value of `False`). + { + None: { '__GNUC__': 4, '__STDC__': 1, '__STRICT_ANSI__': 1 }, + '-std=gnu99': { '__STDC_VERSION__': '199901L', + '__STRICT_ANSI__': False }, + } + With the dict above, invoking the preprocessor with no additional flags + would define __GNUC__, __STDC__ and __STRICT_ANSI__, and with -std=gnu99, + __GNUC__, __STDC__, and __STDC_VERSION__ (__STRICT_ANSI__ would be + unset). + It is also possible to have different symbols depending on the source + file extension. In this case, the key is '*.ext'. e.g. + { + '*.c': { '__STDC__': 1 }, + '*.cpp': { '__cplusplus': '199711L' }, + } + + All the given definitions are merged together. + + A FakeCompiler instance itself can be used as a definition to create + another FakeCompiler. + + For convenience, FakeCompiler instances can be added (+) to one another. + ''' + def __init__(self, *definitions): + for definition in definitions: + if all(not isinstance(d, dict) for d in definition.itervalues()): + definition = {None: definition} + for key, value in definition.iteritems(): + self.setdefault(key, {}).update(value) + + def __call__(self, stdin, args): + files = [arg for arg in args if not arg.startswith('-')] + flags = [arg for arg in args if arg.startswith('-')] + if '-E' in flags: + assert len(files) == 1 + file = files[0] + pp = CompilerPreprocessor(self[None]) + + def apply_defn(defn): + for k, v in defn.iteritems(): + if v is False: + if k in pp.context: + del pp.context[k] + else: + pp.context[k] = v + + for glob, defn in self.iteritems(): + if glob and not glob.startswith('-') and fnmatch(file, glob): + apply_defn(defn) + + for flag in flags: + apply_defn(self.get(flag, {})) + + pp.out = StringIO() + pp.do_include(file) + return 0, pp.out.getvalue(), '' + elif '-c' in flags: + if '-funknown-flag' in flags: + return 1, '', '' + return 0, '', '' + + return 1, '', '' + + def __add__(self, other): + return FakeCompiler(self, other) + + +class TestFakeCompiler(unittest.TestCase): + def test_fake_compiler(self): + with MockedOpen({ + 'file': 'A B C', + 'file.c': 'A B C', + }): + compiler = FakeCompiler({ + 'A': '1', + 'B': '2', + }) + self.assertEquals(compiler(None, ['-E', 'file']), + (0, '1 2 C', '')) + + compiler = FakeCompiler({ + None: { + 'A': '1', + 'B': '2', + }, + '-foo': { + 'C': 'foo', + }, + '-bar': { + 'B': 'bar', + 'C': 'bar', + }, + '-qux': { + 'B': False, + }, + '*.c': { + 'B': '42', + }, + }) + self.assertEquals(compiler(None, ['-E', 'file']), + (0, '1 2 C', '')) + self.assertEquals(compiler(None, ['-E', '-foo', 'file']), + (0, '1 2 foo', '')) + self.assertEquals(compiler(None, ['-E', '-bar', 'file']), + (0, '1 bar bar', '')) + self.assertEquals(compiler(None, ['-E', '-qux', 'file']), + (0, '1 B C', '')) + self.assertEquals(compiler(None, ['-E', '-foo', '-bar', 'file']), + (0, '1 bar bar', '')) + self.assertEquals(compiler(None, ['-E', '-bar', '-foo', 'file']), + (0, '1 bar foo', '')) + self.assertEquals(compiler(None, ['-E', '-bar', '-qux', 'file']), + (0, '1 B bar', '')) + self.assertEquals(compiler(None, ['-E', '-qux', '-bar', 'file']), + (0, '1 bar bar', '')) + self.assertEquals(compiler(None, ['-E', 'file.c']), + (0, '1 42 C', '')) + self.assertEquals(compiler(None, ['-E', '-bar', 'file.c']), + (0, '1 bar bar', '')) + + def test_multiple_definitions(self): + compiler = FakeCompiler({ + 'A': 1, + 'B': 2, + }, { + 'C': 3, + }) + + self.assertEquals(compiler, { + None: { + 'A': 1, + 'B': 2, + 'C': 3, + }, + }) + compiler = FakeCompiler({ + 'A': 1, + 'B': 2, + }, { + 'B': 4, + 'C': 3, + }) + + self.assertEquals(compiler, { + None: { + 'A': 1, + 'B': 4, + 'C': 3, + }, + }) + compiler = FakeCompiler({ + 'A': 1, + 'B': 2, + }, { + None: { + 'B': 4, + 'C': 3, + }, + '-foo': { + 'D': 5, + }, + }) + + self.assertEquals(compiler, { + None: { + 'A': 1, + 'B': 4, + 'C': 3, + }, + '-foo': { + 'D': 5, + }, + }) + + compiler = FakeCompiler({ + None: { + 'A': 1, + 'B': 2, + }, + '-foo': { + 'D': 5, + }, + }, { + '-foo': { + 'D': 5, + }, + '-bar': { + 'E': 6, + }, + }) + + self.assertEquals(compiler, { + None: { + 'A': 1, + 'B': 2, + }, + '-foo': { + 'D': 5, + }, + '-bar': { + 'E': 6, + }, + }) + + +class CompilerResult(ReadOnlyNamespace): + '''Helper of convenience to manipulate toolchain results in unit tests + + When adding a dict, the result is a new CompilerResult with the values + from the dict replacing those from the CompilerResult, except for `flags`, + where the value from the dict extends the `flags` in `self`. + ''' + + def __init__(self, wrapper=None, compiler='', version='', type='', + language='', flags=None): + if flags is None: + flags = [] + if wrapper is None: + wrapper = [] + super(CompilerResult, self).__init__( + flags=flags, + version=version, + type=type, + compiler=mozpath.abspath(compiler), + wrapper=wrapper, + language=language, + ) + + def __add__(self, other): + assert isinstance(other, dict) + result = copy.deepcopy(self.__dict__) + for k, v in other.iteritems(): + if k == 'flags': + result.setdefault(k, []).extend(v) + else: + result[k] = v + return CompilerResult(**result) + + +class TestCompilerResult(unittest.TestCase): + def test_compiler_result(self): + result = CompilerResult() + self.assertEquals(result.__dict__, { + 'wrapper': [], + 'compiler': mozpath.abspath(''), + 'version': '', + 'type': '', + 'language': '', + 'flags': [], + }) + + result = CompilerResult( + compiler='/usr/bin/gcc', + version='4.2.1', + type='gcc', + language='C', + flags=['-std=gnu99'], + ) + self.assertEquals(result.__dict__, { + 'wrapper': [], + 'compiler': mozpath.abspath('/usr/bin/gcc'), + 'version': '4.2.1', + 'type': 'gcc', + 'language': 'C', + 'flags': ['-std=gnu99'], + }) + + result2 = result + {'flags': ['-m32']} + self.assertEquals(result2.__dict__, { + 'wrapper': [], + 'compiler': mozpath.abspath('/usr/bin/gcc'), + 'version': '4.2.1', + 'type': 'gcc', + 'language': 'C', + 'flags': ['-std=gnu99', '-m32'], + }) + # Original flags are untouched. + self.assertEquals(result.flags, ['-std=gnu99']) + + result3 = result + { + 'compiler': '/usr/bin/gcc-4.7', + 'version': '4.7.3', + 'flags': ['-m32'], + } + self.assertEquals(result3.__dict__, { + 'wrapper': [], + 'compiler': mozpath.abspath('/usr/bin/gcc-4.7'), + 'version': '4.7.3', + 'type': 'gcc', + 'language': 'C', + 'flags': ['-std=gnu99', '-m32'], + }) + + +if __name__ == '__main__': + main() diff --git a/python/mozbuild/mozbuild/test/configure/test_toolkit_moz_configure.py b/python/mozbuild/mozbuild/test/configure/test_toolkit_moz_configure.py new file mode 100644 index 000000000..30dc022b7 --- /dev/null +++ b/python/mozbuild/mozbuild/test/configure/test_toolkit_moz_configure.py @@ -0,0 +1,67 @@ +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, You can obtain one at http://mozilla.org/MPL/2.0/. + +from __future__ import absolute_import, print_function, unicode_literals + +import os + +from buildconfig import topsrcdir +from common import BaseConfigureTest +from mozunit import main + + +class TestToolkitMozConfigure(BaseConfigureTest): + def test_necko_protocols(self): + def get_value(arg): + sandbox = self.get_sandbox({}, {}, [arg]) + return sandbox._value_for(sandbox['necko_protocols']) + + default_protocols = get_value('') + self.assertNotEqual(default_protocols, ()) + + # Backwards compatibility + self.assertEqual(get_value('--enable-necko-protocols'), + default_protocols) + + self.assertEqual(get_value('--enable-necko-protocols=yes'), + default_protocols) + + self.assertEqual(get_value('--enable-necko-protocols=all'), + default_protocols) + + self.assertEqual(get_value('--enable-necko-protocols=default'), + default_protocols) + + self.assertEqual(get_value('--enable-necko-protocols='), ()) + + self.assertEqual(get_value('--enable-necko-protocols=no'), ()) + + self.assertEqual(get_value('--enable-necko-protocols=none'), ()) + + self.assertEqual(get_value('--disable-necko-protocols'), ()) + + self.assertEqual(get_value('--enable-necko-protocols=http'), + ('http',)) + + self.assertEqual(get_value('--enable-necko-protocols=http,about'), + ('about', 'http')) + + self.assertEqual(get_value('--enable-necko-protocols=http,none'), ()) + + self.assertEqual(get_value('--enable-necko-protocols=-http'), ()) + + self.assertEqual(get_value('--enable-necko-protocols=none,http'), + ('http',)) + + self.assertEqual( + get_value('--enable-necko-protocols=all,-http,-about'), + tuple(p for p in default_protocols if p not in ('http', 'about'))) + + self.assertEqual( + get_value('--enable-necko-protocols=default,-http,-about'), + tuple(p for p in default_protocols if p not in ('http', 'about'))) + + +if __name__ == '__main__': + main() diff --git a/python/mozbuild/mozbuild/test/configure/test_util.py b/python/mozbuild/mozbuild/test/configure/test_util.py new file mode 100644 index 000000000..38b3c636e --- /dev/null +++ b/python/mozbuild/mozbuild/test/configure/test_util.py @@ -0,0 +1,558 @@ +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, You can obtain one at http://mozilla.org/MPL/2.0/. + +from __future__ import absolute_import, print_function, unicode_literals + +import logging +import os +import tempfile +import textwrap +import unittest +import sys + +from StringIO import StringIO + +from mozunit import main +from mozpack import path as mozpath + +from mozbuild.configure.util import ( + ConfigureOutputHandler, + getpreferredencoding, + LineIO, + Version, +) + +from mozbuild.configure import ( + ConfigureSandbox, +) + +from mozbuild.util import exec_ + +from buildconfig import topsrcdir +from common import ConfigureTestSandbox + + +class TestConfigureOutputHandler(unittest.TestCase): + def test_separation(self): + out = StringIO() + err = StringIO() + name = '%s.test_separation' % self.__class__.__name__ + logger = logging.getLogger(name) + logger.setLevel(logging.DEBUG) + logger.addHandler(ConfigureOutputHandler(out, err)) + + logger.error('foo') + logger.warning('bar') + logger.info('baz') + # DEBUG level is not printed out by this handler + logger.debug('qux') + + self.assertEqual(out.getvalue(), 'baz\n') + self.assertEqual(err.getvalue(), 'foo\nbar\n') + + def test_format(self): + out = StringIO() + err = StringIO() + name = '%s.test_format' % self.__class__.__name__ + logger = logging.getLogger(name) + logger.setLevel(logging.DEBUG) + handler = ConfigureOutputHandler(out, err) + handler.setFormatter(logging.Formatter('%(levelname)s:%(message)s')) + logger.addHandler(handler) + + logger.error('foo') + logger.warning('bar') + logger.info('baz') + # DEBUG level is not printed out by this handler + logger.debug('qux') + + self.assertEqual(out.getvalue(), 'baz\n') + self.assertEqual( + err.getvalue(), + 'ERROR:foo\n' + 'WARNING:bar\n' + ) + + def test_continuation(self): + out = StringIO() + name = '%s.test_continuation' % self.__class__.__name__ + logger = logging.getLogger(name) + logger.setLevel(logging.DEBUG) + handler = ConfigureOutputHandler(out, out) + handler.setFormatter(logging.Formatter('%(levelname)s:%(message)s')) + logger.addHandler(handler) + + logger.info('foo') + logger.info('checking bar... ') + logger.info('yes') + logger.info('qux') + + self.assertEqual( + out.getvalue(), + 'foo\n' + 'checking bar... yes\n' + 'qux\n' + ) + + out.seek(0) + out.truncate() + + logger.info('foo') + logger.info('checking bar... ') + logger.warning('hoge') + logger.info('no') + logger.info('qux') + + self.assertEqual( + out.getvalue(), + 'foo\n' + 'checking bar... \n' + 'WARNING:hoge\n' + ' ... no\n' + 'qux\n' + ) + + out.seek(0) + out.truncate() + + logger.info('foo') + logger.info('checking bar... ') + logger.warning('hoge') + logger.warning('fuga') + logger.info('no') + logger.info('qux') + + self.assertEqual( + out.getvalue(), + 'foo\n' + 'checking bar... \n' + 'WARNING:hoge\n' + 'WARNING:fuga\n' + ' ... no\n' + 'qux\n' + ) + + out.seek(0) + out.truncate() + err = StringIO() + + logger.removeHandler(handler) + handler = ConfigureOutputHandler(out, err) + handler.setFormatter(logging.Formatter('%(levelname)s:%(message)s')) + logger.addHandler(handler) + + logger.info('foo') + logger.info('checking bar... ') + logger.warning('hoge') + logger.warning('fuga') + logger.info('no') + logger.info('qux') + + self.assertEqual( + out.getvalue(), + 'foo\n' + 'checking bar... no\n' + 'qux\n' + ) + + self.assertEqual( + err.getvalue(), + 'WARNING:hoge\n' + 'WARNING:fuga\n' + ) + + def test_queue_debug(self): + out = StringIO() + name = '%s.test_queue_debug' % self.__class__.__name__ + logger = logging.getLogger(name) + logger.setLevel(logging.DEBUG) + handler = ConfigureOutputHandler(out, out, maxlen=3) + handler.setFormatter(logging.Formatter('%(levelname)s:%(message)s')) + logger.addHandler(handler) + + with handler.queue_debug(): + logger.info('checking bar... ') + logger.debug('do foo') + logger.info('yes') + logger.info('qux') + + self.assertEqual( + out.getvalue(), + 'checking bar... yes\n' + 'qux\n' + ) + + out.seek(0) + out.truncate() + + with handler.queue_debug(): + logger.info('checking bar... ') + logger.debug('do foo') + logger.info('no') + logger.error('fail') + + self.assertEqual( + out.getvalue(), + 'checking bar... no\n' + 'DEBUG:do foo\n' + 'ERROR:fail\n' + ) + + out.seek(0) + out.truncate() + + with handler.queue_debug(): + logger.info('checking bar... ') + logger.debug('do foo') + logger.debug('do bar') + logger.debug('do baz') + logger.info('no') + logger.error('fail') + + self.assertEqual( + out.getvalue(), + 'checking bar... no\n' + 'DEBUG:do foo\n' + 'DEBUG:do bar\n' + 'DEBUG:do baz\n' + 'ERROR:fail\n' + ) + + out.seek(0) + out.truncate() + + with handler.queue_debug(): + logger.info('checking bar... ') + logger.debug('do foo') + logger.debug('do bar') + logger.debug('do baz') + logger.debug('do qux') + logger.debug('do hoge') + logger.info('no') + logger.error('fail') + + self.assertEqual( + out.getvalue(), + 'checking bar... no\n' + 'DEBUG:\n' + 'DEBUG:do baz\n' + 'DEBUG:do qux\n' + 'DEBUG:do hoge\n' + 'ERROR:fail\n' + ) + + out.seek(0) + out.truncate() + + try: + with handler.queue_debug(): + logger.info('checking bar... ') + logger.debug('do foo') + logger.debug('do bar') + logger.info('no') + e = Exception('fail') + raise e + except Exception as caught: + self.assertIs(caught, e) + + self.assertEqual( + out.getvalue(), + 'checking bar... no\n' + 'DEBUG:do foo\n' + 'DEBUG:do bar\n' + ) + + def test_queue_debug_reentrant(self): + out = StringIO() + name = '%s.test_queue_debug_reentrant' % self.__class__.__name__ + logger = logging.getLogger(name) + logger.setLevel(logging.DEBUG) + handler = ConfigureOutputHandler(out, out, maxlen=10) + handler.setFormatter(logging.Formatter('%(levelname)s| %(message)s')) + logger.addHandler(handler) + + try: + with handler.queue_debug(): + logger.info('outer info') + logger.debug('outer debug') + with handler.queue_debug(): + logger.info('inner info') + logger.debug('inner debug') + e = Exception('inner exception') + raise e + except Exception as caught: + self.assertIs(caught, e) + + self.assertEqual(out.getvalue(), + 'outer info\n' + 'inner info\n' + 'DEBUG| outer debug\n' + 'DEBUG| inner debug\n') + + out.seek(0) + out.truncate() + + try: + with handler.queue_debug(): + logger.info('outer info') + logger.debug('outer debug') + with handler.queue_debug(): + logger.info('inner info') + logger.debug('inner debug') + e = Exception('outer exception') + raise e + except Exception as caught: + self.assertIs(caught, e) + + self.assertEqual(out.getvalue(), + 'outer info\n' + 'inner info\n' + 'DEBUG| outer debug\n' + 'DEBUG| inner debug\n') + + out.seek(0) + out.truncate() + + with handler.queue_debug(): + logger.info('outer info') + logger.debug('outer debug') + with handler.queue_debug(): + logger.info('inner info') + logger.debug('inner debug') + logger.error('inner error') + self.assertEqual(out.getvalue(), + 'outer info\n' + 'inner info\n' + 'DEBUG| outer debug\n' + 'DEBUG| inner debug\n' + 'ERROR| inner error\n') + + out.seek(0) + out.truncate() + + with handler.queue_debug(): + logger.info('outer info') + logger.debug('outer debug') + with handler.queue_debug(): + logger.info('inner info') + logger.debug('inner debug') + logger.error('outer error') + self.assertEqual(out.getvalue(), + 'outer info\n' + 'inner info\n' + 'DEBUG| outer debug\n' + 'DEBUG| inner debug\n' + 'ERROR| outer error\n') + + def test_is_same_output(self): + fd1 = sys.stderr.fileno() + fd2 = os.dup(fd1) + try: + self.assertTrue(ConfigureOutputHandler._is_same_output(fd1, fd2)) + finally: + os.close(fd2) + + fd2, path = tempfile.mkstemp() + try: + self.assertFalse(ConfigureOutputHandler._is_same_output(fd1, fd2)) + + fd3 = os.dup(fd2) + try: + self.assertTrue(ConfigureOutputHandler._is_same_output(fd2, fd3)) + finally: + os.close(fd3) + + with open(path, 'a') as fh: + fd3 = fh.fileno() + self.assertTrue( + ConfigureOutputHandler._is_same_output(fd2, fd3)) + + finally: + os.close(fd2) + os.remove(path) + + +class TestLineIO(unittest.TestCase): + def test_lineio(self): + lines = [] + l = LineIO(lambda l: lines.append(l)) + + l.write('a') + self.assertEqual(lines, []) + + l.write('b') + self.assertEqual(lines, []) + + l.write('\n') + self.assertEqual(lines, ['ab']) + + l.write('cdef') + self.assertEqual(lines, ['ab']) + + l.write('\n') + self.assertEqual(lines, ['ab', 'cdef']) + + l.write('ghi\njklm') + self.assertEqual(lines, ['ab', 'cdef', 'ghi']) + + l.write('nop\nqrst\nuv\n') + self.assertEqual(lines, ['ab', 'cdef', 'ghi', 'jklmnop', 'qrst', 'uv']) + + l.write('wx\nyz') + self.assertEqual(lines, ['ab', 'cdef', 'ghi', 'jklmnop', 'qrst', 'uv', + 'wx']) + + l.close() + self.assertEqual(lines, ['ab', 'cdef', 'ghi', 'jklmnop', 'qrst', 'uv', + 'wx', 'yz']) + + def test_lineio_contextmanager(self): + lines = [] + with LineIO(lambda l: lines.append(l)) as l: + l.write('a\nb\nc') + + self.assertEqual(lines, ['a', 'b']) + + self.assertEqual(lines, ['a', 'b', 'c']) + + +class TestLogSubprocessOutput(unittest.TestCase): + + def test_non_ascii_subprocess_output(self): + out = StringIO() + sandbox = ConfigureSandbox({}, {}, [], out, out) + + sandbox.include_file(mozpath.join(topsrcdir, 'build', + 'moz.configure', 'util.configure')) + sandbox.include_file(mozpath.join(topsrcdir, 'python', 'mozbuild', + 'mozbuild', 'test', 'configure', + 'data', 'subprocess.configure')) + status = 0 + try: + sandbox.run() + except SystemExit as e: + status = e.code + + self.assertEquals(status, 0) + quote_char = "'" + if getpreferredencoding().lower() == 'utf-8': + quote_char = '\u00B4'.encode('utf-8') + self.assertEquals(out.getvalue().strip(), quote_char) + + +class TestVersion(unittest.TestCase): + def test_version_simple(self): + v = Version('1') + self.assertEqual(v, '1') + self.assertLess(v, '2') + self.assertGreater(v, '0.5') + self.assertEqual(v.major, 1) + self.assertEqual(v.minor, 0) + self.assertEqual(v.patch, 0) + + def test_version_more(self): + v = Version('1.2.3b') + self.assertLess(v, '2') + self.assertEqual(v.major, 1) + self.assertEqual(v.minor, 2) + self.assertEqual(v.patch, 3) + + def test_version_bad(self): + # A version with a letter in the middle doesn't really make sense, + # so everything after it should be ignored. + v = Version('1.2b.3') + self.assertLess(v, '2') + self.assertEqual(v.major, 1) + self.assertEqual(v.minor, 2) + self.assertEqual(v.patch, 0) + + def test_version_badder(self): + v = Version('1b.2.3') + self.assertLess(v, '2') + self.assertEqual(v.major, 1) + self.assertEqual(v.minor, 0) + self.assertEqual(v.patch, 0) + +class TestCheckCmdOutput(unittest.TestCase): + + def get_result(self, command='', paths=None): + paths = paths or {} + config = {} + out = StringIO() + sandbox = ConfigureTestSandbox(paths, config, {}, ['/bin/configure'], + out, out) + sandbox.include_file(mozpath.join(topsrcdir, 'build', + 'moz.configure', 'util.configure')) + status = 0 + try: + exec_(command, sandbox) + sandbox.run() + except SystemExit as e: + status = e.code + return config, out.getvalue(), status + + def test_simple_program(self): + def mock_simple_prog(_, args): + if len(args) == 1 and args[0] == '--help': + return 0, 'simple program help...', '' + self.fail("Unexpected arguments to mock_simple_program: %s" % + args) + prog_path = mozpath.abspath('/simple/prog') + cmd = "log.info(check_cmd_output('%s', '--help'))" % prog_path + config, out, status = self.get_result(cmd, + paths={prog_path: mock_simple_prog}) + self.assertEqual(config, {}) + self.assertEqual(status, 0) + self.assertEqual(out, 'simple program help...\n') + + def test_failing_program(self): + def mock_error_prog(_, args): + if len(args) == 1 and args[0] == '--error': + return (127, 'simple program output', + 'simple program error output') + self.fail("Unexpected arguments to mock_error_program: %s" % + args) + prog_path = mozpath.abspath('/simple/prog') + cmd = "log.info(check_cmd_output('%s', '--error'))" % prog_path + config, out, status = self.get_result(cmd, + paths={prog_path: mock_error_prog}) + self.assertEqual(config, {}) + self.assertEqual(status, 1) + self.assertEqual(out, textwrap.dedent('''\ + DEBUG: Executing: `%s --error` + DEBUG: The command returned non-zero exit status 127. + DEBUG: Its output was: + DEBUG: | simple program output + DEBUG: Its error output was: + DEBUG: | simple program error output + ERROR: Command `%s --error` failed with exit status 127. + ''' % (prog_path, prog_path))) + + def test_error_callback(self): + def mock_error_prog(_, args): + if len(args) == 1 and args[0] == '--error': + return 127, 'simple program error...', '' + self.fail("Unexpected arguments to mock_error_program: %s" % + args) + + prog_path = mozpath.abspath('/simple/prog') + cmd = textwrap.dedent('''\ + check_cmd_output('%s', '--error', + onerror=lambda: die('`prog` produced an error')) + ''' % prog_path) + config, out, status = self.get_result(cmd, + paths={prog_path: mock_error_prog}) + self.assertEqual(config, {}) + self.assertEqual(status, 1) + self.assertEqual(out, textwrap.dedent('''\ + DEBUG: Executing: `%s --error` + DEBUG: The command returned non-zero exit status 127. + DEBUG: Its output was: + DEBUG: | simple program error... + ERROR: `prog` produced an error + ''' % prog_path)) + + +if __name__ == '__main__': + main() diff --git a/python/mozbuild/mozbuild/test/controller/__init__.py b/python/mozbuild/mozbuild/test/controller/__init__.py new file mode 100644 index 000000000..e69de29bb diff --git a/python/mozbuild/mozbuild/test/controller/test_ccachestats.py b/python/mozbuild/mozbuild/test/controller/test_ccachestats.py new file mode 100644 index 000000000..7a6608ec8 --- /dev/null +++ b/python/mozbuild/mozbuild/test/controller/test_ccachestats.py @@ -0,0 +1,208 @@ +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, You can obtain one at http://mozilla.org/MPL/2.0/. + +from __future__ import unicode_literals + +import unittest + +from mozunit import main + +from mozbuild.controller.building import CCacheStats + + +class TestCcacheStats(unittest.TestCase): + STAT_GARBAGE = """A garbage line which should be failed to parse""" + + STAT0 = """ + cache directory /home/tlin/.ccache + cache hit (direct) 0 + cache hit (preprocessed) 0 + cache miss 0 + files in cache 0 + cache size 0 Kbytes + max cache size 16.0 Gbytes""" + + STAT1 = """ + cache directory /home/tlin/.ccache + cache hit (direct) 100 + cache hit (preprocessed) 200 + cache miss 2500 + called for link 180 + called for preprocessing 6 + compile failed 11 + preprocessor error 3 + bad compiler arguments 6 + unsupported source language 9 + autoconf compile/link 60 + unsupported compiler option 2 + no input file 21 + files in cache 7344 + cache size 1.9 Gbytes + max cache size 16.0 Gbytes""" + + STAT2 = """ + cache directory /home/tlin/.ccache + cache hit (direct) 1900 + cache hit (preprocessed) 300 + cache miss 2600 + called for link 361 + called for preprocessing 12 + compile failed 22 + preprocessor error 6 + bad compiler arguments 12 + unsupported source language 18 + autoconf compile/link 120 + unsupported compiler option 4 + no input file 48 + files in cache 7392 + cache size 2.0 Gbytes + max cache size 16.0 Gbytes""" + + STAT3 = """ + cache directory /Users/tlin/.ccache + primary config /Users/tlin/.ccache/ccache.conf + secondary config (readonly) /usr/local/Cellar/ccache/3.2/etc/ccache.conf + cache hit (direct) 12004 + cache hit (preprocessed) 1786 + cache miss 26348 + called for link 2338 + called for preprocessing 6313 + compile failed 399 + preprocessor error 390 + bad compiler arguments 86 + unsupported source language 66 + autoconf compile/link 2439 + unsupported compiler option 187 + no input file 1068 + files in cache 18044 + cache size 7.5 GB + max cache size 8.6 GB + """ + + STAT4 = """ + cache directory /Users/tlin/.ccache + primary config /Users/tlin/.ccache/ccache.conf + secondary config (readonly) /usr/local/Cellar/ccache/3.2.1/etc/ccache.conf + cache hit (direct) 21039 + cache hit (preprocessed) 2315 + cache miss 39370 + called for link 3651 + called for preprocessing 6693 + compile failed 723 + ccache internal error 1 + preprocessor error 588 + bad compiler arguments 128 + unsupported source language 99 + autoconf compile/link 3669 + unsupported compiler option 187 + no input file 1711 + files in cache 18313 + cache size 6.3 GB + max cache size 6.0 GB + """ + + STAT5 = """ + cache directory /Users/tlin/.ccache + primary config /Users/tlin/.ccache/ccache.conf + secondary config (readonly) /usr/local/Cellar/ccache/3.2.1/etc/ccache.conf + cache hit (direct) 21039 + cache hit (preprocessed) 2315 + cache miss 39372 + called for link 3653 + called for preprocessing 6693 + compile failed 723 + ccache internal error 1 + preprocessor error 588 + bad compiler arguments 128 + unsupported source language 99 + autoconf compile/link 3669 + unsupported compiler option 187 + no input file 1711 + files in cache 17411 + cache size 6.0 GB + max cache size 6.0 GB + """ + + STAT6 = """ + cache directory /Users/tlin/.ccache + primary config /Users/tlin/.ccache/ccache.conf + secondary config (readonly) /usr/local/Cellar/ccache/3.3.2/etc/ccache.conf + cache hit (direct) 319287 + cache hit (preprocessed) 125987 + cache miss 749959 + cache hit rate 37.25 % + called for link 87978 + called for preprocessing 418591 + multiple source files 1861 + compiler produced no output 122 + compiler produced empty output 174 + compile failed 14330 + ccache internal error 1 + preprocessor error 9459 + can't use precompiled header 4 + bad compiler arguments 2077 + unsupported source language 18195 + autoconf compile/link 51485 + unsupported compiler option 322 + no input file 309538 + cleanups performed 1 + files in cache 17358 + cache size 15.4 GB + max cache size 17.2 GB + """ + + def test_parse_garbage_stats_message(self): + self.assertRaises(ValueError, CCacheStats, self.STAT_GARBAGE) + + def test_parse_zero_stats_message(self): + stats = CCacheStats(self.STAT0) + self.assertEqual(stats.cache_dir, "/home/tlin/.ccache") + self.assertEqual(stats.hit_rates(), (0, 0, 0)) + + def test_hit_rate_of_diff_stats(self): + stats1 = CCacheStats(self.STAT1) + stats2 = CCacheStats(self.STAT2) + stats_diff = stats2 - stats1 + self.assertEqual(stats_diff.hit_rates(), (0.9, 0.05, 0.05)) + + def test_stats_contains_data(self): + stats0 = CCacheStats(self.STAT0) + stats1 = CCacheStats(self.STAT1) + stats2 = CCacheStats(self.STAT2) + stats_diff_zero = stats1 - stats1 + stats_diff_negative1 = stats0 - stats1 + stats_diff_negative2 = stats1 - stats2 + + self.assertFalse(stats0) + self.assertTrue(stats1) + self.assertTrue(stats2) + self.assertFalse(stats_diff_zero) + self.assertFalse(stats_diff_negative1) + self.assertFalse(stats_diff_negative2) + + def test_stats_version32(self): + stat2 = CCacheStats(self.STAT2) + stat3 = CCacheStats(self.STAT3) + stats_diff = stat3 - stat2 + self.assertTrue(stat3) + self.assertTrue(stats_diff) + + def test_cache_size_shrinking(self): + stat4 = CCacheStats(self.STAT4) + stat5 = CCacheStats(self.STAT5) + stats_diff = stat5 - stat4 + self.assertTrue(stat4) + self.assertTrue(stat5) + self.assertTrue(stats_diff) + + def test_stats_version33(self): + stat3 = CCacheStats(self.STAT3) + stat6 = CCacheStats(self.STAT6) + stats_diff = stat6 - stat3 + self.assertTrue(stat6) + self.assertTrue(stat3) + self.assertTrue(stats_diff) + +if __name__ == '__main__': + main() diff --git a/python/mozbuild/mozbuild/test/controller/test_clobber.py b/python/mozbuild/mozbuild/test/controller/test_clobber.py new file mode 100644 index 000000000..997f467ec --- /dev/null +++ b/python/mozbuild/mozbuild/test/controller/test_clobber.py @@ -0,0 +1,213 @@ +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, You can obtain one at http://mozilla.org/MPL/2.0/. + +from __future__ import unicode_literals + +import os +import shutil +import tempfile +import unittest + +from StringIO import StringIO + +from mozunit import main + +from mozbuild.controller.clobber import Clobberer +from mozbuild.controller.clobber import main as clobber + + +class TestClobberer(unittest.TestCase): + def setUp(self): + self._temp_dirs = [] + + return unittest.TestCase.setUp(self) + + def tearDown(self): + for d in self._temp_dirs: + shutil.rmtree(d, ignore_errors=True) + + return unittest.TestCase.tearDown(self) + + def get_tempdir(self): + t = tempfile.mkdtemp() + self._temp_dirs.append(t) + return t + + def get_topsrcdir(self): + t = self.get_tempdir() + p = os.path.join(t, 'CLOBBER') + with open(p, 'a'): + pass + + return t + + def test_no_objdir(self): + """If topobjdir does not exist, no clobber is needed.""" + + tmp = os.path.join(self.get_tempdir(), 'topobjdir') + self.assertFalse(os.path.exists(tmp)) + + c = Clobberer(self.get_topsrcdir(), tmp) + self.assertFalse(c.clobber_needed()) + + # Side-effect is topobjdir is created with CLOBBER file touched. + required, performed, reason = c.maybe_do_clobber(os.getcwd(), True) + self.assertFalse(required) + self.assertFalse(performed) + self.assertIsNone(reason) + + self.assertTrue(os.path.isdir(tmp)) + self.assertTrue(os.path.exists(os.path.join(tmp, 'CLOBBER'))) + + def test_objdir_no_clobber_file(self): + """If CLOBBER does not exist in topobjdir, treat as empty.""" + + c = Clobberer(self.get_topsrcdir(), self.get_tempdir()) + self.assertFalse(c.clobber_needed()) + + required, performed, reason = c.maybe_do_clobber(os.getcwd(), True) + self.assertFalse(required) + self.assertFalse(performed) + self.assertIsNone(reason) + + self.assertTrue(os.path.exists(os.path.join(c.topobjdir, 'CLOBBER'))) + + def test_objdir_clobber_newer(self): + """If CLOBBER in topobjdir is newer, do nothing.""" + + c = Clobberer(self.get_topsrcdir(), self.get_tempdir()) + with open(c.obj_clobber, 'a'): + pass + + required, performed, reason = c.maybe_do_clobber(os.getcwd(), True) + self.assertFalse(required) + self.assertFalse(performed) + self.assertIsNone(reason) + + def test_objdir_clobber_older(self): + """If CLOBBER in topobjdir is older, we clobber.""" + + c = Clobberer(self.get_topsrcdir(), self.get_tempdir()) + with open(c.obj_clobber, 'a'): + pass + + dummy_path = os.path.join(c.topobjdir, 'foo') + with open(dummy_path, 'a'): + pass + + self.assertTrue(os.path.exists(dummy_path)) + + old_time = os.path.getmtime(c.src_clobber) - 60 + os.utime(c.obj_clobber, (old_time, old_time)) + + self.assertTrue(c.clobber_needed()) + + required, performed, reason = c.maybe_do_clobber(os.getcwd(), True) + self.assertTrue(required) + self.assertTrue(performed) + + self.assertFalse(os.path.exists(dummy_path)) + self.assertTrue(os.path.exists(c.obj_clobber)) + self.assertGreaterEqual(os.path.getmtime(c.obj_clobber), + os.path.getmtime(c.src_clobber)) + + def test_objdir_is_srcdir(self): + """If topobjdir is the topsrcdir, refuse to clobber.""" + + tmp = self.get_topsrcdir() + c = Clobberer(tmp, tmp) + + self.assertFalse(c.clobber_needed()) + + def test_cwd_is_topobjdir(self): + """If cwd is topobjdir, we can still clobber.""" + c = Clobberer(self.get_topsrcdir(), self.get_tempdir()) + + with open(c.obj_clobber, 'a'): + pass + + dummy_file = os.path.join(c.topobjdir, 'dummy_file') + with open(dummy_file, 'a'): + pass + + dummy_dir = os.path.join(c.topobjdir, 'dummy_dir') + os.mkdir(dummy_dir) + + self.assertTrue(os.path.exists(dummy_file)) + self.assertTrue(os.path.isdir(dummy_dir)) + + old_time = os.path.getmtime(c.src_clobber) - 60 + os.utime(c.obj_clobber, (old_time, old_time)) + + self.assertTrue(c.clobber_needed()) + + required, performed, reason = c.maybe_do_clobber(c.topobjdir, True) + self.assertTrue(required) + self.assertTrue(performed) + + self.assertFalse(os.path.exists(dummy_file)) + self.assertFalse(os.path.exists(dummy_dir)) + + def test_cwd_under_topobjdir(self): + """If cwd is under topobjdir, we can't clobber.""" + + c = Clobberer(self.get_topsrcdir(), self.get_tempdir()) + + with open(c.obj_clobber, 'a'): + pass + + old_time = os.path.getmtime(c.src_clobber) - 60 + os.utime(c.obj_clobber, (old_time, old_time)) + + d = os.path.join(c.topobjdir, 'dummy_dir') + os.mkdir(d) + + required, performed, reason = c.maybe_do_clobber(d, True) + self.assertTrue(required) + self.assertFalse(performed) + self.assertIn('Cannot clobber while the shell is inside', reason) + + + def test_mozconfig_opt_in(self): + """Auto clobber iff AUTOCLOBBER is in the environment.""" + + topsrcdir = self.get_topsrcdir() + topobjdir = self.get_tempdir() + + obj_clobber = os.path.join(topobjdir, 'CLOBBER') + with open(obj_clobber, 'a'): + pass + + dummy_file = os.path.join(topobjdir, 'dummy_file') + with open(dummy_file, 'a'): + pass + + self.assertTrue(os.path.exists(dummy_file)) + + old_time = os.path.getmtime(os.path.join(topsrcdir, 'CLOBBER')) - 60 + os.utime(obj_clobber, (old_time, old_time)) + + # Check auto clobber is off by default + env = dict(os.environ) + if env.get('AUTOCLOBBER', False): + del env['AUTOCLOBBER'] + + s = StringIO() + status = clobber([topsrcdir, topobjdir], env, os.getcwd(), s) + self.assertEqual(status, 1) + self.assertIn('Automatic clobbering is not enabled', s.getvalue()) + self.assertTrue(os.path.exists(dummy_file)) + + # Check auto clobber opt-in works + env['AUTOCLOBBER'] = '1' + + s = StringIO() + status = clobber([topsrcdir, topobjdir], env, os.getcwd(), s) + self.assertEqual(status, 0) + self.assertIn('Successfully completed auto clobber', s.getvalue()) + self.assertFalse(os.path.exists(dummy_file)) + + +if __name__ == '__main__': + main() diff --git a/python/mozbuild/mozbuild/test/data/Makefile b/python/mozbuild/mozbuild/test/data/Makefile new file mode 100644 index 000000000..e69de29bb diff --git a/python/mozbuild/mozbuild/test/data/bad.properties b/python/mozbuild/mozbuild/test/data/bad.properties new file mode 100644 index 000000000..d4d8109b6 --- /dev/null +++ b/python/mozbuild/mozbuild/test/data/bad.properties @@ -0,0 +1,12 @@ +# A region.properties file with invalid unicode byte sequences. The +# sequences were cribbed from Markus Kuhn's "UTF-8 decoder capability +# and stress test", available at +# http://www.cl.cam.ac.uk/~mgk25/ucs/examples/UTF-8-test.txt + +# 3.5 Impossible bytes | +# | +# The following two bytes cannot appear in a correct UTF-8 string | +# | +# 3.5.1 fe = "þ" | +# 3.5.2 ff = "ÿ" | +# 3.5.3 fe fe ff ff = "þþÿÿ" | diff --git a/python/mozbuild/mozbuild/test/data/test-dir/Makefile b/python/mozbuild/mozbuild/test/data/test-dir/Makefile new file mode 100644 index 000000000..e69de29bb diff --git a/python/mozbuild/mozbuild/test/data/test-dir/with/Makefile b/python/mozbuild/mozbuild/test/data/test-dir/with/Makefile new file mode 100644 index 000000000..e69de29bb diff --git a/python/mozbuild/mozbuild/test/data/test-dir/with/without/with/Makefile b/python/mozbuild/mozbuild/test/data/test-dir/with/without/with/Makefile new file mode 100644 index 000000000..e69de29bb diff --git a/python/mozbuild/mozbuild/test/data/test-dir/without/with/Makefile b/python/mozbuild/mozbuild/test/data/test-dir/without/with/Makefile new file mode 100644 index 000000000..e69de29bb diff --git a/python/mozbuild/mozbuild/test/data/valid.properties b/python/mozbuild/mozbuild/test/data/valid.properties new file mode 100644 index 000000000..db64bf2ee --- /dev/null +++ b/python/mozbuild/mozbuild/test/data/valid.properties @@ -0,0 +1,11 @@ +# A region.properties file with unicode characters. + +# Danish. +# #### ~~ Søren Munk Skrøder, sskroeder - 2009-05-30 @ #mozmae + +# Korean. +A.title=í•œë©”ì¼ + +# Russian. +list.0 = test +list.1 = Ð¯Ð½Ð´ÐµÐºÑ diff --git a/python/mozbuild/mozbuild/test/frontend/__init__.py b/python/mozbuild/mozbuild/test/frontend/__init__.py new file mode 100644 index 000000000..e69de29bb diff --git a/python/mozbuild/mozbuild/test/frontend/data/android-res-dirs/dir1/foo b/python/mozbuild/mozbuild/test/frontend/data/android-res-dirs/dir1/foo new file mode 100644 index 000000000..e69de29bb diff --git a/python/mozbuild/mozbuild/test/frontend/data/android-res-dirs/moz.build b/python/mozbuild/mozbuild/test/frontend/data/android-res-dirs/moz.build new file mode 100644 index 000000000..242a3628d --- /dev/null +++ b/python/mozbuild/mozbuild/test/frontend/data/android-res-dirs/moz.build @@ -0,0 +1,9 @@ +# -*- Mode: python; indent-tabs-mode: nil; tab-width: 40 -*- +# Any copyright is dedicated to the Public Domain. +# http://creativecommons.org/publicdomain/zero/1.0/ + +ANDROID_RES_DIRS += [ + '/dir1', + '!/dir2', + '%/dir3', +] diff --git a/python/mozbuild/mozbuild/test/frontend/data/binary-components/bar/moz.build b/python/mozbuild/mozbuild/test/frontend/data/binary-components/bar/moz.build new file mode 100644 index 000000000..2946e42aa --- /dev/null +++ b/python/mozbuild/mozbuild/test/frontend/data/binary-components/bar/moz.build @@ -0,0 +1,2 @@ +Component('bar') +NO_COMPONENTS_MANIFEST = True diff --git a/python/mozbuild/mozbuild/test/frontend/data/binary-components/foo/moz.build b/python/mozbuild/mozbuild/test/frontend/data/binary-components/foo/moz.build new file mode 100644 index 000000000..8611a74be --- /dev/null +++ b/python/mozbuild/mozbuild/test/frontend/data/binary-components/foo/moz.build @@ -0,0 +1 @@ +Component('foo') diff --git a/python/mozbuild/mozbuild/test/frontend/data/binary-components/moz.build b/python/mozbuild/mozbuild/test/frontend/data/binary-components/moz.build new file mode 100644 index 000000000..1776d0514 --- /dev/null +++ b/python/mozbuild/mozbuild/test/frontend/data/binary-components/moz.build @@ -0,0 +1,10 @@ +@template +def Component(name): + LIBRARY_NAME = name + FORCE_SHARED_LIB = True + IS_COMPONENT = True + +DIRS += [ + 'foo', + 'bar', +] diff --git a/python/mozbuild/mozbuild/test/frontend/data/branding-files/bar.ico b/python/mozbuild/mozbuild/test/frontend/data/branding-files/bar.ico new file mode 100644 index 000000000..e69de29bb diff --git a/python/mozbuild/mozbuild/test/frontend/data/branding-files/baz.png b/python/mozbuild/mozbuild/test/frontend/data/branding-files/baz.png new file mode 100644 index 000000000..e69de29bb diff --git a/python/mozbuild/mozbuild/test/frontend/data/branding-files/foo.xpm b/python/mozbuild/mozbuild/test/frontend/data/branding-files/foo.xpm new file mode 100644 index 000000000..e69de29bb diff --git a/python/mozbuild/mozbuild/test/frontend/data/branding-files/moz.build b/python/mozbuild/mozbuild/test/frontend/data/branding-files/moz.build new file mode 100644 index 000000000..251bc53ea --- /dev/null +++ b/python/mozbuild/mozbuild/test/frontend/data/branding-files/moz.build @@ -0,0 +1,13 @@ +# Any copyright is dedicated to the Public Domain. +# http://creativecommons.org/publicdomain/zero/1.0/ + +BRANDING_FILES += [ + 'bar.ico', + 'baz.png', + 'foo.xpm', +] + +BRANDING_FILES.icons += [ + 'quux.icns', +] + diff --git a/python/mozbuild/mozbuild/test/frontend/data/branding-files/quux.icns b/python/mozbuild/mozbuild/test/frontend/data/branding-files/quux.icns new file mode 100644 index 000000000..e69de29bb diff --git a/python/mozbuild/mozbuild/test/frontend/data/config-file-substitution/moz.build b/python/mozbuild/mozbuild/test/frontend/data/config-file-substitution/moz.build new file mode 100644 index 000000000..f53dd9454 --- /dev/null +++ b/python/mozbuild/mozbuild/test/frontend/data/config-file-substitution/moz.build @@ -0,0 +1,6 @@ +# -*- Mode: python; indent-tabs-mode: nil; tab-width: 40 -*- +# Any copyright is dedicated to the Public Domain. +# http://creativecommons.org/publicdomain/zero/1.0/ + +CONFIGURE_SUBST_FILES += ['foo'] +CONFIGURE_SUBST_FILES += ['bar'] diff --git a/python/mozbuild/mozbuild/test/frontend/data/crate-dependency-path-resolution/Cargo.toml b/python/mozbuild/mozbuild/test/frontend/data/crate-dependency-path-resolution/Cargo.toml new file mode 100644 index 000000000..99d10b1a6 --- /dev/null +++ b/python/mozbuild/mozbuild/test/frontend/data/crate-dependency-path-resolution/Cargo.toml @@ -0,0 +1,18 @@ +[package] +name = "random-crate" +version = "0.1.0" +authors = [ + "Nobody ", +] + +[lib] +crate-type = ["staticlib"] + +[dependencies] +deep-crate = { version = "0.1.0", path = "the/depths" } + +[profile.dev] +panic = "abort" + +[profile.release] +panic = "abort" diff --git a/python/mozbuild/mozbuild/test/frontend/data/crate-dependency-path-resolution/moz.build b/python/mozbuild/mozbuild/test/frontend/data/crate-dependency-path-resolution/moz.build new file mode 100644 index 000000000..01b3a35a7 --- /dev/null +++ b/python/mozbuild/mozbuild/test/frontend/data/crate-dependency-path-resolution/moz.build @@ -0,0 +1,18 @@ +# Any copyright is dedicated to the Public Domain. +# http://creativecommons.org/publicdomain/zero/1.0/ + +@template +def Library(name): + '''Template for libraries.''' + LIBRARY_NAME = name + + +@template +def RustLibrary(name): + '''Template for Rust libraries.''' + Library(name) + + IS_RUST_LIBRARY = True + + +RustLibrary('random-crate') diff --git a/python/mozbuild/mozbuild/test/frontend/data/crate-dependency-path-resolution/shallow/Cargo.toml b/python/mozbuild/mozbuild/test/frontend/data/crate-dependency-path-resolution/shallow/Cargo.toml new file mode 100644 index 000000000..c347f8c08 --- /dev/null +++ b/python/mozbuild/mozbuild/test/frontend/data/crate-dependency-path-resolution/shallow/Cargo.toml @@ -0,0 +1,6 @@ +[package] +name = "shallow-crate" +version = "0.1.0" +authors = [ + "Nobody ", +] diff --git a/python/mozbuild/mozbuild/test/frontend/data/crate-dependency-path-resolution/the/depths/Cargo.toml b/python/mozbuild/mozbuild/test/frontend/data/crate-dependency-path-resolution/the/depths/Cargo.toml new file mode 100644 index 000000000..10a4ded0a --- /dev/null +++ b/python/mozbuild/mozbuild/test/frontend/data/crate-dependency-path-resolution/the/depths/Cargo.toml @@ -0,0 +1,9 @@ +[package] +name = "deep-crate" +version = "0.1.0" +authors = [ + "Nobody ", +] + +[dependencies] +shallow-crate = { path = "../../shallow" } diff --git a/python/mozbuild/mozbuild/test/frontend/data/defines/moz.build b/python/mozbuild/mozbuild/test/frontend/data/defines/moz.build new file mode 100644 index 000000000..ccb0d5e36 --- /dev/null +++ b/python/mozbuild/mozbuild/test/frontend/data/defines/moz.build @@ -0,0 +1,14 @@ +# Any copyright is dedicated to the Public Domain. +# http://creativecommons.org/publicdomain/zero/1.0/ + +value = 'xyz' +DEFINES = { + 'FOO': True, +} + +DEFINES['BAZ'] = '"abcd"' +DEFINES.update({ + 'BAR': 7, + 'VALUE': value, + 'QUX': False, +}) diff --git a/python/mozbuild/mozbuild/test/frontend/data/dist-files-missing/install.rdf b/python/mozbuild/mozbuild/test/frontend/data/dist-files-missing/install.rdf new file mode 100644 index 000000000..e69de29bb diff --git a/python/mozbuild/mozbuild/test/frontend/data/dist-files-missing/moz.build b/python/mozbuild/mozbuild/test/frontend/data/dist-files-missing/moz.build new file mode 100644 index 000000000..cbd2c942b --- /dev/null +++ b/python/mozbuild/mozbuild/test/frontend/data/dist-files-missing/moz.build @@ -0,0 +1,8 @@ +# -*- Mode: python; indent-tabs-mode: nil; tab-width: 40 -*- +# Any copyright is dedicated to the Public Domain. +# http://creativecommons.org/publicdomain/zero/1.0/ + +FINAL_TARGET_PP_FILES += [ + 'install.rdf', + 'main.js', +] diff --git a/python/mozbuild/mozbuild/test/frontend/data/dist-files/install.rdf b/python/mozbuild/mozbuild/test/frontend/data/dist-files/install.rdf new file mode 100644 index 000000000..e69de29bb diff --git a/python/mozbuild/mozbuild/test/frontend/data/dist-files/main.js b/python/mozbuild/mozbuild/test/frontend/data/dist-files/main.js new file mode 100644 index 000000000..e69de29bb diff --git a/python/mozbuild/mozbuild/test/frontend/data/dist-files/moz.build b/python/mozbuild/mozbuild/test/frontend/data/dist-files/moz.build new file mode 100644 index 000000000..cbd2c942b --- /dev/null +++ b/python/mozbuild/mozbuild/test/frontend/data/dist-files/moz.build @@ -0,0 +1,8 @@ +# -*- Mode: python; indent-tabs-mode: nil; tab-width: 40 -*- +# Any copyright is dedicated to the Public Domain. +# http://creativecommons.org/publicdomain/zero/1.0/ + +FINAL_TARGET_PP_FILES += [ + 'install.rdf', + 'main.js', +] diff --git a/python/mozbuild/mozbuild/test/frontend/data/exports-generated/foo.h b/python/mozbuild/mozbuild/test/frontend/data/exports-generated/foo.h new file mode 100644 index 000000000..e69de29bb diff --git a/python/mozbuild/mozbuild/test/frontend/data/exports-generated/moz.build b/python/mozbuild/mozbuild/test/frontend/data/exports-generated/moz.build new file mode 100644 index 000000000..259d96fcd --- /dev/null +++ b/python/mozbuild/mozbuild/test/frontend/data/exports-generated/moz.build @@ -0,0 +1,8 @@ +# Any copyright is dedicated to the Public Domain. +# http://creativecommons.org/publicdomain/zero/1.0/ + +EXPORTS += ['foo.h'] +EXPORTS.mozilla += ['mozilla1.h'] +EXPORTS.mozilla += ['!mozilla2.h'] + +GENERATED_FILES += ['mozilla2.h'] diff --git a/python/mozbuild/mozbuild/test/frontend/data/exports-generated/mozilla1.h b/python/mozbuild/mozbuild/test/frontend/data/exports-generated/mozilla1.h new file mode 100644 index 000000000..e69de29bb diff --git a/python/mozbuild/mozbuild/test/frontend/data/exports-missing-generated/foo.h b/python/mozbuild/mozbuild/test/frontend/data/exports-missing-generated/foo.h new file mode 100644 index 000000000..e69de29bb diff --git a/python/mozbuild/mozbuild/test/frontend/data/exports-missing-generated/moz.build b/python/mozbuild/mozbuild/test/frontend/data/exports-missing-generated/moz.build new file mode 100644 index 000000000..e0dfce264 --- /dev/null +++ b/python/mozbuild/mozbuild/test/frontend/data/exports-missing-generated/moz.build @@ -0,0 +1,5 @@ +# Any copyright is dedicated to the Public Domain. +# http://creativecommons.org/publicdomain/zero/1.0/ + +EXPORTS += ['foo.h'] +EXPORTS += ['!bar.h'] diff --git a/python/mozbuild/mozbuild/test/frontend/data/exports-missing/foo.h b/python/mozbuild/mozbuild/test/frontend/data/exports-missing/foo.h new file mode 100644 index 000000000..e69de29bb diff --git a/python/mozbuild/mozbuild/test/frontend/data/exports-missing/moz.build b/python/mozbuild/mozbuild/test/frontend/data/exports-missing/moz.build new file mode 100644 index 000000000..e1f93aab5 --- /dev/null +++ b/python/mozbuild/mozbuild/test/frontend/data/exports-missing/moz.build @@ -0,0 +1,6 @@ +# Any copyright is dedicated to the Public Domain. +# http://creativecommons.org/publicdomain/zero/1.0/ + +EXPORTS += ['foo.h'] +EXPORTS.mozilla += ['mozilla1.h'] +EXPORTS.mozilla += ['mozilla2.h'] diff --git a/python/mozbuild/mozbuild/test/frontend/data/exports-missing/mozilla1.h b/python/mozbuild/mozbuild/test/frontend/data/exports-missing/mozilla1.h new file mode 100644 index 000000000..e69de29bb diff --git a/python/mozbuild/mozbuild/test/frontend/data/exports/bar.h b/python/mozbuild/mozbuild/test/frontend/data/exports/bar.h new file mode 100644 index 000000000..e69de29bb diff --git a/python/mozbuild/mozbuild/test/frontend/data/exports/baz.h b/python/mozbuild/mozbuild/test/frontend/data/exports/baz.h new file mode 100644 index 000000000..e69de29bb diff --git a/python/mozbuild/mozbuild/test/frontend/data/exports/dom1.h b/python/mozbuild/mozbuild/test/frontend/data/exports/dom1.h new file mode 100644 index 000000000..e69de29bb diff --git a/python/mozbuild/mozbuild/test/frontend/data/exports/dom2.h b/python/mozbuild/mozbuild/test/frontend/data/exports/dom2.h new file mode 100644 index 000000000..e69de29bb diff --git a/python/mozbuild/mozbuild/test/frontend/data/exports/dom3.h b/python/mozbuild/mozbuild/test/frontend/data/exports/dom3.h new file mode 100644 index 000000000..e69de29bb diff --git a/python/mozbuild/mozbuild/test/frontend/data/exports/foo.h b/python/mozbuild/mozbuild/test/frontend/data/exports/foo.h new file mode 100644 index 000000000..e69de29bb diff --git a/python/mozbuild/mozbuild/test/frontend/data/exports/gfx.h b/python/mozbuild/mozbuild/test/frontend/data/exports/gfx.h new file mode 100644 index 000000000..e69de29bb diff --git a/python/mozbuild/mozbuild/test/frontend/data/exports/mem.h b/python/mozbuild/mozbuild/test/frontend/data/exports/mem.h new file mode 100644 index 000000000..e69de29bb diff --git a/python/mozbuild/mozbuild/test/frontend/data/exports/mem2.h b/python/mozbuild/mozbuild/test/frontend/data/exports/mem2.h new file mode 100644 index 000000000..e69de29bb diff --git a/python/mozbuild/mozbuild/test/frontend/data/exports/moz.build b/python/mozbuild/mozbuild/test/frontend/data/exports/moz.build new file mode 100644 index 000000000..666fbeb81 --- /dev/null +++ b/python/mozbuild/mozbuild/test/frontend/data/exports/moz.build @@ -0,0 +1,13 @@ +# Any copyright is dedicated to the Public Domain. +# http://creativecommons.org/publicdomain/zero/1.0/ + +EXPORTS += ['foo.h'] +EXPORTS += ['bar.h', 'baz.h'] +EXPORTS.mozilla += ['mozilla1.h'] +EXPORTS.mozilla += ['mozilla2.h'] +EXPORTS.mozilla.dom += ['dom1.h'] +EXPORTS.mozilla.dom += ['dom2.h', 'dom3.h'] +EXPORTS.mozilla.gfx += ['gfx.h'] +EXPORTS.vpx = ['mem.h'] +EXPORTS.vpx += ['mem2.h'] +EXPORTS.nspr.private = ['pprio.h', 'pprthred.h'] diff --git a/python/mozbuild/mozbuild/test/frontend/data/exports/mozilla1.h b/python/mozbuild/mozbuild/test/frontend/data/exports/mozilla1.h new file mode 100644 index 000000000..e69de29bb diff --git a/python/mozbuild/mozbuild/test/frontend/data/exports/mozilla2.h b/python/mozbuild/mozbuild/test/frontend/data/exports/mozilla2.h new file mode 100644 index 000000000..e69de29bb diff --git a/python/mozbuild/mozbuild/test/frontend/data/exports/pprio.h b/python/mozbuild/mozbuild/test/frontend/data/exports/pprio.h new file mode 100644 index 000000000..e69de29bb diff --git a/python/mozbuild/mozbuild/test/frontend/data/exports/pprthred.h b/python/mozbuild/mozbuild/test/frontend/data/exports/pprthred.h new file mode 100644 index 000000000..e69de29bb diff --git a/python/mozbuild/mozbuild/test/frontend/data/files-info/bug_component/bad-assignment/moz.build b/python/mozbuild/mozbuild/test/frontend/data/files-info/bug_component/bad-assignment/moz.build new file mode 100644 index 000000000..d6a9799b8 --- /dev/null +++ b/python/mozbuild/mozbuild/test/frontend/data/files-info/bug_component/bad-assignment/moz.build @@ -0,0 +1,2 @@ +with Files('*'): + BUG_COMPONENT = 'bad value' diff --git a/python/mozbuild/mozbuild/test/frontend/data/files-info/bug_component/different-matchers/moz.build b/python/mozbuild/mozbuild/test/frontend/data/files-info/bug_component/different-matchers/moz.build new file mode 100644 index 000000000..990453f7c --- /dev/null +++ b/python/mozbuild/mozbuild/test/frontend/data/files-info/bug_component/different-matchers/moz.build @@ -0,0 +1,4 @@ +with Files('*.jsm'): + BUG_COMPONENT = ('Firefox', 'JS') +with Files('*.cpp'): + BUG_COMPONENT = ('Firefox', 'C++') diff --git a/python/mozbuild/mozbuild/test/frontend/data/files-info/bug_component/final/moz.build b/python/mozbuild/mozbuild/test/frontend/data/files-info/bug_component/final/moz.build new file mode 100644 index 000000000..cee286445 --- /dev/null +++ b/python/mozbuild/mozbuild/test/frontend/data/files-info/bug_component/final/moz.build @@ -0,0 +1,3 @@ +with Files('**/Makefile.in'): + BUG_COMPONENT = ('Core', 'Build Config') + FINAL = True diff --git a/python/mozbuild/mozbuild/test/frontend/data/files-info/bug_component/final/subcomponent/moz.build b/python/mozbuild/mozbuild/test/frontend/data/files-info/bug_component/final/subcomponent/moz.build new file mode 100644 index 000000000..206bf661b --- /dev/null +++ b/python/mozbuild/mozbuild/test/frontend/data/files-info/bug_component/final/subcomponent/moz.build @@ -0,0 +1,2 @@ +with Files('**'): + BUG_COMPONENT = ('Another', 'Component') diff --git a/python/mozbuild/mozbuild/test/frontend/data/files-info/bug_component/moz.build b/python/mozbuild/mozbuild/test/frontend/data/files-info/bug_component/moz.build new file mode 100644 index 000000000..4ecb1112c --- /dev/null +++ b/python/mozbuild/mozbuild/test/frontend/data/files-info/bug_component/moz.build @@ -0,0 +1,2 @@ +with Files('**'): + BUG_COMPONENT = ('default_product', 'default_component') diff --git a/python/mozbuild/mozbuild/test/frontend/data/files-info/bug_component/simple/moz.build b/python/mozbuild/mozbuild/test/frontend/data/files-info/bug_component/simple/moz.build new file mode 100644 index 000000000..7994d4a38 --- /dev/null +++ b/python/mozbuild/mozbuild/test/frontend/data/files-info/bug_component/simple/moz.build @@ -0,0 +1,2 @@ +with Files('*'): + BUG_COMPONENT = ('Core', 'Build Config') diff --git a/python/mozbuild/mozbuild/test/frontend/data/files-info/bug_component/static/moz.build b/python/mozbuild/mozbuild/test/frontend/data/files-info/bug_component/static/moz.build new file mode 100644 index 000000000..0a88e09e7 --- /dev/null +++ b/python/mozbuild/mozbuild/test/frontend/data/files-info/bug_component/static/moz.build @@ -0,0 +1,5 @@ +with Files('foo'): + BUG_COMPONENT = ('FooProduct', 'FooComponent') + +with Files('bar'): + BUG_COMPONENT = ('BarProduct', 'BarComponent') diff --git a/python/mozbuild/mozbuild/test/frontend/data/files-info/moz.build b/python/mozbuild/mozbuild/test/frontend/data/files-info/moz.build new file mode 100644 index 000000000..e69de29bb diff --git a/python/mozbuild/mozbuild/test/frontend/data/files-test-metadata/default/module.js b/python/mozbuild/mozbuild/test/frontend/data/files-test-metadata/default/module.js new file mode 100644 index 000000000..e69de29bb diff --git a/python/mozbuild/mozbuild/test/frontend/data/files-test-metadata/default/moz.build b/python/mozbuild/mozbuild/test/frontend/data/files-test-metadata/default/moz.build new file mode 100644 index 000000000..8915edc12 --- /dev/null +++ b/python/mozbuild/mozbuild/test/frontend/data/files-test-metadata/default/moz.build @@ -0,0 +1,6 @@ +XPCSHELL_TESTS_MANIFESTS += ['tests/xpcshell/xpcshell.ini'] +REFTEST_MANIFESTS += ['tests/reftests/reftest.list'] + +EXTRA_JS_MODULES += [ + 'module.js', +] \ No newline at end of file diff --git a/python/mozbuild/mozbuild/test/frontend/data/files-test-metadata/default/tests/reftests/reftest-stylo.list b/python/mozbuild/mozbuild/test/frontend/data/files-test-metadata/default/tests/reftests/reftest-stylo.list new file mode 100644 index 000000000..252a5b986 --- /dev/null +++ b/python/mozbuild/mozbuild/test/frontend/data/files-test-metadata/default/tests/reftests/reftest-stylo.list @@ -0,0 +1,2 @@ +# DO NOT EDIT! This is a auto-generated temporary list for Stylo testing +== test1.html test1.html diff --git a/python/mozbuild/mozbuild/test/frontend/data/files-test-metadata/default/tests/reftests/reftest.list b/python/mozbuild/mozbuild/test/frontend/data/files-test-metadata/default/tests/reftests/reftest.list new file mode 100644 index 000000000..504d45973 --- /dev/null +++ b/python/mozbuild/mozbuild/test/frontend/data/files-test-metadata/default/tests/reftests/reftest.list @@ -0,0 +1 @@ +== test1.html test1-ref.html \ No newline at end of file diff --git a/python/mozbuild/mozbuild/test/frontend/data/files-test-metadata/default/tests/reftests/test1-ref.html b/python/mozbuild/mozbuild/test/frontend/data/files-test-metadata/default/tests/reftests/test1-ref.html new file mode 100644 index 000000000..e69de29bb diff --git a/python/mozbuild/mozbuild/test/frontend/data/files-test-metadata/default/tests/reftests/test1.html b/python/mozbuild/mozbuild/test/frontend/data/files-test-metadata/default/tests/reftests/test1.html new file mode 100644 index 000000000..e69de29bb diff --git a/python/mozbuild/mozbuild/test/frontend/data/files-test-metadata/default/tests/xpcshell/test_default_mod.js b/python/mozbuild/mozbuild/test/frontend/data/files-test-metadata/default/tests/xpcshell/test_default_mod.js new file mode 100644 index 000000000..e69de29bb diff --git a/python/mozbuild/mozbuild/test/frontend/data/files-test-metadata/default/tests/xpcshell/xpcshell.ini b/python/mozbuild/mozbuild/test/frontend/data/files-test-metadata/default/tests/xpcshell/xpcshell.ini new file mode 100644 index 000000000..55c18a250 --- /dev/null +++ b/python/mozbuild/mozbuild/test/frontend/data/files-test-metadata/default/tests/xpcshell/xpcshell.ini @@ -0,0 +1 @@ +[test_default_mod.js] \ No newline at end of file diff --git a/python/mozbuild/mozbuild/test/frontend/data/files-test-metadata/moz.build b/python/mozbuild/mozbuild/test/frontend/data/files-test-metadata/moz.build new file mode 100644 index 000000000..faff2a173 --- /dev/null +++ b/python/mozbuild/mozbuild/test/frontend/data/files-test-metadata/moz.build @@ -0,0 +1,4 @@ +DIRS += [ + 'default', + 'simple', +] \ No newline at end of file diff --git a/python/mozbuild/mozbuild/test/frontend/data/files-test-metadata/simple/base.cpp b/python/mozbuild/mozbuild/test/frontend/data/files-test-metadata/simple/base.cpp new file mode 100644 index 000000000..e69de29bb diff --git a/python/mozbuild/mozbuild/test/frontend/data/files-test-metadata/simple/browser/browser.ini b/python/mozbuild/mozbuild/test/frontend/data/files-test-metadata/simple/browser/browser.ini new file mode 100644 index 000000000..f284de043 --- /dev/null +++ b/python/mozbuild/mozbuild/test/frontend/data/files-test-metadata/simple/browser/browser.ini @@ -0,0 +1 @@ +[test_mod.js] \ No newline at end of file diff --git a/python/mozbuild/mozbuild/test/frontend/data/files-test-metadata/simple/browser/test_mod.js b/python/mozbuild/mozbuild/test/frontend/data/files-test-metadata/simple/browser/test_mod.js new file mode 100644 index 000000000..e69de29bb diff --git a/python/mozbuild/mozbuild/test/frontend/data/files-test-metadata/simple/moz.build b/python/mozbuild/mozbuild/test/frontend/data/files-test-metadata/simple/moz.build new file mode 100644 index 000000000..cbce16e1d --- /dev/null +++ b/python/mozbuild/mozbuild/test/frontend/data/files-test-metadata/simple/moz.build @@ -0,0 +1,22 @@ +with Files('src/*'): + IMPACTED_TESTS.files += [ + 'tests/test_general.html', + ] + +with Files('src/module.jsm'): + IMPACTED_TESTS.files += [ + 'browser/**.js', + ] + +with Files('base.cpp'): + IMPACTED_TESTS.files += [ + '/default/tests/xpcshell/test_default_mod.js', + 'tests/*', + ] + + +MOCHITEST_MANIFESTS += ['tests/mochitest.ini'] +BROWSER_CHROME_MANIFESTS += ['browser/browser.ini'] + +UNIFIED_SOURCES += ['base.cpp'] +DIRS += ['src'] diff --git a/python/mozbuild/mozbuild/test/frontend/data/files-test-metadata/simple/src/module.jsm b/python/mozbuild/mozbuild/test/frontend/data/files-test-metadata/simple/src/module.jsm new file mode 100644 index 000000000..e69de29bb diff --git a/python/mozbuild/mozbuild/test/frontend/data/files-test-metadata/simple/src/moz.build b/python/mozbuild/mozbuild/test/frontend/data/files-test-metadata/simple/src/moz.build new file mode 100644 index 000000000..e0c49f129 --- /dev/null +++ b/python/mozbuild/mozbuild/test/frontend/data/files-test-metadata/simple/src/moz.build @@ -0,0 +1,3 @@ +EXTRA_JS_MODULES += [ + 'module.jsm', +] \ No newline at end of file diff --git a/python/mozbuild/mozbuild/test/frontend/data/files-test-metadata/simple/tests/mochitest.ini b/python/mozbuild/mozbuild/test/frontend/data/files-test-metadata/simple/tests/mochitest.ini new file mode 100644 index 000000000..662566abd --- /dev/null +++ b/python/mozbuild/mozbuild/test/frontend/data/files-test-metadata/simple/tests/mochitest.ini @@ -0,0 +1,2 @@ +[test_general.html] +[test_specific.html] \ No newline at end of file diff --git a/python/mozbuild/mozbuild/test/frontend/data/files-test-metadata/simple/tests/moz.build b/python/mozbuild/mozbuild/test/frontend/data/files-test-metadata/simple/tests/moz.build new file mode 100644 index 000000000..8ef3a9fd8 --- /dev/null +++ b/python/mozbuild/mozbuild/test/frontend/data/files-test-metadata/simple/tests/moz.build @@ -0,0 +1 @@ +MOCHITEST_MANIFESTS += ['mochitest.ini'] \ No newline at end of file diff --git a/python/mozbuild/mozbuild/test/frontend/data/files-test-metadata/simple/tests/test_general.html b/python/mozbuild/mozbuild/test/frontend/data/files-test-metadata/simple/tests/test_general.html new file mode 100644 index 000000000..e69de29bb diff --git a/python/mozbuild/mozbuild/test/frontend/data/files-test-metadata/simple/tests/test_specific.html b/python/mozbuild/mozbuild/test/frontend/data/files-test-metadata/simple/tests/test_specific.html new file mode 100644 index 000000000..e69de29bb diff --git a/python/mozbuild/mozbuild/test/frontend/data/files-test-metadata/tagged/moz.build b/python/mozbuild/mozbuild/test/frontend/data/files-test-metadata/tagged/moz.build new file mode 100644 index 000000000..0b7ca5a2b --- /dev/null +++ b/python/mozbuild/mozbuild/test/frontend/data/files-test-metadata/tagged/moz.build @@ -0,0 +1,15 @@ +with Files('src/submodule/**'): + IMPACTED_TESTS.tags += [ + 'submodule', + ] + +with Files('src/bar.jsm'): + IMPACTED_TESTS.flavors += [ + 'browser-chrome', + ] + IMPACTED_TESTS.files += [ + '**.js', + ] + +MOCHITEST_MANIFESTS += ['tests/mochitest.ini'] +XPCSHELL_TESTS_MANIFESTS += ['tests/xpcshell.ini'] diff --git a/python/mozbuild/mozbuild/test/frontend/data/files-test-metadata/tagged/src/bar.jsm b/python/mozbuild/mozbuild/test/frontend/data/files-test-metadata/tagged/src/bar.jsm new file mode 100644 index 000000000..e69de29bb diff --git a/python/mozbuild/mozbuild/test/frontend/data/files-test-metadata/tagged/src/submodule/foo.js b/python/mozbuild/mozbuild/test/frontend/data/files-test-metadata/tagged/src/submodule/foo.js new file mode 100644 index 000000000..e69de29bb diff --git a/python/mozbuild/mozbuild/test/frontend/data/files-test-metadata/tagged/tests/mochitest.ini b/python/mozbuild/mozbuild/test/frontend/data/files-test-metadata/tagged/tests/mochitest.ini new file mode 100644 index 000000000..d40ca4d06 --- /dev/null +++ b/python/mozbuild/mozbuild/test/frontend/data/files-test-metadata/tagged/tests/mochitest.ini @@ -0,0 +1,3 @@ +[test_simple.html] +[test_specific.html] +tags = submodule \ No newline at end of file diff --git a/python/mozbuild/mozbuild/test/frontend/data/files-test-metadata/tagged/tests/test_bar.js b/python/mozbuild/mozbuild/test/frontend/data/files-test-metadata/tagged/tests/test_bar.js new file mode 100644 index 000000000..e69de29bb diff --git a/python/mozbuild/mozbuild/test/frontend/data/files-test-metadata/tagged/tests/test_simple.html b/python/mozbuild/mozbuild/test/frontend/data/files-test-metadata/tagged/tests/test_simple.html new file mode 100644 index 000000000..e69de29bb diff --git a/python/mozbuild/mozbuild/test/frontend/data/files-test-metadata/tagged/tests/test_specific.html b/python/mozbuild/mozbuild/test/frontend/data/files-test-metadata/tagged/tests/test_specific.html new file mode 100644 index 000000000..e69de29bb diff --git a/python/mozbuild/mozbuild/test/frontend/data/files-test-metadata/tagged/tests/xpcshell.ini b/python/mozbuild/mozbuild/test/frontend/data/files-test-metadata/tagged/tests/xpcshell.ini new file mode 100644 index 000000000..1275764c4 --- /dev/null +++ b/python/mozbuild/mozbuild/test/frontend/data/files-test-metadata/tagged/tests/xpcshell.ini @@ -0,0 +1 @@ +[test_bar.js] \ No newline at end of file diff --git a/python/mozbuild/mozbuild/test/frontend/data/final-target-pp-files-non-srcdir/moz.build b/python/mozbuild/mozbuild/test/frontend/data/final-target-pp-files-non-srcdir/moz.build new file mode 100644 index 000000000..73132b0cf --- /dev/null +++ b/python/mozbuild/mozbuild/test/frontend/data/final-target-pp-files-non-srcdir/moz.build @@ -0,0 +1,7 @@ +# -*- Mode: python; indent-tabs-mode: nil; tab-width: 40 -*- +# Any copyright is dedicated to the Public Domain. +# http://creativecommons.org/publicdomain/zero/1.0/ + +FINAL_TARGET_PP_FILES += [ + '!foo.js', +] diff --git a/python/mozbuild/mozbuild/test/frontend/data/generated-files-absolute-script/moz.build b/python/mozbuild/mozbuild/test/frontend/data/generated-files-absolute-script/moz.build new file mode 100644 index 000000000..0b694ed84 --- /dev/null +++ b/python/mozbuild/mozbuild/test/frontend/data/generated-files-absolute-script/moz.build @@ -0,0 +1,9 @@ +# -*- Mode: python; indent-tabs-mode: nil; tab-width: 40 -*- +# Any copyright is dedicated to the Public Domain. +# http://creativecommons.org/publicdomain/zero/1.0/ + +GENERATED_FILES += ['bar.c'] + +bar = GENERATED_FILES['bar.c'] +bar.script = '/script.py:make_bar' +bar.inputs = [] diff --git a/python/mozbuild/mozbuild/test/frontend/data/generated-files-absolute-script/script.py b/python/mozbuild/mozbuild/test/frontend/data/generated-files-absolute-script/script.py new file mode 100644 index 000000000..e69de29bb diff --git a/python/mozbuild/mozbuild/test/frontend/data/generated-files-method-names/moz.build b/python/mozbuild/mozbuild/test/frontend/data/generated-files-method-names/moz.build new file mode 100644 index 000000000..e080b47f9 --- /dev/null +++ b/python/mozbuild/mozbuild/test/frontend/data/generated-files-method-names/moz.build @@ -0,0 +1,13 @@ +# -*- Mode: python; indent-tabs-mode: nil; tab-width: 40 -*- +# Any copyright is dedicated to the Public Domain. +# http://creativecommons.org/publicdomain/zero/1.0/ + +GENERATED_FILES += [ 'bar.c', 'foo.c' ] + +bar = GENERATED_FILES['bar.c'] +bar.script = 'script.py:make_bar' +bar.inputs = [] + +foo = GENERATED_FILES['foo.c'] +foo.script = 'script.py' +foo.inputs = [] diff --git a/python/mozbuild/mozbuild/test/frontend/data/generated-files-method-names/script.py b/python/mozbuild/mozbuild/test/frontend/data/generated-files-method-names/script.py new file mode 100644 index 000000000..e69de29bb diff --git a/python/mozbuild/mozbuild/test/frontend/data/generated-files-no-inputs/moz.build b/python/mozbuild/mozbuild/test/frontend/data/generated-files-no-inputs/moz.build new file mode 100644 index 000000000..da96c5fbc --- /dev/null +++ b/python/mozbuild/mozbuild/test/frontend/data/generated-files-no-inputs/moz.build @@ -0,0 +1,9 @@ +# -*- Mode: python; indent-tabs-mode: nil; tab-width: 40 -*- +# Any copyright is dedicated to the Public Domain. +# http://creativecommons.org/publicdomain/zero/1.0/ + +GENERATED_FILES += ['bar.c', 'foo.c'] + +foo = GENERATED_FILES['foo.c'] +foo.script = 'script.py' +foo.inputs = ['datafile'] diff --git a/python/mozbuild/mozbuild/test/frontend/data/generated-files-no-inputs/script.py b/python/mozbuild/mozbuild/test/frontend/data/generated-files-no-inputs/script.py new file mode 100644 index 000000000..e69de29bb diff --git a/python/mozbuild/mozbuild/test/frontend/data/generated-files-no-python-script/moz.build b/python/mozbuild/mozbuild/test/frontend/data/generated-files-no-python-script/moz.build new file mode 100644 index 000000000..080cb2a4e --- /dev/null +++ b/python/mozbuild/mozbuild/test/frontend/data/generated-files-no-python-script/moz.build @@ -0,0 +1,8 @@ +# -*- Mode: python; indent-tabs-mode: nil; tab-width: 40 -*- +# Any copyright is dedicated to the Public Domain. +# http://creativecommons.org/publicdomain/zero/1.0/ + +GENERATED_FILES += ['bar.c', 'foo.c'] + +bar = GENERATED_FILES['bar.c'] +bar.script = 'script.rb' diff --git a/python/mozbuild/mozbuild/test/frontend/data/generated-files-no-python-script/script.rb b/python/mozbuild/mozbuild/test/frontend/data/generated-files-no-python-script/script.rb new file mode 100644 index 000000000..e69de29bb diff --git a/python/mozbuild/mozbuild/test/frontend/data/generated-files-no-script/moz.build b/python/mozbuild/mozbuild/test/frontend/data/generated-files-no-script/moz.build new file mode 100644 index 000000000..90fa17666 --- /dev/null +++ b/python/mozbuild/mozbuild/test/frontend/data/generated-files-no-script/moz.build @@ -0,0 +1,8 @@ +# -*- Mode: python; indent-tabs-mode: nil; tab-width: 40 -*- +# Any copyright is dedicated to the Public Domain. +# http://creativecommons.org/publicdomain/zero/1.0/ + +GENERATED_FILES += [ 'bar.c', 'foo.c' ] + +bar = GENERATED_FILES['bar.c'] +bar.script = 'nonexistent-script.py' diff --git a/python/mozbuild/mozbuild/test/frontend/data/generated-files/moz.build b/python/mozbuild/mozbuild/test/frontend/data/generated-files/moz.build new file mode 100644 index 000000000..1c24113f3 --- /dev/null +++ b/python/mozbuild/mozbuild/test/frontend/data/generated-files/moz.build @@ -0,0 +1,5 @@ +# -*- Mode: python; indent-tabs-mode: nil; tab-width: 40 -*- +# Any copyright is dedicated to the Public Domain. +# http://creativecommons.org/publicdomain/zero/1.0/ + +GENERATED_FILES += [ 'bar.c', 'foo.c', ('xpidllex.py', 'xpidlyacc.py'), ] diff --git a/python/mozbuild/mozbuild/test/frontend/data/generated-sources/a.cpp b/python/mozbuild/mozbuild/test/frontend/data/generated-sources/a.cpp new file mode 100644 index 000000000..e69de29bb diff --git a/python/mozbuild/mozbuild/test/frontend/data/generated-sources/b.cc b/python/mozbuild/mozbuild/test/frontend/data/generated-sources/b.cc new file mode 100644 index 000000000..e69de29bb diff --git a/python/mozbuild/mozbuild/test/frontend/data/generated-sources/c.cxx b/python/mozbuild/mozbuild/test/frontend/data/generated-sources/c.cxx new file mode 100644 index 000000000..e69de29bb diff --git a/python/mozbuild/mozbuild/test/frontend/data/generated-sources/d.c b/python/mozbuild/mozbuild/test/frontend/data/generated-sources/d.c new file mode 100644 index 000000000..e69de29bb diff --git a/python/mozbuild/mozbuild/test/frontend/data/generated-sources/e.m b/python/mozbuild/mozbuild/test/frontend/data/generated-sources/e.m new file mode 100644 index 000000000..e69de29bb diff --git a/python/mozbuild/mozbuild/test/frontend/data/generated-sources/f.mm b/python/mozbuild/mozbuild/test/frontend/data/generated-sources/f.mm new file mode 100644 index 000000000..e69de29bb diff --git a/python/mozbuild/mozbuild/test/frontend/data/generated-sources/g.S b/python/mozbuild/mozbuild/test/frontend/data/generated-sources/g.S new file mode 100644 index 000000000..e69de29bb diff --git a/python/mozbuild/mozbuild/test/frontend/data/generated-sources/h.s b/python/mozbuild/mozbuild/test/frontend/data/generated-sources/h.s new file mode 100644 index 000000000..e69de29bb diff --git a/python/mozbuild/mozbuild/test/frontend/data/generated-sources/i.asm b/python/mozbuild/mozbuild/test/frontend/data/generated-sources/i.asm new file mode 100644 index 000000000..e69de29bb diff --git a/python/mozbuild/mozbuild/test/frontend/data/generated-sources/moz.build b/python/mozbuild/mozbuild/test/frontend/data/generated-sources/moz.build new file mode 100644 index 000000000..12d90b15c --- /dev/null +++ b/python/mozbuild/mozbuild/test/frontend/data/generated-sources/moz.build @@ -0,0 +1,37 @@ +# -*- Mode: python; indent-tabs-mode: nil; tab-width: 40 -*- +# Any copyright is dedicated to the Public Domain. +# http://creativecommons.org/publicdomain/zero/1.0/ + +@template +def Library(name): + '''Template for libraries.''' + LIBRARY_NAME = name + +Library('dummy') + +SOURCES += [ + '!a.cpp', + '!b.cc', + '!c.cxx', +] + +SOURCES += [ + '!d.c', +] + +SOURCES += [ + '!e.m', +] + +SOURCES += [ + '!f.mm', +] + +SOURCES += [ + '!g.S', +] + +SOURCES += [ + '!h.s', + '!i.asm', +] diff --git a/python/mozbuild/mozbuild/test/frontend/data/generated_includes/moz.build b/python/mozbuild/mozbuild/test/frontend/data/generated_includes/moz.build new file mode 100644 index 000000000..14deaf8cf --- /dev/null +++ b/python/mozbuild/mozbuild/test/frontend/data/generated_includes/moz.build @@ -0,0 +1,5 @@ +# -*- Mode: python; indent-tabs-mode: nil; tab-width: 40 -*- +# Any copyright is dedicated to the Public Domain. +# http://creativecommons.org/publicdomain/zero/1.0/ + +LOCAL_INCLUDES += ['!/bar/baz', '!foo'] diff --git a/python/mozbuild/mozbuild/test/frontend/data/host-defines/moz.build b/python/mozbuild/mozbuild/test/frontend/data/host-defines/moz.build new file mode 100644 index 000000000..37628fede --- /dev/null +++ b/python/mozbuild/mozbuild/test/frontend/data/host-defines/moz.build @@ -0,0 +1,14 @@ +# Any copyright is dedicated to the Public Domain. +# http://creativecommons.org/publicdomain/zero/1.0/ + +value = 'xyz' +HOST_DEFINES = { + 'FOO': True, +} + +HOST_DEFINES['BAZ'] = '"abcd"' +HOST_DEFINES.update({ + 'BAR': 7, + 'VALUE': value, + 'QUX': False, +}) diff --git a/python/mozbuild/mozbuild/test/frontend/data/host-sources/a.cpp b/python/mozbuild/mozbuild/test/frontend/data/host-sources/a.cpp new file mode 100644 index 000000000..e69de29bb diff --git a/python/mozbuild/mozbuild/test/frontend/data/host-sources/b.cc b/python/mozbuild/mozbuild/test/frontend/data/host-sources/b.cc new file mode 100644 index 000000000..e69de29bb diff --git a/python/mozbuild/mozbuild/test/frontend/data/host-sources/c.cxx b/python/mozbuild/mozbuild/test/frontend/data/host-sources/c.cxx new file mode 100644 index 000000000..e69de29bb diff --git a/python/mozbuild/mozbuild/test/frontend/data/host-sources/d.c b/python/mozbuild/mozbuild/test/frontend/data/host-sources/d.c new file mode 100644 index 000000000..e69de29bb diff --git a/python/mozbuild/mozbuild/test/frontend/data/host-sources/e.mm b/python/mozbuild/mozbuild/test/frontend/data/host-sources/e.mm new file mode 100644 index 000000000..e69de29bb diff --git a/python/mozbuild/mozbuild/test/frontend/data/host-sources/f.mm b/python/mozbuild/mozbuild/test/frontend/data/host-sources/f.mm new file mode 100644 index 000000000..e69de29bb diff --git a/python/mozbuild/mozbuild/test/frontend/data/host-sources/moz.build b/python/mozbuild/mozbuild/test/frontend/data/host-sources/moz.build new file mode 100644 index 000000000..5a6f0acb6 --- /dev/null +++ b/python/mozbuild/mozbuild/test/frontend/data/host-sources/moz.build @@ -0,0 +1,25 @@ +# -*- Mode: python; indent-tabs-mode: nil; tab-width: 40 -*- +# Any copyright is dedicated to the Public Domain. +# http://creativecommons.org/publicdomain/zero/1.0/ + +@template +def HostLibrary(name): + '''Template for libraries.''' + HOST_LIBRARY_NAME = name + +HostLibrary('dummy') + +HOST_SOURCES += [ + 'a.cpp', + 'b.cc', + 'c.cxx', +] + +HOST_SOURCES += [ + 'd.c', +] + +HOST_SOURCES += [ + 'e.mm', + 'f.mm', +] diff --git a/python/mozbuild/mozbuild/test/frontend/data/include-basic/included.build b/python/mozbuild/mozbuild/test/frontend/data/include-basic/included.build new file mode 100644 index 000000000..bb492a242 --- /dev/null +++ b/python/mozbuild/mozbuild/test/frontend/data/include-basic/included.build @@ -0,0 +1,4 @@ +# Any copyright is dedicated to the Public Domain. +# http://creativecommons.org/publicdomain/zero/1.0/ + +DIRS += ['bar'] diff --git a/python/mozbuild/mozbuild/test/frontend/data/include-basic/moz.build b/python/mozbuild/mozbuild/test/frontend/data/include-basic/moz.build new file mode 100644 index 000000000..8e6a0f338 --- /dev/null +++ b/python/mozbuild/mozbuild/test/frontend/data/include-basic/moz.build @@ -0,0 +1,7 @@ +# -*- Mode: python; indent-tabs-mode: nil; tab-width: 40 -*- +# Any copyright is dedicated to the Public Domain. +# http://creativecommons.org/publicdomain/zero/1.0/ + +DIRS = ['foo'] + +include('included.build') diff --git a/python/mozbuild/mozbuild/test/frontend/data/include-file-stack/included-1.build b/python/mozbuild/mozbuild/test/frontend/data/include-file-stack/included-1.build new file mode 100644 index 000000000..a6a0fd8ea --- /dev/null +++ b/python/mozbuild/mozbuild/test/frontend/data/include-file-stack/included-1.build @@ -0,0 +1,4 @@ +# Any copyright is dedicated to the Public Domain. +# http://creativecommons.org/publicdomain/zero/1.0/ + +include('included-2.build') diff --git a/python/mozbuild/mozbuild/test/frontend/data/include-file-stack/included-2.build b/python/mozbuild/mozbuild/test/frontend/data/include-file-stack/included-2.build new file mode 100644 index 000000000..9bfc65481 --- /dev/null +++ b/python/mozbuild/mozbuild/test/frontend/data/include-file-stack/included-2.build @@ -0,0 +1,4 @@ +# Any copyright is dedicated to the Public Domain. +# http://creativecommons.org/publicdomain/zero/1.0/ + +ILLEGAL = True diff --git a/python/mozbuild/mozbuild/test/frontend/data/include-file-stack/moz.build b/python/mozbuild/mozbuild/test/frontend/data/include-file-stack/moz.build new file mode 100644 index 000000000..7ba111d1f --- /dev/null +++ b/python/mozbuild/mozbuild/test/frontend/data/include-file-stack/moz.build @@ -0,0 +1,5 @@ +# -*- Mode: python; indent-tabs-mode: nil; tab-width: 40 -*- +# Any copyright is dedicated to the Public Domain. +# http://creativecommons.org/publicdomain/zero/1.0/ + +include('included-1.build') diff --git a/python/mozbuild/mozbuild/test/frontend/data/include-missing/moz.build b/python/mozbuild/mozbuild/test/frontend/data/include-missing/moz.build new file mode 100644 index 000000000..d72d47c46 --- /dev/null +++ b/python/mozbuild/mozbuild/test/frontend/data/include-missing/moz.build @@ -0,0 +1,5 @@ +# -*- Mode: python; indent-tabs-mode: nil; tab-width: 40 -*- +# Any copyright is dedicated to the Public Domain. +# http://creativecommons.org/publicdomain/zero/1.0/ + +include('missing.build') diff --git a/python/mozbuild/mozbuild/test/frontend/data/include-outside-topsrcdir/relative.build b/python/mozbuild/mozbuild/test/frontend/data/include-outside-topsrcdir/relative.build new file mode 100644 index 000000000..f8084f0dd --- /dev/null +++ b/python/mozbuild/mozbuild/test/frontend/data/include-outside-topsrcdir/relative.build @@ -0,0 +1,4 @@ +# Any copyright is dedicated to the Public Domain. +# http://creativecommons.org/publicdomain/zero/1.0/ + +include('../moz.build') diff --git a/python/mozbuild/mozbuild/test/frontend/data/include-relative-from-child/child/child.build b/python/mozbuild/mozbuild/test/frontend/data/include-relative-from-child/child/child.build new file mode 100644 index 000000000..446207081 --- /dev/null +++ b/python/mozbuild/mozbuild/test/frontend/data/include-relative-from-child/child/child.build @@ -0,0 +1,4 @@ +# Any copyright is dedicated to the Public Domain. +# http://creativecommons.org/publicdomain/zero/1.0/ + +include('../parent.build') diff --git a/python/mozbuild/mozbuild/test/frontend/data/include-relative-from-child/child/child2.build b/python/mozbuild/mozbuild/test/frontend/data/include-relative-from-child/child/child2.build new file mode 100644 index 000000000..618a75ed0 --- /dev/null +++ b/python/mozbuild/mozbuild/test/frontend/data/include-relative-from-child/child/child2.build @@ -0,0 +1,4 @@ +# Any copyright is dedicated to the Public Domain. +# http://creativecommons.org/publicdomain/zero/1.0/ + +include('grandchild/grandchild.build') diff --git a/python/mozbuild/mozbuild/test/frontend/data/include-relative-from-child/child/grandchild/grandchild.build b/python/mozbuild/mozbuild/test/frontend/data/include-relative-from-child/child/grandchild/grandchild.build new file mode 100644 index 000000000..4d721fde4 --- /dev/null +++ b/python/mozbuild/mozbuild/test/frontend/data/include-relative-from-child/child/grandchild/grandchild.build @@ -0,0 +1,4 @@ +# Any copyright is dedicated to the Public Domain. +# http://creativecommons.org/publicdomain/zero/1.0/ + +include('../../parent.build') diff --git a/python/mozbuild/mozbuild/test/frontend/data/include-relative-from-child/parent.build b/python/mozbuild/mozbuild/test/frontend/data/include-relative-from-child/parent.build new file mode 100644 index 000000000..a2ed3fa49 --- /dev/null +++ b/python/mozbuild/mozbuild/test/frontend/data/include-relative-from-child/parent.build @@ -0,0 +1,4 @@ +# Any copyright is dedicated to the Public Domain. +# http://creativecommons.org/publicdomain/zero/1.0/ + +DIRS = ['foo'] diff --git a/python/mozbuild/mozbuild/test/frontend/data/include-topsrcdir-relative/moz.build b/python/mozbuild/mozbuild/test/frontend/data/include-topsrcdir-relative/moz.build new file mode 100644 index 000000000..f9194c00e --- /dev/null +++ b/python/mozbuild/mozbuild/test/frontend/data/include-topsrcdir-relative/moz.build @@ -0,0 +1,5 @@ +# -*- Mode: python; indent-tabs-mode: nil; tab-width: 40 -*- +# Any copyright is dedicated to the Public Domain. +# http://creativecommons.org/publicdomain/zero/1.0/ + +include('/sibling.build') diff --git a/python/mozbuild/mozbuild/test/frontend/data/include-topsrcdir-relative/sibling.build b/python/mozbuild/mozbuild/test/frontend/data/include-topsrcdir-relative/sibling.build new file mode 100644 index 000000000..a2ed3fa49 --- /dev/null +++ b/python/mozbuild/mozbuild/test/frontend/data/include-topsrcdir-relative/sibling.build @@ -0,0 +1,4 @@ +# Any copyright is dedicated to the Public Domain. +# http://creativecommons.org/publicdomain/zero/1.0/ + +DIRS = ['foo'] diff --git a/python/mozbuild/mozbuild/test/frontend/data/inheriting-variables/bar/moz.build b/python/mozbuild/mozbuild/test/frontend/data/inheriting-variables/bar/moz.build new file mode 100644 index 000000000..568f361a5 --- /dev/null +++ b/python/mozbuild/mozbuild/test/frontend/data/inheriting-variables/bar/moz.build @@ -0,0 +1,5 @@ +# -*- Mode: python; indent-tabs-mode: nil; tab-width: 40 -*- +# vim: set filetype=python: +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, You can obtain one at http://mozilla.org/MPL/2.0/. diff --git a/python/mozbuild/mozbuild/test/frontend/data/inheriting-variables/foo/baz/moz.build b/python/mozbuild/mozbuild/test/frontend/data/inheriting-variables/foo/baz/moz.build new file mode 100644 index 000000000..a1b892e2d --- /dev/null +++ b/python/mozbuild/mozbuild/test/frontend/data/inheriting-variables/foo/baz/moz.build @@ -0,0 +1,7 @@ +# -*- Mode: python; indent-tabs-mode: nil; tab-width: 40 -*- +# vim: set filetype=python: +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, You can obtain one at http://mozilla.org/MPL/2.0/. + +XPIDL_MODULE = 'baz' diff --git a/python/mozbuild/mozbuild/test/frontend/data/inheriting-variables/foo/moz.build b/python/mozbuild/mozbuild/test/frontend/data/inheriting-variables/foo/moz.build new file mode 100644 index 000000000..a06f6d12d --- /dev/null +++ b/python/mozbuild/mozbuild/test/frontend/data/inheriting-variables/foo/moz.build @@ -0,0 +1,7 @@ +# -*- Mode: python; indent-tabs-mode: nil; tab-width: 40 -*- +# vim: set filetype=python: +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, You can obtain one at http://mozilla.org/MPL/2.0/. + +DIRS += ['baz'] diff --git a/python/mozbuild/mozbuild/test/frontend/data/inheriting-variables/moz.build b/python/mozbuild/mozbuild/test/frontend/data/inheriting-variables/moz.build new file mode 100644 index 000000000..2801f105d --- /dev/null +++ b/python/mozbuild/mozbuild/test/frontend/data/inheriting-variables/moz.build @@ -0,0 +1,10 @@ +# -*- Mode: python; indent-tabs-mode: nil; tab-width: 40 -*- +# vim: set filetype=python: +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, You can obtain one at http://mozilla.org/MPL/2.0/. + +XPIDL_MODULE = 'foobar' +export("XPIDL_MODULE") + +DIRS += ['foo', 'bar'] diff --git a/python/mozbuild/mozbuild/test/frontend/data/ipdl_sources/bar/moz.build b/python/mozbuild/mozbuild/test/frontend/data/ipdl_sources/bar/moz.build new file mode 100644 index 000000000..f189212fd --- /dev/null +++ b/python/mozbuild/mozbuild/test/frontend/data/ipdl_sources/bar/moz.build @@ -0,0 +1,10 @@ +# -*- Mode: python; indent-tabs-mode: nil; tab-width: 40 -*- +# vim: set filetype=python: +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, You can obtain one at http://mozilla.org/MPL/2.0/. + +IPDL_SOURCES += [ + 'bar.ipdl', + 'bar2.ipdlh', +] diff --git a/python/mozbuild/mozbuild/test/frontend/data/ipdl_sources/foo/moz.build b/python/mozbuild/mozbuild/test/frontend/data/ipdl_sources/foo/moz.build new file mode 100644 index 000000000..4e1554559 --- /dev/null +++ b/python/mozbuild/mozbuild/test/frontend/data/ipdl_sources/foo/moz.build @@ -0,0 +1,10 @@ +# -*- Mode: python; indent-tabs-mode: nil; tab-width: 40 -*- +# vim: set filetype=python: +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, You can obtain one at http://mozilla.org/MPL/2.0/. + +IPDL_SOURCES += [ + 'foo.ipdl', + 'foo2.ipdlh', +] diff --git a/python/mozbuild/mozbuild/test/frontend/data/ipdl_sources/moz.build b/python/mozbuild/mozbuild/test/frontend/data/ipdl_sources/moz.build new file mode 100644 index 000000000..03cf5e236 --- /dev/null +++ b/python/mozbuild/mozbuild/test/frontend/data/ipdl_sources/moz.build @@ -0,0 +1,10 @@ +# -*- Mode: python; indent-tabs-mode: nil; tab-width: 40 -*- +# vim: set filetype=python: +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, You can obtain one at http://mozilla.org/MPL/2.0/. + +DIRS += [ + 'bar', + 'foo', +] diff --git a/python/mozbuild/mozbuild/test/frontend/data/jar-manifests-multiple-files/moz.build b/python/mozbuild/mozbuild/test/frontend/data/jar-manifests-multiple-files/moz.build new file mode 100644 index 000000000..43789914e --- /dev/null +++ b/python/mozbuild/mozbuild/test/frontend/data/jar-manifests-multiple-files/moz.build @@ -0,0 +1,8 @@ +# -*- Mode: python; indent-tabs-mode: nil; tab-width: 40 -*- +# vim: set filetype=python: +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, You can obtain one at http://mozilla.org/MPL/2.0/. + +JAR_MANIFESTS += ['jar.mn', 'other.jar'] + diff --git a/python/mozbuild/mozbuild/test/frontend/data/jar-manifests/moz.build b/python/mozbuild/mozbuild/test/frontend/data/jar-manifests/moz.build new file mode 100644 index 000000000..aac3a838c --- /dev/null +++ b/python/mozbuild/mozbuild/test/frontend/data/jar-manifests/moz.build @@ -0,0 +1,7 @@ +# -*- Mode: python; indent-tabs-mode: nil; tab-width: 40 -*- +# vim: set filetype=python: +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, You can obtain one at http://mozilla.org/MPL/2.0/. + +JAR_MANIFESTS += ['jar.mn'] diff --git a/python/mozbuild/mozbuild/test/frontend/data/library-defines/liba/moz.build b/python/mozbuild/mozbuild/test/frontend/data/library-defines/liba/moz.build new file mode 100644 index 000000000..5d5e78eed --- /dev/null +++ b/python/mozbuild/mozbuild/test/frontend/data/library-defines/liba/moz.build @@ -0,0 +1,5 @@ +# Any copyright is dedicated to the Public Domain. +# http://creativecommons.org/publicdomain/zero/1.0/ + +Library('liba') +LIBRARY_DEFINES['IN_LIBA'] = True diff --git a/python/mozbuild/mozbuild/test/frontend/data/library-defines/libb/moz.build b/python/mozbuild/mozbuild/test/frontend/data/library-defines/libb/moz.build new file mode 100644 index 000000000..add45f6c1 --- /dev/null +++ b/python/mozbuild/mozbuild/test/frontend/data/library-defines/libb/moz.build @@ -0,0 +1,7 @@ +# Any copyright is dedicated to the Public Domain. +# http://creativecommons.org/publicdomain/zero/1.0/ + +Library('libb') +FINAL_LIBRARY = 'liba' +LIBRARY_DEFINES['IN_LIBB'] = True +USE_LIBS += ['libd'] diff --git a/python/mozbuild/mozbuild/test/frontend/data/library-defines/libc/moz.build b/python/mozbuild/mozbuild/test/frontend/data/library-defines/libc/moz.build new file mode 100644 index 000000000..cf25e2c44 --- /dev/null +++ b/python/mozbuild/mozbuild/test/frontend/data/library-defines/libc/moz.build @@ -0,0 +1,5 @@ +# Any copyright is dedicated to the Public Domain. +# http://creativecommons.org/publicdomain/zero/1.0/ + +Library('libc') +FINAL_LIBRARY = 'libb' diff --git a/python/mozbuild/mozbuild/test/frontend/data/library-defines/libd/moz.build b/python/mozbuild/mozbuild/test/frontend/data/library-defines/libd/moz.build new file mode 100644 index 000000000..dd057c3d7 --- /dev/null +++ b/python/mozbuild/mozbuild/test/frontend/data/library-defines/libd/moz.build @@ -0,0 +1,5 @@ +# Any copyright is dedicated to the Public Domain. +# http://creativecommons.org/publicdomain/zero/1.0/ + +Library('libd') +FORCE_STATIC_LIB = True diff --git a/python/mozbuild/mozbuild/test/frontend/data/library-defines/moz.build b/python/mozbuild/mozbuild/test/frontend/data/library-defines/moz.build new file mode 100644 index 000000000..5f05fcef7 --- /dev/null +++ b/python/mozbuild/mozbuild/test/frontend/data/library-defines/moz.build @@ -0,0 +1,9 @@ +# Any copyright is dedicated to the Public Domain. +# http://creativecommons.org/publicdomain/zero/1.0/ + +@template +def Library(name): + '''Template for libraries.''' + LIBRARY_NAME = name + +DIRS = ['liba', 'libb', 'libc', 'libd'] diff --git a/python/mozbuild/mozbuild/test/frontend/data/local_includes/bar/baz/dummy_file_for_nonempty_directory b/python/mozbuild/mozbuild/test/frontend/data/local_includes/bar/baz/dummy_file_for_nonempty_directory new file mode 100644 index 000000000..e69de29bb diff --git a/python/mozbuild/mozbuild/test/frontend/data/local_includes/foo/dummy_file_for_nonempty_directory b/python/mozbuild/mozbuild/test/frontend/data/local_includes/foo/dummy_file_for_nonempty_directory new file mode 100644 index 000000000..e69de29bb diff --git a/python/mozbuild/mozbuild/test/frontend/data/local_includes/moz.build b/python/mozbuild/mozbuild/test/frontend/data/local_includes/moz.build new file mode 100644 index 000000000..565c2bee6 --- /dev/null +++ b/python/mozbuild/mozbuild/test/frontend/data/local_includes/moz.build @@ -0,0 +1,5 @@ +# -*- Mode: python; indent-tabs-mode: nil; tab-width: 40 -*- +# Any copyright is dedicated to the Public Domain. +# http://creativecommons.org/publicdomain/zero/1.0/ + +LOCAL_INCLUDES += ['/bar/baz', 'foo'] diff --git a/python/mozbuild/mozbuild/test/frontend/data/missing-local-includes/moz.build b/python/mozbuild/mozbuild/test/frontend/data/missing-local-includes/moz.build new file mode 100644 index 000000000..565c2bee6 --- /dev/null +++ b/python/mozbuild/mozbuild/test/frontend/data/missing-local-includes/moz.build @@ -0,0 +1,5 @@ +# -*- Mode: python; indent-tabs-mode: nil; tab-width: 40 -*- +# Any copyright is dedicated to the Public Domain. +# http://creativecommons.org/publicdomain/zero/1.0/ + +LOCAL_INCLUDES += ['/bar/baz', 'foo'] diff --git a/python/mozbuild/mozbuild/test/frontend/data/multiple-rust-libraries/moz.build b/python/mozbuild/mozbuild/test/frontend/data/multiple-rust-libraries/moz.build new file mode 100644 index 000000000..b493ec5b5 --- /dev/null +++ b/python/mozbuild/mozbuild/test/frontend/data/multiple-rust-libraries/moz.build @@ -0,0 +1,27 @@ +# Any copyright is dedicated to the Public Domain. +# http://creativecommons.org/publicdomain/zero/1.0/ + +@template +def Library(name): + '''Template for libraries.''' + LIBRARY_NAME = name + + +@template +def RustLibrary(name): + '''Template for Rust libraries.''' + Library(name) + + IS_RUST_LIBRARY = True + +Library('test') + +DIRS += [ + 'rust1', + 'rust2', +] + +USE_LIBS += [ + 'rust1', + 'rust2', +] diff --git a/python/mozbuild/mozbuild/test/frontend/data/multiple-rust-libraries/rust1/Cargo.toml b/python/mozbuild/mozbuild/test/frontend/data/multiple-rust-libraries/rust1/Cargo.toml new file mode 100644 index 000000000..9037d8f65 --- /dev/null +++ b/python/mozbuild/mozbuild/test/frontend/data/multiple-rust-libraries/rust1/Cargo.toml @@ -0,0 +1,15 @@ +[package] +name = "rust1" +version = "0.1.0" +authors = [ + "Nobody ", +] + +[lib] +crate-type = ["staticlib"] + +[profile.dev] +panic = "abort" + +[profile.release] +panic = "abort" diff --git a/python/mozbuild/mozbuild/test/frontend/data/multiple-rust-libraries/rust1/moz.build b/python/mozbuild/mozbuild/test/frontend/data/multiple-rust-libraries/rust1/moz.build new file mode 100644 index 000000000..7418cca65 --- /dev/null +++ b/python/mozbuild/mozbuild/test/frontend/data/multiple-rust-libraries/rust1/moz.build @@ -0,0 +1,4 @@ +# Any copyright is dedicated to the Public Domain. +# http://creativecommons.org/publicdomain/zero/1.0/ + +RustLibrary('rust1') diff --git a/python/mozbuild/mozbuild/test/frontend/data/multiple-rust-libraries/rust2/Cargo.toml b/python/mozbuild/mozbuild/test/frontend/data/multiple-rust-libraries/rust2/Cargo.toml new file mode 100644 index 000000000..f2001895e --- /dev/null +++ b/python/mozbuild/mozbuild/test/frontend/data/multiple-rust-libraries/rust2/Cargo.toml @@ -0,0 +1,15 @@ +[package] +name = "rust2" +version = "0.1.0" +authors = [ + "Nobody ", +] + +[lib] +crate-type = ["staticlib"] + +[profile.dev] +panic = "abort" + +[profile.release] +panic = "abort" diff --git a/python/mozbuild/mozbuild/test/frontend/data/multiple-rust-libraries/rust2/moz.build b/python/mozbuild/mozbuild/test/frontend/data/multiple-rust-libraries/rust2/moz.build new file mode 100644 index 000000000..abd34e7db --- /dev/null +++ b/python/mozbuild/mozbuild/test/frontend/data/multiple-rust-libraries/rust2/moz.build @@ -0,0 +1,4 @@ +# Any copyright is dedicated to the Public Domain. +# http://creativecommons.org/publicdomain/zero/1.0/ + +RustLibrary('rust2') diff --git a/python/mozbuild/mozbuild/test/frontend/data/program/moz.build b/python/mozbuild/mozbuild/test/frontend/data/program/moz.build new file mode 100644 index 000000000..4c19b90cd --- /dev/null +++ b/python/mozbuild/mozbuild/test/frontend/data/program/moz.build @@ -0,0 +1,15 @@ +# Any copyright is dedicated to the Public Domain. +# http://creativecommons.org/publicdomain/zero/1.0/ + +@template +def Program(name): + PROGRAM = name + + +@template +def SimplePrograms(names): + SIMPLE_PROGRAMS += names + +Program('test_program') + +SimplePrograms([ 'test_program1', 'test_program2' ]) diff --git a/python/mozbuild/mozbuild/test/frontend/data/reader-error-bad-dir/moz.build b/python/mozbuild/mozbuild/test/frontend/data/reader-error-bad-dir/moz.build new file mode 100644 index 000000000..5fac39736 --- /dev/null +++ b/python/mozbuild/mozbuild/test/frontend/data/reader-error-bad-dir/moz.build @@ -0,0 +1,5 @@ +# -*- Mode: python; indent-tabs-mode: nil; tab-width: 40 -*- +# Any copyright is dedicated to the Public Domain. +# http://creativecommons.org/publicdomain/zero/1.0/ + +DIRS = ['foo'] diff --git a/python/mozbuild/mozbuild/test/frontend/data/reader-error-basic/moz.build b/python/mozbuild/mozbuild/test/frontend/data/reader-error-basic/moz.build new file mode 100644 index 000000000..0a91c4692 --- /dev/null +++ b/python/mozbuild/mozbuild/test/frontend/data/reader-error-basic/moz.build @@ -0,0 +1,5 @@ +# -*- Mode: python; indent-tabs-mode: nil; tab-width: 40 -*- +# Any copyright is dedicated to the Public Domain. +# http://creativecommons.org/publicdomain/zero/1.0/ + +ILLEGAL = True diff --git a/python/mozbuild/mozbuild/test/frontend/data/reader-error-empty-list/moz.build b/python/mozbuild/mozbuild/test/frontend/data/reader-error-empty-list/moz.build new file mode 100644 index 000000000..4dfba1c60 --- /dev/null +++ b/python/mozbuild/mozbuild/test/frontend/data/reader-error-empty-list/moz.build @@ -0,0 +1,5 @@ +# -*- Mode: python; indent-tabs-mode: nil; tab-width: 40 -*- +# Any copyright is dedicated to the Public Domain. +# http://creativecommons.org/publicdomain/zero/1.0/ + +DIRS = [] diff --git a/python/mozbuild/mozbuild/test/frontend/data/reader-error-error-func/moz.build b/python/mozbuild/mozbuild/test/frontend/data/reader-error-error-func/moz.build new file mode 100644 index 000000000..84b2cdea4 --- /dev/null +++ b/python/mozbuild/mozbuild/test/frontend/data/reader-error-error-func/moz.build @@ -0,0 +1,6 @@ +# -*- Mode: python; indent-tabs-mode: nil; tab-width: 40 -*- +# Any copyright is dedicated to the Public Domain. +# http://creativecommons.org/publicdomain/zero/1.0/ + +error('Some error.') + diff --git a/python/mozbuild/mozbuild/test/frontend/data/reader-error-included-from/child.build b/python/mozbuild/mozbuild/test/frontend/data/reader-error-included-from/child.build new file mode 100644 index 000000000..9bfc65481 --- /dev/null +++ b/python/mozbuild/mozbuild/test/frontend/data/reader-error-included-from/child.build @@ -0,0 +1,4 @@ +# Any copyright is dedicated to the Public Domain. +# http://creativecommons.org/publicdomain/zero/1.0/ + +ILLEGAL = True diff --git a/python/mozbuild/mozbuild/test/frontend/data/reader-error-included-from/moz.build b/python/mozbuild/mozbuild/test/frontend/data/reader-error-included-from/moz.build new file mode 100644 index 000000000..4a29cae11 --- /dev/null +++ b/python/mozbuild/mozbuild/test/frontend/data/reader-error-included-from/moz.build @@ -0,0 +1,5 @@ +# -*- Mode: python; indent-tabs-mode: nil; tab-width: 40 -*- +# Any copyright is dedicated to the Public Domain. +# http://creativecommons.org/publicdomain/zero/1.0/ + +include('child.build') diff --git a/python/mozbuild/mozbuild/test/frontend/data/reader-error-missing-include/moz.build b/python/mozbuild/mozbuild/test/frontend/data/reader-error-missing-include/moz.build new file mode 100644 index 000000000..d72d47c46 --- /dev/null +++ b/python/mozbuild/mozbuild/test/frontend/data/reader-error-missing-include/moz.build @@ -0,0 +1,5 @@ +# -*- Mode: python; indent-tabs-mode: nil; tab-width: 40 -*- +# Any copyright is dedicated to the Public Domain. +# http://creativecommons.org/publicdomain/zero/1.0/ + +include('missing.build') diff --git a/python/mozbuild/mozbuild/test/frontend/data/reader-error-outside-topsrcdir/moz.build b/python/mozbuild/mozbuild/test/frontend/data/reader-error-outside-topsrcdir/moz.build new file mode 100644 index 000000000..149972edf --- /dev/null +++ b/python/mozbuild/mozbuild/test/frontend/data/reader-error-outside-topsrcdir/moz.build @@ -0,0 +1,5 @@ +# -*- Mode: python; indent-tabs-mode: nil; tab-width: 40 -*- +# Any copyright is dedicated to the Public Domain. +# http://creativecommons.org/publicdomain/zero/1.0/ + +include('../include-basic/moz.build') diff --git a/python/mozbuild/mozbuild/test/frontend/data/reader-error-read-unknown-global/moz.build b/python/mozbuild/mozbuild/test/frontend/data/reader-error-read-unknown-global/moz.build new file mode 100644 index 000000000..6fc10f766 --- /dev/null +++ b/python/mozbuild/mozbuild/test/frontend/data/reader-error-read-unknown-global/moz.build @@ -0,0 +1,5 @@ +# -*- Mode: python; indent-tabs-mode: nil; tab-width: 40 -*- +# Any copyright is dedicated to the Public Domain. +# http://creativecommons.org/publicdomain/zero/1.0/ + +l = FOO diff --git a/python/mozbuild/mozbuild/test/frontend/data/reader-error-repeated-dir/moz.build b/python/mozbuild/mozbuild/test/frontend/data/reader-error-repeated-dir/moz.build new file mode 100644 index 000000000..847f95167 --- /dev/null +++ b/python/mozbuild/mozbuild/test/frontend/data/reader-error-repeated-dir/moz.build @@ -0,0 +1,7 @@ +# -*- Mode: python; indent-tabs-mode: nil; tab-width: 40 -*- +# Any copyright is dedicated to the Public Domain. +# http://creativecommons.org/publicdomain/zero/1.0/ + +DIRS = ['foo'] + +DIRS += ['foo'] diff --git a/python/mozbuild/mozbuild/test/frontend/data/reader-error-script-error/moz.build b/python/mozbuild/mozbuild/test/frontend/data/reader-error-script-error/moz.build new file mode 100644 index 000000000..a91d38b41 --- /dev/null +++ b/python/mozbuild/mozbuild/test/frontend/data/reader-error-script-error/moz.build @@ -0,0 +1,5 @@ +# -*- Mode: python; indent-tabs-mode: nil; tab-width: 40 -*- +# Any copyright is dedicated to the Public Domain. +# http://creativecommons.org/publicdomain/zero/1.0/ + +foo = True + None diff --git a/python/mozbuild/mozbuild/test/frontend/data/reader-error-syntax/moz.build b/python/mozbuild/mozbuild/test/frontend/data/reader-error-syntax/moz.build new file mode 100644 index 000000000..70a0d2c06 --- /dev/null +++ b/python/mozbuild/mozbuild/test/frontend/data/reader-error-syntax/moz.build @@ -0,0 +1,5 @@ +# -*- Mode: python; indent-tabs-mode: nil; tab-width: 40 -*- +# Any copyright is dedicated to the Public Domain. +# http://creativecommons.org/publicdomain/zero/1.0/ + +foo = diff --git a/python/mozbuild/mozbuild/test/frontend/data/reader-error-write-bad-value/moz.build b/python/mozbuild/mozbuild/test/frontend/data/reader-error-write-bad-value/moz.build new file mode 100644 index 000000000..e3d0e656a --- /dev/null +++ b/python/mozbuild/mozbuild/test/frontend/data/reader-error-write-bad-value/moz.build @@ -0,0 +1,5 @@ +# -*- Mode: python; indent-tabs-mode: nil; tab-width: 40 -*- +# Any copyright is dedicated to the Public Domain. +# http://creativecommons.org/publicdomain/zero/1.0/ + +DIRS = 'dir' diff --git a/python/mozbuild/mozbuild/test/frontend/data/reader-error-write-unknown-global/moz.build b/python/mozbuild/mozbuild/test/frontend/data/reader-error-write-unknown-global/moz.build new file mode 100644 index 000000000..34579849d --- /dev/null +++ b/python/mozbuild/mozbuild/test/frontend/data/reader-error-write-unknown-global/moz.build @@ -0,0 +1,7 @@ +# -*- Mode: python; indent-tabs-mode: nil; tab-width: 40 -*- +# Any copyright is dedicated to the Public Domain. +# http://creativecommons.org/publicdomain/zero/1.0/ + +DIRS = ['dir1', 'dir2'] + +FOO = 'bar' diff --git a/python/mozbuild/mozbuild/test/frontend/data/reader-relevant-mozbuild/d1/every-level/a/file b/python/mozbuild/mozbuild/test/frontend/data/reader-relevant-mozbuild/d1/every-level/a/file new file mode 100644 index 000000000..e69de29bb diff --git a/python/mozbuild/mozbuild/test/frontend/data/reader-relevant-mozbuild/d1/every-level/a/moz.build b/python/mozbuild/mozbuild/test/frontend/data/reader-relevant-mozbuild/d1/every-level/a/moz.build new file mode 100644 index 000000000..e69de29bb diff --git a/python/mozbuild/mozbuild/test/frontend/data/reader-relevant-mozbuild/d1/every-level/b/file b/python/mozbuild/mozbuild/test/frontend/data/reader-relevant-mozbuild/d1/every-level/b/file new file mode 100644 index 000000000..e69de29bb diff --git a/python/mozbuild/mozbuild/test/frontend/data/reader-relevant-mozbuild/d1/every-level/b/moz.build b/python/mozbuild/mozbuild/test/frontend/data/reader-relevant-mozbuild/d1/every-level/b/moz.build new file mode 100644 index 000000000..e69de29bb diff --git a/python/mozbuild/mozbuild/test/frontend/data/reader-relevant-mozbuild/d1/every-level/moz.build b/python/mozbuild/mozbuild/test/frontend/data/reader-relevant-mozbuild/d1/every-level/moz.build new file mode 100644 index 000000000..e69de29bb diff --git a/python/mozbuild/mozbuild/test/frontend/data/reader-relevant-mozbuild/d1/file1 b/python/mozbuild/mozbuild/test/frontend/data/reader-relevant-mozbuild/d1/file1 new file mode 100644 index 000000000..e69de29bb diff --git a/python/mozbuild/mozbuild/test/frontend/data/reader-relevant-mozbuild/d1/file2 b/python/mozbuild/mozbuild/test/frontend/data/reader-relevant-mozbuild/d1/file2 new file mode 100644 index 000000000..e69de29bb diff --git a/python/mozbuild/mozbuild/test/frontend/data/reader-relevant-mozbuild/d1/moz.build b/python/mozbuild/mozbuild/test/frontend/data/reader-relevant-mozbuild/d1/moz.build new file mode 100644 index 000000000..e69de29bb diff --git a/python/mozbuild/mozbuild/test/frontend/data/reader-relevant-mozbuild/d1/no-intermediate-moz-build/child/file b/python/mozbuild/mozbuild/test/frontend/data/reader-relevant-mozbuild/d1/no-intermediate-moz-build/child/file new file mode 100644 index 000000000..e69de29bb diff --git a/python/mozbuild/mozbuild/test/frontend/data/reader-relevant-mozbuild/d1/no-intermediate-moz-build/child/moz.build b/python/mozbuild/mozbuild/test/frontend/data/reader-relevant-mozbuild/d1/no-intermediate-moz-build/child/moz.build new file mode 100644 index 000000000..e69de29bb diff --git a/python/mozbuild/mozbuild/test/frontend/data/reader-relevant-mozbuild/d1/parent-is-far/dir1/dir2/dir3/file b/python/mozbuild/mozbuild/test/frontend/data/reader-relevant-mozbuild/d1/parent-is-far/dir1/dir2/dir3/file new file mode 100644 index 000000000..e69de29bb diff --git a/python/mozbuild/mozbuild/test/frontend/data/reader-relevant-mozbuild/d1/parent-is-far/moz.build b/python/mozbuild/mozbuild/test/frontend/data/reader-relevant-mozbuild/d1/parent-is-far/moz.build new file mode 100644 index 000000000..e69de29bb diff --git a/python/mozbuild/mozbuild/test/frontend/data/reader-relevant-mozbuild/d2/dir1/file b/python/mozbuild/mozbuild/test/frontend/data/reader-relevant-mozbuild/d2/dir1/file new file mode 100644 index 000000000..e69de29bb diff --git a/python/mozbuild/mozbuild/test/frontend/data/reader-relevant-mozbuild/d2/dir1/moz.build b/python/mozbuild/mozbuild/test/frontend/data/reader-relevant-mozbuild/d2/dir1/moz.build new file mode 100644 index 000000000..e69de29bb diff --git a/python/mozbuild/mozbuild/test/frontend/data/reader-relevant-mozbuild/d2/dir2/file b/python/mozbuild/mozbuild/test/frontend/data/reader-relevant-mozbuild/d2/dir2/file new file mode 100644 index 000000000..e69de29bb diff --git a/python/mozbuild/mozbuild/test/frontend/data/reader-relevant-mozbuild/d2/dir2/moz.build b/python/mozbuild/mozbuild/test/frontend/data/reader-relevant-mozbuild/d2/dir2/moz.build new file mode 100644 index 000000000..e69de29bb diff --git a/python/mozbuild/mozbuild/test/frontend/data/reader-relevant-mozbuild/d2/moz.build b/python/mozbuild/mozbuild/test/frontend/data/reader-relevant-mozbuild/d2/moz.build new file mode 100644 index 000000000..e69de29bb diff --git a/python/mozbuild/mozbuild/test/frontend/data/reader-relevant-mozbuild/file b/python/mozbuild/mozbuild/test/frontend/data/reader-relevant-mozbuild/file new file mode 100644 index 000000000..e69de29bb diff --git a/python/mozbuild/mozbuild/test/frontend/data/reader-relevant-mozbuild/moz.build b/python/mozbuild/mozbuild/test/frontend/data/reader-relevant-mozbuild/moz.build new file mode 100644 index 000000000..e69de29bb diff --git a/python/mozbuild/mozbuild/test/frontend/data/rust-library-dash-folding/Cargo.toml b/python/mozbuild/mozbuild/test/frontend/data/rust-library-dash-folding/Cargo.toml new file mode 100644 index 000000000..fa122b7ce --- /dev/null +++ b/python/mozbuild/mozbuild/test/frontend/data/rust-library-dash-folding/Cargo.toml @@ -0,0 +1,15 @@ +[package] +name = "random-crate" +version = "0.1.0" +authors = [ + "Nobody ", +] + +[lib] +crate-type = ["staticlib"] + +[profile.dev] +panic = "abort" + +[profile.release] +panic = "abort" diff --git a/python/mozbuild/mozbuild/test/frontend/data/rust-library-dash-folding/moz.build b/python/mozbuild/mozbuild/test/frontend/data/rust-library-dash-folding/moz.build new file mode 100644 index 000000000..01b3a35a7 --- /dev/null +++ b/python/mozbuild/mozbuild/test/frontend/data/rust-library-dash-folding/moz.build @@ -0,0 +1,18 @@ +# Any copyright is dedicated to the Public Domain. +# http://creativecommons.org/publicdomain/zero/1.0/ + +@template +def Library(name): + '''Template for libraries.''' + LIBRARY_NAME = name + + +@template +def RustLibrary(name): + '''Template for Rust libraries.''' + Library(name) + + IS_RUST_LIBRARY = True + + +RustLibrary('random-crate') diff --git a/python/mozbuild/mozbuild/test/frontend/data/rust-library-invalid-crate-type/Cargo.toml b/python/mozbuild/mozbuild/test/frontend/data/rust-library-invalid-crate-type/Cargo.toml new file mode 100644 index 000000000..26c653fde --- /dev/null +++ b/python/mozbuild/mozbuild/test/frontend/data/rust-library-invalid-crate-type/Cargo.toml @@ -0,0 +1,15 @@ +[package] +name = "random-crate" +version = "0.1.0" +authors = [ + "Nobody ", +] + +[lib] +crate-type = ["dylib"] + +[profile.dev] +panic = "abort" + +[profile.release] +panic = "abort" diff --git a/python/mozbuild/mozbuild/test/frontend/data/rust-library-invalid-crate-type/moz.build b/python/mozbuild/mozbuild/test/frontend/data/rust-library-invalid-crate-type/moz.build new file mode 100644 index 000000000..01b3a35a7 --- /dev/null +++ b/python/mozbuild/mozbuild/test/frontend/data/rust-library-invalid-crate-type/moz.build @@ -0,0 +1,18 @@ +# Any copyright is dedicated to the Public Domain. +# http://creativecommons.org/publicdomain/zero/1.0/ + +@template +def Library(name): + '''Template for libraries.''' + LIBRARY_NAME = name + + +@template +def RustLibrary(name): + '''Template for Rust libraries.''' + Library(name) + + IS_RUST_LIBRARY = True + + +RustLibrary('random-crate') diff --git a/python/mozbuild/mozbuild/test/frontend/data/rust-library-name-mismatch/Cargo.toml b/python/mozbuild/mozbuild/test/frontend/data/rust-library-name-mismatch/Cargo.toml new file mode 100644 index 000000000..41a9a7c8f --- /dev/null +++ b/python/mozbuild/mozbuild/test/frontend/data/rust-library-name-mismatch/Cargo.toml @@ -0,0 +1,12 @@ +[package] +name = "deterministic-crate" +version = "0.1.0" +authors = [ + "Nobody ", +] + +[profile.dev] +panic = "abort" + +[profile.release] +panic = "abort" diff --git a/python/mozbuild/mozbuild/test/frontend/data/rust-library-name-mismatch/moz.build b/python/mozbuild/mozbuild/test/frontend/data/rust-library-name-mismatch/moz.build new file mode 100644 index 000000000..01b3a35a7 --- /dev/null +++ b/python/mozbuild/mozbuild/test/frontend/data/rust-library-name-mismatch/moz.build @@ -0,0 +1,18 @@ +# Any copyright is dedicated to the Public Domain. +# http://creativecommons.org/publicdomain/zero/1.0/ + +@template +def Library(name): + '''Template for libraries.''' + LIBRARY_NAME = name + + +@template +def RustLibrary(name): + '''Template for Rust libraries.''' + Library(name) + + IS_RUST_LIBRARY = True + + +RustLibrary('random-crate') diff --git a/python/mozbuild/mozbuild/test/frontend/data/rust-library-no-cargo-toml/moz.build b/python/mozbuild/mozbuild/test/frontend/data/rust-library-no-cargo-toml/moz.build new file mode 100644 index 000000000..01b3a35a7 --- /dev/null +++ b/python/mozbuild/mozbuild/test/frontend/data/rust-library-no-cargo-toml/moz.build @@ -0,0 +1,18 @@ +# Any copyright is dedicated to the Public Domain. +# http://creativecommons.org/publicdomain/zero/1.0/ + +@template +def Library(name): + '''Template for libraries.''' + LIBRARY_NAME = name + + +@template +def RustLibrary(name): + '''Template for Rust libraries.''' + Library(name) + + IS_RUST_LIBRARY = True + + +RustLibrary('random-crate') diff --git a/python/mozbuild/mozbuild/test/frontend/data/rust-library-no-lib-section/Cargo.toml b/python/mozbuild/mozbuild/test/frontend/data/rust-library-no-lib-section/Cargo.toml new file mode 100644 index 000000000..a20b19c62 --- /dev/null +++ b/python/mozbuild/mozbuild/test/frontend/data/rust-library-no-lib-section/Cargo.toml @@ -0,0 +1,12 @@ +[package] +name = "random-crate" +version = "0.1.0" +authors = [ + "Nobody ", +] + +[profile.dev] +panic = "abort" + +[profile.release] +panic = "abort" diff --git a/python/mozbuild/mozbuild/test/frontend/data/rust-library-no-lib-section/moz.build b/python/mozbuild/mozbuild/test/frontend/data/rust-library-no-lib-section/moz.build new file mode 100644 index 000000000..01b3a35a7 --- /dev/null +++ b/python/mozbuild/mozbuild/test/frontend/data/rust-library-no-lib-section/moz.build @@ -0,0 +1,18 @@ +# Any copyright is dedicated to the Public Domain. +# http://creativecommons.org/publicdomain/zero/1.0/ + +@template +def Library(name): + '''Template for libraries.''' + LIBRARY_NAME = name + + +@template +def RustLibrary(name): + '''Template for Rust libraries.''' + Library(name) + + IS_RUST_LIBRARY = True + + +RustLibrary('random-crate') diff --git a/python/mozbuild/mozbuild/test/frontend/data/rust-library-no-profile-section/Cargo.toml b/python/mozbuild/mozbuild/test/frontend/data/rust-library-no-profile-section/Cargo.toml new file mode 100644 index 000000000..2700849db --- /dev/null +++ b/python/mozbuild/mozbuild/test/frontend/data/rust-library-no-profile-section/Cargo.toml @@ -0,0 +1,12 @@ +[package] +name = "random-crate" +version = "0.1.0" +authors = [ + "Nobody ", +] + +[lib] +crate-type = ["staticlib"] + +[profile.release] +panic = "abort" diff --git a/python/mozbuild/mozbuild/test/frontend/data/rust-library-no-profile-section/moz.build b/python/mozbuild/mozbuild/test/frontend/data/rust-library-no-profile-section/moz.build new file mode 100644 index 000000000..01b3a35a7 --- /dev/null +++ b/python/mozbuild/mozbuild/test/frontend/data/rust-library-no-profile-section/moz.build @@ -0,0 +1,18 @@ +# Any copyright is dedicated to the Public Domain. +# http://creativecommons.org/publicdomain/zero/1.0/ + +@template +def Library(name): + '''Template for libraries.''' + LIBRARY_NAME = name + + +@template +def RustLibrary(name): + '''Template for Rust libraries.''' + Library(name) + + IS_RUST_LIBRARY = True + + +RustLibrary('random-crate') diff --git a/python/mozbuild/mozbuild/test/frontend/data/rust-library-non-abort-panic/Cargo.toml b/python/mozbuild/mozbuild/test/frontend/data/rust-library-non-abort-panic/Cargo.toml new file mode 100644 index 000000000..ccdd06243 --- /dev/null +++ b/python/mozbuild/mozbuild/test/frontend/data/rust-library-non-abort-panic/Cargo.toml @@ -0,0 +1,14 @@ +[package] +name = "random-crate" +version = "0.1.0" +authors = [ + "Nobody ", +] + +[lib] +crate-type = ["staticlib"] + +[profile.dev] +panic = "unwind" + +[profile.release] diff --git a/python/mozbuild/mozbuild/test/frontend/data/rust-library-non-abort-panic/moz.build b/python/mozbuild/mozbuild/test/frontend/data/rust-library-non-abort-panic/moz.build new file mode 100644 index 000000000..d3896decc --- /dev/null +++ b/python/mozbuild/mozbuild/test/frontend/data/rust-library-non-abort-panic/moz.build @@ -0,0 +1,18 @@ +# Any copyright is dedicated to the Public Domain. +# http://creativecommons.org/publicdomain/zero/1.0/ + +@template +def Library(name): + '''Template for libraries.''' + LIBRARY_NAME = name + + +@template +def RustLibrary(name): + '''Template for Rust libraries.''' + Library(name) + + IS_RUST_LIBRARY = True + + +RustLibrary('random-crate') \ No newline at end of file diff --git a/python/mozbuild/mozbuild/test/frontend/data/sdk-files/bar.ico b/python/mozbuild/mozbuild/test/frontend/data/sdk-files/bar.ico new file mode 100644 index 000000000..e69de29bb diff --git a/python/mozbuild/mozbuild/test/frontend/data/sdk-files/baz.png b/python/mozbuild/mozbuild/test/frontend/data/sdk-files/baz.png new file mode 100644 index 000000000..e69de29bb diff --git a/python/mozbuild/mozbuild/test/frontend/data/sdk-files/foo.xpm b/python/mozbuild/mozbuild/test/frontend/data/sdk-files/foo.xpm new file mode 100644 index 000000000..e69de29bb diff --git a/python/mozbuild/mozbuild/test/frontend/data/sdk-files/moz.build b/python/mozbuild/mozbuild/test/frontend/data/sdk-files/moz.build new file mode 100644 index 000000000..a2f8ddf9b --- /dev/null +++ b/python/mozbuild/mozbuild/test/frontend/data/sdk-files/moz.build @@ -0,0 +1,12 @@ +# Any copyright is dedicated to the Public Domain. +# http://creativecommons.org/publicdomain/zero/1.0/ + +SDK_FILES += [ + 'bar.ico', + 'baz.png', + 'foo.xpm', +] + +SDK_FILES.icons += [ + 'quux.icns', +] diff --git a/python/mozbuild/mozbuild/test/frontend/data/sdk-files/quux.icns b/python/mozbuild/mozbuild/test/frontend/data/sdk-files/quux.icns new file mode 100644 index 000000000..e69de29bb diff --git a/python/mozbuild/mozbuild/test/frontend/data/sources-just-c/d.c b/python/mozbuild/mozbuild/test/frontend/data/sources-just-c/d.c new file mode 100644 index 000000000..e69de29bb diff --git a/python/mozbuild/mozbuild/test/frontend/data/sources-just-c/e.m b/python/mozbuild/mozbuild/test/frontend/data/sources-just-c/e.m new file mode 100644 index 000000000..e69de29bb diff --git a/python/mozbuild/mozbuild/test/frontend/data/sources-just-c/g.S b/python/mozbuild/mozbuild/test/frontend/data/sources-just-c/g.S new file mode 100644 index 000000000..e69de29bb diff --git a/python/mozbuild/mozbuild/test/frontend/data/sources-just-c/h.s b/python/mozbuild/mozbuild/test/frontend/data/sources-just-c/h.s new file mode 100644 index 000000000..e69de29bb diff --git a/python/mozbuild/mozbuild/test/frontend/data/sources-just-c/i.asm b/python/mozbuild/mozbuild/test/frontend/data/sources-just-c/i.asm new file mode 100644 index 000000000..e69de29bb diff --git a/python/mozbuild/mozbuild/test/frontend/data/sources-just-c/moz.build b/python/mozbuild/mozbuild/test/frontend/data/sources-just-c/moz.build new file mode 100644 index 000000000..8937fc245 --- /dev/null +++ b/python/mozbuild/mozbuild/test/frontend/data/sources-just-c/moz.build @@ -0,0 +1,27 @@ +# -*- Mode: python; indent-tabs-mode: nil; tab-width: 40 -*- +# Any copyright is dedicated to the Public Domain. +# http://creativecommons.org/publicdomain/zero/1.0/ + +@template +def Library(name): + '''Template for libraries.''' + LIBRARY_NAME = name + +Library('dummy') + +SOURCES += [ + 'd.c', +] + +SOURCES += [ + 'e.m', +] + +SOURCES += [ + 'g.S', +] + +SOURCES += [ + 'h.s', + 'i.asm', +] diff --git a/python/mozbuild/mozbuild/test/frontend/data/sources/a.cpp b/python/mozbuild/mozbuild/test/frontend/data/sources/a.cpp new file mode 100644 index 000000000..e69de29bb diff --git a/python/mozbuild/mozbuild/test/frontend/data/sources/b.cc b/python/mozbuild/mozbuild/test/frontend/data/sources/b.cc new file mode 100644 index 000000000..e69de29bb diff --git a/python/mozbuild/mozbuild/test/frontend/data/sources/c.cxx b/python/mozbuild/mozbuild/test/frontend/data/sources/c.cxx new file mode 100644 index 000000000..e69de29bb diff --git a/python/mozbuild/mozbuild/test/frontend/data/sources/d.c b/python/mozbuild/mozbuild/test/frontend/data/sources/d.c new file mode 100644 index 000000000..e69de29bb diff --git a/python/mozbuild/mozbuild/test/frontend/data/sources/e.m b/python/mozbuild/mozbuild/test/frontend/data/sources/e.m new file mode 100644 index 000000000..e69de29bb diff --git a/python/mozbuild/mozbuild/test/frontend/data/sources/f.mm b/python/mozbuild/mozbuild/test/frontend/data/sources/f.mm new file mode 100644 index 000000000..e69de29bb diff --git a/python/mozbuild/mozbuild/test/frontend/data/sources/g.S b/python/mozbuild/mozbuild/test/frontend/data/sources/g.S new file mode 100644 index 000000000..e69de29bb diff --git a/python/mozbuild/mozbuild/test/frontend/data/sources/h.s b/python/mozbuild/mozbuild/test/frontend/data/sources/h.s new file mode 100644 index 000000000..e69de29bb diff --git a/python/mozbuild/mozbuild/test/frontend/data/sources/i.asm b/python/mozbuild/mozbuild/test/frontend/data/sources/i.asm new file mode 100644 index 000000000..e69de29bb diff --git a/python/mozbuild/mozbuild/test/frontend/data/sources/moz.build b/python/mozbuild/mozbuild/test/frontend/data/sources/moz.build new file mode 100644 index 000000000..f9b453238 --- /dev/null +++ b/python/mozbuild/mozbuild/test/frontend/data/sources/moz.build @@ -0,0 +1,37 @@ +# -*- Mode: python; indent-tabs-mode: nil; tab-width: 40 -*- +# Any copyright is dedicated to the Public Domain. +# http://creativecommons.org/publicdomain/zero/1.0/ + +@template +def Library(name): + '''Template for libraries.''' + LIBRARY_NAME = name + +Library('dummy') + +SOURCES += [ + 'a.cpp', + 'b.cc', + 'c.cxx', +] + +SOURCES += [ + 'd.c', +] + +SOURCES += [ + 'e.m', +] + +SOURCES += [ + 'f.mm', +] + +SOURCES += [ + 'g.S', +] + +SOURCES += [ + 'h.s', + 'i.asm', +] diff --git a/python/mozbuild/mozbuild/test/frontend/data/templates/templates.mozbuild b/python/mozbuild/mozbuild/test/frontend/data/templates/templates.mozbuild new file mode 100644 index 000000000..290104bc7 --- /dev/null +++ b/python/mozbuild/mozbuild/test/frontend/data/templates/templates.mozbuild @@ -0,0 +1,21 @@ +@template +def Template(foo, bar=[]): + SOURCES += foo + DIRS += bar + +@template +def TemplateError(foo): + ILLEGAL = foo + +@template +def TemplateGlobalVariable(): + SOURCES += illegal + +@template +def TemplateGlobalUPPERVariable(): + SOURCES += DIRS + +@template +def TemplateInherit(foo): + USE_LIBS += ['foo'] + Template(foo) diff --git a/python/mozbuild/mozbuild/test/frontend/data/test-harness-files-root/moz.build b/python/mozbuild/mozbuild/test/frontend/data/test-harness-files-root/moz.build new file mode 100644 index 000000000..d7f6377d0 --- /dev/null +++ b/python/mozbuild/mozbuild/test/frontend/data/test-harness-files-root/moz.build @@ -0,0 +1,4 @@ +# Any copyright is dedicated to the Public Domain. +# http://creativecommons.org/publicdomain/zero/1.0/ + +TEST_HARNESS_FILES += ["foo.py"] diff --git a/python/mozbuild/mozbuild/test/frontend/data/test-harness-files/mochitest.ini b/python/mozbuild/mozbuild/test/frontend/data/test-harness-files/mochitest.ini new file mode 100644 index 000000000..d87114ac7 --- /dev/null +++ b/python/mozbuild/mozbuild/test/frontend/data/test-harness-files/mochitest.ini @@ -0,0 +1 @@ +# dummy file so the existence checks for TEST_HARNESS_FILES succeed diff --git a/python/mozbuild/mozbuild/test/frontend/data/test-harness-files/mochitest.py b/python/mozbuild/mozbuild/test/frontend/data/test-harness-files/mochitest.py new file mode 100644 index 000000000..d87114ac7 --- /dev/null +++ b/python/mozbuild/mozbuild/test/frontend/data/test-harness-files/mochitest.py @@ -0,0 +1 @@ +# dummy file so the existence checks for TEST_HARNESS_FILES succeed diff --git a/python/mozbuild/mozbuild/test/frontend/data/test-harness-files/moz.build b/python/mozbuild/mozbuild/test/frontend/data/test-harness-files/moz.build new file mode 100644 index 000000000..ff3fed0ee --- /dev/null +++ b/python/mozbuild/mozbuild/test/frontend/data/test-harness-files/moz.build @@ -0,0 +1,7 @@ +# Any copyright is dedicated to the Public Domain. +# http://creativecommons.org/publicdomain/zero/1.0/ + +TEST_HARNESS_FILES.mochitest += ["runtests.py"] +TEST_HARNESS_FILES.mochitest += ["utils.py"] +TEST_HARNESS_FILES.testing.mochitest += ["mochitest.py"] +TEST_HARNESS_FILES.testing.mochitest += ["mochitest.ini"] diff --git a/python/mozbuild/mozbuild/test/frontend/data/test-harness-files/runtests.py b/python/mozbuild/mozbuild/test/frontend/data/test-harness-files/runtests.py new file mode 100644 index 000000000..d87114ac7 --- /dev/null +++ b/python/mozbuild/mozbuild/test/frontend/data/test-harness-files/runtests.py @@ -0,0 +1 @@ +# dummy file so the existence checks for TEST_HARNESS_FILES succeed diff --git a/python/mozbuild/mozbuild/test/frontend/data/test-harness-files/utils.py b/python/mozbuild/mozbuild/test/frontend/data/test-harness-files/utils.py new file mode 100644 index 000000000..d87114ac7 --- /dev/null +++ b/python/mozbuild/mozbuild/test/frontend/data/test-harness-files/utils.py @@ -0,0 +1 @@ +# dummy file so the existence checks for TEST_HARNESS_FILES succeed diff --git a/python/mozbuild/mozbuild/test/frontend/data/test-install-shared-lib/moz.build b/python/mozbuild/mozbuild/test/frontend/data/test-install-shared-lib/moz.build new file mode 100644 index 000000000..bdb209074 --- /dev/null +++ b/python/mozbuild/mozbuild/test/frontend/data/test-install-shared-lib/moz.build @@ -0,0 +1,12 @@ +# Any copyright is dedicated to the Public Domain. +# http://creativecommons.org/publicdomain/zero/1.0/ + +@template +def SharedLibrary(name): + LIBRARY_NAME = name + FORCE_SHARED_LIB = True + +DIST_INSTALL = False +SharedLibrary('foo') + +TEST_HARNESS_FILES.foo.bar += ['!%sfoo%s' % (CONFIG['DLL_PREFIX'], CONFIG['DLL_SUFFIX'])] diff --git a/python/mozbuild/mozbuild/test/frontend/data/test-linkables-cxx-link/moz.build b/python/mozbuild/mozbuild/test/frontend/data/test-linkables-cxx-link/moz.build new file mode 100644 index 000000000..b153dd085 --- /dev/null +++ b/python/mozbuild/mozbuild/test/frontend/data/test-linkables-cxx-link/moz.build @@ -0,0 +1,11 @@ +# Any copyright is dedicated to the Public Domain. +# http://creativecommons.org/publicdomain/zero/1.0/ + +DIRS = ['one','two','three'] +@template +def SharedLibrary(name): + LIBRARY_NAME = name + FORCE_SHARED_LIB = True + +SharedLibrary('cxx_shared') +USE_LIBS += ['cxx_static'] diff --git a/python/mozbuild/mozbuild/test/frontend/data/test-linkables-cxx-link/one/foo.cpp b/python/mozbuild/mozbuild/test/frontend/data/test-linkables-cxx-link/one/foo.cpp new file mode 100644 index 000000000..e69de29bb diff --git a/python/mozbuild/mozbuild/test/frontend/data/test-linkables-cxx-link/one/moz.build b/python/mozbuild/mozbuild/test/frontend/data/test-linkables-cxx-link/one/moz.build new file mode 100644 index 000000000..f66270818 --- /dev/null +++ b/python/mozbuild/mozbuild/test/frontend/data/test-linkables-cxx-link/one/moz.build @@ -0,0 +1,9 @@ +# Any copyright is dedicated to the Public Domain. +# http://creativecommons.org/publicdomain/zero/1.0/ + +@template +def Library(name): + LIBRARY_NAME = name + +Library('cxx_static') +SOURCES += ['foo.cpp'] diff --git a/python/mozbuild/mozbuild/test/frontend/data/test-linkables-cxx-link/three/moz.build b/python/mozbuild/mozbuild/test/frontend/data/test-linkables-cxx-link/three/moz.build new file mode 100644 index 000000000..7b3497be6 --- /dev/null +++ b/python/mozbuild/mozbuild/test/frontend/data/test-linkables-cxx-link/three/moz.build @@ -0,0 +1,5 @@ +# Any copyright is dedicated to the Public Domain. +# http://creativecommons.org/publicdomain/zero/1.0/ + +SharedLibrary('just_c_shared') +USE_LIBS += ['just_c_static'] diff --git a/python/mozbuild/mozbuild/test/frontend/data/test-linkables-cxx-link/two/foo.c b/python/mozbuild/mozbuild/test/frontend/data/test-linkables-cxx-link/two/foo.c new file mode 100644 index 000000000..e69de29bb diff --git a/python/mozbuild/mozbuild/test/frontend/data/test-linkables-cxx-link/two/moz.build b/python/mozbuild/mozbuild/test/frontend/data/test-linkables-cxx-link/two/moz.build new file mode 100644 index 000000000..256642fea --- /dev/null +++ b/python/mozbuild/mozbuild/test/frontend/data/test-linkables-cxx-link/two/moz.build @@ -0,0 +1,9 @@ +# Any copyright is dedicated to the Public Domain. +# http://creativecommons.org/publicdomain/zero/1.0/ + +@template +def Library(name): + LIBRARY_NAME = name + +Library('just_c_static') +SOURCES += ['foo.c'] diff --git a/python/mozbuild/mozbuild/test/frontend/data/test-manifest-absolute-support/absolute-support.ini b/python/mozbuild/mozbuild/test/frontend/data/test-manifest-absolute-support/absolute-support.ini new file mode 100644 index 000000000..900f42158 --- /dev/null +++ b/python/mozbuild/mozbuild/test/frontend/data/test-manifest-absolute-support/absolute-support.ini @@ -0,0 +1,4 @@ +[DEFAULT] +support-files = /.well-known/foo.txt + +[test_file.js] diff --git a/python/mozbuild/mozbuild/test/frontend/data/test-manifest-absolute-support/foo.txt b/python/mozbuild/mozbuild/test/frontend/data/test-manifest-absolute-support/foo.txt new file mode 100644 index 000000000..ce0136250 --- /dev/null +++ b/python/mozbuild/mozbuild/test/frontend/data/test-manifest-absolute-support/foo.txt @@ -0,0 +1 @@ +hello diff --git a/python/mozbuild/mozbuild/test/frontend/data/test-manifest-absolute-support/moz.build b/python/mozbuild/mozbuild/test/frontend/data/test-manifest-absolute-support/moz.build new file mode 100644 index 000000000..87b20c6b1 --- /dev/null +++ b/python/mozbuild/mozbuild/test/frontend/data/test-manifest-absolute-support/moz.build @@ -0,0 +1,4 @@ +# Any copyright is dedicated to the Public Domain. +# http://creativecommons.org/publicdomain/zero/1.0/ + +MOCHITEST_MANIFESTS += ['absolute-support.ini'] diff --git a/python/mozbuild/mozbuild/test/frontend/data/test-manifest-absolute-support/test_file.js b/python/mozbuild/mozbuild/test/frontend/data/test-manifest-absolute-support/test_file.js new file mode 100644 index 000000000..e69de29bb diff --git a/python/mozbuild/mozbuild/test/frontend/data/test-manifest-dupes/bar.js b/python/mozbuild/mozbuild/test/frontend/data/test-manifest-dupes/bar.js new file mode 100644 index 000000000..e69de29bb diff --git a/python/mozbuild/mozbuild/test/frontend/data/test-manifest-dupes/foo.js b/python/mozbuild/mozbuild/test/frontend/data/test-manifest-dupes/foo.js new file mode 100644 index 000000000..e69de29bb diff --git a/python/mozbuild/mozbuild/test/frontend/data/test-manifest-dupes/mochitest.ini b/python/mozbuild/mozbuild/test/frontend/data/test-manifest-dupes/mochitest.ini new file mode 100644 index 000000000..2f1fc406a --- /dev/null +++ b/python/mozbuild/mozbuild/test/frontend/data/test-manifest-dupes/mochitest.ini @@ -0,0 +1,7 @@ +# Any copyright is dedicated to the Public Domain. +# http://creativecommons.org/publicdomain/zero/1.0/ + +[DEFAULT] +support-files = bar.js foo.js bar.js + +[test_baz.js] diff --git a/python/mozbuild/mozbuild/test/frontend/data/test-manifest-dupes/moz.build b/python/mozbuild/mozbuild/test/frontend/data/test-manifest-dupes/moz.build new file mode 100644 index 000000000..4e7e9ff4e --- /dev/null +++ b/python/mozbuild/mozbuild/test/frontend/data/test-manifest-dupes/moz.build @@ -0,0 +1,4 @@ +# Any copyright is dedicated to the Public Domain. +# http://creativecommons.org/publicdomain/zero/1.0/ + +MOCHITEST_MANIFESTS += ['mochitest.ini'] diff --git a/python/mozbuild/mozbuild/test/frontend/data/test-manifest-dupes/test_baz.js b/python/mozbuild/mozbuild/test/frontend/data/test-manifest-dupes/test_baz.js new file mode 100644 index 000000000..e69de29bb diff --git a/python/mozbuild/mozbuild/test/frontend/data/test-manifest-emitted-includes/included-reftest.list b/python/mozbuild/mozbuild/test/frontend/data/test-manifest-emitted-includes/included-reftest.list new file mode 100644 index 000000000..1caf9cc39 --- /dev/null +++ b/python/mozbuild/mozbuild/test/frontend/data/test-manifest-emitted-includes/included-reftest.list @@ -0,0 +1 @@ +!= reftest2.html reftest2-ref.html \ No newline at end of file diff --git a/python/mozbuild/mozbuild/test/frontend/data/test-manifest-emitted-includes/moz.build b/python/mozbuild/mozbuild/test/frontend/data/test-manifest-emitted-includes/moz.build new file mode 100644 index 000000000..39ad44c28 --- /dev/null +++ b/python/mozbuild/mozbuild/test/frontend/data/test-manifest-emitted-includes/moz.build @@ -0,0 +1 @@ +REFTEST_MANIFESTS += ['reftest.list'] \ No newline at end of file diff --git a/python/mozbuild/mozbuild/test/frontend/data/test-manifest-emitted-includes/reftest-stylo.list b/python/mozbuild/mozbuild/test/frontend/data/test-manifest-emitted-includes/reftest-stylo.list new file mode 100644 index 000000000..237aea0e0 --- /dev/null +++ b/python/mozbuild/mozbuild/test/frontend/data/test-manifest-emitted-includes/reftest-stylo.list @@ -0,0 +1,3 @@ +# DO NOT EDIT! This is a auto-generated temporary list for Stylo testing +== reftest1.html reftest1.html +include included-reftest-stylo.list diff --git a/python/mozbuild/mozbuild/test/frontend/data/test-manifest-emitted-includes/reftest.list b/python/mozbuild/mozbuild/test/frontend/data/test-manifest-emitted-includes/reftest.list new file mode 100644 index 000000000..80caf8ffa --- /dev/null +++ b/python/mozbuild/mozbuild/test/frontend/data/test-manifest-emitted-includes/reftest.list @@ -0,0 +1,2 @@ +== reftest1.html reftest1-ref.html +include included-reftest.list diff --git a/python/mozbuild/mozbuild/test/frontend/data/test-manifest-empty/empty.ini b/python/mozbuild/mozbuild/test/frontend/data/test-manifest-empty/empty.ini new file mode 100644 index 000000000..83a0cec0c --- /dev/null +++ b/python/mozbuild/mozbuild/test/frontend/data/test-manifest-empty/empty.ini @@ -0,0 +1,2 @@ +[DEFAULT] +foo = bar diff --git a/python/mozbuild/mozbuild/test/frontend/data/test-manifest-empty/moz.build b/python/mozbuild/mozbuild/test/frontend/data/test-manifest-empty/moz.build new file mode 100644 index 000000000..edfaf435f --- /dev/null +++ b/python/mozbuild/mozbuild/test/frontend/data/test-manifest-empty/moz.build @@ -0,0 +1,4 @@ +# Any copyright is dedicated to the Public Domain. +# http://creativecommons.org/publicdomain/zero/1.0/ + +MOCHITEST_MANIFESTS += ['empty.ini'] diff --git a/python/mozbuild/mozbuild/test/frontend/data/test-manifest-inactive-ignored/test_inactive.html b/python/mozbuild/mozbuild/test/frontend/data/test-manifest-inactive-ignored/test_inactive.html new file mode 100644 index 000000000..e69de29bb diff --git a/python/mozbuild/mozbuild/test/frontend/data/test-manifest-install-includes/common.ini b/python/mozbuild/mozbuild/test/frontend/data/test-manifest-install-includes/common.ini new file mode 100644 index 000000000..753cd0ec0 --- /dev/null +++ b/python/mozbuild/mozbuild/test/frontend/data/test-manifest-install-includes/common.ini @@ -0,0 +1 @@ +[test_foo.html] diff --git a/python/mozbuild/mozbuild/test/frontend/data/test-manifest-install-includes/mochitest.ini b/python/mozbuild/mozbuild/test/frontend/data/test-manifest-install-includes/mochitest.ini new file mode 100644 index 000000000..b8d4e123d --- /dev/null +++ b/python/mozbuild/mozbuild/test/frontend/data/test-manifest-install-includes/mochitest.ini @@ -0,0 +1,4 @@ +[DEFAULT] +install-to-subdir = subdir + +[include:common.ini] diff --git a/python/mozbuild/mozbuild/test/frontend/data/test-manifest-install-includes/moz.build b/python/mozbuild/mozbuild/test/frontend/data/test-manifest-install-includes/moz.build new file mode 100644 index 000000000..4e7e9ff4e --- /dev/null +++ b/python/mozbuild/mozbuild/test/frontend/data/test-manifest-install-includes/moz.build @@ -0,0 +1,4 @@ +# Any copyright is dedicated to the Public Domain. +# http://creativecommons.org/publicdomain/zero/1.0/ + +MOCHITEST_MANIFESTS += ['mochitest.ini'] diff --git a/python/mozbuild/mozbuild/test/frontend/data/test-manifest-install-includes/test_foo.html b/python/mozbuild/mozbuild/test/frontend/data/test-manifest-install-includes/test_foo.html new file mode 100644 index 000000000..18ecdcb79 --- /dev/null +++ b/python/mozbuild/mozbuild/test/frontend/data/test-manifest-install-includes/test_foo.html @@ -0,0 +1 @@ + diff --git a/python/mozbuild/mozbuild/test/frontend/data/test-manifest-install-subdir/moz.build b/python/mozbuild/mozbuild/test/frontend/data/test-manifest-install-subdir/moz.build new file mode 100644 index 000000000..9e4d7b21c --- /dev/null +++ b/python/mozbuild/mozbuild/test/frontend/data/test-manifest-install-subdir/moz.build @@ -0,0 +1,4 @@ +# Any copyright is dedicated to the Public Domain. +# http://creativecommons.org/publicdomain/zero/1.0/ + +MOCHITEST_MANIFESTS += ['subdir.ini'] diff --git a/python/mozbuild/mozbuild/test/frontend/data/test-manifest-install-subdir/subdir.ini b/python/mozbuild/mozbuild/test/frontend/data/test-manifest-install-subdir/subdir.ini new file mode 100644 index 000000000..6b320c2d5 --- /dev/null +++ b/python/mozbuild/mozbuild/test/frontend/data/test-manifest-install-subdir/subdir.ini @@ -0,0 +1,5 @@ +[DEFAULT] +install-to-subdir = subdir +support-files = support.txt + +[test_foo.html] diff --git a/python/mozbuild/mozbuild/test/frontend/data/test-manifest-install-subdir/test_foo.html b/python/mozbuild/mozbuild/test/frontend/data/test-manifest-install-subdir/test_foo.html new file mode 100644 index 000000000..18ecdcb79 --- /dev/null +++ b/python/mozbuild/mozbuild/test/frontend/data/test-manifest-install-subdir/test_foo.html @@ -0,0 +1 @@ + diff --git a/python/mozbuild/mozbuild/test/frontend/data/test-manifest-just-support/foo.txt b/python/mozbuild/mozbuild/test/frontend/data/test-manifest-just-support/foo.txt new file mode 100644 index 000000000..ce0136250 --- /dev/null +++ b/python/mozbuild/mozbuild/test/frontend/data/test-manifest-just-support/foo.txt @@ -0,0 +1 @@ +hello diff --git a/python/mozbuild/mozbuild/test/frontend/data/test-manifest-just-support/just-support.ini b/python/mozbuild/mozbuild/test/frontend/data/test-manifest-just-support/just-support.ini new file mode 100644 index 000000000..efa2d4bc0 --- /dev/null +++ b/python/mozbuild/mozbuild/test/frontend/data/test-manifest-just-support/just-support.ini @@ -0,0 +1,2 @@ +[DEFAULT] +support-files = foo.txt diff --git a/python/mozbuild/mozbuild/test/frontend/data/test-manifest-just-support/moz.build b/python/mozbuild/mozbuild/test/frontend/data/test-manifest-just-support/moz.build new file mode 100644 index 000000000..80a038d42 --- /dev/null +++ b/python/mozbuild/mozbuild/test/frontend/data/test-manifest-just-support/moz.build @@ -0,0 +1,4 @@ +# Any copyright is dedicated to the Public Domain. +# http://creativecommons.org/publicdomain/zero/1.0/ + +MOCHITEST_MANIFESTS += ['just-support.ini'] diff --git a/python/mozbuild/mozbuild/test/frontend/data/test-manifest-keys-extracted/a11y-support/dir1/bar b/python/mozbuild/mozbuild/test/frontend/data/test-manifest-keys-extracted/a11y-support/dir1/bar new file mode 100644 index 000000000..e69de29bb diff --git a/python/mozbuild/mozbuild/test/frontend/data/test-manifest-keys-extracted/a11y-support/foo b/python/mozbuild/mozbuild/test/frontend/data/test-manifest-keys-extracted/a11y-support/foo new file mode 100644 index 000000000..e69de29bb diff --git a/python/mozbuild/mozbuild/test/frontend/data/test-manifest-keys-extracted/a11y.ini b/python/mozbuild/mozbuild/test/frontend/data/test-manifest-keys-extracted/a11y.ini new file mode 100644 index 000000000..9cf798918 --- /dev/null +++ b/python/mozbuild/mozbuild/test/frontend/data/test-manifest-keys-extracted/a11y.ini @@ -0,0 +1,4 @@ +[DEFAULT] +support-files = a11y-support/** + +[test_a11y.js] diff --git a/python/mozbuild/mozbuild/test/frontend/data/test-manifest-keys-extracted/browser.ini b/python/mozbuild/mozbuild/test/frontend/data/test-manifest-keys-extracted/browser.ini new file mode 100644 index 000000000..a81ee3acb --- /dev/null +++ b/python/mozbuild/mozbuild/test/frontend/data/test-manifest-keys-extracted/browser.ini @@ -0,0 +1,4 @@ +[DEFAULT] +support-files = support1 support2 + +[test_browser.js] diff --git a/python/mozbuild/mozbuild/test/frontend/data/test-manifest-keys-extracted/chrome.ini b/python/mozbuild/mozbuild/test/frontend/data/test-manifest-keys-extracted/chrome.ini new file mode 100644 index 000000000..1070c7853 --- /dev/null +++ b/python/mozbuild/mozbuild/test/frontend/data/test-manifest-keys-extracted/chrome.ini @@ -0,0 +1,4 @@ +[DEFAULT] +skip-if = buildapp == 'b2g' + +[test_chrome.js] diff --git a/python/mozbuild/mozbuild/test/frontend/data/test-manifest-keys-extracted/crashtest.list b/python/mozbuild/mozbuild/test/frontend/data/test-manifest-keys-extracted/crashtest.list new file mode 100644 index 000000000..b9d7f2685 --- /dev/null +++ b/python/mozbuild/mozbuild/test/frontend/data/test-manifest-keys-extracted/crashtest.list @@ -0,0 +1 @@ +== crashtest1.html crashtest1-ref.html diff --git a/python/mozbuild/mozbuild/test/frontend/data/test-manifest-keys-extracted/metro.ini b/python/mozbuild/mozbuild/test/frontend/data/test-manifest-keys-extracted/metro.ini new file mode 100644 index 000000000..a7eb6def4 --- /dev/null +++ b/python/mozbuild/mozbuild/test/frontend/data/test-manifest-keys-extracted/metro.ini @@ -0,0 +1,3 @@ +[DEFAULT] + +[test_metro.js] diff --git a/python/mozbuild/mozbuild/test/frontend/data/test-manifest-keys-extracted/mochitest.ini b/python/mozbuild/mozbuild/test/frontend/data/test-manifest-keys-extracted/mochitest.ini new file mode 100644 index 000000000..69fd71de0 --- /dev/null +++ b/python/mozbuild/mozbuild/test/frontend/data/test-manifest-keys-extracted/mochitest.ini @@ -0,0 +1,5 @@ +[DEFAULT] +support-files = external1 external2 +generated-files = external1 external2 + +[test_mochitest.js] diff --git a/python/mozbuild/mozbuild/test/frontend/data/test-manifest-keys-extracted/moz.build b/python/mozbuild/mozbuild/test/frontend/data/test-manifest-keys-extracted/moz.build new file mode 100644 index 000000000..33839d9e3 --- /dev/null +++ b/python/mozbuild/mozbuild/test/frontend/data/test-manifest-keys-extracted/moz.build @@ -0,0 +1,12 @@ +# Any copyright is dedicated to the Public Domain. +# http://creativecommons.org/publicdomain/zero/1.0/ + +A11Y_MANIFESTS += ['a11y.ini'] +BROWSER_CHROME_MANIFESTS += ['browser.ini'] +METRO_CHROME_MANIFESTS += ['metro.ini'] +MOCHITEST_MANIFESTS += ['mochitest.ini'] +MOCHITEST_CHROME_MANIFESTS += ['chrome.ini'] +XPCSHELL_TESTS_MANIFESTS += ['xpcshell.ini'] +REFTEST_MANIFESTS += ['reftest.list'] +CRASHTEST_MANIFESTS += ['crashtest.list'] +PYTHON_UNIT_TESTS += ['test_foo.py'] diff --git a/python/mozbuild/mozbuild/test/frontend/data/test-manifest-keys-extracted/reftest-stylo.list b/python/mozbuild/mozbuild/test/frontend/data/test-manifest-keys-extracted/reftest-stylo.list new file mode 100644 index 000000000..bd7b4f9cb --- /dev/null +++ b/python/mozbuild/mozbuild/test/frontend/data/test-manifest-keys-extracted/reftest-stylo.list @@ -0,0 +1,2 @@ +# DO NOT EDIT! This is a auto-generated temporary list for Stylo testing +== reftest1.html reftest1.html diff --git a/python/mozbuild/mozbuild/test/frontend/data/test-manifest-keys-extracted/reftest.list b/python/mozbuild/mozbuild/test/frontend/data/test-manifest-keys-extracted/reftest.list new file mode 100644 index 000000000..3fc25b296 --- /dev/null +++ b/python/mozbuild/mozbuild/test/frontend/data/test-manifest-keys-extracted/reftest.list @@ -0,0 +1 @@ +== reftest1.html reftest1-ref.html diff --git a/python/mozbuild/mozbuild/test/frontend/data/test-manifest-keys-extracted/test_a11y.js b/python/mozbuild/mozbuild/test/frontend/data/test-manifest-keys-extracted/test_a11y.js new file mode 100644 index 000000000..e69de29bb diff --git a/python/mozbuild/mozbuild/test/frontend/data/test-manifest-keys-extracted/test_browser.js b/python/mozbuild/mozbuild/test/frontend/data/test-manifest-keys-extracted/test_browser.js new file mode 100644 index 000000000..e69de29bb diff --git a/python/mozbuild/mozbuild/test/frontend/data/test-manifest-keys-extracted/test_chrome.js b/python/mozbuild/mozbuild/test/frontend/data/test-manifest-keys-extracted/test_chrome.js new file mode 100644 index 000000000..e69de29bb diff --git a/python/mozbuild/mozbuild/test/frontend/data/test-manifest-keys-extracted/test_foo.py b/python/mozbuild/mozbuild/test/frontend/data/test-manifest-keys-extracted/test_foo.py new file mode 100644 index 000000000..e69de29bb diff --git a/python/mozbuild/mozbuild/test/frontend/data/test-manifest-keys-extracted/test_metro.js b/python/mozbuild/mozbuild/test/frontend/data/test-manifest-keys-extracted/test_metro.js new file mode 100644 index 000000000..e69de29bb diff --git a/python/mozbuild/mozbuild/test/frontend/data/test-manifest-keys-extracted/test_mochitest.js b/python/mozbuild/mozbuild/test/frontend/data/test-manifest-keys-extracted/test_mochitest.js new file mode 100644 index 000000000..e69de29bb diff --git a/python/mozbuild/mozbuild/test/frontend/data/test-manifest-keys-extracted/test_xpcshell.js b/python/mozbuild/mozbuild/test/frontend/data/test-manifest-keys-extracted/test_xpcshell.js new file mode 100644 index 000000000..e69de29bb diff --git a/python/mozbuild/mozbuild/test/frontend/data/test-manifest-keys-extracted/xpcshell.ini b/python/mozbuild/mozbuild/test/frontend/data/test-manifest-keys-extracted/xpcshell.ini new file mode 100644 index 000000000..fb3005434 --- /dev/null +++ b/python/mozbuild/mozbuild/test/frontend/data/test-manifest-keys-extracted/xpcshell.ini @@ -0,0 +1,6 @@ +[DEFAULT] +head = head1 head2 +tail = tail1 tail2 +dupe-manifest = + +[test_xpcshell.js] diff --git a/python/mozbuild/mozbuild/test/frontend/data/test-manifest-missing-manifest/moz.build b/python/mozbuild/mozbuild/test/frontend/data/test-manifest-missing-manifest/moz.build new file mode 100644 index 000000000..45edcc027 --- /dev/null +++ b/python/mozbuild/mozbuild/test/frontend/data/test-manifest-missing-manifest/moz.build @@ -0,0 +1,4 @@ +# Any copyright is dedicated to the Public Domain. +# http://creativecommons.org/publicdomain/zero/1.0/ + +XPCSHELL_TESTS_MANIFESTS += ['does_not_exist.ini'] diff --git a/python/mozbuild/mozbuild/test/frontend/data/test-manifest-missing-test-file-unfiltered/moz.build b/python/mozbuild/mozbuild/test/frontend/data/test-manifest-missing-test-file-unfiltered/moz.build new file mode 100644 index 000000000..09c51cbb8 --- /dev/null +++ b/python/mozbuild/mozbuild/test/frontend/data/test-manifest-missing-test-file-unfiltered/moz.build @@ -0,0 +1,4 @@ +# Any copyright is dedicated to the Public Domain. +# http://creativecommons.org/publicdomain/zero/1.0/ + +XPCSHELL_TESTS_MANIFESTS += ['xpcshell.ini'] diff --git a/python/mozbuild/mozbuild/test/frontend/data/test-manifest-missing-test-file-unfiltered/xpcshell.ini b/python/mozbuild/mozbuild/test/frontend/data/test-manifest-missing-test-file-unfiltered/xpcshell.ini new file mode 100644 index 000000000..9ab85c0ce --- /dev/null +++ b/python/mozbuild/mozbuild/test/frontend/data/test-manifest-missing-test-file-unfiltered/xpcshell.ini @@ -0,0 +1,4 @@ +[DEFAULT] +support-files = support/** + +[missing.js] diff --git a/python/mozbuild/mozbuild/test/frontend/data/test-manifest-missing-test-file/mochitest.ini b/python/mozbuild/mozbuild/test/frontend/data/test-manifest-missing-test-file/mochitest.ini new file mode 100644 index 000000000..e3ef6216b --- /dev/null +++ b/python/mozbuild/mozbuild/test/frontend/data/test-manifest-missing-test-file/mochitest.ini @@ -0,0 +1 @@ +[test_missing.html] diff --git a/python/mozbuild/mozbuild/test/frontend/data/test-manifest-missing-test-file/moz.build b/python/mozbuild/mozbuild/test/frontend/data/test-manifest-missing-test-file/moz.build new file mode 100644 index 000000000..4e7e9ff4e --- /dev/null +++ b/python/mozbuild/mozbuild/test/frontend/data/test-manifest-missing-test-file/moz.build @@ -0,0 +1,4 @@ +# Any copyright is dedicated to the Public Domain. +# http://creativecommons.org/publicdomain/zero/1.0/ + +MOCHITEST_MANIFESTS += ['mochitest.ini'] diff --git a/python/mozbuild/mozbuild/test/frontend/data/test-manifest-parent-support-files-dir/child/mochitest.ini b/python/mozbuild/mozbuild/test/frontend/data/test-manifest-parent-support-files-dir/child/mochitest.ini new file mode 100644 index 000000000..c78822429 --- /dev/null +++ b/python/mozbuild/mozbuild/test/frontend/data/test-manifest-parent-support-files-dir/child/mochitest.ini @@ -0,0 +1,4 @@ +[DEFAULT] +support-files = ../support-file.txt + +[test_foo.js] diff --git a/python/mozbuild/mozbuild/test/frontend/data/test-manifest-parent-support-files-dir/child/test_foo.js b/python/mozbuild/mozbuild/test/frontend/data/test-manifest-parent-support-files-dir/child/test_foo.js new file mode 100644 index 000000000..e69de29bb diff --git a/python/mozbuild/mozbuild/test/frontend/data/test-manifest-parent-support-files-dir/moz.build b/python/mozbuild/mozbuild/test/frontend/data/test-manifest-parent-support-files-dir/moz.build new file mode 100644 index 000000000..a40e25625 --- /dev/null +++ b/python/mozbuild/mozbuild/test/frontend/data/test-manifest-parent-support-files-dir/moz.build @@ -0,0 +1,4 @@ +# Any copyright is dedicated to the Public Domain. +# http://creativecommons.org/publicdomain/zero/1.0/ + +MOCHITEST_MANIFESTS += ['child/mochitest.ini'] diff --git a/python/mozbuild/mozbuild/test/frontend/data/test-manifest-parent-support-files-dir/support-file.txt b/python/mozbuild/mozbuild/test/frontend/data/test-manifest-parent-support-files-dir/support-file.txt new file mode 100644 index 000000000..e69de29bb diff --git a/python/mozbuild/mozbuild/test/frontend/data/test-manifest-shared-missing/child/another-file.sjs b/python/mozbuild/mozbuild/test/frontend/data/test-manifest-shared-missing/child/another-file.sjs new file mode 100644 index 000000000..e69de29bb diff --git a/python/mozbuild/mozbuild/test/frontend/data/test-manifest-shared-missing/child/browser.ini b/python/mozbuild/mozbuild/test/frontend/data/test-manifest-shared-missing/child/browser.ini new file mode 100644 index 000000000..4f1335d6b --- /dev/null +++ b/python/mozbuild/mozbuild/test/frontend/data/test-manifest-shared-missing/child/browser.ini @@ -0,0 +1,6 @@ +[DEFAULT] +support-files = + another-file.sjs + data/** + +[test_sub.js] \ No newline at end of file diff --git a/python/mozbuild/mozbuild/test/frontend/data/test-manifest-shared-missing/child/data/one.txt b/python/mozbuild/mozbuild/test/frontend/data/test-manifest-shared-missing/child/data/one.txt new file mode 100644 index 000000000..e69de29bb diff --git a/python/mozbuild/mozbuild/test/frontend/data/test-manifest-shared-missing/child/data/two.txt b/python/mozbuild/mozbuild/test/frontend/data/test-manifest-shared-missing/child/data/two.txt new file mode 100644 index 000000000..e69de29bb diff --git a/python/mozbuild/mozbuild/test/frontend/data/test-manifest-shared-missing/child/test_sub.js b/python/mozbuild/mozbuild/test/frontend/data/test-manifest-shared-missing/child/test_sub.js new file mode 100644 index 000000000..e69de29bb diff --git a/python/mozbuild/mozbuild/test/frontend/data/test-manifest-shared-missing/mochitest.ini b/python/mozbuild/mozbuild/test/frontend/data/test-manifest-shared-missing/mochitest.ini new file mode 100644 index 000000000..ada59d387 --- /dev/null +++ b/python/mozbuild/mozbuild/test/frontend/data/test-manifest-shared-missing/mochitest.ini @@ -0,0 +1,9 @@ +[DEFAULT] +support-files = + support-file.txt + !/child/test_sub.js + !/child/another-file.sjs + !/child/data/** + !/does/not/exist.sjs + +[test_foo.js] diff --git a/python/mozbuild/mozbuild/test/frontend/data/test-manifest-shared-missing/moz.build b/python/mozbuild/mozbuild/test/frontend/data/test-manifest-shared-missing/moz.build new file mode 100644 index 000000000..1c1d064ea --- /dev/null +++ b/python/mozbuild/mozbuild/test/frontend/data/test-manifest-shared-missing/moz.build @@ -0,0 +1,5 @@ +# Any copyright is dedicated to the Public Domain. +# http://creativecommons.org/publicdomain/zero/1.0/ + +MOCHITEST_MANIFESTS += ['mochitest.ini'] +BROWSER_CHROME_MANIFESTS += ['child/browser.ini'] diff --git a/python/mozbuild/mozbuild/test/frontend/data/test-manifest-shared-missing/support-file.txt b/python/mozbuild/mozbuild/test/frontend/data/test-manifest-shared-missing/support-file.txt new file mode 100644 index 000000000..e69de29bb diff --git a/python/mozbuild/mozbuild/test/frontend/data/test-manifest-shared-missing/test_foo.js b/python/mozbuild/mozbuild/test/frontend/data/test-manifest-shared-missing/test_foo.js new file mode 100644 index 000000000..e69de29bb diff --git a/python/mozbuild/mozbuild/test/frontend/data/test-manifest-shared-support/child/another-file.sjs b/python/mozbuild/mozbuild/test/frontend/data/test-manifest-shared-support/child/another-file.sjs new file mode 100644 index 000000000..e69de29bb diff --git a/python/mozbuild/mozbuild/test/frontend/data/test-manifest-shared-support/child/browser.ini b/python/mozbuild/mozbuild/test/frontend/data/test-manifest-shared-support/child/browser.ini new file mode 100644 index 000000000..4f1335d6b --- /dev/null +++ b/python/mozbuild/mozbuild/test/frontend/data/test-manifest-shared-support/child/browser.ini @@ -0,0 +1,6 @@ +[DEFAULT] +support-files = + another-file.sjs + data/** + +[test_sub.js] \ No newline at end of file diff --git a/python/mozbuild/mozbuild/test/frontend/data/test-manifest-shared-support/child/data/one.txt b/python/mozbuild/mozbuild/test/frontend/data/test-manifest-shared-support/child/data/one.txt new file mode 100644 index 000000000..e69de29bb diff --git a/python/mozbuild/mozbuild/test/frontend/data/test-manifest-shared-support/child/data/two.txt b/python/mozbuild/mozbuild/test/frontend/data/test-manifest-shared-support/child/data/two.txt new file mode 100644 index 000000000..e69de29bb diff --git a/python/mozbuild/mozbuild/test/frontend/data/test-manifest-shared-support/child/test_sub.js b/python/mozbuild/mozbuild/test/frontend/data/test-manifest-shared-support/child/test_sub.js new file mode 100644 index 000000000..e69de29bb diff --git a/python/mozbuild/mozbuild/test/frontend/data/test-manifest-shared-support/mochitest.ini b/python/mozbuild/mozbuild/test/frontend/data/test-manifest-shared-support/mochitest.ini new file mode 100644 index 000000000..a9860f3de --- /dev/null +++ b/python/mozbuild/mozbuild/test/frontend/data/test-manifest-shared-support/mochitest.ini @@ -0,0 +1,8 @@ +[DEFAULT] +support-files = + support-file.txt + !/child/test_sub.js + !/child/another-file.sjs + !/child/data/** + +[test_foo.js] diff --git a/python/mozbuild/mozbuild/test/frontend/data/test-manifest-shared-support/moz.build b/python/mozbuild/mozbuild/test/frontend/data/test-manifest-shared-support/moz.build new file mode 100644 index 000000000..1c1d064ea --- /dev/null +++ b/python/mozbuild/mozbuild/test/frontend/data/test-manifest-shared-support/moz.build @@ -0,0 +1,5 @@ +# Any copyright is dedicated to the Public Domain. +# http://creativecommons.org/publicdomain/zero/1.0/ + +MOCHITEST_MANIFESTS += ['mochitest.ini'] +BROWSER_CHROME_MANIFESTS += ['child/browser.ini'] diff --git a/python/mozbuild/mozbuild/test/frontend/data/test-manifest-shared-support/support-file.txt b/python/mozbuild/mozbuild/test/frontend/data/test-manifest-shared-support/support-file.txt new file mode 100644 index 000000000..e69de29bb diff --git a/python/mozbuild/mozbuild/test/frontend/data/test-manifest-shared-support/test_foo.js b/python/mozbuild/mozbuild/test/frontend/data/test-manifest-shared-support/test_foo.js new file mode 100644 index 000000000..e69de29bb diff --git a/python/mozbuild/mozbuild/test/frontend/data/test-manifest-unmatched-generated/moz.build b/python/mozbuild/mozbuild/test/frontend/data/test-manifest-unmatched-generated/moz.build new file mode 100644 index 000000000..281dee610 --- /dev/null +++ b/python/mozbuild/mozbuild/test/frontend/data/test-manifest-unmatched-generated/moz.build @@ -0,0 +1,4 @@ +# Any copyright is dedicated to the Public Domain. +# http://creativecommons.org/publicdomain/zero/1.0/ + +MOCHITEST_MANIFESTS += ['test.ini'] diff --git a/python/mozbuild/mozbuild/test/frontend/data/test-manifest-unmatched-generated/test.ini b/python/mozbuild/mozbuild/test/frontend/data/test-manifest-unmatched-generated/test.ini new file mode 100644 index 000000000..caf391186 --- /dev/null +++ b/python/mozbuild/mozbuild/test/frontend/data/test-manifest-unmatched-generated/test.ini @@ -0,0 +1,4 @@ +[DEFAULT] +generated-files = does_not_exist + +[test_foo] diff --git a/python/mozbuild/mozbuild/test/frontend/data/test-manifest-unmatched-generated/test_foo b/python/mozbuild/mozbuild/test/frontend/data/test-manifest-unmatched-generated/test_foo new file mode 100644 index 000000000..e69de29bb diff --git a/python/mozbuild/mozbuild/test/frontend/data/test-python-unit-test-missing/moz.build b/python/mozbuild/mozbuild/test/frontend/data/test-python-unit-test-missing/moz.build new file mode 100644 index 000000000..c9d769802 --- /dev/null +++ b/python/mozbuild/mozbuild/test/frontend/data/test-python-unit-test-missing/moz.build @@ -0,0 +1,4 @@ +# Any copyright is dedicated to the Public Domain. +# http://creativecommons.org/publicdomain/zero/1.0/ + +PYTHON_UNIT_TESTS += ['test_foo.py'] diff --git a/python/mozbuild/mozbuild/test/frontend/data/test-symbols-file-objdir-missing-generated/moz.build b/python/mozbuild/mozbuild/test/frontend/data/test-symbols-file-objdir-missing-generated/moz.build new file mode 100644 index 000000000..9d35a8ccc --- /dev/null +++ b/python/mozbuild/mozbuild/test/frontend/data/test-symbols-file-objdir-missing-generated/moz.build @@ -0,0 +1,10 @@ +# Any copyright is dedicated to the Public Domain. +# http://creativecommons.org/publicdomain/zero/1.0/ + +@template +def SharedLibrary(name): + LIBRARY_NAME = name + FORCE_SHARED_LIB = True + +SharedLibrary('foo') +SYMBOLS_FILE = '!foo.symbols' diff --git a/python/mozbuild/mozbuild/test/frontend/data/test-symbols-file-objdir/foo.py b/python/mozbuild/mozbuild/test/frontend/data/test-symbols-file-objdir/foo.py new file mode 100644 index 000000000..e69de29bb diff --git a/python/mozbuild/mozbuild/test/frontend/data/test-symbols-file-objdir/moz.build b/python/mozbuild/mozbuild/test/frontend/data/test-symbols-file-objdir/moz.build new file mode 100644 index 000000000..fe227224d --- /dev/null +++ b/python/mozbuild/mozbuild/test/frontend/data/test-symbols-file-objdir/moz.build @@ -0,0 +1,13 @@ +# Any copyright is dedicated to the Public Domain. +# http://creativecommons.org/publicdomain/zero/1.0/ + +@template +def SharedLibrary(name): + LIBRARY_NAME = name + FORCE_SHARED_LIB = True + +SharedLibrary('foo') +SYMBOLS_FILE = '!foo.symbols' + +GENERATED_FILES += ['foo.symbols'] +GENERATED_FILES['foo.symbols'].script = 'foo.py' diff --git a/python/mozbuild/mozbuild/test/frontend/data/test-symbols-file/foo.symbols b/python/mozbuild/mozbuild/test/frontend/data/test-symbols-file/foo.symbols new file mode 100644 index 000000000..257cc5642 --- /dev/null +++ b/python/mozbuild/mozbuild/test/frontend/data/test-symbols-file/foo.symbols @@ -0,0 +1 @@ +foo diff --git a/python/mozbuild/mozbuild/test/frontend/data/test-symbols-file/moz.build b/python/mozbuild/mozbuild/test/frontend/data/test-symbols-file/moz.build new file mode 100644 index 000000000..d69333ea4 --- /dev/null +++ b/python/mozbuild/mozbuild/test/frontend/data/test-symbols-file/moz.build @@ -0,0 +1,10 @@ +# Any copyright is dedicated to the Public Domain. +# http://creativecommons.org/publicdomain/zero/1.0/ + +@template +def SharedLibrary(name): + LIBRARY_NAME = name + FORCE_SHARED_LIB = True + +SharedLibrary('foo') +SYMBOLS_FILE = 'foo.symbols' diff --git a/python/mozbuild/mozbuild/test/frontend/data/traversal-all-vars/moz.build b/python/mozbuild/mozbuild/test/frontend/data/traversal-all-vars/moz.build new file mode 100644 index 000000000..73045dd43 --- /dev/null +++ b/python/mozbuild/mozbuild/test/frontend/data/traversal-all-vars/moz.build @@ -0,0 +1,6 @@ +# -*- Mode: python; indent-tabs-mode: nil; tab-width: 40 -*- +# Any copyright is dedicated to the Public Domain. +# http://creativecommons.org/publicdomain/zero/1.0/ + +DIRS += ['regular'] +TEST_DIRS += ['test'] diff --git a/python/mozbuild/mozbuild/test/frontend/data/traversal-all-vars/parallel/moz.build b/python/mozbuild/mozbuild/test/frontend/data/traversal-all-vars/parallel/moz.build new file mode 100644 index 000000000..e69de29bb diff --git a/python/mozbuild/mozbuild/test/frontend/data/traversal-all-vars/regular/moz.build b/python/mozbuild/mozbuild/test/frontend/data/traversal-all-vars/regular/moz.build new file mode 100644 index 000000000..e69de29bb diff --git a/python/mozbuild/mozbuild/test/frontend/data/traversal-all-vars/test/moz.build b/python/mozbuild/mozbuild/test/frontend/data/traversal-all-vars/test/moz.build new file mode 100644 index 000000000..e69de29bb diff --git a/python/mozbuild/mozbuild/test/frontend/data/traversal-outside-topsrcdir/moz.build b/python/mozbuild/mozbuild/test/frontend/data/traversal-outside-topsrcdir/moz.build new file mode 100644 index 000000000..92ceb7f3b --- /dev/null +++ b/python/mozbuild/mozbuild/test/frontend/data/traversal-outside-topsrcdir/moz.build @@ -0,0 +1,5 @@ +# -*- Mode: python; indent-tabs-mode: nil; tab-width: 40 -*- +# Any copyright is dedicated to the Public Domain. +# http://creativecommons.org/publicdomain/zero/1.0/ + +DIRS = ['../../foo'] diff --git a/python/mozbuild/mozbuild/test/frontend/data/traversal-relative-dirs/bar/moz.build b/python/mozbuild/mozbuild/test/frontend/data/traversal-relative-dirs/bar/moz.build new file mode 100644 index 000000000..e69de29bb diff --git a/python/mozbuild/mozbuild/test/frontend/data/traversal-relative-dirs/foo/moz.build b/python/mozbuild/mozbuild/test/frontend/data/traversal-relative-dirs/foo/moz.build new file mode 100644 index 000000000..ca1a429d9 --- /dev/null +++ b/python/mozbuild/mozbuild/test/frontend/data/traversal-relative-dirs/foo/moz.build @@ -0,0 +1,5 @@ +# -*- Mode: python; indent-tabs-mode: nil; tab-width: 40 -*- +# Any copyright is dedicated to the Public Domain. +# http://creativecommons.org/publicdomain/zero/1.0/ + +DIRS = ['../bar'] diff --git a/python/mozbuild/mozbuild/test/frontend/data/traversal-relative-dirs/moz.build b/python/mozbuild/mozbuild/test/frontend/data/traversal-relative-dirs/moz.build new file mode 100644 index 000000000..5fac39736 --- /dev/null +++ b/python/mozbuild/mozbuild/test/frontend/data/traversal-relative-dirs/moz.build @@ -0,0 +1,5 @@ +# -*- Mode: python; indent-tabs-mode: nil; tab-width: 40 -*- +# Any copyright is dedicated to the Public Domain. +# http://creativecommons.org/publicdomain/zero/1.0/ + +DIRS = ['foo'] diff --git a/python/mozbuild/mozbuild/test/frontend/data/traversal-repeated-dirs/bar/moz.build b/python/mozbuild/mozbuild/test/frontend/data/traversal-repeated-dirs/bar/moz.build new file mode 100644 index 000000000..f06edcd36 --- /dev/null +++ b/python/mozbuild/mozbuild/test/frontend/data/traversal-repeated-dirs/bar/moz.build @@ -0,0 +1,5 @@ +# -*- Mode: python; indent-tabs-mode: nil; tab-width: 40 -*- +# Any copyright is dedicated to the Public Domain. +# http://creativecommons.org/publicdomain/zero/1.0/ + +DIRS = ['../foo'] diff --git a/python/mozbuild/mozbuild/test/frontend/data/traversal-repeated-dirs/foo/moz.build b/python/mozbuild/mozbuild/test/frontend/data/traversal-repeated-dirs/foo/moz.build new file mode 100644 index 000000000..ca1a429d9 --- /dev/null +++ b/python/mozbuild/mozbuild/test/frontend/data/traversal-repeated-dirs/foo/moz.build @@ -0,0 +1,5 @@ +# -*- Mode: python; indent-tabs-mode: nil; tab-width: 40 -*- +# Any copyright is dedicated to the Public Domain. +# http://creativecommons.org/publicdomain/zero/1.0/ + +DIRS = ['../bar'] diff --git a/python/mozbuild/mozbuild/test/frontend/data/traversal-repeated-dirs/moz.build b/python/mozbuild/mozbuild/test/frontend/data/traversal-repeated-dirs/moz.build new file mode 100644 index 000000000..924f667d9 --- /dev/null +++ b/python/mozbuild/mozbuild/test/frontend/data/traversal-repeated-dirs/moz.build @@ -0,0 +1,5 @@ +# -*- Mode: python; indent-tabs-mode: nil; tab-width: 40 -*- +# Any copyright is dedicated to the Public Domain. +# http://creativecommons.org/publicdomain/zero/1.0/ + +DIRS = ['foo', 'bar'] diff --git a/python/mozbuild/mozbuild/test/frontend/data/traversal-simple/bar/moz.build b/python/mozbuild/mozbuild/test/frontend/data/traversal-simple/bar/moz.build new file mode 100644 index 000000000..e69de29bb diff --git a/python/mozbuild/mozbuild/test/frontend/data/traversal-simple/foo/biz/moz.build b/python/mozbuild/mozbuild/test/frontend/data/traversal-simple/foo/biz/moz.build new file mode 100644 index 000000000..e69de29bb diff --git a/python/mozbuild/mozbuild/test/frontend/data/traversal-simple/foo/moz.build b/python/mozbuild/mozbuild/test/frontend/data/traversal-simple/foo/moz.build new file mode 100644 index 000000000..182541efd --- /dev/null +++ b/python/mozbuild/mozbuild/test/frontend/data/traversal-simple/foo/moz.build @@ -0,0 +1,2 @@ +# -*- Mode: python; indent-tabs-mode: nil; tab-width: 40 -*- +DIRS = ['biz'] diff --git a/python/mozbuild/mozbuild/test/frontend/data/traversal-simple/moz.build b/python/mozbuild/mozbuild/test/frontend/data/traversal-simple/moz.build new file mode 100644 index 000000000..924f667d9 --- /dev/null +++ b/python/mozbuild/mozbuild/test/frontend/data/traversal-simple/moz.build @@ -0,0 +1,5 @@ +# -*- Mode: python; indent-tabs-mode: nil; tab-width: 40 -*- +# Any copyright is dedicated to the Public Domain. +# http://creativecommons.org/publicdomain/zero/1.0/ + +DIRS = ['foo', 'bar'] diff --git a/python/mozbuild/mozbuild/test/frontend/data/unified-sources-non-unified/bar.cxx b/python/mozbuild/mozbuild/test/frontend/data/unified-sources-non-unified/bar.cxx new file mode 100644 index 000000000..e69de29bb diff --git a/python/mozbuild/mozbuild/test/frontend/data/unified-sources-non-unified/c1.c b/python/mozbuild/mozbuild/test/frontend/data/unified-sources-non-unified/c1.c new file mode 100644 index 000000000..e69de29bb diff --git a/python/mozbuild/mozbuild/test/frontend/data/unified-sources-non-unified/c2.c b/python/mozbuild/mozbuild/test/frontend/data/unified-sources-non-unified/c2.c new file mode 100644 index 000000000..e69de29bb diff --git a/python/mozbuild/mozbuild/test/frontend/data/unified-sources-non-unified/foo.cpp b/python/mozbuild/mozbuild/test/frontend/data/unified-sources-non-unified/foo.cpp new file mode 100644 index 000000000..e69de29bb diff --git a/python/mozbuild/mozbuild/test/frontend/data/unified-sources-non-unified/moz.build b/python/mozbuild/mozbuild/test/frontend/data/unified-sources-non-unified/moz.build new file mode 100644 index 000000000..a3660222d --- /dev/null +++ b/python/mozbuild/mozbuild/test/frontend/data/unified-sources-non-unified/moz.build @@ -0,0 +1,28 @@ +# -*- Mode: python; indent-tabs-mode: nil; tab-width: 40 -*- +# Any copyright is dedicated to the Public Domain. +# http://creativecommons.org/publicdomain/zero/1.0/ + +@template +def Library(name): + '''Template for libraries.''' + LIBRARY_NAME = name + +Library('dummy') + +UNIFIED_SOURCES += [ + 'bar.cxx', + 'foo.cpp', + 'quux.cc', +] + +UNIFIED_SOURCES += [ + 'objc1.mm', + 'objc2.mm', +] + +UNIFIED_SOURCES += [ + 'c1.c', + 'c2.c', +] + +FILES_PER_UNIFIED_FILE = 1 diff --git a/python/mozbuild/mozbuild/test/frontend/data/unified-sources-non-unified/objc1.mm b/python/mozbuild/mozbuild/test/frontend/data/unified-sources-non-unified/objc1.mm new file mode 100644 index 000000000..e69de29bb diff --git a/python/mozbuild/mozbuild/test/frontend/data/unified-sources-non-unified/objc2.mm b/python/mozbuild/mozbuild/test/frontend/data/unified-sources-non-unified/objc2.mm new file mode 100644 index 000000000..e69de29bb diff --git a/python/mozbuild/mozbuild/test/frontend/data/unified-sources-non-unified/quux.cc b/python/mozbuild/mozbuild/test/frontend/data/unified-sources-non-unified/quux.cc new file mode 100644 index 000000000..e69de29bb diff --git a/python/mozbuild/mozbuild/test/frontend/data/unified-sources/bar.cxx b/python/mozbuild/mozbuild/test/frontend/data/unified-sources/bar.cxx new file mode 100644 index 000000000..e69de29bb diff --git a/python/mozbuild/mozbuild/test/frontend/data/unified-sources/c1.c b/python/mozbuild/mozbuild/test/frontend/data/unified-sources/c1.c new file mode 100644 index 000000000..e69de29bb diff --git a/python/mozbuild/mozbuild/test/frontend/data/unified-sources/c2.c b/python/mozbuild/mozbuild/test/frontend/data/unified-sources/c2.c new file mode 100644 index 000000000..e69de29bb diff --git a/python/mozbuild/mozbuild/test/frontend/data/unified-sources/foo.cpp b/python/mozbuild/mozbuild/test/frontend/data/unified-sources/foo.cpp new file mode 100644 index 000000000..e69de29bb diff --git a/python/mozbuild/mozbuild/test/frontend/data/unified-sources/moz.build b/python/mozbuild/mozbuild/test/frontend/data/unified-sources/moz.build new file mode 100644 index 000000000..5d1d89fb4 --- /dev/null +++ b/python/mozbuild/mozbuild/test/frontend/data/unified-sources/moz.build @@ -0,0 +1,28 @@ +# -*- Mode: python; indent-tabs-mode: nil; tab-width: 40 -*- +# Any copyright is dedicated to the Public Domain. +# http://creativecommons.org/publicdomain/zero/1.0/ + +@template +def Library(name): + '''Template for libraries.''' + LIBRARY_NAME = name + +Library('dummy') + +UNIFIED_SOURCES += [ + 'bar.cxx', + 'foo.cpp', + 'quux.cc', +] + +UNIFIED_SOURCES += [ + 'objc1.mm', + 'objc2.mm', +] + +UNIFIED_SOURCES += [ + 'c1.c', + 'c2.c', +] + +FILES_PER_UNIFIED_FILE = 32 diff --git a/python/mozbuild/mozbuild/test/frontend/data/unified-sources/objc1.mm b/python/mozbuild/mozbuild/test/frontend/data/unified-sources/objc1.mm new file mode 100644 index 000000000..e69de29bb diff --git a/python/mozbuild/mozbuild/test/frontend/data/unified-sources/objc2.mm b/python/mozbuild/mozbuild/test/frontend/data/unified-sources/objc2.mm new file mode 100644 index 000000000..e69de29bb diff --git a/python/mozbuild/mozbuild/test/frontend/data/unified-sources/quux.cc b/python/mozbuild/mozbuild/test/frontend/data/unified-sources/quux.cc new file mode 100644 index 000000000..e69de29bb diff --git a/python/mozbuild/mozbuild/test/frontend/data/use-yasm/moz.build b/python/mozbuild/mozbuild/test/frontend/data/use-yasm/moz.build new file mode 100644 index 000000000..11f45953d --- /dev/null +++ b/python/mozbuild/mozbuild/test/frontend/data/use-yasm/moz.build @@ -0,0 +1,5 @@ +# -*- Mode: python; indent-tabs-mode: nil; tab-width: 40 -*- +# Any copyright is dedicated to the Public Domain. +# http://creativecommons.org/publicdomain/zero/1.0/ + +USE_YASM = True diff --git a/python/mozbuild/mozbuild/test/frontend/data/variable-passthru/bans.S b/python/mozbuild/mozbuild/test/frontend/data/variable-passthru/bans.S new file mode 100644 index 000000000..e69de29bb diff --git a/python/mozbuild/mozbuild/test/frontend/data/variable-passthru/moz.build b/python/mozbuild/mozbuild/test/frontend/data/variable-passthru/moz.build new file mode 100644 index 000000000..e85e6ff5d --- /dev/null +++ b/python/mozbuild/mozbuild/test/frontend/data/variable-passthru/moz.build @@ -0,0 +1,25 @@ +# -*- Mode: python; indent-tabs-mode: nil; tab-width: 40 -*- +# Any copyright is dedicated to the Public Domain. +# http://creativecommons.org/publicdomain/zero/1.0/ + +DIST_INSTALL = False + +NO_VISIBILITY_FLAGS = True + +DELAYLOAD_DLLS = ['foo.dll', 'bar.dll'] + +RCFILE = 'foo.rc' +RESFILE = 'bar.res' +RCINCLUDE = 'bar.rc' +DEFFILE = 'baz.def' + +CFLAGS += ['-fno-exceptions', '-w'] +CXXFLAGS += ['-fcxx-exceptions', '-include foo.h'] +LDFLAGS += ['-framework Foo', '-x'] +HOST_CFLAGS += ['-funroll-loops', '-wall'] +HOST_CXXFLAGS += ['-funroll-loops-harder', '-wall-day-everyday'] +WIN32_EXE_LDFLAGS += ['-subsystem:console'] + +DISABLE_STL_WRAPPING = True + +ALLOW_COMPILER_WARNINGS = True diff --git a/python/mozbuild/mozbuild/test/frontend/data/variable-passthru/test1.c b/python/mozbuild/mozbuild/test/frontend/data/variable-passthru/test1.c new file mode 100644 index 000000000..e69de29bb diff --git a/python/mozbuild/mozbuild/test/frontend/data/variable-passthru/test1.cpp b/python/mozbuild/mozbuild/test/frontend/data/variable-passthru/test1.cpp new file mode 100644 index 000000000..e69de29bb diff --git a/python/mozbuild/mozbuild/test/frontend/data/variable-passthru/test1.mm b/python/mozbuild/mozbuild/test/frontend/data/variable-passthru/test1.mm new file mode 100644 index 000000000..e69de29bb diff --git a/python/mozbuild/mozbuild/test/frontend/data/variable-passthru/test2.c b/python/mozbuild/mozbuild/test/frontend/data/variable-passthru/test2.c new file mode 100644 index 000000000..e69de29bb diff --git a/python/mozbuild/mozbuild/test/frontend/data/variable-passthru/test2.cpp b/python/mozbuild/mozbuild/test/frontend/data/variable-passthru/test2.cpp new file mode 100644 index 000000000..e69de29bb diff --git a/python/mozbuild/mozbuild/test/frontend/data/variable-passthru/test2.mm b/python/mozbuild/mozbuild/test/frontend/data/variable-passthru/test2.mm new file mode 100644 index 000000000..e69de29bb diff --git a/python/mozbuild/mozbuild/test/frontend/data/xpidl-module-no-sources/moz.build b/python/mozbuild/mozbuild/test/frontend/data/xpidl-module-no-sources/moz.build new file mode 100644 index 000000000..60f061d5c --- /dev/null +++ b/python/mozbuild/mozbuild/test/frontend/data/xpidl-module-no-sources/moz.build @@ -0,0 +1,5 @@ +# -*- Mode: python; indent-tabs-mode: nil; tab-width: 40 -*- +# Any copyright is dedicated to the Public Domain. +# http://creativecommons.org/publicdomain/zero/1.0/ + +XPIDL_MODULE = 'xpidl_module' diff --git a/python/mozbuild/mozbuild/test/frontend/test_context.py b/python/mozbuild/mozbuild/test/frontend/test_context.py new file mode 100644 index 000000000..070cfad67 --- /dev/null +++ b/python/mozbuild/mozbuild/test/frontend/test_context.py @@ -0,0 +1,721 @@ +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, You can obtain one at http://mozilla.org/MPL/2.0/. + +import os +import unittest + +from mozunit import main + +from mozbuild.frontend.context import ( + AbsolutePath, + Context, + ContextDerivedTypedHierarchicalStringList, + ContextDerivedTypedList, + ContextDerivedTypedListWithItems, + ContextDerivedTypedRecord, + Files, + FUNCTIONS, + ObjDirPath, + Path, + SourcePath, + SPECIAL_VARIABLES, + SUBCONTEXTS, + VARIABLES, +) + +from mozbuild.util import StrictOrderingOnAppendListWithFlagsFactory +from mozpack import path as mozpath + + +class TestContext(unittest.TestCase): + def test_defaults(self): + test = Context({ + 'foo': (int, int, ''), + 'bar': (bool, bool, ''), + 'baz': (dict, dict, ''), + }) + + self.assertEqual(test.keys(), []) + + self.assertEqual(test['foo'], 0) + + self.assertEqual(set(test.keys()), { 'foo' }) + + self.assertEqual(test['bar'], False) + + self.assertEqual(set(test.keys()), { 'foo', 'bar' }) + + self.assertEqual(test['baz'], {}) + + self.assertEqual(set(test.keys()), { 'foo', 'bar', 'baz' }) + + with self.assertRaises(KeyError): + test['qux'] + + self.assertEqual(set(test.keys()), { 'foo', 'bar', 'baz' }) + + def test_type_check(self): + test = Context({ + 'foo': (int, int, ''), + 'baz': (dict, list, ''), + }) + + test['foo'] = 5 + + self.assertEqual(test['foo'], 5) + + with self.assertRaises(ValueError): + test['foo'] = {} + + self.assertEqual(test['foo'], 5) + + with self.assertRaises(KeyError): + test['bar'] = True + + test['baz'] = [('a', 1), ('b', 2)] + + self.assertEqual(test['baz'], { 'a': 1, 'b': 2 }) + + def test_update(self): + test = Context({ + 'foo': (int, int, ''), + 'bar': (bool, bool, ''), + 'baz': (dict, list, ''), + }) + + self.assertEqual(test.keys(), []) + + with self.assertRaises(ValueError): + test.update(bar=True, foo={}) + + self.assertEqual(test.keys(), []) + + test.update(bar=True, foo=1) + + self.assertEqual(set(test.keys()), { 'foo', 'bar' }) + self.assertEqual(test['foo'], 1) + self.assertEqual(test['bar'], True) + + test.update([('bar', False), ('foo', 2)]) + self.assertEqual(test['foo'], 2) + self.assertEqual(test['bar'], False) + + test.update([('foo', 0), ('baz', { 'a': 1, 'b': 2 })]) + self.assertEqual(test['foo'], 0) + self.assertEqual(test['baz'], { 'a': 1, 'b': 2 }) + + test.update([('foo', 42), ('baz', [('c', 3), ('d', 4)])]) + self.assertEqual(test['foo'], 42) + self.assertEqual(test['baz'], { 'c': 3, 'd': 4 }) + + def test_context_paths(self): + test = Context() + + # Newly created context has no paths. + self.assertIsNone(test.main_path) + self.assertIsNone(test.current_path) + self.assertEqual(test.all_paths, set()) + self.assertEqual(test.source_stack, []) + + foo = os.path.abspath('foo') + test.add_source(foo) + + # Adding the first source makes it the main and current path. + self.assertEqual(test.main_path, foo) + self.assertEqual(test.current_path, foo) + self.assertEqual(test.all_paths, set([foo])) + self.assertEqual(test.source_stack, [foo]) + + bar = os.path.abspath('bar') + test.add_source(bar) + + # Adding the second source makes leaves main and current paths alone. + self.assertEqual(test.main_path, foo) + self.assertEqual(test.current_path, foo) + self.assertEqual(test.all_paths, set([bar, foo])) + self.assertEqual(test.source_stack, [foo]) + + qux = os.path.abspath('qux') + test.push_source(qux) + + # Pushing a source makes it the current path + self.assertEqual(test.main_path, foo) + self.assertEqual(test.current_path, qux) + self.assertEqual(test.all_paths, set([bar, foo, qux])) + self.assertEqual(test.source_stack, [foo, qux]) + + hoge = os.path.abspath('hoge') + test.push_source(hoge) + self.assertEqual(test.main_path, foo) + self.assertEqual(test.current_path, hoge) + self.assertEqual(test.all_paths, set([bar, foo, hoge, qux])) + self.assertEqual(test.source_stack, [foo, qux, hoge]) + + fuga = os.path.abspath('fuga') + + # Adding a source after pushing doesn't change the source stack + test.add_source(fuga) + self.assertEqual(test.main_path, foo) + self.assertEqual(test.current_path, hoge) + self.assertEqual(test.all_paths, set([bar, foo, fuga, hoge, qux])) + self.assertEqual(test.source_stack, [foo, qux, hoge]) + + # Adding a source twice doesn't change anything + test.add_source(qux) + self.assertEqual(test.main_path, foo) + self.assertEqual(test.current_path, hoge) + self.assertEqual(test.all_paths, set([bar, foo, fuga, hoge, qux])) + self.assertEqual(test.source_stack, [foo, qux, hoge]) + + last = test.pop_source() + + # Popping a source returns the last pushed one, not the last added one. + self.assertEqual(last, hoge) + self.assertEqual(test.main_path, foo) + self.assertEqual(test.current_path, qux) + self.assertEqual(test.all_paths, set([bar, foo, fuga, hoge, qux])) + self.assertEqual(test.source_stack, [foo, qux]) + + last = test.pop_source() + self.assertEqual(last, qux) + self.assertEqual(test.main_path, foo) + self.assertEqual(test.current_path, foo) + self.assertEqual(test.all_paths, set([bar, foo, fuga, hoge, qux])) + self.assertEqual(test.source_stack, [foo]) + + # Popping the main path is allowed. + last = test.pop_source() + self.assertEqual(last, foo) + self.assertEqual(test.main_path, foo) + self.assertIsNone(test.current_path) + self.assertEqual(test.all_paths, set([bar, foo, fuga, hoge, qux])) + self.assertEqual(test.source_stack, []) + + # Popping past the main path asserts. + with self.assertRaises(AssertionError): + test.pop_source() + + # Pushing after the main path was popped asserts. + with self.assertRaises(AssertionError): + test.push_source(foo) + + test = Context() + test.push_source(foo) + test.push_source(bar) + + # Pushing the same file twice is allowed. + test.push_source(bar) + test.push_source(foo) + self.assertEqual(last, foo) + self.assertEqual(test.main_path, foo) + self.assertEqual(test.current_path, foo) + self.assertEqual(test.all_paths, set([bar, foo])) + self.assertEqual(test.source_stack, [foo, bar, bar, foo]) + + def test_context_dirs(self): + class Config(object): pass + config = Config() + config.topsrcdir = mozpath.abspath(os.curdir) + config.topobjdir = mozpath.abspath('obj') + test = Context(config=config) + foo = mozpath.abspath('foo') + test.push_source(foo) + + self.assertEqual(test.srcdir, config.topsrcdir) + self.assertEqual(test.relsrcdir, '') + self.assertEqual(test.objdir, config.topobjdir) + self.assertEqual(test.relobjdir, '') + + foobar = os.path.abspath('foo/bar') + test.push_source(foobar) + self.assertEqual(test.srcdir, mozpath.join(config.topsrcdir, 'foo')) + self.assertEqual(test.relsrcdir, 'foo') + self.assertEqual(test.objdir, config.topobjdir) + self.assertEqual(test.relobjdir, '') + + +class TestSymbols(unittest.TestCase): + def _verify_doc(self, doc): + # Documentation should be of the format: + # """SUMMARY LINE + # + # EXTRA PARAGRAPHS + # """ + + self.assertNotIn('\r', doc) + + lines = doc.split('\n') + + # No trailing whitespace. + for line in lines[0:-1]: + self.assertEqual(line, line.rstrip()) + + self.assertGreater(len(lines), 0) + self.assertGreater(len(lines[0].strip()), 0) + + # Last line should be empty. + self.assertEqual(lines[-1].strip(), '') + + def test_documentation_formatting(self): + for typ, inp, doc in VARIABLES.values(): + self._verify_doc(doc) + + for attr, args, doc in FUNCTIONS.values(): + self._verify_doc(doc) + + for func, typ, doc in SPECIAL_VARIABLES.values(): + self._verify_doc(doc) + + for name, cls in SUBCONTEXTS.items(): + self._verify_doc(cls.__doc__) + + for name, v in cls.VARIABLES.items(): + self._verify_doc(v[2]) + + +class TestPaths(unittest.TestCase): + @classmethod + def setUpClass(cls): + class Config(object): pass + cls.config = config = Config() + config.topsrcdir = mozpath.abspath(os.curdir) + config.topobjdir = mozpath.abspath('obj') + config.external_source_dir = None + + def test_path(self): + config = self.config + ctxt1 = Context(config=config) + ctxt1.push_source(mozpath.join(config.topsrcdir, 'foo', 'moz.build')) + ctxt2 = Context(config=config) + ctxt2.push_source(mozpath.join(config.topsrcdir, 'bar', 'moz.build')) + + path1 = Path(ctxt1, 'qux') + self.assertIsInstance(path1, SourcePath) + self.assertEqual(path1, 'qux') + self.assertEqual(path1.full_path, + mozpath.join(config.topsrcdir, 'foo', 'qux')) + + path2 = Path(ctxt2, '../foo/qux') + self.assertIsInstance(path2, SourcePath) + self.assertEqual(path2, '../foo/qux') + self.assertEqual(path2.full_path, + mozpath.join(config.topsrcdir, 'foo', 'qux')) + + self.assertEqual(path1, path2) + + self.assertEqual(path1.join('../../bar/qux').full_path, + mozpath.join(config.topsrcdir, 'bar', 'qux')) + + path1 = Path(ctxt1, '/qux/qux') + self.assertIsInstance(path1, SourcePath) + self.assertEqual(path1, '/qux/qux') + self.assertEqual(path1.full_path, + mozpath.join(config.topsrcdir, 'qux', 'qux')) + + path2 = Path(ctxt2, '/qux/qux') + self.assertIsInstance(path2, SourcePath) + self.assertEqual(path2, '/qux/qux') + self.assertEqual(path2.full_path, + mozpath.join(config.topsrcdir, 'qux', 'qux')) + + self.assertEqual(path1, path2) + + path1 = Path(ctxt1, '!qux') + self.assertIsInstance(path1, ObjDirPath) + self.assertEqual(path1, '!qux') + self.assertEqual(path1.full_path, + mozpath.join(config.topobjdir, 'foo', 'qux')) + + path2 = Path(ctxt2, '!../foo/qux') + self.assertIsInstance(path2, ObjDirPath) + self.assertEqual(path2, '!../foo/qux') + self.assertEqual(path2.full_path, + mozpath.join(config.topobjdir, 'foo', 'qux')) + + self.assertEqual(path1, path2) + + path1 = Path(ctxt1, '!/qux/qux') + self.assertIsInstance(path1, ObjDirPath) + self.assertEqual(path1, '!/qux/qux') + self.assertEqual(path1.full_path, + mozpath.join(config.topobjdir, 'qux', 'qux')) + + path2 = Path(ctxt2, '!/qux/qux') + self.assertIsInstance(path2, ObjDirPath) + self.assertEqual(path2, '!/qux/qux') + self.assertEqual(path2.full_path, + mozpath.join(config.topobjdir, 'qux', 'qux')) + + self.assertEqual(path1, path2) + + path1 = Path(ctxt1, path1) + self.assertIsInstance(path1, ObjDirPath) + self.assertEqual(path1, '!/qux/qux') + self.assertEqual(path1.full_path, + mozpath.join(config.topobjdir, 'qux', 'qux')) + + path2 = Path(ctxt2, path2) + self.assertIsInstance(path2, ObjDirPath) + self.assertEqual(path2, '!/qux/qux') + self.assertEqual(path2.full_path, + mozpath.join(config.topobjdir, 'qux', 'qux')) + + self.assertEqual(path1, path2) + + path1 = Path(path1) + self.assertIsInstance(path1, ObjDirPath) + self.assertEqual(path1, '!/qux/qux') + self.assertEqual(path1.full_path, + mozpath.join(config.topobjdir, 'qux', 'qux')) + + self.assertEqual(path1, path2) + + path2 = Path(path2) + self.assertIsInstance(path2, ObjDirPath) + self.assertEqual(path2, '!/qux/qux') + self.assertEqual(path2.full_path, + mozpath.join(config.topobjdir, 'qux', 'qux')) + + self.assertEqual(path1, path2) + + def test_source_path(self): + config = self.config + ctxt = Context(config=config) + ctxt.push_source(mozpath.join(config.topsrcdir, 'foo', 'moz.build')) + + path = SourcePath(ctxt, 'qux') + self.assertEqual(path, 'qux') + self.assertEqual(path.full_path, + mozpath.join(config.topsrcdir, 'foo', 'qux')) + self.assertEqual(path.translated, + mozpath.join(config.topobjdir, 'foo', 'qux')) + + path = SourcePath(ctxt, '../bar/qux') + self.assertEqual(path, '../bar/qux') + self.assertEqual(path.full_path, + mozpath.join(config.topsrcdir, 'bar', 'qux')) + self.assertEqual(path.translated, + mozpath.join(config.topobjdir, 'bar', 'qux')) + + path = SourcePath(ctxt, '/qux/qux') + self.assertEqual(path, '/qux/qux') + self.assertEqual(path.full_path, + mozpath.join(config.topsrcdir, 'qux', 'qux')) + self.assertEqual(path.translated, + mozpath.join(config.topobjdir, 'qux', 'qux')) + + with self.assertRaises(ValueError): + SourcePath(ctxt, '!../bar/qux') + + with self.assertRaises(ValueError): + SourcePath(ctxt, '!/qux/qux') + + path = SourcePath(path) + self.assertIsInstance(path, SourcePath) + self.assertEqual(path, '/qux/qux') + self.assertEqual(path.full_path, + mozpath.join(config.topsrcdir, 'qux', 'qux')) + self.assertEqual(path.translated, + mozpath.join(config.topobjdir, 'qux', 'qux')) + + path = Path(path) + self.assertIsInstance(path, SourcePath) + + def test_objdir_path(self): + config = self.config + ctxt = Context(config=config) + ctxt.push_source(mozpath.join(config.topsrcdir, 'foo', 'moz.build')) + + path = ObjDirPath(ctxt, '!qux') + self.assertEqual(path, '!qux') + self.assertEqual(path.full_path, + mozpath.join(config.topobjdir, 'foo', 'qux')) + + path = ObjDirPath(ctxt, '!../bar/qux') + self.assertEqual(path, '!../bar/qux') + self.assertEqual(path.full_path, + mozpath.join(config.topobjdir, 'bar', 'qux')) + + path = ObjDirPath(ctxt, '!/qux/qux') + self.assertEqual(path, '!/qux/qux') + self.assertEqual(path.full_path, + mozpath.join(config.topobjdir, 'qux', 'qux')) + + with self.assertRaises(ValueError): + path = ObjDirPath(ctxt, '../bar/qux') + + with self.assertRaises(ValueError): + path = ObjDirPath(ctxt, '/qux/qux') + + path = ObjDirPath(path) + self.assertIsInstance(path, ObjDirPath) + self.assertEqual(path, '!/qux/qux') + self.assertEqual(path.full_path, + mozpath.join(config.topobjdir, 'qux', 'qux')) + + path = Path(path) + self.assertIsInstance(path, ObjDirPath) + + def test_absolute_path(self): + config = self.config + ctxt = Context(config=config) + ctxt.push_source(mozpath.join(config.topsrcdir, 'foo', 'moz.build')) + + path = AbsolutePath(ctxt, '%/qux') + self.assertEqual(path, '%/qux') + self.assertEqual(path.full_path, '/qux') + + with self.assertRaises(ValueError): + path = AbsolutePath(ctxt, '%qux') + + def test_path_with_mixed_contexts(self): + config = self.config + ctxt1 = Context(config=config) + ctxt1.push_source(mozpath.join(config.topsrcdir, 'foo', 'moz.build')) + ctxt2 = Context(config=config) + ctxt2.push_source(mozpath.join(config.topsrcdir, 'bar', 'moz.build')) + + path1 = Path(ctxt1, 'qux') + path2 = Path(ctxt2, path1) + self.assertEqual(path2, path1) + self.assertEqual(path2, 'qux') + self.assertEqual(path2.context, ctxt1) + self.assertEqual(path2.full_path, + mozpath.join(config.topsrcdir, 'foo', 'qux')) + + path1 = Path(ctxt1, '../bar/qux') + path2 = Path(ctxt2, path1) + self.assertEqual(path2, path1) + self.assertEqual(path2, '../bar/qux') + self.assertEqual(path2.context, ctxt1) + self.assertEqual(path2.full_path, + mozpath.join(config.topsrcdir, 'bar', 'qux')) + + path1 = Path(ctxt1, '/qux/qux') + path2 = Path(ctxt2, path1) + self.assertEqual(path2, path1) + self.assertEqual(path2, '/qux/qux') + self.assertEqual(path2.context, ctxt1) + self.assertEqual(path2.full_path, + mozpath.join(config.topsrcdir, 'qux', 'qux')) + + path1 = Path(ctxt1, '!qux') + path2 = Path(ctxt2, path1) + self.assertEqual(path2, path1) + self.assertEqual(path2, '!qux') + self.assertEqual(path2.context, ctxt1) + self.assertEqual(path2.full_path, + mozpath.join(config.topobjdir, 'foo', 'qux')) + + path1 = Path(ctxt1, '!../bar/qux') + path2 = Path(ctxt2, path1) + self.assertEqual(path2, path1) + self.assertEqual(path2, '!../bar/qux') + self.assertEqual(path2.context, ctxt1) + self.assertEqual(path2.full_path, + mozpath.join(config.topobjdir, 'bar', 'qux')) + + path1 = Path(ctxt1, '!/qux/qux') + path2 = Path(ctxt2, path1) + self.assertEqual(path2, path1) + self.assertEqual(path2, '!/qux/qux') + self.assertEqual(path2.context, ctxt1) + self.assertEqual(path2.full_path, + mozpath.join(config.topobjdir, 'qux', 'qux')) + + def test_path_typed_list(self): + config = self.config + ctxt1 = Context(config=config) + ctxt1.push_source(mozpath.join(config.topsrcdir, 'foo', 'moz.build')) + ctxt2 = Context(config=config) + ctxt2.push_source(mozpath.join(config.topsrcdir, 'bar', 'moz.build')) + + paths = [ + '!../bar/qux', + '!/qux/qux', + '!qux', + '../bar/qux', + '/qux/qux', + 'qux', + ] + + MyList = ContextDerivedTypedList(Path) + l = MyList(ctxt1) + l += paths + + for p_str, p_path in zip(paths, l): + self.assertEqual(p_str, p_path) + self.assertEqual(p_path, Path(ctxt1, p_str)) + self.assertEqual(p_path.join('foo'), + Path(ctxt1, mozpath.join(p_str, 'foo'))) + + l2 = MyList(ctxt2) + l2 += paths + + for p_str, p_path in zip(paths, l2): + self.assertEqual(p_str, p_path) + self.assertEqual(p_path, Path(ctxt2, p_str)) + + # Assigning with Paths from another context doesn't rebase them + l2 = MyList(ctxt2) + l2 += l + + for p_str, p_path in zip(paths, l2): + self.assertEqual(p_str, p_path) + self.assertEqual(p_path, Path(ctxt1, p_str)) + + MyListWithFlags = ContextDerivedTypedListWithItems( + Path, StrictOrderingOnAppendListWithFlagsFactory({ + 'foo': bool, + })) + l = MyListWithFlags(ctxt1) + l += paths + + for p in paths: + l[p].foo = True + + for p_str, p_path in zip(paths, l): + self.assertEqual(p_str, p_path) + self.assertEqual(p_path, Path(ctxt1, p_str)) + self.assertEqual(l[p_str].foo, True) + self.assertEqual(l[p_path].foo, True) + + def test_path_typed_hierarchy_list(self): + config = self.config + ctxt1 = Context(config=config) + ctxt1.push_source(mozpath.join(config.topsrcdir, 'foo', 'moz.build')) + ctxt2 = Context(config=config) + ctxt2.push_source(mozpath.join(config.topsrcdir, 'bar', 'moz.build')) + + paths = [ + '!../bar/qux', + '!/qux/qux', + '!qux', + '../bar/qux', + '/qux/qux', + 'qux', + ] + + MyList = ContextDerivedTypedHierarchicalStringList(Path) + l = MyList(ctxt1) + l += paths + l.subdir += paths + + for _, files in l.walk(): + for p_str, p_path in zip(paths, files): + self.assertEqual(p_str, p_path) + self.assertEqual(p_path, Path(ctxt1, p_str)) + self.assertEqual(p_path.join('foo'), + Path(ctxt1, mozpath.join(p_str, 'foo'))) + + l2 = MyList(ctxt2) + l2 += paths + l2.subdir += paths + + for _, files in l2.walk(): + for p_str, p_path in zip(paths, files): + self.assertEqual(p_str, p_path) + self.assertEqual(p_path, Path(ctxt2, p_str)) + + # Assigning with Paths from another context doesn't rebase them + l2 = MyList(ctxt2) + l2 += l + + for _, files in l2.walk(): + for p_str, p_path in zip(paths, files): + self.assertEqual(p_str, p_path) + self.assertEqual(p_path, Path(ctxt1, p_str)) + + +class TestTypedRecord(unittest.TestCase): + + def test_fields(self): + T = ContextDerivedTypedRecord(('field1', unicode), + ('field2', list)) + inst = T(None) + self.assertEqual(inst.field1, '') + self.assertEqual(inst.field2, []) + + inst.field1 = 'foo' + inst.field2 += ['bar'] + + self.assertEqual(inst.field1, 'foo') + self.assertEqual(inst.field2, ['bar']) + + with self.assertRaises(AttributeError): + inst.field3 = [] + + def test_coercion(self): + T = ContextDerivedTypedRecord(('field1', unicode), + ('field2', list)) + inst = T(None) + inst.field1 = 3 + inst.field2 += ('bar',) + self.assertEqual(inst.field1, '3') + self.assertEqual(inst.field2, ['bar']) + + with self.assertRaises(TypeError): + inst.field2 = object() + + +class TestFiles(unittest.TestCase): + def test_aggregate_empty(self): + c = Context({}) + + files = {'moz.build': Files(c, pattern='**')} + + self.assertEqual(Files.aggregate(files), { + 'bug_component_counts': [], + 'recommended_bug_component': None, + }) + + def test_single_bug_component(self): + c = Context({}) + f = Files(c, pattern='**') + f['BUG_COMPONENT'] = (u'Product1', u'Component1') + + files = {'moz.build': f} + self.assertEqual(Files.aggregate(files), { + 'bug_component_counts': [((u'Product1', u'Component1'), 1)], + 'recommended_bug_component': (u'Product1', u'Component1'), + }) + + def test_multiple_bug_components(self): + c = Context({}) + f1 = Files(c, pattern='**') + f1['BUG_COMPONENT'] = (u'Product1', u'Component1') + + f2 = Files(c, pattern='**') + f2['BUG_COMPONENT'] = (u'Product2', u'Component2') + + files = {'a': f1, 'b': f2, 'c': f1} + self.assertEqual(Files.aggregate(files), { + 'bug_component_counts': [ + ((u'Product1', u'Component1'), 2), + ((u'Product2', u'Component2'), 1), + ], + 'recommended_bug_component': (u'Product1', u'Component1'), + }) + + def test_no_recommended_bug_component(self): + """If there is no clear count winner, we don't recommend a bug component.""" + c = Context({}) + f1 = Files(c, pattern='**') + f1['BUG_COMPONENT'] = (u'Product1', u'Component1') + + f2 = Files(c, pattern='**') + f2['BUG_COMPONENT'] = (u'Product2', u'Component2') + + files = {'a': f1, 'b': f2} + self.assertEqual(Files.aggregate(files), { + 'bug_component_counts': [ + ((u'Product1', u'Component1'), 1), + ((u'Product2', u'Component2'), 1), + ], + 'recommended_bug_component': None, + }) + + +if __name__ == '__main__': + main() diff --git a/python/mozbuild/mozbuild/test/frontend/test_emitter.py b/python/mozbuild/mozbuild/test/frontend/test_emitter.py new file mode 100644 index 000000000..6ac4e0aac --- /dev/null +++ b/python/mozbuild/mozbuild/test/frontend/test_emitter.py @@ -0,0 +1,1172 @@ +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, You can obtain one at http://mozilla.org/MPL/2.0/. + +from __future__ import unicode_literals + +import os +import unittest + +from mozunit import main + +from mozbuild.frontend.context import ( + ObjDirPath, + Path, +) +from mozbuild.frontend.data import ( + AndroidResDirs, + BrandingFiles, + ChromeManifestEntry, + ConfigFileSubstitution, + Defines, + DirectoryTraversal, + Exports, + FinalTargetPreprocessedFiles, + GeneratedFile, + GeneratedSources, + HostDefines, + HostSources, + IPDLFile, + JARManifest, + LinkageMultipleRustLibrariesError, + LocalInclude, + Program, + RustLibrary, + SdkFiles, + SharedLibrary, + SimpleProgram, + Sources, + StaticLibrary, + TestHarnessFiles, + TestManifest, + UnifiedSources, + VariablePassthru, +) +from mozbuild.frontend.emitter import TreeMetadataEmitter +from mozbuild.frontend.reader import ( + BuildReader, + BuildReaderError, + SandboxValidationError, +) +from mozpack.chrome import manifest + +from mozbuild.test.common import MockConfig + +import mozpack.path as mozpath + + +data_path = mozpath.abspath(mozpath.dirname(__file__)) +data_path = mozpath.join(data_path, 'data') + + +class TestEmitterBasic(unittest.TestCase): + def setUp(self): + self._old_env = dict(os.environ) + os.environ.pop('MOZ_OBJDIR', None) + + def tearDown(self): + os.environ.clear() + os.environ.update(self._old_env) + + def reader(self, name, enable_tests=False, extra_substs=None): + substs = dict( + ENABLE_TESTS='1' if enable_tests else '', + BIN_SUFFIX='.prog', + OS_TARGET='WINNT', + COMPILE_ENVIRONMENT='1', + ) + if extra_substs: + substs.update(extra_substs) + config = MockConfig(mozpath.join(data_path, name), extra_substs=substs) + + return BuildReader(config) + + def read_topsrcdir(self, reader, filter_common=True): + emitter = TreeMetadataEmitter(reader.config) + objs = list(emitter.emit(reader.read_topsrcdir())) + self.assertGreater(len(objs), 0) + + filtered = [] + for obj in objs: + if filter_common and isinstance(obj, DirectoryTraversal): + continue + + filtered.append(obj) + + return filtered + + def test_dirs_traversal_simple(self): + reader = self.reader('traversal-simple') + objs = self.read_topsrcdir(reader, filter_common=False) + self.assertEqual(len(objs), 4) + + for o in objs: + self.assertIsInstance(o, DirectoryTraversal) + self.assertTrue(os.path.isabs(o.context_main_path)) + self.assertEqual(len(o.context_all_paths), 1) + + reldirs = [o.relativedir for o in objs] + self.assertEqual(reldirs, ['', 'foo', 'foo/biz', 'bar']) + + dirs = [[d.full_path for d in o.dirs] for o in objs] + self.assertEqual(dirs, [ + [ + mozpath.join(reader.config.topsrcdir, 'foo'), + mozpath.join(reader.config.topsrcdir, 'bar') + ], [ + mozpath.join(reader.config.topsrcdir, 'foo', 'biz') + ], [], []]) + + def test_traversal_all_vars(self): + reader = self.reader('traversal-all-vars') + objs = self.read_topsrcdir(reader, filter_common=False) + self.assertEqual(len(objs), 2) + + for o in objs: + self.assertIsInstance(o, DirectoryTraversal) + + reldirs = set([o.relativedir for o in objs]) + self.assertEqual(reldirs, set(['', 'regular'])) + + for o in objs: + reldir = o.relativedir + + if reldir == '': + self.assertEqual([d.full_path for d in o.dirs], [ + mozpath.join(reader.config.topsrcdir, 'regular')]) + + def test_traversal_all_vars_enable_tests(self): + reader = self.reader('traversal-all-vars', enable_tests=True) + objs = self.read_topsrcdir(reader, filter_common=False) + self.assertEqual(len(objs), 3) + + for o in objs: + self.assertIsInstance(o, DirectoryTraversal) + + reldirs = set([o.relativedir for o in objs]) + self.assertEqual(reldirs, set(['', 'regular', 'test'])) + + for o in objs: + reldir = o.relativedir + + if reldir == '': + self.assertEqual([d.full_path for d in o.dirs], [ + mozpath.join(reader.config.topsrcdir, 'regular'), + mozpath.join(reader.config.topsrcdir, 'test')]) + + def test_config_file_substitution(self): + reader = self.reader('config-file-substitution') + objs = self.read_topsrcdir(reader) + self.assertEqual(len(objs), 2) + + self.assertIsInstance(objs[0], ConfigFileSubstitution) + self.assertIsInstance(objs[1], ConfigFileSubstitution) + + topobjdir = mozpath.abspath(reader.config.topobjdir) + self.assertEqual(objs[0].relpath, 'foo') + self.assertEqual(mozpath.normpath(objs[0].output_path), + mozpath.normpath(mozpath.join(topobjdir, 'foo'))) + self.assertEqual(mozpath.normpath(objs[1].output_path), + mozpath.normpath(mozpath.join(topobjdir, 'bar'))) + + def test_variable_passthru(self): + reader = self.reader('variable-passthru') + objs = self.read_topsrcdir(reader) + + self.assertEqual(len(objs), 1) + self.assertIsInstance(objs[0], VariablePassthru) + + wanted = { + 'ALLOW_COMPILER_WARNINGS': True, + 'DISABLE_STL_WRAPPING': True, + 'NO_DIST_INSTALL': True, + 'VISIBILITY_FLAGS': '', + 'RCFILE': 'foo.rc', + 'RESFILE': 'bar.res', + 'RCINCLUDE': 'bar.rc', + 'DEFFILE': 'baz.def', + 'MOZBUILD_CFLAGS': ['-fno-exceptions', '-w'], + 'MOZBUILD_CXXFLAGS': ['-fcxx-exceptions', '-include foo.h'], + 'MOZBUILD_LDFLAGS': ['-framework Foo', '-x', '-DELAYLOAD:foo.dll', + '-DELAYLOAD:bar.dll'], + 'MOZBUILD_HOST_CFLAGS': ['-funroll-loops', '-wall'], + 'MOZBUILD_HOST_CXXFLAGS': ['-funroll-loops-harder', + '-wall-day-everyday'], + 'WIN32_EXE_LDFLAGS': ['-subsystem:console'], + } + + variables = objs[0].variables + maxDiff = self.maxDiff + self.maxDiff = None + self.assertEqual(wanted, variables) + self.maxDiff = maxDiff + + def test_use_yasm(self): + # When yasm is not available, this should raise. + reader = self.reader('use-yasm') + with self.assertRaisesRegexp(SandboxValidationError, + 'yasm is not available'): + self.read_topsrcdir(reader) + + # When yasm is available, this should work. + reader = self.reader('use-yasm', + extra_substs=dict( + YASM='yasm', + YASM_ASFLAGS='-foo', + )) + objs = self.read_topsrcdir(reader) + + self.assertEqual(len(objs), 1) + self.assertIsInstance(objs[0], VariablePassthru) + maxDiff = self.maxDiff + self.maxDiff = None + self.assertEqual(objs[0].variables, + {'AS': 'yasm', + 'ASFLAGS': '-foo', + 'AS_DASH_C_FLAG': ''}) + self.maxDiff = maxDiff + + + def test_generated_files(self): + reader = self.reader('generated-files') + objs = self.read_topsrcdir(reader) + + self.assertEqual(len(objs), 3) + for o in objs: + self.assertIsInstance(o, GeneratedFile) + + expected = ['bar.c', 'foo.c', ('xpidllex.py', 'xpidlyacc.py'), ] + for o, f in zip(objs, expected): + expected_filename = f if isinstance(f, tuple) else (f,) + self.assertEqual(o.outputs, expected_filename) + self.assertEqual(o.script, None) + self.assertEqual(o.method, None) + self.assertEqual(o.inputs, []) + + def test_generated_files_method_names(self): + reader = self.reader('generated-files-method-names') + objs = self.read_topsrcdir(reader) + + self.assertEqual(len(objs), 2) + for o in objs: + self.assertIsInstance(o, GeneratedFile) + + expected = ['bar.c', 'foo.c'] + expected_method_names = ['make_bar', 'main'] + for o, expected_filename, expected_method in zip(objs, expected, expected_method_names): + self.assertEqual(o.outputs, (expected_filename,)) + self.assertEqual(o.method, expected_method) + self.assertEqual(o.inputs, []) + + def test_generated_files_absolute_script(self): + reader = self.reader('generated-files-absolute-script') + objs = self.read_topsrcdir(reader) + + self.assertEqual(len(objs), 1) + + o = objs[0] + self.assertIsInstance(o, GeneratedFile) + self.assertEqual(o.outputs, ('bar.c',)) + self.assertRegexpMatches(o.script, 'script.py$') + self.assertEqual(o.method, 'make_bar') + self.assertEqual(o.inputs, []) + + def test_generated_files_no_script(self): + reader = self.reader('generated-files-no-script') + with self.assertRaisesRegexp(SandboxValidationError, + 'Script for generating bar.c does not exist'): + self.read_topsrcdir(reader) + + def test_generated_files_no_inputs(self): + reader = self.reader('generated-files-no-inputs') + with self.assertRaisesRegexp(SandboxValidationError, + 'Input for generating foo.c does not exist'): + self.read_topsrcdir(reader) + + def test_generated_files_no_python_script(self): + reader = self.reader('generated-files-no-python-script') + with self.assertRaisesRegexp(SandboxValidationError, + 'Script for generating bar.c does not end in .py'): + self.read_topsrcdir(reader) + + def test_exports(self): + reader = self.reader('exports') + objs = self.read_topsrcdir(reader) + + self.assertEqual(len(objs), 1) + self.assertIsInstance(objs[0], Exports) + + expected = [ + ('', ['foo.h', 'bar.h', 'baz.h']), + ('mozilla', ['mozilla1.h', 'mozilla2.h']), + ('mozilla/dom', ['dom1.h', 'dom2.h', 'dom3.h']), + ('mozilla/gfx', ['gfx.h']), + ('nspr/private', ['pprio.h', 'pprthred.h']), + ('vpx', ['mem.h', 'mem2.h']), + ] + for (expect_path, expect_headers), (actual_path, actual_headers) in \ + zip(expected, [(path, list(seq)) for path, seq in objs[0].files.walk()]): + self.assertEqual(expect_path, actual_path) + self.assertEqual(expect_headers, actual_headers) + + def test_exports_missing(self): + ''' + Missing files in EXPORTS is an error. + ''' + reader = self.reader('exports-missing') + with self.assertRaisesRegexp(SandboxValidationError, + 'File listed in EXPORTS does not exist:'): + self.read_topsrcdir(reader) + + def test_exports_missing_generated(self): + ''' + An objdir file in EXPORTS that is not in GENERATED_FILES is an error. + ''' + reader = self.reader('exports-missing-generated') + with self.assertRaisesRegexp(SandboxValidationError, + 'Objdir file listed in EXPORTS not in GENERATED_FILES:'): + self.read_topsrcdir(reader) + + def test_exports_generated(self): + reader = self.reader('exports-generated') + objs = self.read_topsrcdir(reader) + + self.assertEqual(len(objs), 2) + self.assertIsInstance(objs[0], GeneratedFile) + self.assertIsInstance(objs[1], Exports) + exports = [(path, list(seq)) for path, seq in objs[1].files.walk()] + self.assertEqual(exports, + [('', ['foo.h']), + ('mozilla', ['mozilla1.h', '!mozilla2.h'])]) + path, files = exports[1] + self.assertIsInstance(files[1], ObjDirPath) + + def test_test_harness_files(self): + reader = self.reader('test-harness-files') + objs = self.read_topsrcdir(reader) + + self.assertEqual(len(objs), 1) + self.assertIsInstance(objs[0], TestHarnessFiles) + + expected = { + 'mochitest': ['runtests.py', 'utils.py'], + 'testing/mochitest': ['mochitest.py', 'mochitest.ini'], + } + + for path, strings in objs[0].files.walk(): + self.assertTrue(path in expected) + basenames = sorted(mozpath.basename(s) for s in strings) + self.assertEqual(sorted(expected[path]), basenames) + + def test_test_harness_files_root(self): + reader = self.reader('test-harness-files-root') + with self.assertRaisesRegexp(SandboxValidationError, + 'Cannot install files to the root of TEST_HARNESS_FILES'): + self.read_topsrcdir(reader) + + def test_branding_files(self): + reader = self.reader('branding-files') + objs = self.read_topsrcdir(reader) + + self.assertEqual(len(objs), 1) + self.assertIsInstance(objs[0], BrandingFiles) + + files = objs[0].files + + self.assertEqual(files._strings, ['bar.ico', 'baz.png', 'foo.xpm']) + + self.assertIn('icons', files._children) + icons = files._children['icons'] + + self.assertEqual(icons._strings, ['quux.icns']) + + def test_sdk_files(self): + reader = self.reader('sdk-files') + objs = self.read_topsrcdir(reader) + + self.assertEqual(len(objs), 1) + self.assertIsInstance(objs[0], SdkFiles) + + files = objs[0].files + + self.assertEqual(files._strings, ['bar.ico', 'baz.png', 'foo.xpm']) + + self.assertIn('icons', files._children) + icons = files._children['icons'] + + self.assertEqual(icons._strings, ['quux.icns']) + + def test_program(self): + reader = self.reader('program') + objs = self.read_topsrcdir(reader) + + self.assertEqual(len(objs), 3) + self.assertIsInstance(objs[0], Program) + self.assertIsInstance(objs[1], SimpleProgram) + self.assertIsInstance(objs[2], SimpleProgram) + + self.assertEqual(objs[0].program, 'test_program.prog') + self.assertEqual(objs[1].program, 'test_program1.prog') + self.assertEqual(objs[2].program, 'test_program2.prog') + + def test_test_manifest_missing_manifest(self): + """A missing manifest file should result in an error.""" + reader = self.reader('test-manifest-missing-manifest') + + with self.assertRaisesRegexp(BuildReaderError, 'IOError: Missing files'): + self.read_topsrcdir(reader) + + def test_empty_test_manifest_rejected(self): + """A test manifest without any entries is rejected.""" + reader = self.reader('test-manifest-empty') + + with self.assertRaisesRegexp(SandboxValidationError, 'Empty test manifest'): + self.read_topsrcdir(reader) + + + def test_test_manifest_just_support_files(self): + """A test manifest with no tests but support-files is not supported.""" + reader = self.reader('test-manifest-just-support') + + with self.assertRaisesRegexp(SandboxValidationError, 'Empty test manifest'): + self.read_topsrcdir(reader) + + def test_test_manifest_dupe_support_files(self): + """A test manifest with dupe support-files in a single test is not + supported. + """ + reader = self.reader('test-manifest-dupes') + + with self.assertRaisesRegexp(SandboxValidationError, 'bar.js appears multiple times ' + 'in a test manifest under a support-files field, please omit the duplicate entry.'): + self.read_topsrcdir(reader) + + def test_test_manifest_absolute_support_files(self): + """Support files starting with '/' are placed relative to the install root""" + reader = self.reader('test-manifest-absolute-support') + + objs = self.read_topsrcdir(reader) + self.assertEqual(len(objs), 1) + o = objs[0] + self.assertEqual(len(o.installs), 3) + expected = [ + mozpath.normpath(mozpath.join(o.install_prefix, "../.well-known/foo.txt")), + mozpath.join(o.install_prefix, "absolute-support.ini"), + mozpath.join(o.install_prefix, "test_file.js"), + ] + paths = sorted([v[0] for v in o.installs.values()]) + self.assertEqual(paths, expected) + + @unittest.skip('Bug 1304316 - Items in the second set but not the first') + def test_test_manifest_shared_support_files(self): + """Support files starting with '!' are given separate treatment, so their + installation can be resolved when running tests. + """ + reader = self.reader('test-manifest-shared-support') + supported, child = self.read_topsrcdir(reader) + + expected_deferred_installs = { + '!/child/test_sub.js', + '!/child/another-file.sjs', + '!/child/data/**', + } + + self.assertEqual(len(supported.installs), 3) + self.assertEqual(set(supported.deferred_installs), + expected_deferred_installs) + self.assertEqual(len(child.installs), 3) + self.assertEqual(len(child.pattern_installs), 1) + + def test_test_manifest_deffered_install_missing(self): + """A non-existent shared support file reference produces an error.""" + reader = self.reader('test-manifest-shared-missing') + + with self.assertRaisesRegexp(SandboxValidationError, + 'entry in support-files not present in the srcdir'): + self.read_topsrcdir(reader) + + def test_test_manifest_install_to_subdir(self): + """ """ + reader = self.reader('test-manifest-install-subdir') + + objs = self.read_topsrcdir(reader) + self.assertEqual(len(objs), 1) + o = objs[0] + self.assertEqual(len(o.installs), 3) + self.assertEqual(o.manifest_relpath, "subdir.ini") + self.assertEqual(o.manifest_obj_relpath, "subdir/subdir.ini") + expected = [ + mozpath.normpath(mozpath.join(o.install_prefix, "subdir/subdir.ini")), + mozpath.normpath(mozpath.join(o.install_prefix, "subdir/support.txt")), + mozpath.normpath(mozpath.join(o.install_prefix, "subdir/test_foo.html")), + ] + paths = sorted([v[0] for v in o.installs.values()]) + self.assertEqual(paths, expected) + + def test_test_manifest_install_includes(self): + """Ensure that any [include:foo.ini] are copied to the objdir.""" + reader = self.reader('test-manifest-install-includes') + + objs = self.read_topsrcdir(reader) + self.assertEqual(len(objs), 1) + o = objs[0] + self.assertEqual(len(o.installs), 3) + self.assertEqual(o.manifest_relpath, "mochitest.ini") + self.assertEqual(o.manifest_obj_relpath, "subdir/mochitest.ini") + expected = [ + mozpath.normpath(mozpath.join(o.install_prefix, "subdir/common.ini")), + mozpath.normpath(mozpath.join(o.install_prefix, "subdir/mochitest.ini")), + mozpath.normpath(mozpath.join(o.install_prefix, "subdir/test_foo.html")), + ] + paths = sorted([v[0] for v in o.installs.values()]) + self.assertEqual(paths, expected) + + def test_test_manifest_includes(self): + """Ensure that manifest objects from the emitter list a correct manifest. + """ + reader = self.reader('test-manifest-emitted-includes') + [obj] = self.read_topsrcdir(reader) + + # Expected manifest leafs for our tests. + expected_manifests = { + 'reftest1.html': 'reftest.list', + 'reftest1-ref.html': 'reftest.list', + 'reftest2.html': 'included-reftest.list', + 'reftest2-ref.html': 'included-reftest.list', + } + + for t in obj.tests: + self.assertTrue(t['manifest'].endswith(expected_manifests[t['name']])) + + def test_python_unit_test_missing(self): + """Missing files in PYTHON_UNIT_TESTS should raise.""" + reader = self.reader('test-python-unit-test-missing') + with self.assertRaisesRegexp(SandboxValidationError, + 'Path specified in PYTHON_UNIT_TESTS does not exist:'): + self.read_topsrcdir(reader) + + def test_test_manifest_keys_extracted(self): + """Ensure all metadata from test manifests is extracted.""" + reader = self.reader('test-manifest-keys-extracted') + + objs = [o for o in self.read_topsrcdir(reader) + if isinstance(o, TestManifest)] + + self.assertEqual(len(objs), 9) + + metadata = { + 'a11y.ini': { + 'flavor': 'a11y', + 'installs': { + 'a11y.ini': False, + 'test_a11y.js': True, + }, + 'pattern-installs': 1, + }, + 'browser.ini': { + 'flavor': 'browser-chrome', + 'installs': { + 'browser.ini': False, + 'test_browser.js': True, + 'support1': False, + 'support2': False, + }, + }, + 'metro.ini': { + 'flavor': 'metro-chrome', + 'installs': { + 'metro.ini': False, + 'test_metro.js': True, + }, + }, + 'mochitest.ini': { + 'flavor': 'mochitest', + 'installs': { + 'mochitest.ini': False, + 'test_mochitest.js': True, + }, + 'external': { + 'external1', + 'external2', + }, + }, + 'chrome.ini': { + 'flavor': 'chrome', + 'installs': { + 'chrome.ini': False, + 'test_chrome.js': True, + }, + }, + 'xpcshell.ini': { + 'flavor': 'xpcshell', + 'dupe': True, + 'installs': { + 'xpcshell.ini': False, + 'test_xpcshell.js': True, + 'head1': False, + 'head2': False, + 'tail1': False, + 'tail2': False, + }, + }, + 'reftest.list': { + 'flavor': 'reftest', + 'installs': {}, + }, + 'crashtest.list': { + 'flavor': 'crashtest', + 'installs': {}, + }, + 'moz.build': { + 'flavor': 'python', + 'installs': {}, + } + } + + for o in objs: + m = metadata[mozpath.basename(o.manifest_relpath)] + + self.assertTrue(o.path.startswith(o.directory)) + self.assertEqual(o.flavor, m['flavor']) + self.assertEqual(o.dupe_manifest, m.get('dupe', False)) + + external_normalized = set(mozpath.basename(p) for p in + o.external_installs) + self.assertEqual(external_normalized, m.get('external', set())) + + self.assertEqual(len(o.installs), len(m['installs'])) + for path in o.installs.keys(): + self.assertTrue(path.startswith(o.directory)) + relpath = path[len(o.directory)+1:] + + self.assertIn(relpath, m['installs']) + self.assertEqual(o.installs[path][1], m['installs'][relpath]) + + if 'pattern-installs' in m: + self.assertEqual(len(o.pattern_installs), m['pattern-installs']) + + def test_test_manifest_unmatched_generated(self): + reader = self.reader('test-manifest-unmatched-generated') + + with self.assertRaisesRegexp(SandboxValidationError, + 'entry in generated-files not present elsewhere'): + self.read_topsrcdir(reader), + + def test_test_manifest_parent_support_files_dir(self): + """support-files referencing a file in a parent directory works.""" + reader = self.reader('test-manifest-parent-support-files-dir') + + objs = [o for o in self.read_topsrcdir(reader) + if isinstance(o, TestManifest)] + + self.assertEqual(len(objs), 1) + + o = objs[0] + + expected = mozpath.join(o.srcdir, 'support-file.txt') + self.assertIn(expected, o.installs) + self.assertEqual(o.installs[expected], + ('testing/mochitest/tests/child/support-file.txt', False)) + + def test_test_manifest_missing_test_error(self): + """Missing test files should result in error.""" + reader = self.reader('test-manifest-missing-test-file') + + with self.assertRaisesRegexp(SandboxValidationError, + 'lists test that does not exist: test_missing.html'): + self.read_topsrcdir(reader) + + def test_test_manifest_missing_test_error_unfiltered(self): + """Missing test files should result in error, even when the test list is not filtered.""" + reader = self.reader('test-manifest-missing-test-file-unfiltered') + + with self.assertRaisesRegexp(SandboxValidationError, + 'lists test that does not exist: missing.js'): + self.read_topsrcdir(reader) + + def test_ipdl_sources(self): + reader = self.reader('ipdl_sources') + objs = self.read_topsrcdir(reader) + + ipdls = [] + for o in objs: + if isinstance(o, IPDLFile): + ipdls.append('%s/%s' % (o.relativedir, o.basename)) + + expected = [ + 'bar/bar.ipdl', + 'bar/bar2.ipdlh', + 'foo/foo.ipdl', + 'foo/foo2.ipdlh', + ] + + self.assertEqual(ipdls, expected) + + def test_local_includes(self): + """Test that LOCAL_INCLUDES is emitted correctly.""" + reader = self.reader('local_includes') + objs = self.read_topsrcdir(reader) + + local_includes = [o.path for o in objs if isinstance(o, LocalInclude)] + expected = [ + '/bar/baz', + 'foo', + ] + + self.assertEqual(local_includes, expected) + + local_includes = [o.path.full_path + for o in objs if isinstance(o, LocalInclude)] + expected = [ + mozpath.join(reader.config.topsrcdir, 'bar/baz'), + mozpath.join(reader.config.topsrcdir, 'foo'), + ] + + self.assertEqual(local_includes, expected) + + def test_generated_includes(self): + """Test that GENERATED_INCLUDES is emitted correctly.""" + reader = self.reader('generated_includes') + objs = self.read_topsrcdir(reader) + + generated_includes = [o.path for o in objs if isinstance(o, LocalInclude)] + expected = [ + '!/bar/baz', + '!foo', + ] + + self.assertEqual(generated_includes, expected) + + generated_includes = [o.path.full_path + for o in objs if isinstance(o, LocalInclude)] + expected = [ + mozpath.join(reader.config.topobjdir, 'bar/baz'), + mozpath.join(reader.config.topobjdir, 'foo'), + ] + + self.assertEqual(generated_includes, expected) + + def test_defines(self): + reader = self.reader('defines') + objs = self.read_topsrcdir(reader) + + defines = {} + for o in objs: + if isinstance(o, Defines): + defines = o.defines + + expected = { + 'BAR': 7, + 'BAZ': '"abcd"', + 'FOO': True, + 'VALUE': 'xyz', + 'QUX': False, + } + + self.assertEqual(defines, expected) + + def test_host_defines(self): + reader = self.reader('host-defines') + objs = self.read_topsrcdir(reader) + + defines = {} + for o in objs: + if isinstance(o, HostDefines): + defines = o.defines + + expected = { + 'BAR': 7, + 'BAZ': '"abcd"', + 'FOO': True, + 'VALUE': 'xyz', + 'QUX': False, + } + + self.assertEqual(defines, expected) + + def test_jar_manifests(self): + reader = self.reader('jar-manifests') + objs = self.read_topsrcdir(reader) + + self.assertEqual(len(objs), 1) + for obj in objs: + self.assertIsInstance(obj, JARManifest) + self.assertIsInstance(obj.path, Path) + + def test_jar_manifests_multiple_files(self): + with self.assertRaisesRegexp(SandboxValidationError, 'limited to one value'): + reader = self.reader('jar-manifests-multiple-files') + self.read_topsrcdir(reader) + + def test_xpidl_module_no_sources(self): + """XPIDL_MODULE without XPIDL_SOURCES should be rejected.""" + with self.assertRaisesRegexp(SandboxValidationError, 'XPIDL_MODULE ' + 'cannot be defined'): + reader = self.reader('xpidl-module-no-sources') + self.read_topsrcdir(reader) + + def test_missing_local_includes(self): + """LOCAL_INCLUDES containing non-existent directories should be rejected.""" + with self.assertRaisesRegexp(SandboxValidationError, 'Path specified in ' + 'LOCAL_INCLUDES does not exist'): + reader = self.reader('missing-local-includes') + self.read_topsrcdir(reader) + + def test_library_defines(self): + """Test that LIBRARY_DEFINES is propagated properly.""" + reader = self.reader('library-defines') + objs = self.read_topsrcdir(reader) + + libraries = [o for o in objs if isinstance(o,StaticLibrary)] + expected = { + 'liba': '-DIN_LIBA', + 'libb': '-DIN_LIBA -DIN_LIBB', + 'libc': '-DIN_LIBA -DIN_LIBB', + 'libd': '' + } + defines = {} + for lib in libraries: + defines[lib.basename] = ' '.join(lib.lib_defines.get_defines()) + self.assertEqual(expected, defines) + + def test_sources(self): + """Test that SOURCES works properly.""" + reader = self.reader('sources') + objs = self.read_topsrcdir(reader) + + # The last object is a Linkable. + linkable = objs.pop() + self.assertTrue(linkable.cxx_link) + self.assertEqual(len(objs), 6) + for o in objs: + self.assertIsInstance(o, Sources) + + suffix_map = {obj.canonical_suffix: obj for obj in objs} + self.assertEqual(len(suffix_map), 6) + + expected = { + '.cpp': ['a.cpp', 'b.cc', 'c.cxx'], + '.c': ['d.c'], + '.m': ['e.m'], + '.mm': ['f.mm'], + '.S': ['g.S'], + '.s': ['h.s', 'i.asm'], + } + for suffix, files in expected.items(): + sources = suffix_map[suffix] + self.assertEqual( + sources.files, + [mozpath.join(reader.config.topsrcdir, f) for f in files]) + + def test_sources_just_c(self): + """Test that a linkable with no C++ sources doesn't have cxx_link set.""" + reader = self.reader('sources-just-c') + objs = self.read_topsrcdir(reader) + + # The last object is a Linkable. + linkable = objs.pop() + self.assertFalse(linkable.cxx_link) + + def test_linkables_cxx_link(self): + """Test that linkables transitively set cxx_link properly.""" + reader = self.reader('test-linkables-cxx-link') + got_results = 0 + for obj in self.read_topsrcdir(reader): + if isinstance(obj, SharedLibrary): + if obj.basename == 'cxx_shared': + self.assertTrue(obj.cxx_link) + got_results += 1 + elif obj.basename == 'just_c_shared': + self.assertFalse(obj.cxx_link) + got_results += 1 + self.assertEqual(got_results, 2) + + def test_generated_sources(self): + """Test that GENERATED_SOURCES works properly.""" + reader = self.reader('generated-sources') + objs = self.read_topsrcdir(reader) + + # The last object is a Linkable. + linkable = objs.pop() + self.assertTrue(linkable.cxx_link) + self.assertEqual(len(objs), 6) + + generated_sources = [o for o in objs if isinstance(o, GeneratedSources)] + self.assertEqual(len(generated_sources), 6) + + suffix_map = {obj.canonical_suffix: obj for obj in generated_sources} + self.assertEqual(len(suffix_map), 6) + + expected = { + '.cpp': ['a.cpp', 'b.cc', 'c.cxx'], + '.c': ['d.c'], + '.m': ['e.m'], + '.mm': ['f.mm'], + '.S': ['g.S'], + '.s': ['h.s', 'i.asm'], + } + for suffix, files in expected.items(): + sources = suffix_map[suffix] + self.assertEqual( + sources.files, + [mozpath.join(reader.config.topobjdir, f) for f in files]) + + def test_host_sources(self): + """Test that HOST_SOURCES works properly.""" + reader = self.reader('host-sources') + objs = self.read_topsrcdir(reader) + + # The last object is a Linkable + linkable = objs.pop() + self.assertTrue(linkable.cxx_link) + self.assertEqual(len(objs), 3) + for o in objs: + self.assertIsInstance(o, HostSources) + + suffix_map = {obj.canonical_suffix: obj for obj in objs} + self.assertEqual(len(suffix_map), 3) + + expected = { + '.cpp': ['a.cpp', 'b.cc', 'c.cxx'], + '.c': ['d.c'], + '.mm': ['e.mm', 'f.mm'], + } + for suffix, files in expected.items(): + sources = suffix_map[suffix] + self.assertEqual( + sources.files, + [mozpath.join(reader.config.topsrcdir, f) for f in files]) + + def test_unified_sources(self): + """Test that UNIFIED_SOURCES works properly.""" + reader = self.reader('unified-sources') + objs = self.read_topsrcdir(reader) + + # The last object is a Linkable, ignore it + objs = objs[:-1] + self.assertEqual(len(objs), 3) + for o in objs: + self.assertIsInstance(o, UnifiedSources) + + suffix_map = {obj.canonical_suffix: obj for obj in objs} + self.assertEqual(len(suffix_map), 3) + + expected = { + '.cpp': ['bar.cxx', 'foo.cpp', 'quux.cc'], + '.mm': ['objc1.mm', 'objc2.mm'], + '.c': ['c1.c', 'c2.c'], + } + for suffix, files in expected.items(): + sources = suffix_map[suffix] + self.assertEqual( + sources.files, + [mozpath.join(reader.config.topsrcdir, f) for f in files]) + self.assertTrue(sources.have_unified_mapping) + + def test_unified_sources_non_unified(self): + """Test that UNIFIED_SOURCES with FILES_PER_UNIFIED_FILE=1 works properly.""" + reader = self.reader('unified-sources-non-unified') + objs = self.read_topsrcdir(reader) + + # The last object is a Linkable, ignore it + objs = objs[:-1] + self.assertEqual(len(objs), 3) + for o in objs: + self.assertIsInstance(o, UnifiedSources) + + suffix_map = {obj.canonical_suffix: obj for obj in objs} + self.assertEqual(len(suffix_map), 3) + + expected = { + '.cpp': ['bar.cxx', 'foo.cpp', 'quux.cc'], + '.mm': ['objc1.mm', 'objc2.mm'], + '.c': ['c1.c', 'c2.c'], + } + for suffix, files in expected.items(): + sources = suffix_map[suffix] + self.assertEqual( + sources.files, + [mozpath.join(reader.config.topsrcdir, f) for f in files]) + self.assertFalse(sources.have_unified_mapping) + + def test_final_target_pp_files(self): + """Test that FINAL_TARGET_PP_FILES works properly.""" + reader = self.reader('dist-files') + objs = self.read_topsrcdir(reader) + + self.assertEqual(len(objs), 1) + self.assertIsInstance(objs[0], FinalTargetPreprocessedFiles) + + # Ideally we'd test hierarchies, but that would just be testing + # the HierarchicalStringList class, which we test separately. + for path, files in objs[0].files.walk(): + self.assertEqual(path, '') + self.assertEqual(len(files), 2) + + expected = {'install.rdf', 'main.js'} + for f in files: + self.assertTrue(unicode(f) in expected) + + def test_missing_final_target_pp_files(self): + """Test that FINAL_TARGET_PP_FILES with missing files throws errors.""" + with self.assertRaisesRegexp(SandboxValidationError, 'File listed in ' + 'FINAL_TARGET_PP_FILES does not exist'): + reader = self.reader('dist-files-missing') + self.read_topsrcdir(reader) + + def test_final_target_pp_files_non_srcdir(self): + '''Test that non-srcdir paths in FINAL_TARGET_PP_FILES throws errors.''' + reader = self.reader('final-target-pp-files-non-srcdir') + with self.assertRaisesRegexp(SandboxValidationError, + 'Only source directory paths allowed in FINAL_TARGET_PP_FILES:'): + self.read_topsrcdir(reader) + + def test_rust_library_no_cargo_toml(self): + '''Test that defining a RustLibrary without a Cargo.toml fails.''' + reader = self.reader('rust-library-no-cargo-toml') + with self.assertRaisesRegexp(SandboxValidationError, + 'No Cargo.toml file found'): + self.read_topsrcdir(reader) + + def test_rust_library_name_mismatch(self): + '''Test that defining a RustLibrary that doesn't match Cargo.toml fails.''' + reader = self.reader('rust-library-name-mismatch') + with self.assertRaisesRegexp(SandboxValidationError, + 'library.*does not match Cargo.toml-defined package'): + self.read_topsrcdir(reader) + + def test_rust_library_no_lib_section(self): + '''Test that a RustLibrary Cargo.toml with no [lib] section fails.''' + reader = self.reader('rust-library-no-lib-section') + with self.assertRaisesRegexp(SandboxValidationError, + 'Cargo.toml for.* has no \\[lib\\] section'): + self.read_topsrcdir(reader) + + def test_rust_library_no_profile_section(self): + '''Test that a RustLibrary Cargo.toml with no [profile] section fails.''' + reader = self.reader('rust-library-no-profile-section') + with self.assertRaisesRegexp(SandboxValidationError, + 'Cargo.toml for.* has no \\[profile\\.dev\\] section'): + self.read_topsrcdir(reader) + + def test_rust_library_invalid_crate_type(self): + '''Test that a RustLibrary Cargo.toml has a permitted crate-type.''' + reader = self.reader('rust-library-invalid-crate-type') + with self.assertRaisesRegexp(SandboxValidationError, + 'crate-type.* is not permitted'): + self.read_topsrcdir(reader) + + def test_rust_library_non_abort_panic(self): + '''Test that a RustLibrary Cargo.toml has `panic = "abort" set''' + reader = self.reader('rust-library-non-abort-panic') + with self.assertRaisesRegexp(SandboxValidationError, + 'does not specify `panic = "abort"`'): + self.read_topsrcdir(reader) + + def test_rust_library_dash_folding(self): + '''Test that on-disk names of RustLibrary objects convert dashes to underscores.''' + reader = self.reader('rust-library-dash-folding', + extra_substs=dict(RUST_TARGET='i686-pc-windows-msvc')) + objs = self.read_topsrcdir(reader) + + self.assertEqual(len(objs), 1) + lib = objs[0] + self.assertIsInstance(lib, RustLibrary) + self.assertRegexpMatches(lib.lib_name, "random_crate") + self.assertRegexpMatches(lib.import_name, "random_crate") + self.assertRegexpMatches(lib.basename, "random-crate") + + def test_multiple_rust_libraries(self): + '''Test that linking multiple Rust libraries throws an error''' + reader = self.reader('multiple-rust-libraries', + extra_substs=dict(RUST_TARGET='i686-pc-windows-msvc')) + with self.assertRaisesRegexp(LinkageMultipleRustLibrariesError, + 'Cannot link multiple Rust libraries'): + self.read_topsrcdir(reader) + + def test_crate_dependency_path_resolution(self): + '''Test recursive dependencies resolve with the correct paths.''' + reader = self.reader('crate-dependency-path-resolution', + extra_substs=dict(RUST_TARGET='i686-pc-windows-msvc')) + objs = self.read_topsrcdir(reader) + + self.assertEqual(len(objs), 1) + self.assertIsInstance(objs[0], RustLibrary) + + def test_android_res_dirs(self): + """Test that ANDROID_RES_DIRS works properly.""" + reader = self.reader('android-res-dirs') + objs = self.read_topsrcdir(reader) + + self.assertEqual(len(objs), 1) + self.assertIsInstance(objs[0], AndroidResDirs) + + # Android resource directories are ordered. + expected = [ + mozpath.join(reader.config.topsrcdir, 'dir1'), + mozpath.join(reader.config.topobjdir, 'dir2'), + '/dir3', + ] + self.assertEquals([p.full_path for p in objs[0].paths], expected) + + def test_binary_components(self): + """Test that IS_COMPONENT/NO_COMPONENTS_MANIFEST work properly.""" + reader = self.reader('binary-components') + objs = self.read_topsrcdir(reader) + + self.assertEqual(len(objs), 3) + self.assertIsInstance(objs[0], ChromeManifestEntry) + self.assertEqual(objs[0].path, + 'dist/bin/components/components.manifest') + self.assertIsInstance(objs[0].entry, manifest.ManifestBinaryComponent) + self.assertEqual(objs[0].entry.base, 'dist/bin/components') + self.assertEqual(objs[0].entry.relpath, objs[1].lib_name) + self.assertIsInstance(objs[1], SharedLibrary) + self.assertEqual(objs[1].basename, 'foo') + self.assertIsInstance(objs[2], SharedLibrary) + self.assertEqual(objs[2].basename, 'bar') + + def test_install_shared_lib(self): + """Test that we can install a shared library with TEST_HARNESS_FILES""" + reader = self.reader('test-install-shared-lib') + objs = self.read_topsrcdir(reader) + self.assertIsInstance(objs[0], TestHarnessFiles) + self.assertIsInstance(objs[1], VariablePassthru) + self.assertIsInstance(objs[2], SharedLibrary) + for path, files in objs[0].files.walk(): + for f in files: + self.assertEqual(str(f), '!libfoo.so') + self.assertEqual(path, 'foo/bar') + + def test_symbols_file(self): + """Test that SYMBOLS_FILE works""" + reader = self.reader('test-symbols-file') + genfile, shlib = self.read_topsrcdir(reader) + self.assertIsInstance(genfile, GeneratedFile) + self.assertIsInstance(shlib, SharedLibrary) + # This looks weird but MockConfig sets DLL_{PREFIX,SUFFIX} and + # the reader method in this class sets OS_TARGET=WINNT. + self.assertEqual(shlib.symbols_file, 'libfoo.so.def') + + def test_symbols_file_objdir(self): + """Test that a SYMBOLS_FILE in the objdir works""" + reader = self.reader('test-symbols-file-objdir') + genfile, shlib = self.read_topsrcdir(reader) + self.assertIsInstance(genfile, GeneratedFile) + self.assertEqual(genfile.script, + mozpath.join(reader.config.topsrcdir, 'foo.py')) + self.assertIsInstance(shlib, SharedLibrary) + self.assertEqual(shlib.symbols_file, 'foo.symbols') + + def test_symbols_file_objdir_missing_generated(self): + """Test that a SYMBOLS_FILE in the objdir that's missing + from GENERATED_FILES is an error. + """ + reader = self.reader('test-symbols-file-objdir-missing-generated') + with self.assertRaisesRegexp(SandboxValidationError, + 'Objdir file specified in SYMBOLS_FILE not in GENERATED_FILES:'): + self.read_topsrcdir(reader) + + +if __name__ == '__main__': + main() diff --git a/python/mozbuild/mozbuild/test/frontend/test_namespaces.py b/python/mozbuild/mozbuild/test/frontend/test_namespaces.py new file mode 100644 index 000000000..71cc634e1 --- /dev/null +++ b/python/mozbuild/mozbuild/test/frontend/test_namespaces.py @@ -0,0 +1,207 @@ +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, You can obtain one at http://mozilla.org/MPL/2.0/. + +from __future__ import unicode_literals + +import unittest + +from mozunit import main + +from mozbuild.frontend.context import ( + Context, + ContextDerivedValue, + ContextDerivedTypedList, + ContextDerivedTypedListWithItems, +) + +from mozbuild.util import ( + StrictOrderingOnAppendList, + StrictOrderingOnAppendListWithFlagsFactory, + UnsortedError, +) + + +class Fuga(object): + def __init__(self, value): + self.value = value + + +class Piyo(ContextDerivedValue): + def __init__(self, context, value): + if not isinstance(value, unicode): + raise ValueError + self.context = context + self.value = value + + def lower(self): + return self.value.lower() + + def __str__(self): + return self.value + + def __cmp__(self, other): + return cmp(self.value, str(other)) + + def __hash__(self): + return hash(self.value) + + +VARIABLES = { + 'HOGE': (unicode, unicode, None), + 'FUGA': (Fuga, unicode, None), + 'PIYO': (Piyo, unicode, None), + 'HOGERA': (ContextDerivedTypedList(Piyo, StrictOrderingOnAppendList), + list, None), + 'HOGEHOGE': (ContextDerivedTypedListWithItems( + Piyo, + StrictOrderingOnAppendListWithFlagsFactory({ + 'foo': bool, + })), list, None), +} + +class TestContext(unittest.TestCase): + def test_key_rejection(self): + # Lowercase keys should be rejected during normal operation. + ns = Context(allowed_variables=VARIABLES) + + with self.assertRaises(KeyError) as ke: + ns['foo'] = True + + e = ke.exception.args + self.assertEqual(e[0], 'global_ns') + self.assertEqual(e[1], 'set_unknown') + self.assertEqual(e[2], 'foo') + self.assertTrue(e[3]) + + # Unknown uppercase keys should be rejected. + with self.assertRaises(KeyError) as ke: + ns['FOO'] = True + + e = ke.exception.args + self.assertEqual(e[0], 'global_ns') + self.assertEqual(e[1], 'set_unknown') + self.assertEqual(e[2], 'FOO') + self.assertTrue(e[3]) + + def test_allowed_set(self): + self.assertIn('HOGE', VARIABLES) + + ns = Context(allowed_variables=VARIABLES) + + ns['HOGE'] = 'foo' + self.assertEqual(ns['HOGE'], 'foo') + + def test_value_checking(self): + ns = Context(allowed_variables=VARIABLES) + + # Setting to a non-allowed type should not work. + with self.assertRaises(ValueError) as ve: + ns['HOGE'] = True + + e = ve.exception.args + self.assertEqual(e[0], 'global_ns') + self.assertEqual(e[1], 'set_type') + self.assertEqual(e[2], 'HOGE') + self.assertEqual(e[3], True) + self.assertEqual(e[4], unicode) + + def test_key_checking(self): + # Checking for existence of a key should not populate the key if it + # doesn't exist. + g = Context(allowed_variables=VARIABLES) + + self.assertFalse('HOGE' in g) + self.assertFalse('HOGE' in g) + + def test_coercion(self): + ns = Context(allowed_variables=VARIABLES) + + # Setting to a type different from the allowed input type should not + # work. + with self.assertRaises(ValueError) as ve: + ns['FUGA'] = False + + e = ve.exception.args + self.assertEqual(e[0], 'global_ns') + self.assertEqual(e[1], 'set_type') + self.assertEqual(e[2], 'FUGA') + self.assertEqual(e[3], False) + self.assertEqual(e[4], unicode) + + ns['FUGA'] = 'fuga' + self.assertIsInstance(ns['FUGA'], Fuga) + self.assertEqual(ns['FUGA'].value, 'fuga') + + ns['FUGA'] = Fuga('hoge') + self.assertIsInstance(ns['FUGA'], Fuga) + self.assertEqual(ns['FUGA'].value, 'hoge') + + def test_context_derived_coercion(self): + ns = Context(allowed_variables=VARIABLES) + + # Setting to a type different from the allowed input type should not + # work. + with self.assertRaises(ValueError) as ve: + ns['PIYO'] = False + + e = ve.exception.args + self.assertEqual(e[0], 'global_ns') + self.assertEqual(e[1], 'set_type') + self.assertEqual(e[2], 'PIYO') + self.assertEqual(e[3], False) + self.assertEqual(e[4], unicode) + + ns['PIYO'] = 'piyo' + self.assertIsInstance(ns['PIYO'], Piyo) + self.assertEqual(ns['PIYO'].value, 'piyo') + self.assertEqual(ns['PIYO'].context, ns) + + ns['PIYO'] = Piyo(ns, 'fuga') + self.assertIsInstance(ns['PIYO'], Piyo) + self.assertEqual(ns['PIYO'].value, 'fuga') + self.assertEqual(ns['PIYO'].context, ns) + + def test_context_derived_typed_list(self): + ns = Context(allowed_variables=VARIABLES) + + # Setting to a type that's rejected by coercion should not work. + with self.assertRaises(ValueError): + ns['HOGERA'] = [False] + + ns['HOGERA'] += ['a', 'b', 'c'] + + self.assertIsInstance(ns['HOGERA'], VARIABLES['HOGERA'][0]) + for n in range(0, 3): + self.assertIsInstance(ns['HOGERA'][n], Piyo) + self.assertEqual(ns['HOGERA'][n].value, ['a', 'b', 'c'][n]) + self.assertEqual(ns['HOGERA'][n].context, ns) + + with self.assertRaises(UnsortedError): + ns['HOGERA'] += ['f', 'e', 'd'] + + def test_context_derived_typed_list_with_items(self): + ns = Context(allowed_variables=VARIABLES) + + # Setting to a type that's rejected by coercion should not work. + with self.assertRaises(ValueError): + ns['HOGEHOGE'] = [False] + + values = ['a', 'b', 'c'] + ns['HOGEHOGE'] += values + + self.assertIsInstance(ns['HOGEHOGE'], VARIABLES['HOGEHOGE'][0]) + for v in values: + ns['HOGEHOGE'][v].foo = True + + for v, item in zip(values, ns['HOGEHOGE']): + self.assertIsInstance(item, Piyo) + self.assertEqual(v, item) + self.assertEqual(ns['HOGEHOGE'][v].foo, True) + self.assertEqual(ns['HOGEHOGE'][item].foo, True) + + with self.assertRaises(UnsortedError): + ns['HOGEHOGE'] += ['f', 'e', 'd'] + +if __name__ == '__main__': + main() diff --git a/python/mozbuild/mozbuild/test/frontend/test_reader.py b/python/mozbuild/mozbuild/test/frontend/test_reader.py new file mode 100644 index 000000000..7c2aed9df --- /dev/null +++ b/python/mozbuild/mozbuild/test/frontend/test_reader.py @@ -0,0 +1,485 @@ +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, You can obtain one at http://mozilla.org/MPL/2.0/. + +from __future__ import unicode_literals + +import os +import sys +import unittest + +from mozunit import main + +from mozbuild.frontend.context import BugzillaComponent +from mozbuild.frontend.reader import ( + BuildReaderError, + BuildReader, +) + +from mozbuild.test.common import MockConfig + +import mozpack.path as mozpath + + +if sys.version_info.major == 2: + text_type = 'unicode' +else: + text_type = 'str' + +data_path = mozpath.abspath(mozpath.dirname(__file__)) +data_path = mozpath.join(data_path, 'data') + + +class TestBuildReader(unittest.TestCase): + def setUp(self): + self._old_env = dict(os.environ) + os.environ.pop('MOZ_OBJDIR', None) + + def tearDown(self): + os.environ.clear() + os.environ.update(self._old_env) + + def config(self, name, **kwargs): + path = mozpath.join(data_path, name) + + return MockConfig(path, **kwargs) + + def reader(self, name, enable_tests=False, error_is_fatal=True, **kwargs): + extra = {} + if enable_tests: + extra['ENABLE_TESTS'] = '1' + config = self.config(name, + extra_substs=extra, + error_is_fatal=error_is_fatal) + + return BuildReader(config, **kwargs) + + def file_path(self, name, *args): + return mozpath.join(data_path, name, *args) + + def test_dirs_traversal_simple(self): + reader = self.reader('traversal-simple') + + contexts = list(reader.read_topsrcdir()) + + self.assertEqual(len(contexts), 4) + + def test_dirs_traversal_no_descend(self): + reader = self.reader('traversal-simple') + + path = mozpath.join(reader.config.topsrcdir, 'moz.build') + self.assertTrue(os.path.exists(path)) + + contexts = list(reader.read_mozbuild(path, reader.config, + descend=False)) + + self.assertEqual(len(contexts), 1) + + def test_dirs_traversal_all_variables(self): + reader = self.reader('traversal-all-vars') + + contexts = list(reader.read_topsrcdir()) + self.assertEqual(len(contexts), 2) + + reader = self.reader('traversal-all-vars', enable_tests=True) + + contexts = list(reader.read_topsrcdir()) + self.assertEqual(len(contexts), 3) + + def test_relative_dirs(self): + # Ensure relative directories are traversed. + reader = self.reader('traversal-relative-dirs') + + contexts = list(reader.read_topsrcdir()) + self.assertEqual(len(contexts), 3) + + def test_repeated_dirs_ignored(self): + # Ensure repeated directories are ignored. + reader = self.reader('traversal-repeated-dirs') + + contexts = list(reader.read_topsrcdir()) + self.assertEqual(len(contexts), 3) + + def test_outside_topsrcdir(self): + # References to directories outside the topsrcdir should fail. + reader = self.reader('traversal-outside-topsrcdir') + + with self.assertRaises(Exception): + list(reader.read_topsrcdir()) + + def test_error_basic(self): + reader = self.reader('reader-error-basic') + + with self.assertRaises(BuildReaderError) as bre: + list(reader.read_topsrcdir()) + + e = bre.exception + self.assertEqual(e.actual_file, self.file_path('reader-error-basic', + 'moz.build')) + + self.assertIn('The error occurred while processing the', str(e)) + + def test_error_included_from(self): + reader = self.reader('reader-error-included-from') + + with self.assertRaises(BuildReaderError) as bre: + list(reader.read_topsrcdir()) + + e = bre.exception + self.assertEqual(e.actual_file, + self.file_path('reader-error-included-from', 'child.build')) + self.assertEqual(e.main_file, + self.file_path('reader-error-included-from', 'moz.build')) + + self.assertIn('This file was included as part of processing', str(e)) + + def test_error_syntax_error(self): + reader = self.reader('reader-error-syntax') + + with self.assertRaises(BuildReaderError) as bre: + list(reader.read_topsrcdir()) + + e = bre.exception + self.assertIn('Python syntax error on line 5', str(e)) + self.assertIn(' foo =', str(e)) + self.assertIn(' ^', str(e)) + + def test_error_read_unknown_global(self): + reader = self.reader('reader-error-read-unknown-global') + + with self.assertRaises(BuildReaderError) as bre: + list(reader.read_topsrcdir()) + + e = bre.exception + self.assertIn('The error was triggered on line 5', str(e)) + self.assertIn('The underlying problem is an attempt to read', str(e)) + self.assertIn(' FOO', str(e)) + + def test_error_write_unknown_global(self): + reader = self.reader('reader-error-write-unknown-global') + + with self.assertRaises(BuildReaderError) as bre: + list(reader.read_topsrcdir()) + + e = bre.exception + self.assertIn('The error was triggered on line 7', str(e)) + self.assertIn('The underlying problem is an attempt to write', str(e)) + self.assertIn(' FOO', str(e)) + + def test_error_write_bad_value(self): + reader = self.reader('reader-error-write-bad-value') + + with self.assertRaises(BuildReaderError) as bre: + list(reader.read_topsrcdir()) + + e = bre.exception + self.assertIn('The error was triggered on line 5', str(e)) + self.assertIn('is an attempt to write an illegal value to a special', + str(e)) + + self.assertIn('variable whose value was rejected is:\n\n DIRS', + str(e)) + + self.assertIn('written to it was of the following type:\n\n %s' % text_type, + str(e)) + + self.assertIn('expects the following type(s):\n\n list', str(e)) + + def test_error_illegal_path(self): + reader = self.reader('reader-error-outside-topsrcdir') + + with self.assertRaises(BuildReaderError) as bre: + list(reader.read_topsrcdir()) + + e = bre.exception + self.assertIn('The underlying problem is an illegal file access', + str(e)) + + def test_error_missing_include_path(self): + reader = self.reader('reader-error-missing-include') + + with self.assertRaises(BuildReaderError) as bre: + list(reader.read_topsrcdir()) + + e = bre.exception + self.assertIn('we referenced a path that does not exist', str(e)) + + def test_error_script_error(self): + reader = self.reader('reader-error-script-error') + + with self.assertRaises(BuildReaderError) as bre: + list(reader.read_topsrcdir()) + + e = bre.exception + self.assertIn('The error appears to be the fault of the script', + str(e)) + self.assertIn(' ["TypeError: unsupported operand', str(e)) + + def test_error_bad_dir(self): + reader = self.reader('reader-error-bad-dir') + + with self.assertRaises(BuildReaderError) as bre: + list(reader.read_topsrcdir()) + + e = bre.exception + self.assertIn('we referenced a path that does not exist', str(e)) + + def test_error_repeated_dir(self): + reader = self.reader('reader-error-repeated-dir') + + with self.assertRaises(BuildReaderError) as bre: + list(reader.read_topsrcdir()) + + e = bre.exception + self.assertIn('Directory (foo) registered multiple times', str(e)) + + def test_error_error_func(self): + reader = self.reader('reader-error-error-func') + + with self.assertRaises(BuildReaderError) as bre: + list(reader.read_topsrcdir()) + + e = bre.exception + self.assertIn('A moz.build file called the error() function.', str(e)) + self.assertIn(' Some error.', str(e)) + + def test_error_error_func_ok(self): + reader = self.reader('reader-error-error-func', error_is_fatal=False) + + contexts = list(reader.read_topsrcdir()) + + def test_error_empty_list(self): + reader = self.reader('reader-error-empty-list') + + with self.assertRaises(BuildReaderError) as bre: + list(reader.read_topsrcdir()) + + e = bre.exception + self.assertIn('Variable DIRS assigned an empty value.', str(e)) + + def test_inheriting_variables(self): + reader = self.reader('inheriting-variables') + + contexts = list(reader.read_topsrcdir()) + + self.assertEqual(len(contexts), 4) + self.assertEqual([context.relsrcdir for context in contexts], + ['', 'foo', 'foo/baz', 'bar']) + self.assertEqual([context['XPIDL_MODULE'] for context in contexts], + ['foobar', 'foobar', 'baz', 'foobar']) + + def test_find_relevant_mozbuilds(self): + reader = self.reader('reader-relevant-mozbuild') + + # Absolute paths outside topsrcdir are rejected. + with self.assertRaises(Exception): + reader._find_relevant_mozbuilds(['/foo']) + + # File in root directory. + paths = reader._find_relevant_mozbuilds(['file']) + self.assertEqual(paths, {'file': ['moz.build']}) + + # File in child directory. + paths = reader._find_relevant_mozbuilds(['d1/file1']) + self.assertEqual(paths, {'d1/file1': ['moz.build', 'd1/moz.build']}) + + # Multiple files in same directory. + paths = reader._find_relevant_mozbuilds(['d1/file1', 'd1/file2']) + self.assertEqual(paths, { + 'd1/file1': ['moz.build', 'd1/moz.build'], + 'd1/file2': ['moz.build', 'd1/moz.build']}) + + # Missing moz.build from missing intermediate directory. + paths = reader._find_relevant_mozbuilds( + ['d1/no-intermediate-moz-build/child/file']) + self.assertEqual(paths, { + 'd1/no-intermediate-moz-build/child/file': [ + 'moz.build', 'd1/moz.build', 'd1/no-intermediate-moz-build/child/moz.build']}) + + # Lots of empty directories. + paths = reader._find_relevant_mozbuilds([ + 'd1/parent-is-far/dir1/dir2/dir3/file']) + self.assertEqual(paths, { + 'd1/parent-is-far/dir1/dir2/dir3/file': + ['moz.build', 'd1/moz.build', 'd1/parent-is-far/moz.build']}) + + # Lots of levels. + paths = reader._find_relevant_mozbuilds([ + 'd1/every-level/a/file', 'd1/every-level/b/file']) + self.assertEqual(paths, { + 'd1/every-level/a/file': [ + 'moz.build', + 'd1/moz.build', + 'd1/every-level/moz.build', + 'd1/every-level/a/moz.build', + ], + 'd1/every-level/b/file': [ + 'moz.build', + 'd1/moz.build', + 'd1/every-level/moz.build', + 'd1/every-level/b/moz.build', + ], + }) + + # Different root directories. + paths = reader._find_relevant_mozbuilds(['d1/file', 'd2/file', 'file']) + self.assertEqual(paths, { + 'file': ['moz.build'], + 'd1/file': ['moz.build', 'd1/moz.build'], + 'd2/file': ['moz.build', 'd2/moz.build'], + }) + + def test_read_relevant_mozbuilds(self): + reader = self.reader('reader-relevant-mozbuild') + + paths, contexts = reader.read_relevant_mozbuilds(['d1/every-level/a/file', + 'd1/every-level/b/file', 'd2/file']) + self.assertEqual(len(paths), 3) + self.assertEqual(len(contexts), 6) + + self.assertEqual([ctx.relsrcdir for ctx in paths['d1/every-level/a/file']], + ['', 'd1', 'd1/every-level', 'd1/every-level/a']) + self.assertEqual([ctx.relsrcdir for ctx in paths['d1/every-level/b/file']], + ['', 'd1', 'd1/every-level', 'd1/every-level/b']) + self.assertEqual([ctx.relsrcdir for ctx in paths['d2/file']], + ['', 'd2']) + + def test_files_bad_bug_component(self): + reader = self.reader('files-info') + + with self.assertRaises(BuildReaderError): + reader.files_info(['bug_component/bad-assignment/moz.build']) + + def test_files_bug_component_static(self): + reader = self.reader('files-info') + + v = reader.files_info(['bug_component/static/foo', + 'bug_component/static/bar', + 'bug_component/static/foo/baz']) + self.assertEqual(len(v), 3) + self.assertEqual(v['bug_component/static/foo']['BUG_COMPONENT'], + BugzillaComponent('FooProduct', 'FooComponent')) + self.assertEqual(v['bug_component/static/bar']['BUG_COMPONENT'], + BugzillaComponent('BarProduct', 'BarComponent')) + self.assertEqual(v['bug_component/static/foo/baz']['BUG_COMPONENT'], + BugzillaComponent('default_product', 'default_component')) + + def test_files_bug_component_simple(self): + reader = self.reader('files-info') + + v = reader.files_info(['bug_component/simple/moz.build']) + self.assertEqual(len(v), 1) + flags = v['bug_component/simple/moz.build'] + self.assertEqual(flags['BUG_COMPONENT'].product, 'Core') + self.assertEqual(flags['BUG_COMPONENT'].component, 'Build Config') + + def test_files_bug_component_different_matchers(self): + reader = self.reader('files-info') + + v = reader.files_info([ + 'bug_component/different-matchers/foo.jsm', + 'bug_component/different-matchers/bar.cpp', + 'bug_component/different-matchers/baz.misc']) + self.assertEqual(len(v), 3) + + js_flags = v['bug_component/different-matchers/foo.jsm'] + cpp_flags = v['bug_component/different-matchers/bar.cpp'] + misc_flags = v['bug_component/different-matchers/baz.misc'] + + self.assertEqual(js_flags['BUG_COMPONENT'], BugzillaComponent('Firefox', 'JS')) + self.assertEqual(cpp_flags['BUG_COMPONENT'], BugzillaComponent('Firefox', 'C++')) + self.assertEqual(misc_flags['BUG_COMPONENT'], BugzillaComponent('default_product', 'default_component')) + + def test_files_bug_component_final(self): + reader = self.reader('files-info') + + v = reader.files_info([ + 'bug_component/final/foo', + 'bug_component/final/Makefile.in', + 'bug_component/final/subcomponent/Makefile.in', + 'bug_component/final/subcomponent/bar']) + + self.assertEqual(v['bug_component/final/foo']['BUG_COMPONENT'], + BugzillaComponent('default_product', 'default_component')) + self.assertEqual(v['bug_component/final/Makefile.in']['BUG_COMPONENT'], + BugzillaComponent('Core', 'Build Config')) + self.assertEqual(v['bug_component/final/subcomponent/Makefile.in']['BUG_COMPONENT'], + BugzillaComponent('Core', 'Build Config')) + self.assertEqual(v['bug_component/final/subcomponent/bar']['BUG_COMPONENT'], + BugzillaComponent('Another', 'Component')) + + def test_file_test_deps(self): + reader = self.reader('files-test-metadata') + + expected = { + 'simple/src/module.jsm': set(['simple/tests/test_general.html', + 'simple/browser/**.js']), + 'simple/base.cpp': set(['simple/tests/*', + 'default/tests/xpcshell/test_default_mod.js']), + } + + v = reader.files_info([ + 'simple/src/module.jsm', + 'simple/base.cpp', + ]) + + for path, pattern_set in expected.items(): + self.assertEqual(v[path].test_files, + expected[path]) + + def test_file_test_deps_default(self): + reader = self.reader('files-test-metadata') + v = reader.files_info([ + 'default/module.js', + ]) + + expected = { + 'default/module.js': set(['default/tests/xpcshell/**', + 'default/tests/reftests/**']), + } + + for path, pattern_set in expected.items(): + self.assertEqual(v[path].test_files, + expected[path]) + + def test_file_test_deps_tags(self): + reader = self.reader('files-test-metadata') + v = reader.files_info([ + 'tagged/src/bar.jsm', + 'tagged/src/submodule/foo.js', + ]) + + expected_patterns = { + 'tagged/src/submodule/foo.js': set([]), + 'tagged/src/bar.jsm': set(['tagged/**.js']), + } + + for path, pattern_set in expected_patterns.items(): + self.assertEqual(v[path].test_files, + expected_patterns[path]) + + expected_tags = { + 'tagged/src/submodule/foo.js': set(['submodule']), + 'tagged/src/bar.jsm': set([]), + } + for path, pattern_set in expected_tags.items(): + self.assertEqual(v[path].test_tags, + expected_tags[path]) + + expected_flavors = { + 'tagged/src/bar.jsm': set(['browser-chrome']), + 'tagged/src/submodule/foo.js': set([]), + } + for path, pattern_set in expected_flavors.items(): + self.assertEqual(v[path].test_flavors, + expected_flavors[path]) + + def test_invalid_flavor(self): + reader = self.reader('invalid-files-flavor') + + with self.assertRaises(BuildReaderError): + reader.files_info(['foo.js']) + + +if __name__ == '__main__': + main() diff --git a/python/mozbuild/mozbuild/test/frontend/test_sandbox.py b/python/mozbuild/mozbuild/test/frontend/test_sandbox.py new file mode 100644 index 000000000..d24c5d9ea --- /dev/null +++ b/python/mozbuild/mozbuild/test/frontend/test_sandbox.py @@ -0,0 +1,534 @@ +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, You can obtain one at http://mozilla.org/MPL/2.0/. + +from __future__ import unicode_literals + +import os +import shutil +import unittest + +from mozunit import main + +from mozbuild.frontend.reader import ( + MozbuildSandbox, + SandboxCalledError, +) + +from mozbuild.frontend.sandbox import ( + Sandbox, + SandboxExecutionError, + SandboxLoadError, +) + +from mozbuild.frontend.context import ( + Context, + FUNCTIONS, + SourcePath, + SPECIAL_VARIABLES, + VARIABLES, +) + +from mozbuild.test.common import MockConfig +from types import StringTypes + +import mozpack.path as mozpath + +test_data_path = mozpath.abspath(mozpath.dirname(__file__)) +test_data_path = mozpath.join(test_data_path, 'data') + + +class TestSandbox(unittest.TestCase): + def sandbox(self): + return Sandbox(Context({ + 'DIRS': (list, list, None), + })) + + def test_exec_source_success(self): + sandbox = self.sandbox() + context = sandbox._context + + sandbox.exec_source('foo = True', mozpath.abspath('foo.py')) + + self.assertNotIn('foo', context) + self.assertEqual(context.main_path, mozpath.abspath('foo.py')) + self.assertEqual(context.all_paths, set([mozpath.abspath('foo.py')])) + + def test_exec_compile_error(self): + sandbox = self.sandbox() + + with self.assertRaises(SandboxExecutionError) as se: + sandbox.exec_source('2f23;k;asfj', mozpath.abspath('foo.py')) + + self.assertEqual(se.exception.file_stack, [mozpath.abspath('foo.py')]) + self.assertIsInstance(se.exception.exc_value, SyntaxError) + self.assertEqual(sandbox._context.main_path, mozpath.abspath('foo.py')) + + def test_exec_import_denied(self): + sandbox = self.sandbox() + + with self.assertRaises(SandboxExecutionError) as se: + sandbox.exec_source('import sys') + + self.assertIsInstance(se.exception, SandboxExecutionError) + self.assertEqual(se.exception.exc_type, ImportError) + + def test_exec_source_multiple(self): + sandbox = self.sandbox() + + sandbox.exec_source('DIRS = ["foo"]') + sandbox.exec_source('DIRS += ["bar"]') + + self.assertEqual(sandbox['DIRS'], ['foo', 'bar']) + + def test_exec_source_illegal_key_set(self): + sandbox = self.sandbox() + + with self.assertRaises(SandboxExecutionError) as se: + sandbox.exec_source('ILLEGAL = True') + + e = se.exception + self.assertIsInstance(e.exc_value, KeyError) + + e = se.exception.exc_value + self.assertEqual(e.args[0], 'global_ns') + self.assertEqual(e.args[1], 'set_unknown') + + def test_exec_source_reassign(self): + sandbox = self.sandbox() + + sandbox.exec_source('DIRS = ["foo"]') + with self.assertRaises(SandboxExecutionError) as se: + sandbox.exec_source('DIRS = ["bar"]') + + self.assertEqual(sandbox['DIRS'], ['foo']) + e = se.exception + self.assertIsInstance(e.exc_value, KeyError) + + e = se.exception.exc_value + self.assertEqual(e.args[0], 'global_ns') + self.assertEqual(e.args[1], 'reassign') + self.assertEqual(e.args[2], 'DIRS') + + def test_exec_source_reassign_builtin(self): + sandbox = self.sandbox() + + with self.assertRaises(SandboxExecutionError) as se: + sandbox.exec_source('True = 1') + + e = se.exception + self.assertIsInstance(e.exc_value, KeyError) + + e = se.exception.exc_value + self.assertEqual(e.args[0], 'Cannot reassign builtins') + + +class TestedSandbox(MozbuildSandbox): + '''Version of MozbuildSandbox with a little more convenience for testing. + + It automatically normalizes paths given to exec_file and exec_source. This + helps simplify the test code. + ''' + def normalize_path(self, path): + return mozpath.normpath( + mozpath.join(self._context.config.topsrcdir, path)) + + def source_path(self, path): + return SourcePath(self._context, path) + + def exec_file(self, path): + super(TestedSandbox, self).exec_file(self.normalize_path(path)) + + def exec_source(self, source, path=''): + super(TestedSandbox, self).exec_source(source, + self.normalize_path(path) if path else '') + + +class TestMozbuildSandbox(unittest.TestCase): + def sandbox(self, data_path=None, metadata={}): + config = None + + if data_path is not None: + config = MockConfig(mozpath.join(test_data_path, data_path)) + else: + config = MockConfig() + + return TestedSandbox(Context(VARIABLES, config), metadata) + + def test_default_state(self): + sandbox = self.sandbox() + sandbox._context.add_source(sandbox.normalize_path('moz.build')) + config = sandbox._context.config + + self.assertEqual(sandbox['TOPSRCDIR'], config.topsrcdir) + self.assertEqual(sandbox['TOPOBJDIR'], config.topobjdir) + self.assertEqual(sandbox['RELATIVEDIR'], '') + self.assertEqual(sandbox['SRCDIR'], config.topsrcdir) + self.assertEqual(sandbox['OBJDIR'], config.topobjdir) + + def test_symbol_presence(self): + # Ensure no discrepancies between the master symbol table and what's in + # the sandbox. + sandbox = self.sandbox() + sandbox._context.add_source(sandbox.normalize_path('moz.build')) + + all_symbols = set() + all_symbols |= set(FUNCTIONS.keys()) + all_symbols |= set(SPECIAL_VARIABLES.keys()) + + for symbol in all_symbols: + self.assertIsNotNone(sandbox[symbol]) + + def test_path_calculation(self): + sandbox = self.sandbox() + sandbox._context.add_source(sandbox.normalize_path('foo/bar/moz.build')) + config = sandbox._context.config + + self.assertEqual(sandbox['TOPSRCDIR'], config.topsrcdir) + self.assertEqual(sandbox['TOPOBJDIR'], config.topobjdir) + self.assertEqual(sandbox['RELATIVEDIR'], 'foo/bar') + self.assertEqual(sandbox['SRCDIR'], + mozpath.join(config.topsrcdir, 'foo/bar')) + self.assertEqual(sandbox['OBJDIR'], + mozpath.join(config.topobjdir, 'foo/bar')) + + def test_config_access(self): + sandbox = self.sandbox() + config = sandbox._context.config + + self.assertEqual(sandbox['CONFIG']['MOZ_TRUE'], '1') + self.assertEqual(sandbox['CONFIG']['MOZ_FOO'], config.substs['MOZ_FOO']) + + # Access to an undefined substitution should return None. + self.assertNotIn('MISSING', sandbox['CONFIG']) + self.assertIsNone(sandbox['CONFIG']['MISSING']) + + # Should shouldn't be allowed to assign to the config. + with self.assertRaises(Exception): + sandbox['CONFIG']['FOO'] = '' + + def test_special_variables(self): + sandbox = self.sandbox() + sandbox._context.add_source(sandbox.normalize_path('moz.build')) + + for k in SPECIAL_VARIABLES: + with self.assertRaises(KeyError): + sandbox[k] = 0 + + def test_exec_source_reassign_exported(self): + template_sandbox = self.sandbox(data_path='templates') + + # Templates need to be defined in actual files because of + # inspect.getsourcelines. + template_sandbox.exec_file('templates.mozbuild') + + config = MockConfig() + + exports = {'DIST_SUBDIR': 'browser'} + + sandbox = TestedSandbox(Context(VARIABLES, config), metadata={ + 'exports': exports, + 'templates': template_sandbox.templates, + }) + + self.assertEqual(sandbox['DIST_SUBDIR'], 'browser') + + # Templates should not interfere + sandbox.exec_source('Template([])', 'foo.mozbuild') + + sandbox.exec_source('DIST_SUBDIR = "foo"') + with self.assertRaises(SandboxExecutionError) as se: + sandbox.exec_source('DIST_SUBDIR = "bar"') + + self.assertEqual(sandbox['DIST_SUBDIR'], 'foo') + e = se.exception + self.assertIsInstance(e.exc_value, KeyError) + + e = se.exception.exc_value + self.assertEqual(e.args[0], 'global_ns') + self.assertEqual(e.args[1], 'reassign') + self.assertEqual(e.args[2], 'DIST_SUBDIR') + + def test_include_basic(self): + sandbox = self.sandbox(data_path='include-basic') + + sandbox.exec_file('moz.build') + + self.assertEqual(sandbox['DIRS'], [ + sandbox.source_path('foo'), + sandbox.source_path('bar'), + ]) + self.assertEqual(sandbox._context.main_path, + sandbox.normalize_path('moz.build')) + self.assertEqual(len(sandbox._context.all_paths), 2) + + def test_include_outside_topsrcdir(self): + sandbox = self.sandbox(data_path='include-outside-topsrcdir') + + with self.assertRaises(SandboxLoadError) as se: + sandbox.exec_file('relative.build') + + self.assertEqual(se.exception.illegal_path, + sandbox.normalize_path('../moz.build')) + + def test_include_error_stack(self): + # Ensure the path stack is reported properly in exceptions. + sandbox = self.sandbox(data_path='include-file-stack') + + with self.assertRaises(SandboxExecutionError) as se: + sandbox.exec_file('moz.build') + + e = se.exception + self.assertIsInstance(e.exc_value, KeyError) + + args = e.exc_value.args + self.assertEqual(args[0], 'global_ns') + self.assertEqual(args[1], 'set_unknown') + self.assertEqual(args[2], 'ILLEGAL') + + expected_stack = [mozpath.join(sandbox._context.config.topsrcdir, p) for p in [ + 'moz.build', 'included-1.build', 'included-2.build']] + + self.assertEqual(e.file_stack, expected_stack) + + def test_include_missing(self): + sandbox = self.sandbox(data_path='include-missing') + + with self.assertRaises(SandboxLoadError) as sle: + sandbox.exec_file('moz.build') + + self.assertIsNotNone(sle.exception.read_error) + + def test_include_relative_from_child_dir(self): + # A relative path from a subdirectory should be relative from that + # child directory. + sandbox = self.sandbox(data_path='include-relative-from-child') + sandbox.exec_file('child/child.build') + self.assertEqual(sandbox['DIRS'], [sandbox.source_path('../foo')]) + + sandbox = self.sandbox(data_path='include-relative-from-child') + sandbox.exec_file('child/child2.build') + self.assertEqual(sandbox['DIRS'], [sandbox.source_path('../foo')]) + + def test_include_topsrcdir_relative(self): + # An absolute path for include() is relative to topsrcdir. + + sandbox = self.sandbox(data_path='include-topsrcdir-relative') + sandbox.exec_file('moz.build') + + self.assertEqual(sandbox['DIRS'], [sandbox.source_path('foo')]) + + def test_error(self): + sandbox = self.sandbox() + + with self.assertRaises(SandboxCalledError) as sce: + sandbox.exec_source('error("This is an error.")') + + e = sce.exception + self.assertEqual(e.message, 'This is an error.') + + def test_substitute_config_files(self): + sandbox = self.sandbox() + sandbox._context.add_source(sandbox.normalize_path('moz.build')) + + sandbox.exec_source('CONFIGURE_SUBST_FILES += ["bar", "foo"]') + self.assertEqual(sandbox['CONFIGURE_SUBST_FILES'], ['bar', 'foo']) + for item in sandbox['CONFIGURE_SUBST_FILES']: + self.assertIsInstance(item, SourcePath) + + def test_invalid_utf8_substs(self): + """Ensure invalid UTF-8 in substs is converted with an error.""" + + # This is really mbcs. It's a bunch of invalid UTF-8. + config = MockConfig(extra_substs={'BAD_UTF8': b'\x83\x81\x83\x82\x3A'}) + + sandbox = MozbuildSandbox(Context(VARIABLES, config)) + + self.assertEqual(sandbox['CONFIG']['BAD_UTF8'], + u'\ufffd\ufffd\ufffd\ufffd:') + + def test_invalid_exports_set_base(self): + sandbox = self.sandbox() + + with self.assertRaises(SandboxExecutionError) as se: + sandbox.exec_source('EXPORTS = "foo.h"') + + self.assertEqual(se.exception.exc_type, ValueError) + + def test_templates(self): + sandbox = self.sandbox(data_path='templates') + + # Templates need to be defined in actual files because of + # inspect.getsourcelines. + sandbox.exec_file('templates.mozbuild') + + sandbox2 = self.sandbox(metadata={'templates': sandbox.templates}) + source = ''' +Template([ + 'foo.cpp', +]) +''' + sandbox2.exec_source(source, 'foo.mozbuild') + + self.assertEqual(sandbox2._context, { + 'SOURCES': ['foo.cpp'], + 'DIRS': [], + }) + + sandbox2 = self.sandbox(metadata={'templates': sandbox.templates}) + source = ''' +SOURCES += ['qux.cpp'] +Template([ + 'bar.cpp', + 'foo.cpp', +],[ + 'foo', +]) +SOURCES += ['hoge.cpp'] +''' + sandbox2.exec_source(source, 'foo.mozbuild') + + self.assertEqual(sandbox2._context, { + 'SOURCES': ['qux.cpp', 'bar.cpp', 'foo.cpp', 'hoge.cpp'], + 'DIRS': [sandbox2.source_path('foo')], + }) + + sandbox2 = self.sandbox(metadata={'templates': sandbox.templates}) + source = ''' +TemplateError([ + 'foo.cpp', +]) +''' + with self.assertRaises(SandboxExecutionError) as se: + sandbox2.exec_source(source, 'foo.mozbuild') + + e = se.exception + self.assertIsInstance(e.exc_value, KeyError) + + e = se.exception.exc_value + self.assertEqual(e.args[0], 'global_ns') + self.assertEqual(e.args[1], 'set_unknown') + + # TemplateGlobalVariable tries to access 'illegal' but that is expected + # to throw. + sandbox2 = self.sandbox(metadata={'templates': sandbox.templates}) + source = ''' +illegal = True +TemplateGlobalVariable() +''' + with self.assertRaises(SandboxExecutionError) as se: + sandbox2.exec_source(source, 'foo.mozbuild') + + e = se.exception + self.assertIsInstance(e.exc_value, NameError) + + # TemplateGlobalUPPERVariable sets SOURCES with DIRS, but the context + # used when running the template is not expected to access variables + # from the global context. + sandbox2 = self.sandbox(metadata={'templates': sandbox.templates}) + source = ''' +DIRS += ['foo'] +TemplateGlobalUPPERVariable() +''' + sandbox2.exec_source(source, 'foo.mozbuild') + self.assertEqual(sandbox2._context, { + 'SOURCES': [], + 'DIRS': [sandbox2.source_path('foo')], + }) + + # However, the result of the template is mixed with the global + # context. + sandbox2 = self.sandbox(metadata={'templates': sandbox.templates}) + source = ''' +SOURCES += ['qux.cpp'] +TemplateInherit([ + 'bar.cpp', + 'foo.cpp', +]) +SOURCES += ['hoge.cpp'] +''' + sandbox2.exec_source(source, 'foo.mozbuild') + + self.assertEqual(sandbox2._context, { + 'SOURCES': ['qux.cpp', 'bar.cpp', 'foo.cpp', 'hoge.cpp'], + 'USE_LIBS': ['foo'], + 'DIRS': [], + }) + + # Template names must be CamelCase. Here, we can define the template + # inline because the error happens before inspect.getsourcelines. + sandbox2 = self.sandbox(metadata={'templates': sandbox.templates}) + source = ''' +@template +def foo(): + pass +''' + + with self.assertRaises(SandboxExecutionError) as se: + sandbox2.exec_source(source, 'foo.mozbuild') + + e = se.exception + self.assertIsInstance(e.exc_value, NameError) + + e = se.exception.exc_value + self.assertEqual(e.message, + 'Template function names must be CamelCase.') + + # Template names must not already be registered. + sandbox2 = self.sandbox(metadata={'templates': sandbox.templates}) + source = ''' +@template +def Template(): + pass +''' + with self.assertRaises(SandboxExecutionError) as se: + sandbox2.exec_source(source, 'foo.mozbuild') + + e = se.exception + self.assertIsInstance(e.exc_value, KeyError) + + e = se.exception.exc_value + self.assertEqual(e.message, + 'A template named "Template" was already declared in %s.' % + sandbox.normalize_path('templates.mozbuild')) + + def test_function_args(self): + class Foo(int): pass + + def foo(a, b): + return type(a), type(b) + + FUNCTIONS.update({ + 'foo': (lambda self: foo, (Foo, int), ''), + }) + + try: + sandbox = self.sandbox() + source = 'foo("a", "b")' + + with self.assertRaises(SandboxExecutionError) as se: + sandbox.exec_source(source, 'foo.mozbuild') + + e = se.exception + self.assertIsInstance(e.exc_value, ValueError) + + sandbox = self.sandbox() + source = 'foo(1, "b")' + + with self.assertRaises(SandboxExecutionError) as se: + sandbox.exec_source(source, 'foo.mozbuild') + + e = se.exception + self.assertIsInstance(e.exc_value, ValueError) + + sandbox = self.sandbox() + source = 'a = foo(1, 2)' + sandbox.exec_source(source, 'foo.mozbuild') + + self.assertEquals(sandbox['a'], (Foo, int)) + finally: + del FUNCTIONS['foo'] + + +if __name__ == '__main__': + main() diff --git a/python/mozbuild/mozbuild/test/test_android_version_code.py b/python/mozbuild/mozbuild/test/test_android_version_code.py new file mode 100644 index 000000000..059f4588c --- /dev/null +++ b/python/mozbuild/mozbuild/test/test_android_version_code.py @@ -0,0 +1,63 @@ +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, You can obtain one at http://mozilla.org/MPL/2.0/. + +from mozunit import main +import unittest + +from mozbuild.android_version_code import ( + android_version_code_v0, + android_version_code_v1, +) + +class TestAndroidVersionCode(unittest.TestCase): + def test_android_version_code_v0(self): + # From https://treeherder.mozilla.org/#/jobs?repo=mozilla-central&revision=e25de9972a77. + buildid = '20150708104620' + arm_api9 = 2015070819 + arm_api11 = 2015070821 + x86_api9 = 2015070822 + self.assertEqual(android_version_code_v0(buildid, cpu_arch='armeabi', min_sdk=9, max_sdk=None), arm_api9) + self.assertEqual(android_version_code_v0(buildid, cpu_arch='armeabi-v7a', min_sdk=11, max_sdk=None), arm_api11) + self.assertEqual(android_version_code_v0(buildid, cpu_arch='x86', min_sdk=9, max_sdk=None), x86_api9) + + def test_android_version_code_v1(self): + buildid = '20150825141628' + arm_api15 = 0b01111000001000000001001001110001 + x86_api9 = 0b01111000001000000001001001110100 + self.assertEqual(android_version_code_v1(buildid, cpu_arch='armeabi-v7a', min_sdk=15, max_sdk=None), arm_api15) + self.assertEqual(android_version_code_v1(buildid, cpu_arch='x86', min_sdk=9, max_sdk=None), x86_api9) + + def test_android_version_code_v1_underflow(self): + '''Verify that it is an error to ask for v1 codes predating the cutoff.''' + buildid = '201508010000' # Earliest possible. + arm_api9 = 0b01111000001000000000000000000000 + self.assertEqual(android_version_code_v1(buildid, cpu_arch='armeabi', min_sdk=9, max_sdk=None), arm_api9) + with self.assertRaises(ValueError) as cm: + underflow = '201507310000' # Latest possible (valid) underflowing date. + android_version_code_v1(underflow, cpu_arch='armeabi', min_sdk=9, max_sdk=None) + self.assertTrue('underflow' in cm.exception.message) + + def test_android_version_code_v1_running_low(self): + '''Verify there is an informative message if one asks for v1 codes that are close to overflow.''' + with self.assertRaises(ValueError) as cm: + overflow = '20290801000000' + android_version_code_v1(overflow, cpu_arch='armeabi', min_sdk=9, max_sdk=None) + self.assertTrue('Running out of low order bits' in cm.exception.message) + + def test_android_version_code_v1_overflow(self): + '''Verify that it is an error to ask for v1 codes that actually does overflow.''' + with self.assertRaises(ValueError) as cm: + overflow = '20310801000000' + android_version_code_v1(overflow, cpu_arch='armeabi', min_sdk=9, max_sdk=None) + self.assertTrue('overflow' in cm.exception.message) + + def test_android_version_code_v0_relative_v1(self): + '''Verify that the first v1 code is greater than the equivalent v0 code.''' + buildid = '20150801000000' + self.assertGreater(android_version_code_v1(buildid, cpu_arch='armeabi', min_sdk=9, max_sdk=None), + android_version_code_v0(buildid, cpu_arch='armeabi', min_sdk=9, max_sdk=None)) + + +if __name__ == '__main__': + main() diff --git a/python/mozbuild/mozbuild/test/test_base.py b/python/mozbuild/mozbuild/test/test_base.py new file mode 100644 index 000000000..87f0db85b --- /dev/null +++ b/python/mozbuild/mozbuild/test/test_base.py @@ -0,0 +1,410 @@ +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, You can obtain one at http://mozilla.org/MPL/2.0/. + +from __future__ import unicode_literals + +import json +import os +import shutil +import subprocess +import sys +import tempfile +import unittest + +from cStringIO import StringIO +from mozfile.mozfile import NamedTemporaryFile + +from mozunit import main + +from mach.logging import LoggingManager + +from mozbuild.base import ( + BadEnvironmentException, + MachCommandBase, + MozbuildObject, + ObjdirMismatchException, + PathArgument, +) + +from mozbuild.backend.configenvironment import ConfigEnvironment +from buildconfig import topsrcdir, topobjdir +import mozpack.path as mozpath + + +curdir = os.path.dirname(__file__) +log_manager = LoggingManager() + + +class TestMozbuildObject(unittest.TestCase): + def setUp(self): + self._old_cwd = os.getcwd() + self._old_env = dict(os.environ) + os.environ.pop('MOZCONFIG', None) + os.environ.pop('MOZ_OBJDIR', None) + os.environ.pop('MOZ_CURRENT_PROJECT', None) + + def tearDown(self): + os.chdir(self._old_cwd) + os.environ.clear() + os.environ.update(self._old_env) + + def get_base(self, topobjdir=None): + return MozbuildObject(topsrcdir, None, log_manager, topobjdir=topobjdir) + + def test_objdir_config_guess(self): + base = self.get_base() + + with NamedTemporaryFile() as mozconfig: + os.environ[b'MOZCONFIG'] = mozconfig.name + + self.assertIsNotNone(base.topobjdir) + self.assertEqual(len(base.topobjdir.split()), 1) + config_guess = base.resolve_config_guess() + self.assertTrue(base.topobjdir.endswith(config_guess)) + self.assertTrue(os.path.isabs(base.topobjdir)) + self.assertTrue(base.topobjdir.startswith(base.topsrcdir)) + + def test_objdir_trailing_slash(self): + """Trailing slashes in topobjdir should be removed.""" + base = self.get_base() + + with NamedTemporaryFile() as mozconfig: + mozconfig.write('mk_add_options MOZ_OBJDIR=@TOPSRCDIR@/foo/') + mozconfig.flush() + os.environ[b'MOZCONFIG'] = mozconfig.name + + self.assertEqual(base.topobjdir, mozpath.join(base.topsrcdir, + 'foo')) + self.assertTrue(base.topobjdir.endswith('foo')) + + def test_objdir_config_status(self): + """Ensure @CONFIG_GUESS@ is handled when loading mozconfig.""" + base = self.get_base() + cmd = base._normalize_command( + [os.path.join(topsrcdir, 'build', 'autoconf', 'config.guess')], + True) + guess = subprocess.check_output(cmd, cwd=topsrcdir).strip() + + # There may be symlinks involved, so we use real paths to ensure + # path consistency. + d = os.path.realpath(tempfile.mkdtemp()) + try: + mozconfig = os.path.join(d, 'mozconfig') + with open(mozconfig, 'wt') as fh: + fh.write('mk_add_options MOZ_OBJDIR=@TOPSRCDIR@/foo/@CONFIG_GUESS@') + print('Wrote mozconfig %s' % mozconfig) + + topobjdir = os.path.join(d, 'foo', guess) + os.makedirs(topobjdir) + + # Create a fake topsrcdir. + guess_path = os.path.join(d, 'build', 'autoconf', 'config.guess') + os.makedirs(os.path.dirname(guess_path)) + shutil.copy(os.path.join(topsrcdir, 'build', 'autoconf', + 'config.guess',), guess_path) + + mozinfo = os.path.join(topobjdir, 'mozinfo.json') + with open(mozinfo, 'wt') as fh: + json.dump(dict( + topsrcdir=d, + mozconfig=mozconfig, + ), fh) + + os.environ[b'MOZCONFIG'] = mozconfig.encode('utf-8') + os.chdir(topobjdir) + + obj = MozbuildObject.from_environment( + detect_virtualenv_mozinfo=False) + + self.assertEqual(obj.topobjdir, mozpath.normsep(topobjdir)) + finally: + os.chdir(self._old_cwd) + shutil.rmtree(d) + + def test_relative_objdir(self): + """Relative defined objdirs are loaded properly.""" + d = os.path.realpath(tempfile.mkdtemp()) + try: + mozconfig = os.path.join(d, 'mozconfig') + with open(mozconfig, 'wt') as fh: + fh.write('mk_add_options MOZ_OBJDIR=./objdir') + + topobjdir = mozpath.join(d, 'objdir') + os.mkdir(topobjdir) + + mozinfo = os.path.join(topobjdir, 'mozinfo.json') + with open(mozinfo, 'wt') as fh: + json.dump(dict( + topsrcdir=d, + mozconfig=mozconfig, + ), fh) + + os.environ[b'MOZCONFIG'] = mozconfig.encode('utf-8') + child = os.path.join(topobjdir, 'foo', 'bar') + os.makedirs(child) + os.chdir(child) + + obj = MozbuildObject.from_environment( + detect_virtualenv_mozinfo=False) + + self.assertEqual(obj.topobjdir, topobjdir) + + finally: + os.chdir(self._old_cwd) + shutil.rmtree(d) + + @unittest.skipIf(not hasattr(os, 'symlink'), 'symlinks not available.') + def test_symlink_objdir(self): + """Objdir that is a symlink is loaded properly.""" + d = os.path.realpath(tempfile.mkdtemp()) + try: + topobjdir_real = os.path.join(d, 'objdir') + topobjdir_link = os.path.join(d, 'objlink') + + os.mkdir(topobjdir_real) + os.symlink(topobjdir_real, topobjdir_link) + + mozconfig = os.path.join(d, 'mozconfig') + with open(mozconfig, 'wt') as fh: + fh.write('mk_add_options MOZ_OBJDIR=%s' % topobjdir_link) + + mozinfo = os.path.join(topobjdir_real, 'mozinfo.json') + with open(mozinfo, 'wt') as fh: + json.dump(dict( + topsrcdir=d, + mozconfig=mozconfig, + ), fh) + + os.chdir(topobjdir_link) + obj = MozbuildObject.from_environment(detect_virtualenv_mozinfo=False) + self.assertEqual(obj.topobjdir, topobjdir_real) + + os.chdir(topobjdir_real) + obj = MozbuildObject.from_environment(detect_virtualenv_mozinfo=False) + self.assertEqual(obj.topobjdir, topobjdir_real) + + finally: + os.chdir(self._old_cwd) + shutil.rmtree(d) + + def test_mach_command_base_inside_objdir(self): + """Ensure a MachCommandBase constructed from inside the objdir works.""" + + d = os.path.realpath(tempfile.mkdtemp()) + + try: + topobjdir = os.path.join(d, 'objdir') + os.makedirs(topobjdir) + + topsrcdir = os.path.join(d, 'srcdir') + os.makedirs(topsrcdir) + + mozinfo = os.path.join(topobjdir, 'mozinfo.json') + with open(mozinfo, 'wt') as fh: + json.dump(dict( + topsrcdir=topsrcdir, + ), fh) + + os.chdir(topobjdir) + + class MockMachContext(object): + pass + + context = MockMachContext() + context.cwd = topobjdir + context.topdir = topsrcdir + context.settings = None + context.log_manager = None + context.detect_virtualenv_mozinfo=False + + o = MachCommandBase(context) + + self.assertEqual(o.topobjdir, mozpath.normsep(topobjdir)) + self.assertEqual(o.topsrcdir, mozpath.normsep(topsrcdir)) + + finally: + os.chdir(self._old_cwd) + shutil.rmtree(d) + + def test_objdir_is_srcdir_rejected(self): + """Ensure the srcdir configurations are rejected.""" + d = os.path.realpath(tempfile.mkdtemp()) + + try: + # The easiest way to do this is to create a mozinfo.json with data + # that will never happen. + mozinfo = os.path.join(d, 'mozinfo.json') + with open(mozinfo, 'wt') as fh: + json.dump({'topsrcdir': d}, fh) + + os.chdir(d) + + with self.assertRaises(BadEnvironmentException): + MozbuildObject.from_environment(detect_virtualenv_mozinfo=False) + + finally: + os.chdir(self._old_cwd) + shutil.rmtree(d) + + def test_objdir_mismatch(self): + """Ensure MachCommandBase throwing on objdir mismatch.""" + d = os.path.realpath(tempfile.mkdtemp()) + + try: + real_topobjdir = os.path.join(d, 'real-objdir') + os.makedirs(real_topobjdir) + + topobjdir = os.path.join(d, 'objdir') + os.makedirs(topobjdir) + + topsrcdir = os.path.join(d, 'srcdir') + os.makedirs(topsrcdir) + + mozconfig = os.path.join(d, 'mozconfig') + with open(mozconfig, 'wt') as fh: + fh.write('mk_add_options MOZ_OBJDIR=%s' % real_topobjdir) + + mozinfo = os.path.join(topobjdir, 'mozinfo.json') + with open(mozinfo, 'wt') as fh: + json.dump(dict( + topsrcdir=topsrcdir, + mozconfig=mozconfig, + ), fh) + + os.chdir(topobjdir) + + class MockMachContext(object): + pass + + context = MockMachContext() + context.cwd = topobjdir + context.topdir = topsrcdir + context.settings = None + context.log_manager = None + context.detect_virtualenv_mozinfo=False + + stdout = sys.stdout + sys.stdout = StringIO() + try: + with self.assertRaises(SystemExit): + MachCommandBase(context) + + self.assertTrue(sys.stdout.getvalue().startswith( + 'Ambiguous object directory detected.')) + finally: + sys.stdout = stdout + + finally: + os.chdir(self._old_cwd) + shutil.rmtree(d) + + def test_config_environment(self): + base = self.get_base(topobjdir=topobjdir) + + ce = base.config_environment + self.assertIsInstance(ce, ConfigEnvironment) + + self.assertEqual(base.defines, ce.defines) + self.assertEqual(base.substs, ce.substs) + + self.assertIsInstance(base.defines, dict) + self.assertIsInstance(base.substs, dict) + + def test_get_binary_path(self): + base = self.get_base(topobjdir=topobjdir) + + platform = sys.platform + + # We should ideally use the config.status from the build. Let's install + # a fake one. + substs = [ + ('MOZ_APP_NAME', 'awesomeapp'), + ('MOZ_BUILD_APP', 'awesomeapp'), + ] + if sys.platform.startswith('darwin'): + substs.append(('OS_ARCH', 'Darwin')) + substs.append(('BIN_SUFFIX', '')) + substs.append(('MOZ_MACBUNDLE_NAME', 'Nightly.app')) + elif sys.platform.startswith(('win32', 'cygwin')): + substs.append(('OS_ARCH', 'WINNT')) + substs.append(('BIN_SUFFIX', '.exe')) + else: + substs.append(('OS_ARCH', 'something')) + substs.append(('BIN_SUFFIX', '')) + + base._config_environment = ConfigEnvironment(base.topsrcdir, + base.topobjdir, substs=substs) + + p = base.get_binary_path('xpcshell', False) + if platform.startswith('darwin'): + self.assertTrue(p.endswith('Contents/MacOS/xpcshell')) + elif platform.startswith(('win32', 'cygwin')): + self.assertTrue(p.endswith('xpcshell.exe')) + else: + self.assertTrue(p.endswith('dist/bin/xpcshell')) + + p = base.get_binary_path(validate_exists=False) + if platform.startswith('darwin'): + self.assertTrue(p.endswith('Contents/MacOS/awesomeapp')) + elif platform.startswith(('win32', 'cygwin')): + self.assertTrue(p.endswith('awesomeapp.exe')) + else: + self.assertTrue(p.endswith('dist/bin/awesomeapp')) + + p = base.get_binary_path(validate_exists=False, where="staged-package") + if platform.startswith('darwin'): + self.assertTrue(p.endswith('awesomeapp/Nightly.app/Contents/MacOS/awesomeapp')) + elif platform.startswith(('win32', 'cygwin')): + self.assertTrue(p.endswith('awesomeapp\\awesomeapp.exe')) + else: + self.assertTrue(p.endswith('awesomeapp/awesomeapp')) + + self.assertRaises(Exception, base.get_binary_path, where="somewhere") + + p = base.get_binary_path('foobar', validate_exists=False) + if platform.startswith('win32'): + self.assertTrue(p.endswith('foobar.exe')) + else: + self.assertTrue(p.endswith('foobar')) + +class TestPathArgument(unittest.TestCase): + def test_path_argument(self): + # Absolute path + p = PathArgument("/obj/foo", "/src", "/obj", "/src") + self.assertEqual(p.relpath(), "foo") + self.assertEqual(p.srcdir_path(), "/src/foo") + self.assertEqual(p.objdir_path(), "/obj/foo") + + # Relative path within srcdir + p = PathArgument("foo", "/src", "/obj", "/src") + self.assertEqual(p.relpath(), "foo") + self.assertEqual(p.srcdir_path(), "/src/foo") + self.assertEqual(p.objdir_path(), "/obj/foo") + + # Relative path within subdirectory + p = PathArgument("bar", "/src", "/obj", "/src/foo") + self.assertEqual(p.relpath(), "foo/bar") + self.assertEqual(p.srcdir_path(), "/src/foo/bar") + self.assertEqual(p.objdir_path(), "/obj/foo/bar") + + # Relative path within objdir + p = PathArgument("foo", "/src", "/obj", "/obj") + self.assertEqual(p.relpath(), "foo") + self.assertEqual(p.srcdir_path(), "/src/foo") + self.assertEqual(p.objdir_path(), "/obj/foo") + + # "." path + p = PathArgument(".", "/src", "/obj", "/src/foo") + self.assertEqual(p.relpath(), "foo") + self.assertEqual(p.srcdir_path(), "/src/foo") + self.assertEqual(p.objdir_path(), "/obj/foo") + + # Nested src/obj directories + p = PathArgument("bar", "/src", "/src/obj", "/src/obj/foo") + self.assertEqual(p.relpath(), "foo/bar") + self.assertEqual(p.srcdir_path(), "/src/foo/bar") + self.assertEqual(p.objdir_path(), "/src/obj/foo/bar") + +if __name__ == '__main__': + main() diff --git a/python/mozbuild/mozbuild/test/test_containers.py b/python/mozbuild/mozbuild/test/test_containers.py new file mode 100644 index 000000000..3d46f86a9 --- /dev/null +++ b/python/mozbuild/mozbuild/test/test_containers.py @@ -0,0 +1,224 @@ +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this file, +# You can obtain one at http://mozilla.org/MPL/2.0/. + +import unittest + +from mozunit import main + +from mozbuild.util import ( + KeyedDefaultDict, + List, + OrderedDefaultDict, + ReadOnlyNamespace, + ReadOnlyDefaultDict, + ReadOnlyDict, + ReadOnlyKeyedDefaultDict, +) + +from collections import OrderedDict + + +class TestReadOnlyNamespace(unittest.TestCase): + def test_basic(self): + test = ReadOnlyNamespace(foo=1, bar=2) + + self.assertEqual(test.foo, 1) + self.assertEqual(test.bar, 2) + self.assertEqual( + sorted(i for i in dir(test) if not i.startswith('__')), + ['bar', 'foo']) + + with self.assertRaises(AttributeError): + value = test.missing + + with self.assertRaises(Exception): + test.foo = 2 + + with self.assertRaises(Exception): + del test.foo + + self.assertEqual(test, test) + self.assertEqual(test, ReadOnlyNamespace(foo=1, bar=2)) + self.assertNotEqual(test, ReadOnlyNamespace(foo='1', bar=2)) + self.assertNotEqual(test, ReadOnlyNamespace(foo=1, bar=2, qux=3)) + self.assertNotEqual(test, ReadOnlyNamespace(foo=1, qux=3)) + self.assertNotEqual(test, ReadOnlyNamespace(foo=3, bar='42')) + + +class TestReadOnlyDict(unittest.TestCase): + def test_basic(self): + original = {'foo': 1, 'bar': 2} + + test = ReadOnlyDict(original) + + self.assertEqual(original, test) + self.assertEqual(test['foo'], 1) + + with self.assertRaises(KeyError): + value = test['missing'] + + with self.assertRaises(Exception): + test['baz'] = True + + def test_update(self): + original = {'foo': 1, 'bar': 2} + + test = ReadOnlyDict(original) + + with self.assertRaises(Exception): + test.update(foo=2) + + self.assertEqual(original, test) + + def test_del(self): + original = {'foo': 1, 'bar': 2} + + test = ReadOnlyDict(original) + + with self.assertRaises(Exception): + del test['foo'] + + self.assertEqual(original, test) + + +class TestReadOnlyDefaultDict(unittest.TestCase): + def test_simple(self): + original = {'foo': 1, 'bar': 2} + + test = ReadOnlyDefaultDict(bool, original) + + self.assertEqual(original, test) + + self.assertEqual(test['foo'], 1) + + def test_assignment(self): + test = ReadOnlyDefaultDict(bool, {}) + + with self.assertRaises(Exception): + test['foo'] = True + + def test_defaults(self): + test = ReadOnlyDefaultDict(bool, {'foo': 1}) + + self.assertEqual(test['foo'], 1) + + self.assertEqual(test['qux'], False) + + +class TestList(unittest.TestCase): + def test_add_list(self): + test = List([1, 2, 3]) + + test += [4, 5, 6] + self.assertIsInstance(test, List) + self.assertEqual(test, [1, 2, 3, 4, 5, 6]) + + test = test + [7, 8] + self.assertIsInstance(test, List) + self.assertEqual(test, [1, 2, 3, 4, 5, 6, 7, 8]) + + def test_add_string(self): + test = List([1, 2, 3]) + + with self.assertRaises(ValueError): + test += 'string' + + def test_none(self): + """As a special exception, we allow None to be treated as an empty + list.""" + test = List([1, 2, 3]) + + test += None + self.assertEqual(test, [1, 2, 3]) + + test = test + None + self.assertIsInstance(test, List) + self.assertEqual(test, [1, 2, 3]) + + with self.assertRaises(ValueError): + test += False + + with self.assertRaises(ValueError): + test = test + False + +class TestOrderedDefaultDict(unittest.TestCase): + def test_simple(self): + original = OrderedDict(foo=1, bar=2) + + test = OrderedDefaultDict(bool, original) + + self.assertEqual(original, test) + + self.assertEqual(test['foo'], 1) + + self.assertEqual(test.keys(), ['foo', 'bar' ]) + + def test_defaults(self): + test = OrderedDefaultDict(bool, {'foo': 1 }) + + self.assertEqual(test['foo'], 1) + + self.assertEqual(test['qux'], False) + + self.assertEqual(test.keys(), ['foo', 'qux' ]) + + +class TestKeyedDefaultDict(unittest.TestCase): + def test_simple(self): + original = {'foo': 1, 'bar': 2 } + + test = KeyedDefaultDict(lambda x: x, original) + + self.assertEqual(original, test) + + self.assertEqual(test['foo'], 1) + + def test_defaults(self): + test = KeyedDefaultDict(lambda x: x, {'foo': 1 }) + + self.assertEqual(test['foo'], 1) + + self.assertEqual(test['qux'], 'qux') + + self.assertEqual(test['bar'], 'bar') + + test['foo'] = 2 + test['qux'] = None + test['baz'] = 'foo' + + self.assertEqual(test['foo'], 2) + + self.assertEqual(test['qux'], None) + + self.assertEqual(test['baz'], 'foo') + + +class TestReadOnlyKeyedDefaultDict(unittest.TestCase): + def test_defaults(self): + test = ReadOnlyKeyedDefaultDict(lambda x: x, {'foo': 1 }) + + self.assertEqual(test['foo'], 1) + + self.assertEqual(test['qux'], 'qux') + + self.assertEqual(test['bar'], 'bar') + + copy = dict(test) + + with self.assertRaises(Exception): + test['foo'] = 2 + + with self.assertRaises(Exception): + test['qux'] = None + + with self.assertRaises(Exception): + test['baz'] = 'foo' + + self.assertEqual(test, copy) + + self.assertEqual(len(test), 3) + + +if __name__ == '__main__': + main() diff --git a/python/mozbuild/mozbuild/test/test_dotproperties.py b/python/mozbuild/mozbuild/test/test_dotproperties.py new file mode 100644 index 000000000..a03f85b0d --- /dev/null +++ b/python/mozbuild/mozbuild/test/test_dotproperties.py @@ -0,0 +1,178 @@ +# -*- coding: utf-8 -*- + +from __future__ import unicode_literals + +import os +import unittest + +from StringIO import StringIO + +import mozpack.path as mozpath + +from mozbuild.dotproperties import ( + DotProperties, +) + +from mozunit import ( + main, +) + +test_data_path = mozpath.abspath(mozpath.dirname(__file__)) +test_data_path = mozpath.join(test_data_path, 'data') + + +class TestDotProperties(unittest.TestCase): + def test_get(self): + contents = StringIO(''' +key=value +''') + p = DotProperties(contents) + self.assertEqual(p.get('missing'), None) + self.assertEqual(p.get('missing', 'default'), 'default') + self.assertEqual(p.get('key'), 'value') + + + def test_update(self): + contents = StringIO(''' +old=old value +key=value +''') + p = DotProperties(contents) + self.assertEqual(p.get('old'), 'old value') + self.assertEqual(p.get('key'), 'value') + + new_contents = StringIO(''' +key=new value +''') + p.update(new_contents) + self.assertEqual(p.get('old'), 'old value') + self.assertEqual(p.get('key'), 'new value') + + + def test_get_list(self): + contents = StringIO(''' +list.0=A +list.1=B +list.2=C + +order.1=B +order.0=A +order.2=C +''') + p = DotProperties(contents) + self.assertEqual(p.get_list('missing'), []) + self.assertEqual(p.get_list('list'), ['A', 'B', 'C']) + self.assertEqual(p.get_list('order'), ['A', 'B', 'C']) + + + def test_get_list_with_shared_prefix(self): + contents = StringIO(''' +list.0=A +list.1=B +list.2=C + +list.sublist.1=E +list.sublist.0=D +list.sublist.2=F + +list.sublist.second.0=G + +list.other.0=H +''') + p = DotProperties(contents) + self.assertEqual(p.get_list('list'), ['A', 'B', 'C']) + self.assertEqual(p.get_list('list.sublist'), ['D', 'E', 'F']) + self.assertEqual(p.get_list('list.sublist.second'), ['G']) + self.assertEqual(p.get_list('list.other'), ['H']) + + + def test_get_dict(self): + contents = StringIO(''' +A.title=title A + +B.title=title B +B.url=url B + +C=value +''') + p = DotProperties(contents) + self.assertEqual(p.get_dict('missing'), {}) + self.assertEqual(p.get_dict('A'), {'title': 'title A'}) + self.assertEqual(p.get_dict('B'), {'title': 'title B', 'url': 'url B'}) + with self.assertRaises(ValueError): + p.get_dict('A', required_keys=['title', 'url']) + with self.assertRaises(ValueError): + p.get_dict('missing', required_keys=['key']) + # A key=value pair is considered to root an empty dict. + self.assertEqual(p.get_dict('C'), {}) + with self.assertRaises(ValueError): + p.get_dict('C', required_keys=['missing_key']) + + + def test_get_dict_with_shared_prefix(self): + contents = StringIO(''' +A.title=title A +A.subdict.title=title A subdict + +B.title=title B +B.url=url B +B.subdict.title=title B subdict +B.subdict.url=url B subdict +''') + p = DotProperties(contents) + self.assertEqual(p.get_dict('A'), {'title': 'title A'}) + self.assertEqual(p.get_dict('B'), {'title': 'title B', 'url': 'url B'}) + self.assertEqual(p.get_dict('A.subdict'), + {'title': 'title A subdict'}) + self.assertEqual(p.get_dict('B.subdict'), + {'title': 'title B subdict', 'url': 'url B subdict'}) + + def test_get_dict_with_value_prefix(self): + contents = StringIO(''' +A.default=A +A.default.B=B +A.default.B.ignored=B ignored +A.default.C=C +A.default.C.ignored=C ignored +''') + p = DotProperties(contents) + self.assertEqual(p.get('A.default'), 'A') + # This enumerates the properties. + self.assertEqual(p.get_dict('A.default'), {'B': 'B', 'C': 'C'}) + # They can still be fetched directly. + self.assertEqual(p.get('A.default.B'), 'B') + self.assertEqual(p.get('A.default.C'), 'C') + + + def test_unicode(self): + contents = StringIO(''' +# Danish. +# #### ~~ Søren Munk Skrøder, sskroeder - 2009-05-30 @ #mozmae + +# Korean. +A.title=í•œë©”ì¼ + +# Russian. +list.0 = test +list.1 = Ð¯Ð½Ð´ÐµÐºÑ +''') + p = DotProperties(contents) + self.assertEqual(p.get_dict('A'), {'title': '한메ì¼'}) + self.assertEqual(p.get_list('list'), ['test', 'ЯндекÑ']) + + def test_valid_unicode_from_file(self): + # The contents of valid.properties is identical to the contents of the + # test above. This specifically exercises reading from a file. + p = DotProperties(os.path.join(test_data_path, 'valid.properties')) + self.assertEqual(p.get_dict('A'), {'title': '한메ì¼'}) + self.assertEqual(p.get_list('list'), ['test', 'ЯндекÑ']) + + def test_bad_unicode_from_file(self): + # The contents of bad.properties is not valid Unicode; see the comments + # in the file itself for details. + with self.assertRaises(UnicodeDecodeError): + DotProperties(os.path.join(test_data_path, 'bad.properties')) + + +if __name__ == '__main__': + main() diff --git a/python/mozbuild/mozbuild/test/test_expression.py b/python/mozbuild/mozbuild/test/test_expression.py new file mode 100644 index 000000000..fb3c45894 --- /dev/null +++ b/python/mozbuild/mozbuild/test/test_expression.py @@ -0,0 +1,82 @@ +import unittest + +import sys +import os.path +import mozunit + +from mozbuild.preprocessor import Expression, Context + +class TestContext(unittest.TestCase): + """ + Unit tests for the Context class + """ + + def setUp(self): + self.c = Context() + self.c['FAIL'] = 'PASS' + + def test_string_literal(self): + """test string literal, fall-through for undefined var in a Context""" + self.assertEqual(self.c['PASS'], 'PASS') + + def test_variable(self): + """test value for defined var in the Context class""" + self.assertEqual(self.c['FAIL'], 'PASS') + + def test_in(self): + """test 'var in context' to not fall for fallback""" + self.assert_('FAIL' in self.c) + self.assert_('PASS' not in self.c) + +class TestExpression(unittest.TestCase): + """ + Unit tests for the Expression class + evaluate() is called with a context {FAIL: 'PASS'} + """ + + def setUp(self): + self.c = Context() + self.c['FAIL'] = 'PASS' + + def test_string_literal(self): + """Test for a string literal in an Expression""" + self.assertEqual(Expression('PASS').evaluate(self.c), 'PASS') + + def test_variable(self): + """Test for variable value in an Expression""" + self.assertEqual(Expression('FAIL').evaluate(self.c), 'PASS') + + def test_not(self): + """Test for the ! operator""" + self.assert_(Expression('!0').evaluate(self.c)) + self.assert_(not Expression('!1').evaluate(self.c)) + + def test_equals(self): + """ Test for the == operator""" + self.assert_(Expression('FAIL == PASS').evaluate(self.c)) + + def test_notequals(self): + """ Test for the != operator""" + self.assert_(Expression('FAIL != 1').evaluate(self.c)) + + def test_logical_and(self): + """ Test for the && operator""" + self.assertTrue(Expression('PASS == PASS && PASS != NOTPASS').evaluate(self.c)) + + def test_logical_or(self): + """ Test for the || operator""" + self.assertTrue(Expression('PASS == NOTPASS || PASS != NOTPASS').evaluate(self.c)) + + def test_logical_ops(self): + """ Test for the && and || operators precedence""" + # Would evaluate to false if precedence was wrong + self.assertTrue(Expression('PASS == PASS || PASS != NOTPASS && PASS == NOTPASS').evaluate(self.c)) + + def test_defined(self): + """ Test for the defined() value""" + self.assertTrue(Expression('defined(FAIL)').evaluate(self.c)) + self.assertTrue(Expression('!defined(PASS)').evaluate(self.c)) + + +if __name__ == '__main__': + mozunit.main() diff --git a/python/mozbuild/mozbuild/test/test_jarmaker.py b/python/mozbuild/mozbuild/test/test_jarmaker.py new file mode 100644 index 000000000..a4d4156a7 --- /dev/null +++ b/python/mozbuild/mozbuild/test/test_jarmaker.py @@ -0,0 +1,367 @@ +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, You can obtain one at http://mozilla.org/MPL/2.0/. + +from __future__ import print_function +import unittest + +import os, sys, os.path, time, inspect +from filecmp import dircmp +from tempfile import mkdtemp +from shutil import rmtree, copy2 +from StringIO import StringIO +from zipfile import ZipFile +import mozunit + +from mozbuild.jar import JarMaker + + +if sys.platform == "win32": + import ctypes + from ctypes import POINTER, WinError + DWORD = ctypes.c_ulong + LPDWORD = POINTER(DWORD) + HANDLE = ctypes.c_void_p + GENERIC_READ = 0x80000000 + FILE_SHARE_READ = 0x00000001 + OPEN_EXISTING = 3 + MAX_PATH = 260 + + class FILETIME(ctypes.Structure): + _fields_ = [("dwLowDateTime", DWORD), + ("dwHighDateTime", DWORD)] + + class BY_HANDLE_FILE_INFORMATION(ctypes.Structure): + _fields_ = [("dwFileAttributes", DWORD), + ("ftCreationTime", FILETIME), + ("ftLastAccessTime", FILETIME), + ("ftLastWriteTime", FILETIME), + ("dwVolumeSerialNumber", DWORD), + ("nFileSizeHigh", DWORD), + ("nFileSizeLow", DWORD), + ("nNumberOfLinks", DWORD), + ("nFileIndexHigh", DWORD), + ("nFileIndexLow", DWORD)] + + # http://msdn.microsoft.com/en-us/library/aa363858 + CreateFile = ctypes.windll.kernel32.CreateFileA + CreateFile.argtypes = [ctypes.c_char_p, DWORD, DWORD, ctypes.c_void_p, + DWORD, DWORD, HANDLE] + CreateFile.restype = HANDLE + + # http://msdn.microsoft.com/en-us/library/aa364952 + GetFileInformationByHandle = ctypes.windll.kernel32.GetFileInformationByHandle + GetFileInformationByHandle.argtypes = [HANDLE, POINTER(BY_HANDLE_FILE_INFORMATION)] + GetFileInformationByHandle.restype = ctypes.c_int + + # http://msdn.microsoft.com/en-us/library/aa364996 + GetVolumePathName = ctypes.windll.kernel32.GetVolumePathNameA + GetVolumePathName.argtypes = [ctypes.c_char_p, ctypes.c_char_p, DWORD] + GetVolumePathName.restype = ctypes.c_int + + # http://msdn.microsoft.com/en-us/library/aa364993 + GetVolumeInformation = ctypes.windll.kernel32.GetVolumeInformationA + GetVolumeInformation.argtypes = [ctypes.c_char_p, ctypes.c_char_p, DWORD, + LPDWORD, LPDWORD, LPDWORD, ctypes.c_char_p, + DWORD] + GetVolumeInformation.restype = ctypes.c_int + +def symlinks_supported(path): + if sys.platform == "win32": + # Add 1 for a trailing backslash if necessary, and 1 for the terminating + # null character. + volpath = ctypes.create_string_buffer(len(path) + 2) + rv = GetVolumePathName(path, volpath, len(volpath)) + if rv == 0: + raise WinError() + + fsname = ctypes.create_string_buffer(MAX_PATH + 1) + rv = GetVolumeInformation(volpath, None, 0, None, None, None, fsname, + len(fsname)) + if rv == 0: + raise WinError() + + # Return true only if the fsname is NTFS + return fsname.value == "NTFS" + else: + return True + +def _getfileinfo(path): + """Return information for the given file. This only works on Windows.""" + fh = CreateFile(path, GENERIC_READ, FILE_SHARE_READ, None, OPEN_EXISTING, 0, None) + if fh is None: + raise WinError() + info = BY_HANDLE_FILE_INFORMATION() + rv = GetFileInformationByHandle(fh, info) + if rv == 0: + raise WinError() + return info + +def is_symlink_to(dest, src): + if sys.platform == "win32": + # Check if both are on the same volume and have the same file ID + destinfo = _getfileinfo(dest) + srcinfo = _getfileinfo(src) + return (destinfo.dwVolumeSerialNumber == srcinfo.dwVolumeSerialNumber and + destinfo.nFileIndexHigh == srcinfo.nFileIndexHigh and + destinfo.nFileIndexLow == srcinfo.nFileIndexLow) + else: + # Read the link and check if it is correct + if not os.path.islink(dest): + return False + target = os.path.abspath(os.readlink(dest)) + abssrc = os.path.abspath(src) + return target == abssrc + +class _TreeDiff(dircmp): + """Helper to report rich results on difference between two directories. + """ + def _fillDiff(self, dc, rv, basepath="{0}"): + rv['right_only'] += map(lambda l: basepath.format(l), dc.right_only) + rv['left_only'] += map(lambda l: basepath.format(l), dc.left_only) + rv['diff_files'] += map(lambda l: basepath.format(l), dc.diff_files) + rv['funny'] += map(lambda l: basepath.format(l), dc.common_funny) + rv['funny'] += map(lambda l: basepath.format(l), dc.funny_files) + for subdir, _dc in dc.subdirs.iteritems(): + self._fillDiff(_dc, rv, basepath.format(subdir + "/{0}")) + def allResults(self, left, right): + rv = {'right_only':[], 'left_only':[], + 'diff_files':[], 'funny': []} + self._fillDiff(self, rv) + chunks = [] + if rv['right_only']: + chunks.append('{0} only in {1}'.format(', '.join(rv['right_only']), + right)) + if rv['left_only']: + chunks.append('{0} only in {1}'.format(', '.join(rv['left_only']), + left)) + if rv['diff_files']: + chunks.append('{0} differ'.format(', '.join(rv['diff_files']))) + if rv['funny']: + chunks.append("{0} don't compare".format(', '.join(rv['funny']))) + return '; '.join(chunks) + +class TestJarMaker(unittest.TestCase): + """ + Unit tests for JarMaker.py + """ + debug = False # set to True to debug failing tests on disk + def setUp(self): + self.tmpdir = mkdtemp() + self.srcdir = os.path.join(self.tmpdir, 'src') + os.mkdir(self.srcdir) + self.builddir = os.path.join(self.tmpdir, 'build') + os.mkdir(self.builddir) + self.refdir = os.path.join(self.tmpdir, 'ref') + os.mkdir(self.refdir) + self.stagedir = os.path.join(self.tmpdir, 'stage') + os.mkdir(self.stagedir) + + def tearDown(self): + if self.debug: + print(self.tmpdir) + elif sys.platform != "win32": + # can't clean up on windows + rmtree(self.tmpdir) + + def _jar_and_compare(self, infile, **kwargs): + jm = JarMaker(outputFormat='jar') + if 'topsourcedir' not in kwargs: + kwargs['topsourcedir'] = self.srcdir + for attr in ('topsourcedir', 'sourcedirs'): + if attr in kwargs: + setattr(jm, attr, kwargs[attr]) + jm.makeJar(infile, self.builddir) + cwd = os.getcwd() + os.chdir(self.builddir) + try: + # expand build to stage + for path, dirs, files in os.walk('.'): + stagedir = os.path.join(self.stagedir, path) + if not os.path.isdir(stagedir): + os.mkdir(stagedir) + for file in files: + if file.endswith('.jar'): + # expand jar + stagepath = os.path.join(stagedir, file) + os.mkdir(stagepath) + zf = ZipFile(os.path.join(path, file)) + # extractall is only in 2.6, do this manually :-( + for entry_name in zf.namelist(): + segs = entry_name.split('/') + fname = segs.pop() + dname = os.path.join(stagepath, *segs) + if not os.path.isdir(dname): + os.makedirs(dname) + if not fname: + # directory, we're done + continue + _c = zf.read(entry_name) + open(os.path.join(dname, fname), 'wb').write(_c) + zf.close() + else: + copy2(os.path.join(path, file), stagedir) + # compare both dirs + os.chdir('..') + td = _TreeDiff('ref', 'stage') + return td.allResults('reference', 'build') + finally: + os.chdir(cwd) + + def _create_simple_setup(self): + # create src content + jarf = open(os.path.join(self.srcdir, 'jar.mn'), 'w') + jarf.write('''test.jar: + dir/foo (bar) +''') + jarf.close() + open(os.path.join(self.srcdir,'bar'),'w').write('content\n') + # create reference + refpath = os.path.join(self.refdir, 'chrome', 'test.jar', 'dir') + os.makedirs(refpath) + open(os.path.join(refpath, 'foo'), 'w').write('content\n') + + def test_a_simple_jar(self): + '''Test a simple jar.mn''' + self._create_simple_setup() + # call JarMaker + rv = self._jar_and_compare(os.path.join(self.srcdir,'jar.mn'), + sourcedirs = [self.srcdir]) + self.assertTrue(not rv, rv) + + def test_a_simple_symlink(self): + '''Test a simple jar.mn with a symlink''' + if not symlinks_supported(self.srcdir): + raise unittest.SkipTest('symlinks not supported') + + self._create_simple_setup() + jm = JarMaker(outputFormat='symlink') + jm.sourcedirs = [self.srcdir] + jm.topsourcedir = self.srcdir + jm.makeJar(os.path.join(self.srcdir,'jar.mn'), self.builddir) + # All we do is check that srcdir/bar points to builddir/chrome/test/dir/foo + srcbar = os.path.join(self.srcdir, 'bar') + destfoo = os.path.join(self.builddir, 'chrome', 'test', 'dir', 'foo') + self.assertTrue(is_symlink_to(destfoo, srcbar), + "{0} is not a symlink to {1}".format(destfoo, srcbar)) + + def _create_wildcard_setup(self): + # create src content + jarf = open(os.path.join(self.srcdir, 'jar.mn'), 'w') + jarf.write('''test.jar: + dir/bar (*.js) + dir/hoge (qux/*) +''') + jarf.close() + open(os.path.join(self.srcdir,'foo.js'),'w').write('foo.js\n') + open(os.path.join(self.srcdir,'bar.js'),'w').write('bar.js\n') + os.makedirs(os.path.join(self.srcdir, 'qux', 'foo')) + open(os.path.join(self.srcdir,'qux', 'foo', '1'),'w').write('1\n') + open(os.path.join(self.srcdir,'qux', 'foo', '2'),'w').write('2\n') + open(os.path.join(self.srcdir,'qux', 'baz'),'w').write('baz\n') + # create reference + refpath = os.path.join(self.refdir, 'chrome', 'test.jar', 'dir') + os.makedirs(os.path.join(refpath, 'bar')) + os.makedirs(os.path.join(refpath, 'hoge', 'foo')) + open(os.path.join(refpath, 'bar', 'foo.js'), 'w').write('foo.js\n') + open(os.path.join(refpath, 'bar', 'bar.js'), 'w').write('bar.js\n') + open(os.path.join(refpath, 'hoge', 'foo', '1'), 'w').write('1\n') + open(os.path.join(refpath, 'hoge', 'foo', '2'), 'w').write('2\n') + open(os.path.join(refpath, 'hoge', 'baz'), 'w').write('baz\n') + + def test_a_wildcard_jar(self): + '''Test a wildcard in jar.mn''' + self._create_wildcard_setup() + # call JarMaker + rv = self._jar_and_compare(os.path.join(self.srcdir,'jar.mn'), + sourcedirs = [self.srcdir]) + self.assertTrue(not rv, rv) + + def test_a_wildcard_symlink(self): + '''Test a wildcard in jar.mn with symlinks''' + if not symlinks_supported(self.srcdir): + raise unittest.SkipTest('symlinks not supported') + + self._create_wildcard_setup() + jm = JarMaker(outputFormat='symlink') + jm.sourcedirs = [self.srcdir] + jm.topsourcedir = self.srcdir + jm.makeJar(os.path.join(self.srcdir,'jar.mn'), self.builddir) + + expected_symlinks = { + ('bar', 'foo.js'): ('foo.js',), + ('bar', 'bar.js'): ('bar.js',), + ('hoge', 'foo', '1'): ('qux', 'foo', '1'), + ('hoge', 'foo', '2'): ('qux', 'foo', '2'), + ('hoge', 'baz'): ('qux', 'baz'), + } + for dest, src in expected_symlinks.iteritems(): + srcpath = os.path.join(self.srcdir, *src) + destpath = os.path.join(self.builddir, 'chrome', 'test', 'dir', + *dest) + self.assertTrue(is_symlink_to(destpath, srcpath), + "{0} is not a symlink to {1}".format(destpath, + srcpath)) + + +class Test_relativesrcdir(unittest.TestCase): + def setUp(self): + self.jm = JarMaker() + self.jm.topsourcedir = '/TOPSOURCEDIR' + self.jm.relativesrcdir = 'browser/locales' + self.fake_empty_file = StringIO() + self.fake_empty_file.name = 'fake_empty_file' + def tearDown(self): + del self.jm + del self.fake_empty_file + def test_en_US(self): + jm = self.jm + jm.makeJar(self.fake_empty_file, '/NO_OUTPUT_REQUIRED') + self.assertEquals(jm.localedirs, + [ + os.path.join(os.path.abspath('/TOPSOURCEDIR'), + 'browser/locales', 'en-US') + ]) + def test_l10n_no_merge(self): + jm = self.jm + jm.l10nbase = '/L10N_BASE' + jm.makeJar(self.fake_empty_file, '/NO_OUTPUT_REQUIRED') + self.assertEquals(jm.localedirs, [os.path.join('/L10N_BASE', 'browser')]) + def test_l10n_merge(self): + jm = self.jm + jm.l10nbase = '/L10N_BASE' + jm.l10nmerge = '/L10N_MERGE' + jm.makeJar(self.fake_empty_file, '/NO_OUTPUT_REQUIRED') + self.assertEquals(jm.localedirs, + [os.path.join('/L10N_MERGE', 'browser'), + os.path.join('/L10N_BASE', 'browser'), + os.path.join(os.path.abspath('/TOPSOURCEDIR'), + 'browser/locales', 'en-US') + ]) + def test_override(self): + jm = self.jm + jm.outputFormat = 'flat' # doesn't touch chrome dir without files + jarcontents = StringIO('''en-US.jar: +relativesrcdir dom/locales: +''') + jarcontents.name = 'override.mn' + jm.makeJar(jarcontents, '/NO_OUTPUT_REQUIRED') + self.assertEquals(jm.localedirs, + [ + os.path.join(os.path.abspath('/TOPSOURCEDIR'), + 'dom/locales', 'en-US') + ]) + def test_override_l10n(self): + jm = self.jm + jm.l10nbase = '/L10N_BASE' + jm.outputFormat = 'flat' # doesn't touch chrome dir without files + jarcontents = StringIO('''en-US.jar: +relativesrcdir dom/locales: +''') + jarcontents.name = 'override.mn' + jm.makeJar(jarcontents, '/NO_OUTPUT_REQUIRED') + self.assertEquals(jm.localedirs, [os.path.join('/L10N_BASE', 'dom')]) + + +if __name__ == '__main__': + mozunit.main() diff --git a/python/mozbuild/mozbuild/test/test_line_endings.py b/python/mozbuild/mozbuild/test/test_line_endings.py new file mode 100644 index 000000000..565abc8c9 --- /dev/null +++ b/python/mozbuild/mozbuild/test/test_line_endings.py @@ -0,0 +1,46 @@ +import unittest + +from StringIO import StringIO +import os +import sys +import os.path +import mozunit + +from mozbuild.preprocessor import Preprocessor + +class TestLineEndings(unittest.TestCase): + """ + Unit tests for the Context class + """ + + def setUp(self): + self.pp = Preprocessor() + self.pp.out = StringIO() + self.tempnam = os.tempnam('.') + + def tearDown(self): + os.remove(self.tempnam) + + def createFile(self, lineendings): + f = open(self.tempnam, 'wb') + for line, ending in zip(['a', '#literal b', 'c'], lineendings): + f.write(line+ending) + f.close() + + def testMac(self): + self.createFile(['\x0D']*3) + self.pp.do_include(self.tempnam) + self.assertEquals(self.pp.out.getvalue(), 'a\nb\nc\n') + + def testUnix(self): + self.createFile(['\x0A']*3) + self.pp.do_include(self.tempnam) + self.assertEquals(self.pp.out.getvalue(), 'a\nb\nc\n') + + def testWindows(self): + self.createFile(['\x0D\x0A']*3) + self.pp.do_include(self.tempnam) + self.assertEquals(self.pp.out.getvalue(), 'a\nb\nc\n') + +if __name__ == '__main__': + mozunit.main() diff --git a/python/mozbuild/mozbuild/test/test_makeutil.py b/python/mozbuild/mozbuild/test/test_makeutil.py new file mode 100644 index 000000000..6fffa0e0e --- /dev/null +++ b/python/mozbuild/mozbuild/test/test_makeutil.py @@ -0,0 +1,165 @@ +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, You can obtain one at http://mozilla.org/MPL/2.0/. + +from mozbuild.makeutil import ( + Makefile, + read_dep_makefile, + Rule, + write_dep_makefile, +) +from mozunit import main +import os +import unittest +from StringIO import StringIO + + +class TestMakefile(unittest.TestCase): + def test_rule(self): + out = StringIO() + rule = Rule() + rule.dump(out) + self.assertEqual(out.getvalue(), '') + out.truncate(0) + + rule.add_targets(['foo', 'bar']) + rule.dump(out) + self.assertEqual(out.getvalue(), 'foo bar:\n') + out.truncate(0) + + rule.add_targets(['baz']) + rule.add_dependencies(['qux', 'hoge', 'piyo']) + rule.dump(out) + self.assertEqual(out.getvalue(), 'foo bar baz: qux hoge piyo\n') + out.truncate(0) + + rule = Rule(['foo', 'bar']) + rule.add_dependencies(['baz']) + rule.add_commands(['echo $@']) + rule.add_commands(['$(BAZ) -o $@ $<', '$(TOUCH) $@']) + rule.dump(out) + self.assertEqual(out.getvalue(), + 'foo bar: baz\n' + + '\techo $@\n' + + '\t$(BAZ) -o $@ $<\n' + + '\t$(TOUCH) $@\n') + out.truncate(0) + + rule = Rule(['foo']) + rule.add_dependencies(['bar', 'foo', 'baz']) + rule.dump(out) + self.assertEqual(out.getvalue(), 'foo: bar baz\n') + out.truncate(0) + + rule.add_targets(['bar']) + rule.dump(out) + self.assertEqual(out.getvalue(), 'foo bar: baz\n') + out.truncate(0) + + rule.add_targets(['bar']) + rule.dump(out) + self.assertEqual(out.getvalue(), 'foo bar: baz\n') + out.truncate(0) + + rule.add_dependencies(['bar']) + rule.dump(out) + self.assertEqual(out.getvalue(), 'foo bar: baz\n') + out.truncate(0) + + rule.add_dependencies(['qux']) + rule.dump(out) + self.assertEqual(out.getvalue(), 'foo bar: baz qux\n') + out.truncate(0) + + rule.add_dependencies(['qux']) + rule.dump(out) + self.assertEqual(out.getvalue(), 'foo bar: baz qux\n') + out.truncate(0) + + rule.add_dependencies(['hoge', 'hoge']) + rule.dump(out) + self.assertEqual(out.getvalue(), 'foo bar: baz qux hoge\n') + out.truncate(0) + + rule.add_targets(['fuga', 'fuga']) + rule.dump(out) + self.assertEqual(out.getvalue(), 'foo bar fuga: baz qux hoge\n') + + def test_makefile(self): + out = StringIO() + mk = Makefile() + rule = mk.create_rule(['foo']) + rule.add_dependencies(['bar', 'baz', 'qux']) + rule.add_commands(['echo foo']) + rule = mk.create_rule().add_targets(['bar', 'baz']) + rule.add_dependencies(['hoge']) + rule.add_commands(['echo $@']) + mk.dump(out, removal_guard=False) + self.assertEqual(out.getvalue(), + 'foo: bar baz qux\n' + + '\techo foo\n' + + 'bar baz: hoge\n' + + '\techo $@\n') + out.truncate(0) + + mk.dump(out) + self.assertEqual(out.getvalue(), + 'foo: bar baz qux\n' + + '\techo foo\n' + + 'bar baz: hoge\n' + + '\techo $@\n' + + 'hoge qux:\n') + + def test_statement(self): + out = StringIO() + mk = Makefile() + mk.create_rule(['foo']).add_dependencies(['bar']) \ + .add_commands(['echo foo']) + mk.add_statement('BAR = bar') + mk.create_rule(['$(BAR)']).add_commands(['echo $@']) + mk.dump(out, removal_guard=False) + self.assertEqual(out.getvalue(), + 'foo: bar\n' + + '\techo foo\n' + + 'BAR = bar\n' + + '$(BAR):\n' + + '\techo $@\n') + + @unittest.skipIf(os.name != 'nt', 'Test only applicable on Windows.') + def test_path_normalization(self): + out = StringIO() + mk = Makefile() + rule = mk.create_rule(['c:\\foo']) + rule.add_dependencies(['c:\\bar', 'c:\\baz\\qux']) + rule.add_commands(['echo c:\\foo']) + mk.dump(out) + self.assertEqual(out.getvalue(), + 'c:/foo: c:/bar c:/baz/qux\n' + + '\techo c:\\foo\n' + + 'c:/bar c:/baz/qux:\n') + + def test_read_dep_makefile(self): + input = StringIO( + os.path.abspath('foo') + ': bar\n' + + 'baz qux: \\ \n' + + 'hoge \\\n' + + 'piyo \\\n' + + 'fuga\n' + + 'fuga:\n' + ) + result = list(read_dep_makefile(input)) + self.assertEqual(len(result), 2) + self.assertEqual(list(result[0].targets()), [os.path.abspath('foo').replace(os.sep, '/')]) + self.assertEqual(list(result[0].dependencies()), ['bar']) + self.assertEqual(list(result[1].targets()), ['baz', 'qux']) + self.assertEqual(list(result[1].dependencies()), ['hoge', 'piyo', 'fuga']) + + def test_write_dep_makefile(self): + out = StringIO() + write_dep_makefile(out, 'target', ['b', 'c', 'a']) + self.assertEqual(out.getvalue(), + 'target: b c a\n' + + 'a b c:\n') + +if __name__ == '__main__': + main() diff --git a/python/mozbuild/mozbuild/test/test_mozconfig.py b/python/mozbuild/mozbuild/test/test_mozconfig.py new file mode 100644 index 000000000..0cd125912 --- /dev/null +++ b/python/mozbuild/mozbuild/test/test_mozconfig.py @@ -0,0 +1,489 @@ +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, You can obtain one at http://mozilla.org/MPL/2.0/. + +from __future__ import unicode_literals + +import os +import unittest + +from shutil import rmtree + +from tempfile import ( + gettempdir, + mkdtemp, +) + +from mozfile.mozfile import NamedTemporaryFile + +from mozunit import main + +from mozbuild.mozconfig import ( + MozconfigFindException, + MozconfigLoadException, + MozconfigLoader, +) + + +class TestMozconfigLoader(unittest.TestCase): + def setUp(self): + self._old_env = dict(os.environ) + os.environ.pop('MOZCONFIG', None) + os.environ.pop('MOZ_OBJDIR', None) + os.environ.pop('CC', None) + os.environ.pop('CXX', None) + self._temp_dirs = set() + + def tearDown(self): + os.environ.clear() + os.environ.update(self._old_env) + + for d in self._temp_dirs: + rmtree(d) + + def get_loader(self): + return MozconfigLoader(self.get_temp_dir()) + + def get_temp_dir(self): + d = mkdtemp() + self._temp_dirs.add(d) + + return d + + def test_find_legacy_env(self): + """Ensure legacy mozconfig path definitions result in error.""" + + os.environ[b'MOZ_MYCONFIG'] = '/foo' + + with self.assertRaises(MozconfigFindException) as e: + self.get_loader().find_mozconfig() + + self.assertTrue(e.exception.message.startswith('The MOZ_MYCONFIG')) + + def test_find_multiple_configs(self): + """Ensure multiple relative-path MOZCONFIGs result in error.""" + relative_mozconfig = '.mconfig' + os.environ[b'MOZCONFIG'] = relative_mozconfig + + srcdir = self.get_temp_dir() + curdir = self.get_temp_dir() + dirs = [srcdir, curdir] + loader = MozconfigLoader(srcdir) + for d in dirs: + path = os.path.join(d, relative_mozconfig) + with open(path, 'wb') as f: + f.write(path) + + orig_dir = os.getcwd() + try: + os.chdir(curdir) + with self.assertRaises(MozconfigFindException) as e: + loader.find_mozconfig() + finally: + os.chdir(orig_dir) + + self.assertIn('exists in more than one of', e.exception.message) + for d in dirs: + self.assertIn(d, e.exception.message) + + def test_find_multiple_but_identical_configs(self): + """Ensure multiple relative-path MOZCONFIGs pointing at the same file are OK.""" + relative_mozconfig = '../src/.mconfig' + os.environ[b'MOZCONFIG'] = relative_mozconfig + + topdir = self.get_temp_dir() + srcdir = os.path.join(topdir, 'src') + os.mkdir(srcdir) + curdir = os.path.join(topdir, 'obj') + os.mkdir(curdir) + + loader = MozconfigLoader(srcdir) + path = os.path.join(srcdir, relative_mozconfig) + with open(path, 'w'): + pass + + orig_dir = os.getcwd() + try: + os.chdir(curdir) + self.assertEqual(os.path.realpath(loader.find_mozconfig()), + os.path.realpath(path)) + finally: + os.chdir(orig_dir) + + def test_find_no_relative_configs(self): + """Ensure a missing relative-path MOZCONFIG is detected.""" + relative_mozconfig = '.mconfig' + os.environ[b'MOZCONFIG'] = relative_mozconfig + + srcdir = self.get_temp_dir() + curdir = self.get_temp_dir() + dirs = [srcdir, curdir] + loader = MozconfigLoader(srcdir) + + orig_dir = os.getcwd() + try: + os.chdir(curdir) + with self.assertRaises(MozconfigFindException) as e: + loader.find_mozconfig() + finally: + os.chdir(orig_dir) + + self.assertIn('does not exist in any of', e.exception.message) + for d in dirs: + self.assertIn(d, e.exception.message) + + def test_find_relative_mozconfig(self): + """Ensure a relative MOZCONFIG can be found in the srcdir.""" + relative_mozconfig = '.mconfig' + os.environ[b'MOZCONFIG'] = relative_mozconfig + + srcdir = self.get_temp_dir() + curdir = self.get_temp_dir() + dirs = [srcdir, curdir] + loader = MozconfigLoader(srcdir) + + path = os.path.join(srcdir, relative_mozconfig) + with open(path, 'w'): + pass + + orig_dir = os.getcwd() + try: + os.chdir(curdir) + self.assertEqual(os.path.normpath(loader.find_mozconfig()), + os.path.normpath(path)) + finally: + os.chdir(orig_dir) + + def test_find_abs_path_not_exist(self): + """Ensure a missing absolute path is detected.""" + os.environ[b'MOZCONFIG'] = '/foo/bar/does/not/exist' + + with self.assertRaises(MozconfigFindException) as e: + self.get_loader().find_mozconfig() + + self.assertIn('path that does not exist', e.exception.message) + self.assertTrue(e.exception.message.endswith('/foo/bar/does/not/exist')) + + def test_find_path_not_file(self): + """Ensure non-file paths are detected.""" + + os.environ[b'MOZCONFIG'] = gettempdir() + + with self.assertRaises(MozconfigFindException) as e: + self.get_loader().find_mozconfig() + + self.assertIn('refers to a non-file', e.exception.message) + self.assertTrue(e.exception.message.endswith(gettempdir())) + + def test_find_default_files(self): + """Ensure default paths are used when present.""" + for p in MozconfigLoader.DEFAULT_TOPSRCDIR_PATHS: + d = self.get_temp_dir() + path = os.path.join(d, p) + + with open(path, 'w'): + pass + + self.assertEqual(MozconfigLoader(d).find_mozconfig(), path) + + def test_find_multiple_defaults(self): + """Ensure we error when multiple default files are present.""" + self.assertGreater(len(MozconfigLoader.DEFAULT_TOPSRCDIR_PATHS), 1) + + d = self.get_temp_dir() + for p in MozconfigLoader.DEFAULT_TOPSRCDIR_PATHS: + with open(os.path.join(d, p), 'w'): + pass + + with self.assertRaises(MozconfigFindException) as e: + MozconfigLoader(d).find_mozconfig() + + self.assertIn('Multiple default mozconfig files present', + e.exception.message) + + def test_find_deprecated_path_srcdir(self): + """Ensure we error when deprecated path locations are present.""" + for p in MozconfigLoader.DEPRECATED_TOPSRCDIR_PATHS: + d = self.get_temp_dir() + with open(os.path.join(d, p), 'w'): + pass + + with self.assertRaises(MozconfigFindException) as e: + MozconfigLoader(d).find_mozconfig() + + self.assertIn('This implicit location is no longer', + e.exception.message) + self.assertIn(d, e.exception.message) + + def test_find_deprecated_home_paths(self): + """Ensure we error when deprecated home directory paths are present.""" + + for p in MozconfigLoader.DEPRECATED_HOME_PATHS: + home = self.get_temp_dir() + os.environ[b'HOME'] = home + path = os.path.join(home, p) + + with open(path, 'w'): + pass + + with self.assertRaises(MozconfigFindException) as e: + self.get_loader().find_mozconfig() + + self.assertIn('This implicit location is no longer', + e.exception.message) + self.assertIn(path, e.exception.message) + + def test_read_no_mozconfig(self): + # This is basically to ensure changes to defaults incur a test failure. + result = self.get_loader().read_mozconfig() + + self.assertEqual(result, { + 'path': None, + 'topobjdir': None, + 'configure_args': None, + 'make_flags': None, + 'make_extra': None, + 'env': None, + 'vars': None, + }) + + def test_read_empty_mozconfig(self): + with NamedTemporaryFile(mode='w') as mozconfig: + result = self.get_loader().read_mozconfig(mozconfig.name) + + self.assertEqual(result['path'], mozconfig.name) + self.assertIsNone(result['topobjdir']) + self.assertEqual(result['configure_args'], []) + self.assertEqual(result['make_flags'], []) + self.assertEqual(result['make_extra'], []) + + for f in ('added', 'removed', 'modified'): + self.assertEqual(len(result['vars'][f]), 0) + self.assertEqual(len(result['env'][f]), 0) + + self.assertEqual(result['env']['unmodified'], {}) + + def test_read_capture_ac_options(self): + """Ensures ac_add_options calls are captured.""" + with NamedTemporaryFile(mode='w') as mozconfig: + mozconfig.write('ac_add_options --enable-debug\n') + mozconfig.write('ac_add_options --disable-tests --enable-foo\n') + mozconfig.write('ac_add_options --foo="bar baz"\n') + mozconfig.flush() + + result = self.get_loader().read_mozconfig(mozconfig.name) + self.assertEqual(result['configure_args'], [ + '--enable-debug', '--disable-tests', '--enable-foo', + '--foo=bar baz']) + + def test_read_ac_options_substitution(self): + """Ensure ac_add_options values are substituted.""" + with NamedTemporaryFile(mode='w') as mozconfig: + mozconfig.write('ac_add_options --foo=@TOPSRCDIR@\n') + mozconfig.flush() + + loader = self.get_loader() + result = loader.read_mozconfig(mozconfig.name) + self.assertEqual(result['configure_args'], [ + '--foo=%s' % loader.topsrcdir]) + + def test_read_ac_app_options(self): + with NamedTemporaryFile(mode='w') as mozconfig: + mozconfig.write('ac_add_options --foo=@TOPSRCDIR@\n') + mozconfig.write('ac_add_app_options app1 --bar=@TOPSRCDIR@\n') + mozconfig.write('ac_add_app_options app2 --bar=x\n') + mozconfig.flush() + + loader = self.get_loader() + result = loader.read_mozconfig(mozconfig.name, moz_build_app='app1') + self.assertEqual(result['configure_args'], [ + '--foo=%s' % loader.topsrcdir, + '--bar=%s' % loader.topsrcdir]) + + result = loader.read_mozconfig(mozconfig.name, moz_build_app='app2') + self.assertEqual(result['configure_args'], [ + '--foo=%s' % loader.topsrcdir, + '--bar=x']) + + def test_read_capture_mk_options(self): + """Ensures mk_add_options calls are captured.""" + with NamedTemporaryFile(mode='w') as mozconfig: + mozconfig.write('mk_add_options MOZ_OBJDIR=/foo/bar\n') + mozconfig.write('mk_add_options MOZ_MAKE_FLAGS="-j8 -s"\n') + mozconfig.write('mk_add_options FOO="BAR BAZ"\n') + mozconfig.write('mk_add_options BIZ=1\n') + mozconfig.flush() + + result = self.get_loader().read_mozconfig(mozconfig.name) + self.assertEqual(result['topobjdir'], '/foo/bar') + self.assertEqual(result['make_flags'], ['-j8', '-s']) + self.assertEqual(result['make_extra'], ['FOO=BAR BAZ', 'BIZ=1']) + + vars = result['vars']['added'] + for var in ('MOZ_OBJDIR', 'MOZ_MAKE_FLAGS', 'FOO', 'BIZ'): + self.assertEqual(vars.get('%s_IS_SET' % var), '1') + + def test_read_empty_mozconfig_objdir_environ(self): + os.environ[b'MOZ_OBJDIR'] = b'obj-firefox' + with NamedTemporaryFile(mode='w') as mozconfig: + result = self.get_loader().read_mozconfig(mozconfig.name) + self.assertEqual(result['topobjdir'], 'obj-firefox') + + def test_read_capture_mk_options_objdir_environ(self): + """Ensures mk_add_options calls are captured and override the environ.""" + os.environ[b'MOZ_OBJDIR'] = b'obj-firefox' + with NamedTemporaryFile(mode='w') as mozconfig: + mozconfig.write('mk_add_options MOZ_OBJDIR=/foo/bar\n') + mozconfig.flush() + + result = self.get_loader().read_mozconfig(mozconfig.name) + self.assertEqual(result['topobjdir'], '/foo/bar') + + def test_read_moz_objdir_substitution(self): + """Ensure @TOPSRCDIR@ substitution is recognized in MOZ_OBJDIR.""" + with NamedTemporaryFile(mode='w') as mozconfig: + mozconfig.write('mk_add_options MOZ_OBJDIR=@TOPSRCDIR@/some-objdir') + mozconfig.flush() + + loader = self.get_loader() + result = loader.read_mozconfig(mozconfig.name) + + self.assertEqual(result['topobjdir'], '%s/some-objdir' % + loader.topsrcdir) + + def test_read_new_variables(self): + """New variables declared in mozconfig file are detected.""" + with NamedTemporaryFile(mode='w') as mozconfig: + mozconfig.write('CC=/usr/local/bin/clang\n') + mozconfig.write('CXX=/usr/local/bin/clang++\n') + mozconfig.flush() + + result = self.get_loader().read_mozconfig(mozconfig.name) + + self.assertEqual(result['vars']['added'], { + 'CC': '/usr/local/bin/clang', + 'CXX': '/usr/local/bin/clang++'}) + self.assertEqual(result['env']['added'], {}) + + def test_read_exported_variables(self): + """Exported variables are caught as new variables.""" + with NamedTemporaryFile(mode='w') as mozconfig: + mozconfig.write('export MY_EXPORTED=woot\n') + mozconfig.flush() + + result = self.get_loader().read_mozconfig(mozconfig.name) + + self.assertEqual(result['vars']['added'], {}) + self.assertEqual(result['env']['added'], { + 'MY_EXPORTED': 'woot'}) + + def test_read_modify_variables(self): + """Variables modified by mozconfig are detected.""" + old_path = os.path.realpath(b'/usr/bin/gcc') + new_path = os.path.realpath(b'/usr/local/bin/clang') + os.environ[b'CC'] = old_path + + with NamedTemporaryFile(mode='w') as mozconfig: + mozconfig.write('CC="%s"\n' % new_path) + mozconfig.flush() + + result = self.get_loader().read_mozconfig(mozconfig.name) + + self.assertEqual(result['vars']['modified'], {}) + self.assertEqual(result['env']['modified'], { + 'CC': (old_path, new_path) + }) + + def test_read_unmodified_variables(self): + """Variables modified by mozconfig are detected.""" + cc_path = os.path.realpath(b'/usr/bin/gcc') + os.environ[b'CC'] = cc_path + + with NamedTemporaryFile(mode='w') as mozconfig: + mozconfig.flush() + + result = self.get_loader().read_mozconfig(mozconfig.name) + + self.assertEqual(result['vars']['unmodified'], {}) + self.assertEqual(result['env']['unmodified'], { + 'CC': cc_path + }) + + def test_read_removed_variables(self): + """Variables unset by the mozconfig are detected.""" + cc_path = os.path.realpath(b'/usr/bin/clang') + os.environ[b'CC'] = cc_path + + with NamedTemporaryFile(mode='w') as mozconfig: + mozconfig.write('unset CC\n') + mozconfig.flush() + + result = self.get_loader().read_mozconfig(mozconfig.name) + + self.assertEqual(result['vars']['removed'], {}) + self.assertEqual(result['env']['removed'], { + 'CC': cc_path}) + + def test_read_multiline_variables(self): + """Ensure multi-line variables are captured properly.""" + with NamedTemporaryFile(mode='w') as mozconfig: + mozconfig.write('multi="foo\nbar"\n') + mozconfig.write('single=1\n') + mozconfig.flush() + + result = self.get_loader().read_mozconfig(mozconfig.name) + + self.assertEqual(result['vars']['added'], { + 'multi': 'foo\nbar', + 'single': '1' + }) + self.assertEqual(result['env']['added'], {}) + + def test_read_topsrcdir_defined(self): + """Ensure $topsrcdir references work as expected.""" + with NamedTemporaryFile(mode='w') as mozconfig: + mozconfig.write('TEST=$topsrcdir') + mozconfig.flush() + + loader = self.get_loader() + result = loader.read_mozconfig(mozconfig.name) + + self.assertEqual(result['vars']['added']['TEST'], + loader.topsrcdir.replace(os.sep, '/')) + self.assertEqual(result['env']['added'], {}) + + def test_read_empty_variable_value(self): + """Ensure empty variable values are parsed properly.""" + with NamedTemporaryFile(mode='w') as mozconfig: + mozconfig.write('EMPTY=\n') + mozconfig.write('export EXPORT_EMPTY=\n') + mozconfig.flush() + + result = self.get_loader().read_mozconfig(mozconfig.name) + + self.assertEqual(result['vars']['added'], { + 'EMPTY': '', + }) + self.assertEqual(result['env']['added'], { + 'EXPORT_EMPTY': '' + }) + + def test_read_load_exception(self): + """Ensure non-0 exit codes in mozconfigs are handled properly.""" + with NamedTemporaryFile(mode='w') as mozconfig: + mozconfig.write('echo "hello world"\n') + mozconfig.write('exit 1\n') + mozconfig.flush() + + with self.assertRaises(MozconfigLoadException) as e: + self.get_loader().read_mozconfig(mozconfig.name) + + self.assertTrue(e.exception.message.startswith( + 'Evaluation of your mozconfig exited with an error')) + self.assertEquals(e.exception.path, + mozconfig.name.replace(os.sep, '/')) + self.assertEquals(e.exception.output, ['hello world']) + + +if __name__ == '__main__': + main() diff --git a/python/mozbuild/mozbuild/test/test_mozinfo.py b/python/mozbuild/mozbuild/test/test_mozinfo.py new file mode 100755 index 000000000..1a4194cb5 --- /dev/null +++ b/python/mozbuild/mozbuild/test/test_mozinfo.py @@ -0,0 +1,278 @@ +#!/usr/bin/env python +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this file, +# You can obtain one at http://mozilla.org/MPL/2.0/. + +import json +import os +import tempfile +import unittest + +from StringIO import StringIO + +import mozunit + +from mozbuild.backend.configenvironment import ConfigEnvironment + +from mozbuild.mozinfo import ( + build_dict, + write_mozinfo, +) + +from mozfile.mozfile import NamedTemporaryFile + + +class Base(object): + def _config(self, substs={}): + d = os.path.dirname(__file__) + return ConfigEnvironment(d, d, substs=substs) + + +class TestBuildDict(unittest.TestCase, Base): + def test_missing(self): + """ + Test that missing required values raises. + """ + + with self.assertRaises(Exception): + build_dict(self._config(substs=dict(OS_TARGET='foo'))) + + with self.assertRaises(Exception): + build_dict(self._config(substs=dict(TARGET_CPU='foo'))) + + with self.assertRaises(Exception): + build_dict(self._config(substs=dict(MOZ_WIDGET_TOOLKIT='foo'))) + + def test_win(self): + d = build_dict(self._config(dict( + OS_TARGET='WINNT', + TARGET_CPU='i386', + MOZ_WIDGET_TOOLKIT='windows', + ))) + self.assertEqual('win', d['os']) + self.assertEqual('x86', d['processor']) + self.assertEqual('windows', d['toolkit']) + self.assertEqual(32, d['bits']) + + def test_linux(self): + d = build_dict(self._config(dict( + OS_TARGET='Linux', + TARGET_CPU='i386', + MOZ_WIDGET_TOOLKIT='gtk2', + ))) + self.assertEqual('linux', d['os']) + self.assertEqual('x86', d['processor']) + self.assertEqual('gtk2', d['toolkit']) + self.assertEqual(32, d['bits']) + + d = build_dict(self._config(dict( + OS_TARGET='Linux', + TARGET_CPU='x86_64', + MOZ_WIDGET_TOOLKIT='gtk2', + ))) + self.assertEqual('linux', d['os']) + self.assertEqual('x86_64', d['processor']) + self.assertEqual('gtk2', d['toolkit']) + self.assertEqual(64, d['bits']) + + def test_mac(self): + d = build_dict(self._config(dict( + OS_TARGET='Darwin', + TARGET_CPU='i386', + MOZ_WIDGET_TOOLKIT='cocoa', + ))) + self.assertEqual('mac', d['os']) + self.assertEqual('x86', d['processor']) + self.assertEqual('cocoa', d['toolkit']) + self.assertEqual(32, d['bits']) + + d = build_dict(self._config(dict( + OS_TARGET='Darwin', + TARGET_CPU='x86_64', + MOZ_WIDGET_TOOLKIT='cocoa', + ))) + self.assertEqual('mac', d['os']) + self.assertEqual('x86_64', d['processor']) + self.assertEqual('cocoa', d['toolkit']) + self.assertEqual(64, d['bits']) + + def test_mac_universal(self): + d = build_dict(self._config(dict( + OS_TARGET='Darwin', + TARGET_CPU='i386', + MOZ_WIDGET_TOOLKIT='cocoa', + UNIVERSAL_BINARY='1', + ))) + self.assertEqual('mac', d['os']) + self.assertEqual('universal-x86-x86_64', d['processor']) + self.assertEqual('cocoa', d['toolkit']) + self.assertFalse('bits' in d) + + d = build_dict(self._config(dict( + OS_TARGET='Darwin', + TARGET_CPU='x86_64', + MOZ_WIDGET_TOOLKIT='cocoa', + UNIVERSAL_BINARY='1', + ))) + self.assertEqual('mac', d['os']) + self.assertEqual('universal-x86-x86_64', d['processor']) + self.assertEqual('cocoa', d['toolkit']) + self.assertFalse('bits' in d) + + def test_android(self): + d = build_dict(self._config(dict( + OS_TARGET='Android', + TARGET_CPU='arm', + MOZ_WIDGET_TOOLKIT='android', + ))) + self.assertEqual('android', d['os']) + self.assertEqual('arm', d['processor']) + self.assertEqual('android', d['toolkit']) + self.assertEqual(32, d['bits']) + + def test_x86(self): + """ + Test that various i?86 values => x86. + """ + d = build_dict(self._config(dict( + OS_TARGET='WINNT', + TARGET_CPU='i486', + MOZ_WIDGET_TOOLKIT='windows', + ))) + self.assertEqual('x86', d['processor']) + + d = build_dict(self._config(dict( + OS_TARGET='WINNT', + TARGET_CPU='i686', + MOZ_WIDGET_TOOLKIT='windows', + ))) + self.assertEqual('x86', d['processor']) + + def test_arm(self): + """ + Test that all arm CPU architectures => arm. + """ + d = build_dict(self._config(dict( + OS_TARGET='Linux', + TARGET_CPU='arm', + MOZ_WIDGET_TOOLKIT='gtk2', + ))) + self.assertEqual('arm', d['processor']) + + d = build_dict(self._config(dict( + OS_TARGET='Linux', + TARGET_CPU='armv7', + MOZ_WIDGET_TOOLKIT='gtk2', + ))) + self.assertEqual('arm', d['processor']) + + def test_unknown(self): + """ + Test that unknown values pass through okay. + """ + d = build_dict(self._config(dict( + OS_TARGET='RandOS', + TARGET_CPU='cptwo', + MOZ_WIDGET_TOOLKIT='foobar', + ))) + self.assertEqual("randos", d["os"]) + self.assertEqual("cptwo", d["processor"]) + self.assertEqual("foobar", d["toolkit"]) + # unknown CPUs should not get a bits value + self.assertFalse("bits" in d) + + def test_debug(self): + """ + Test that debug values are properly detected. + """ + d = build_dict(self._config(dict( + OS_TARGET='Linux', + TARGET_CPU='i386', + MOZ_WIDGET_TOOLKIT='gtk2', + ))) + self.assertEqual(False, d['debug']) + + d = build_dict(self._config(dict( + OS_TARGET='Linux', + TARGET_CPU='i386', + MOZ_WIDGET_TOOLKIT='gtk2', + MOZ_DEBUG='1', + ))) + self.assertEqual(True, d['debug']) + + def test_crashreporter(self): + """ + Test that crashreporter values are properly detected. + """ + d = build_dict(self._config(dict( + OS_TARGET='Linux', + TARGET_CPU='i386', + MOZ_WIDGET_TOOLKIT='gtk2', + ))) + self.assertEqual(False, d['crashreporter']) + + d = build_dict(self._config(dict( + OS_TARGET='Linux', + TARGET_CPU='i386', + MOZ_WIDGET_TOOLKIT='gtk2', + MOZ_CRASHREPORTER='1', + ))) + self.assertEqual(True, d['crashreporter']) + + +class TestWriteMozinfo(unittest.TestCase, Base): + """ + Test the write_mozinfo function. + """ + def setUp(self): + fd, self.f = tempfile.mkstemp() + os.close(fd) + + def tearDown(self): + os.unlink(self.f) + + def test_basic(self): + """ + Test that writing to a file produces correct output. + """ + c = self._config(dict( + OS_TARGET='WINNT', + TARGET_CPU='i386', + MOZ_WIDGET_TOOLKIT='windows', + )) + tempdir = tempfile.tempdir + c.topsrcdir = tempdir + with NamedTemporaryFile(dir=os.path.normpath(c.topsrcdir)) as mozconfig: + mozconfig.write('unused contents') + mozconfig.flush() + c.mozconfig = mozconfig.name + write_mozinfo(self.f, c) + with open(self.f) as f: + d = json.load(f) + self.assertEqual('win', d['os']) + self.assertEqual('x86', d['processor']) + self.assertEqual('windows', d['toolkit']) + self.assertEqual(tempdir, d['topsrcdir']) + self.assertEqual(mozconfig.name, d['mozconfig']) + self.assertEqual(32, d['bits']) + + def test_fileobj(self): + """ + Test that writing to a file-like object produces correct output. + """ + s = StringIO() + c = self._config(dict( + OS_TARGET='WINNT', + TARGET_CPU='i386', + MOZ_WIDGET_TOOLKIT='windows', + )) + write_mozinfo(s, c) + d = json.loads(s.getvalue()) + self.assertEqual('win', d['os']) + self.assertEqual('x86', d['processor']) + self.assertEqual('windows', d['toolkit']) + self.assertEqual(32, d['bits']) + + +if __name__ == '__main__': + mozunit.main() diff --git a/python/mozbuild/mozbuild/test/test_preprocessor.py b/python/mozbuild/mozbuild/test/test_preprocessor.py new file mode 100644 index 000000000..9aba94853 --- /dev/null +++ b/python/mozbuild/mozbuild/test/test_preprocessor.py @@ -0,0 +1,646 @@ +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, You can obtain one at http://mozilla.org/MPL/2.0/. + +import unittest + +from StringIO import StringIO +import os +import shutil + +from tempfile import mkdtemp + +from mozunit import main, MockedOpen + +from mozbuild.preprocessor import Preprocessor + + +class TestPreprocessor(unittest.TestCase): + """ + Unit tests for the Context class + """ + + def setUp(self): + self.pp = Preprocessor() + self.pp.out = StringIO() + + def do_include_compare(self, content_lines, expected_lines): + content = '%s' % '\n'.join(content_lines) + expected = '%s'.rstrip() % '\n'.join(expected_lines) + + with MockedOpen({'dummy': content}): + self.pp.do_include('dummy') + self.assertEqual(self.pp.out.getvalue().rstrip('\n'), expected) + + def do_include_pass(self, content_lines): + self.do_include_compare(content_lines, ['PASS']) + + def test_conditional_if_0(self): + self.do_include_pass([ + '#if 0', + 'FAIL', + '#else', + 'PASS', + '#endif', + ]) + + def test_no_marker(self): + lines = [ + '#if 0', + 'PASS', + '#endif', + ] + self.pp.setMarker(None) + self.do_include_compare(lines, lines) + + def test_string_value(self): + self.do_include_compare([ + '#define FOO STRING', + '#if FOO', + 'string value is true', + '#else', + 'string value is false', + '#endif', + ], ['string value is false']) + + def test_number_value(self): + self.do_include_compare([ + '#define FOO 1', + '#if FOO', + 'number value is true', + '#else', + 'number value is false', + '#endif', + ], ['number value is true']) + + def test_conditional_if_0_elif_1(self): + self.do_include_pass([ + '#if 0', + '#elif 1', + 'PASS', + '#else', + 'FAIL', + '#endif', + ]) + + def test_conditional_if_1(self): + self.do_include_pass([ + '#if 1', + 'PASS', + '#else', + 'FAIL', + '#endif', + ]) + + def test_conditional_if_0_or_1(self): + self.do_include_pass([ + '#if 0 || 1', + 'PASS', + '#else', + 'FAIL', + '#endif', + ]) + + def test_conditional_if_1_elif_1_else(self): + self.do_include_pass([ + '#if 1', + 'PASS', + '#elif 1', + 'FAIL', + '#else', + 'FAIL', + '#endif', + ]) + + def test_conditional_if_1_if_1(self): + self.do_include_pass([ + '#if 1', + '#if 1', + 'PASS', + '#else', + 'FAIL', + '#endif', + '#else', + 'FAIL', + '#endif', + ]) + + def test_conditional_not_0(self): + self.do_include_pass([ + '#if !0', + 'PASS', + '#else', + 'FAIL', + '#endif', + ]) + + def test_conditional_not_0_and_1(self): + self.do_include_pass([ + '#if !0 && !1', + 'FAIL', + '#else', + 'PASS', + '#endif', + ]) + + def test_conditional_not_1(self): + self.do_include_pass([ + '#if !1', + 'FAIL', + '#else', + 'PASS', + '#endif', + ]) + + def test_conditional_not_emptyval(self): + self.do_include_compare([ + '#define EMPTYVAL', + '#ifndef EMPTYVAL', + 'FAIL', + '#else', + 'PASS', + '#endif', + '#ifdef EMPTYVAL', + 'PASS', + '#else', + 'FAIL', + '#endif', + ], ['PASS', 'PASS']) + + def test_conditional_not_nullval(self): + self.do_include_pass([ + '#define NULLVAL 0', + '#if !NULLVAL', + 'PASS', + '#else', + 'FAIL', + '#endif', + ]) + + def test_expand(self): + self.do_include_pass([ + '#define ASVAR AS', + '#expand P__ASVAR__S', + ]) + + def test_undef_defined(self): + self.do_include_compare([ + '#define BAR', + '#undef BAR', + 'BAR', + ], ['BAR']) + + def test_undef_undefined(self): + self.do_include_compare([ + '#undef BAR', + ], []) + + def test_filter_attemptSubstitution(self): + self.do_include_compare([ + '#filter attemptSubstitution', + '@PASS@', + '#unfilter attemptSubstitution', + ], ['@PASS@']) + + def test_filter_emptyLines(self): + self.do_include_compare([ + 'lines with a', + '', + 'blank line', + '#filter emptyLines', + 'lines with', + '', + 'no blank lines', + '#unfilter emptyLines', + 'yet more lines with', + '', + 'blank lines', + ], [ + 'lines with a', + '', + 'blank line', + 'lines with', + 'no blank lines', + 'yet more lines with', + '', + 'blank lines', + ]) + + def test_filter_slashslash(self): + self.do_include_compare([ + '#filter slashslash', + 'PASS//FAIL // FAIL', + '#unfilter slashslash', + 'PASS // PASS', + ], [ + 'PASS', + 'PASS // PASS', + ]) + + def test_filter_spaces(self): + self.do_include_compare([ + '#filter spaces', + 'You should see two nice ascii tables', + ' +-+-+-+', + ' | | | |', + ' +-+-+-+', + '#unfilter spaces', + '+-+---+', + '| | |', + '+-+---+', + ], [ + 'You should see two nice ascii tables', + '+-+-+-+', + '| | | |', + '+-+-+-+', + '+-+---+', + '| | |', + '+-+---+', + ]) + + def test_filter_substitution(self): + self.do_include_pass([ + '#define VAR ASS', + '#filter substitution', + 'P@VAR@', + '#unfilter substitution', + ]) + + def test_error(self): + with MockedOpen({'f': '#error spit this message out\n'}): + with self.assertRaises(Preprocessor.Error) as e: + self.pp.do_include('f') + self.assertEqual(e.args[0][-1], 'spit this message out') + + def test_javascript_line(self): + # The preprocessor is reading the filename from somewhere not caught + # by MockedOpen. + tmpdir = mkdtemp() + try: + full = os.path.join(tmpdir, 'javascript_line.js.in') + with open(full, 'w') as fh: + fh.write('\n'.join([ + '// Line 1', + '#if 0', + '// line 3', + '#endif', + '// line 5', + '# comment', + '// line 7', + '// line 8', + '// line 9', + '# another comment', + '// line 11', + '#define LINE 1', + '// line 13, given line number overwritten with 2', + '', + ])) + + self.pp.do_include(full) + out = '\n'.join([ + '// Line 1', + '//@line 5 "CWDjavascript_line.js.in"', + '// line 5', + '//@line 7 "CWDjavascript_line.js.in"', + '// line 7', + '// line 8', + '// line 9', + '//@line 11 "CWDjavascript_line.js.in"', + '// line 11', + '//@line 2 "CWDjavascript_line.js.in"', + '// line 13, given line number overwritten with 2', + '', + ]) + out = out.replace('CWD', tmpdir + os.path.sep) + self.assertEqual(self.pp.out.getvalue(), out) + finally: + shutil.rmtree(tmpdir) + + def test_literal(self): + self.do_include_pass([ + '#literal PASS', + ]) + + def test_var_directory(self): + self.do_include_pass([ + '#ifdef DIRECTORY', + 'PASS', + '#else', + 'FAIL', + '#endif', + ]) + + def test_var_file(self): + self.do_include_pass([ + '#ifdef FILE', + 'PASS', + '#else', + 'FAIL', + '#endif', + ]) + + def test_var_if_0(self): + self.do_include_pass([ + '#define VAR 0', + '#if VAR', + 'FAIL', + '#else', + 'PASS', + '#endif', + ]) + + def test_var_if_0_elifdef(self): + self.do_include_pass([ + '#if 0', + '#elifdef FILE', + 'PASS', + '#else', + 'FAIL', + '#endif', + ]) + + def test_var_if_0_elifndef(self): + self.do_include_pass([ + '#if 0', + '#elifndef VAR', + 'PASS', + '#else', + 'FAIL', + '#endif', + ]) + + def test_var_ifdef_0(self): + self.do_include_pass([ + '#define VAR 0', + '#ifdef VAR', + 'PASS', + '#else', + 'FAIL', + '#endif', + ]) + + def test_var_ifdef_1_or_undef(self): + self.do_include_pass([ + '#define FOO 1', + '#if defined(FOO) || defined(BAR)', + 'PASS', + '#else', + 'FAIL', + '#endif', + ]) + + def test_var_ifdef_undef(self): + self.do_include_pass([ + '#define VAR 0', + '#undef VAR', + '#ifdef VAR', + 'FAIL', + '#else', + 'PASS', + '#endif', + ]) + + def test_var_ifndef_0(self): + self.do_include_pass([ + '#define VAR 0', + '#ifndef VAR', + 'FAIL', + '#else', + 'PASS', + '#endif', + ]) + + def test_var_ifndef_0_and_undef(self): + self.do_include_pass([ + '#define FOO 0', + '#if !defined(FOO) && !defined(BAR)', + 'FAIL', + '#else', + 'PASS', + '#endif', + ]) + + def test_var_ifndef_undef(self): + self.do_include_pass([ + '#define VAR 0', + '#undef VAR', + '#ifndef VAR', + 'PASS', + '#else', + 'FAIL', + '#endif', + ]) + + def test_var_line(self): + self.do_include_pass([ + '#ifdef LINE', + 'PASS', + '#else', + 'FAIL', + '#endif', + ]) + + def test_filterDefine(self): + self.do_include_pass([ + '#filter substitution', + '#define VAR AS', + '#define VAR2 P@VAR@', + '@VAR2@S', + ]) + + def test_number_value_equals(self): + self.do_include_pass([ + '#define FOO 1000', + '#if FOO == 1000', + 'PASS', + '#else', + 'FAIL', + '#endif', + ]) + + def test_default_defines(self): + self.pp.handleCommandLine(["-DFOO"]) + self.do_include_pass([ + '#if FOO == 1', + 'PASS', + '#else', + 'FAIL', + ]) + + def test_number_value_equals_defines(self): + self.pp.handleCommandLine(["-DFOO=1000"]) + self.do_include_pass([ + '#if FOO == 1000', + 'PASS', + '#else', + 'FAIL', + ]) + + def test_octal_value_equals(self): + self.do_include_pass([ + '#define FOO 0100', + '#if FOO == 0100', + 'PASS', + '#else', + 'FAIL', + '#endif', + ]) + + def test_octal_value_equals_defines(self): + self.pp.handleCommandLine(["-DFOO=0100"]) + self.do_include_pass([ + '#if FOO == 0100', + 'PASS', + '#else', + 'FAIL', + '#endif', + ]) + + def test_value_quoted_expansion(self): + """ + Quoted values on the commandline don't currently have quotes stripped. + Pike says this is for compat reasons. + """ + self.pp.handleCommandLine(['-DFOO="ABCD"']) + self.do_include_compare([ + '#filter substitution', + '@FOO@', + ], ['"ABCD"']) + + def test_octal_value_quoted_expansion(self): + self.pp.handleCommandLine(['-DFOO="0100"']) + self.do_include_compare([ + '#filter substitution', + '@FOO@', + ], ['"0100"']) + + def test_number_value_not_equals_quoted_defines(self): + self.pp.handleCommandLine(['-DFOO="1000"']) + self.do_include_pass([ + '#if FOO == 1000', + 'FAIL', + '#else', + 'PASS', + '#endif', + ]) + + def test_octal_value_not_equals_quoted_defines(self): + self.pp.handleCommandLine(['-DFOO="0100"']) + self.do_include_pass([ + '#if FOO == 0100', + 'FAIL', + '#else', + 'PASS', + '#endif', + ]) + + def test_undefined_variable(self): + with MockedOpen({'f': '#filter substitution\n@foo@'}): + with self.assertRaises(Preprocessor.Error) as e: + self.pp.do_include('f') + self.assertEqual(e.key, 'UNDEFINED_VAR') + + def test_include(self): + files = { + 'foo/test': '\n'.join([ + '#define foo foobarbaz', + '#include @inc@', + '@bar@', + '', + ]), + 'bar': '\n'.join([ + '#define bar barfoobaz', + '@foo@', + '', + ]), + 'f': '\n'.join([ + '#filter substitution', + '#define inc ../bar', + '#include foo/test', + '', + ]), + } + + with MockedOpen(files): + self.pp.do_include('f') + self.assertEqual(self.pp.out.getvalue(), 'foobarbaz\nbarfoobaz\n') + + def test_include_line(self): + files = { + 'test.js': '\n'.join([ + '#define foo foobarbaz', + '#include @inc@', + '@bar@', + '', + ]), + 'bar.js': '\n'.join([ + '#define bar barfoobaz', + '@foo@', + '', + ]), + 'foo.js': '\n'.join([ + 'bazfoobar', + '#include bar.js', + 'bazbarfoo', + '', + ]), + 'baz.js': 'baz\n', + 'f.js': '\n'.join([ + '#include foo.js', + '#filter substitution', + '#define inc bar.js', + '#include test.js', + '#include baz.js', + 'fin', + '', + ]), + } + + with MockedOpen(files): + self.pp.do_include('f.js') + self.assertEqual(self.pp.out.getvalue(), + ('//@line 1 "CWD/foo.js"\n' + 'bazfoobar\n' + '//@line 2 "CWD/bar.js"\n' + '@foo@\n' + '//@line 3 "CWD/foo.js"\n' + 'bazbarfoo\n' + '//@line 2 "CWD/bar.js"\n' + 'foobarbaz\n' + '//@line 3 "CWD/test.js"\n' + 'barfoobaz\n' + '//@line 1 "CWD/baz.js"\n' + 'baz\n' + '//@line 6 "CWD/f.js"\n' + 'fin\n').replace('CWD/', + os.getcwd() + os.path.sep)) + + def test_include_missing_file(self): + with MockedOpen({'f': '#include foo\n'}): + with self.assertRaises(Preprocessor.Error) as e: + self.pp.do_include('f') + self.assertEqual(e.exception.key, 'FILE_NOT_FOUND') + + def test_include_undefined_variable(self): + with MockedOpen({'f': '#filter substitution\n#include @foo@\n'}): + with self.assertRaises(Preprocessor.Error) as e: + self.pp.do_include('f') + self.assertEqual(e.exception.key, 'UNDEFINED_VAR') + + def test_include_literal_at(self): + files = { + '@foo@': '#define foo foobarbaz\n', + 'f': '#include @foo@\n#filter substitution\n@foo@\n', + } + + with MockedOpen(files): + self.pp.do_include('f') + self.assertEqual(self.pp.out.getvalue(), 'foobarbaz\n') + + def test_command_line_literal_at(self): + with MockedOpen({"@foo@.in": '@foo@\n'}): + self.pp.handleCommandLine(['-Fsubstitution', '-Dfoo=foobarbaz', '@foo@.in']) + self.assertEqual(self.pp.out.getvalue(), 'foobarbaz\n') + +if __name__ == '__main__': + main() diff --git a/python/mozbuild/mozbuild/test/test_pythonutil.py b/python/mozbuild/mozbuild/test/test_pythonutil.py new file mode 100644 index 000000000..87399b3f5 --- /dev/null +++ b/python/mozbuild/mozbuild/test/test_pythonutil.py @@ -0,0 +1,23 @@ +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, You can obtain one at http://mozilla.org/MPL/2.0/. + +from mozbuild.pythonutil import iter_modules_in_path +from mozunit import main +import os +import unittest + + +class TestIterModules(unittest.TestCase): + def test_iter_modules_in_path(self): + mozbuild_path = os.path.normcase(os.path.dirname(os.path.dirname(__file__))) + paths = list(iter_modules_in_path(mozbuild_path)) + self.assertEquals(sorted(paths), [ + os.path.join(os.path.abspath(mozbuild_path), '__init__.py'), + os.path.join(os.path.abspath(mozbuild_path), 'pythonutil.py'), + os.path.join(os.path.abspath(mozbuild_path), 'test', 'test_pythonutil.py'), + ]) + + +if __name__ == '__main__': + main() diff --git a/python/mozbuild/mozbuild/test/test_testing.py b/python/mozbuild/mozbuild/test/test_testing.py new file mode 100644 index 000000000..e71892e24 --- /dev/null +++ b/python/mozbuild/mozbuild/test/test_testing.py @@ -0,0 +1,332 @@ +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, You can obtain one at http://mozilla.org/MPL/2.0/. + +from __future__ import unicode_literals + +import cPickle as pickle +import os +import shutil +import tempfile +import unittest + +import mozpack.path as mozpath + +from mozfile import NamedTemporaryFile +from mozunit import main + +from mozbuild.base import MozbuildObject +from mozbuild.testing import ( + TestMetadata, + TestResolver, +) + + +ALL_TESTS = { + "accessible/tests/mochitest/actions/test_anchors.html": [ + { + "dir_relpath": "accessible/tests/mochitest/actions", + "expected": "pass", + "file_relpath": "accessible/tests/mochitest/actions/test_anchors.html", + "flavor": "a11y", + "here": "/Users/gps/src/firefox/accessible/tests/mochitest/actions", + "manifest": "/Users/gps/src/firefox/accessible/tests/mochitest/actions/a11y.ini", + "name": "test_anchors.html", + "path": "/Users/gps/src/firefox/accessible/tests/mochitest/actions/test_anchors.html", + "relpath": "test_anchors.html" + } + ], + "services/common/tests/unit/test_async_chain.js": [ + { + "dir_relpath": "services/common/tests/unit", + "file_relpath": "services/common/tests/unit/test_async_chain.js", + "firefox-appdir": "browser", + "flavor": "xpcshell", + "head": "head_global.js head_helpers.js head_http.js", + "here": "/Users/gps/src/firefox/services/common/tests/unit", + "manifest": "/Users/gps/src/firefox/services/common/tests/unit/xpcshell.ini", + "name": "test_async_chain.js", + "path": "/Users/gps/src/firefox/services/common/tests/unit/test_async_chain.js", + "relpath": "test_async_chain.js", + "tail": "" + } + ], + "services/common/tests/unit/test_async_querySpinningly.js": [ + { + "dir_relpath": "services/common/tests/unit", + "file_relpath": "services/common/tests/unit/test_async_querySpinningly.js", + "firefox-appdir": "browser", + "flavor": "xpcshell", + "head": "head_global.js head_helpers.js head_http.js", + "here": "/Users/gps/src/firefox/services/common/tests/unit", + "manifest": "/Users/gps/src/firefox/services/common/tests/unit/xpcshell.ini", + "name": "test_async_querySpinningly.js", + "path": "/Users/gps/src/firefox/services/common/tests/unit/test_async_querySpinningly.js", + "relpath": "test_async_querySpinningly.js", + "tail": "" + } + ], + "toolkit/mozapps/update/test/unit/test_0201_app_launch_apply_update.js": [ + { + "dir_relpath": "toolkit/mozapps/update/test/unit", + "file_relpath": "toolkit/mozapps/update/test/unit/test_0201_app_launch_apply_update.js", + "flavor": "xpcshell", + "generated-files": "head_update.js", + "head": "head_update.js", + "here": "/Users/gps/src/firefox/toolkit/mozapps/update/test/unit", + "manifest": "/Users/gps/src/firefox/toolkit/mozapps/update/test/unit/xpcshell_updater.ini", + "name": "test_0201_app_launch_apply_update.js", + "path": "/Users/gps/src/firefox/toolkit/mozapps/update/test/unit/test_0201_app_launch_apply_update.js", + "reason": "bug 820380", + "relpath": "test_0201_app_launch_apply_update.js", + "run-sequentially": "Launches application.", + "skip-if": "toolkit == 'gonk' || os == 'android'", + "tail": "" + }, + { + "dir_relpath": "toolkit/mozapps/update/test/unit", + "file_relpath": "toolkit/mozapps/update/test/unit/test_0201_app_launch_apply_update.js", + "flavor": "xpcshell", + "generated-files": "head_update.js", + "head": "head_update.js head2.js", + "here": "/Users/gps/src/firefox/toolkit/mozapps/update/test/unit", + "manifest": "/Users/gps/src/firefox/toolkit/mozapps/update/test/unit/xpcshell_updater.ini", + "name": "test_0201_app_launch_apply_update.js", + "path": "/Users/gps/src/firefox/toolkit/mozapps/update/test/unit/test_0201_app_launch_apply_update.js", + "reason": "bug 820380", + "relpath": "test_0201_app_launch_apply_update.js", + "run-sequentially": "Launches application.", + "skip-if": "toolkit == 'gonk' || os == 'android'", + "tail": "" + } + ], + "mobile/android/tests/background/junit3/src/common/TestAndroidLogWriters.java": [ + { + "dir_relpath": "mobile/android/tests/background/junit3/src/common", + "file_relpath": "mobile/android/tests/background/junit3/src/common/TestAndroidLogWriters.java", + "flavor": "instrumentation", + "here": "/Users/nalexander/Mozilla/gecko-dev/mobile/android/tests/background/junit3", + "manifest": "/Users/nalexander/Mozilla/gecko-dev/mobile/android/tests/background/junit3/instrumentation.ini", + "name": "src/common/TestAndroidLogWriters.java", + "path": "/Users/nalexander/Mozilla/gecko-dev/mobile/android/tests/background/junit3/src/common/TestAndroidLogWriters.java", + "relpath": "src/common/TestAndroidLogWriters.java", + "subsuite": "background" + } + ], + "mobile/android/tests/browser/junit3/src/TestDistribution.java": [ + { + "dir_relpath": "mobile/android/tests/browser/junit3/src", + "file_relpath": "mobile/android/tests/browser/junit3/src/TestDistribution.java", + "flavor": "instrumentation", + "here": "/Users/nalexander/Mozilla/gecko-dev/mobile/android/tests/browser/junit3", + "manifest": "/Users/nalexander/Mozilla/gecko-dev/mobile/android/tests/browser/junit3/instrumentation.ini", + "name": "src/TestDistribution.java", + "path": "/Users/nalexander/Mozilla/gecko-dev/mobile/android/tests/browser/junit3/src/TestDistribution.java", + "relpath": "src/TestDistribution.java", + "subsuite": "browser" + } + ], + "image/test/browser/browser_bug666317.js": [ + { + "dir_relpath": "image/test/browser", + "file_relpath": "image/test/browser/browser_bug666317.js", + "flavor": "browser-chrome", + "here": "/home/chris/m-c/obj-dbg/_tests/testing/mochitest/browser/image/test/browser", + "manifest": "/home/chris/m-c/image/test/browser/browser.ini", + "name": "browser_bug666317.js", + "path": "/home/chris/m-c/obj-dbg/_tests/testing/mochitest/browser/image/test/browser/browser_bug666317.js", + "relpath": "image/test/browser/browser_bug666317.js", + "skip-if": "e10s # Bug 948194 - Decoded Images seem to not be discarded on memory-pressure notification with e10s enabled", + "subsuite": "" + } + ], + "devtools/client/markupview/test/browser_markupview_copy_image_data.js": [ + { + "dir_relpath": "devtools/client/markupview/test", + "file_relpath": "devtools/client/markupview/test/browser_markupview_copy_image_data.js", + "flavor": "browser-chrome", + "here": "/home/chris/m-c/obj-dbg/_tests/testing/mochitest/browser/devtools/client/markupview/test", + "manifest": "/home/chris/m-c/devtools/client/markupview/test/browser.ini", + "name": "browser_markupview_copy_image_data.js", + "path": "/home/chris/m-c/obj-dbg/_tests/testing/mochitest/browser/devtools/client/markupview/test/browser_markupview_copy_image_data.js", + "relpath": "devtools/client/markupview/test/browser_markupview_copy_image_data.js", + "subsuite": "devtools", + "tags": "devtools" + } + ] +} + +TEST_DEFAULTS = { + "/Users/gps/src/firefox/toolkit/mozapps/update/test/unit/xpcshell_updater.ini": {"support-files": "\ndata/**\nxpcshell_updater.ini"} +} + + +class Base(unittest.TestCase): + def setUp(self): + self._temp_files = [] + + def tearDown(self): + for f in self._temp_files: + del f + + self._temp_files = [] + + def _get_test_metadata(self): + all_tests = NamedTemporaryFile(mode='wb') + pickle.dump(ALL_TESTS, all_tests) + all_tests.flush() + self._temp_files.append(all_tests) + + test_defaults = NamedTemporaryFile(mode='wb') + pickle.dump(TEST_DEFAULTS, test_defaults) + test_defaults.flush() + self._temp_files.append(test_defaults) + + return TestMetadata(all_tests.name, test_defaults=test_defaults.name) + + +class TestTestMetadata(Base): + def test_load(self): + t = self._get_test_metadata() + self.assertEqual(len(t._tests_by_path), 8) + + self.assertEqual(len(list(t.tests_with_flavor('xpcshell'))), 3) + self.assertEqual(len(list(t.tests_with_flavor('mochitest-plain'))), 0) + + def test_resolve_all(self): + t = self._get_test_metadata() + self.assertEqual(len(list(t.resolve_tests())), 9) + + def test_resolve_filter_flavor(self): + t = self._get_test_metadata() + self.assertEqual(len(list(t.resolve_tests(flavor='xpcshell'))), 4) + + def test_resolve_by_dir(self): + t = self._get_test_metadata() + self.assertEqual(len(list(t.resolve_tests(paths=['services/common']))), 2) + + def test_resolve_under_path(self): + t = self._get_test_metadata() + self.assertEqual(len(list(t.resolve_tests(under_path='services'))), 2) + + self.assertEqual(len(list(t.resolve_tests(flavor='xpcshell', + under_path='services'))), 2) + + def test_resolve_multiple_paths(self): + t = self._get_test_metadata() + result = list(t.resolve_tests(paths=['services', 'toolkit'])) + self.assertEqual(len(result), 4) + + def test_resolve_support_files(self): + expected_support_files = "\ndata/**\nxpcshell_updater.ini" + t = self._get_test_metadata() + result = list(t.resolve_tests(paths=['toolkit'])) + self.assertEqual(len(result), 2) + + for test in result: + self.assertEqual(test['support-files'], + expected_support_files) + + def test_resolve_path_prefix(self): + t = self._get_test_metadata() + result = list(t.resolve_tests(paths=['image'])) + self.assertEqual(len(result), 1) + + +class TestTestResolver(Base): + FAKE_TOPSRCDIR = '/Users/gps/src/firefox' + + def setUp(self): + Base.setUp(self) + + self._temp_dirs = [] + + def tearDown(self): + Base.tearDown(self) + + for d in self._temp_dirs: + shutil.rmtree(d) + + def _get_resolver(self): + topobjdir = tempfile.mkdtemp() + self._temp_dirs.append(topobjdir) + + with open(os.path.join(topobjdir, 'all-tests.pkl'), 'wb') as fh: + pickle.dump(ALL_TESTS, fh) + with open(os.path.join(topobjdir, 'test-defaults.pkl'), 'wb') as fh: + pickle.dump(TEST_DEFAULTS, fh) + + o = MozbuildObject(self.FAKE_TOPSRCDIR, None, None, topobjdir=topobjdir) + + # Monkey patch the test resolver to avoid tests failing to find make + # due to our fake topscrdir. + TestResolver._run_make = lambda *a, **b: None + + return o._spawn(TestResolver) + + def test_cwd_children_only(self): + """If cwd is defined, only resolve tests under the specified cwd.""" + r = self._get_resolver() + + # Pretend we're under '/services' and ask for 'common'. This should + # pick up all tests from '/services/common' + tests = list(r.resolve_tests(paths=['common'], cwd=os.path.join(r.topsrcdir, + 'services'))) + + self.assertEqual(len(tests), 2) + + # Tests should be rewritten to objdir. + for t in tests: + self.assertEqual(t['here'], mozpath.join(r.topobjdir, + '_tests/xpcshell/services/common/tests/unit')) + + def test_various_cwd(self): + """Test various cwd conditions are all equal.""" + + r = self._get_resolver() + + expected = list(r.resolve_tests(paths=['services'])) + actual = list(r.resolve_tests(paths=['services'], cwd='/')) + self.assertEqual(actual, expected) + + actual = list(r.resolve_tests(paths=['services'], cwd=r.topsrcdir)) + self.assertEqual(actual, expected) + + actual = list(r.resolve_tests(paths=['services'], cwd=r.topobjdir)) + self.assertEqual(actual, expected) + + def test_subsuites(self): + """Test filtering by subsuite.""" + + r = self._get_resolver() + + tests = list(r.resolve_tests(paths=['mobile'])) + self.assertEqual(len(tests), 2) + + tests = list(r.resolve_tests(paths=['mobile'], subsuite='browser')) + self.assertEqual(len(tests), 1) + self.assertEqual(tests[0]['name'], 'src/TestDistribution.java') + + tests = list(r.resolve_tests(paths=['mobile'], subsuite='background')) + self.assertEqual(len(tests), 1) + self.assertEqual(tests[0]['name'], 'src/common/TestAndroidLogWriters.java') + + def test_wildcard_patterns(self): + """Test matching paths by wildcard.""" + + r = self._get_resolver() + + tests = list(r.resolve_tests(paths=['mobile/**'])) + self.assertEqual(len(tests), 2) + for t in tests: + self.assertTrue(t['file_relpath'].startswith('mobile')) + + tests = list(r.resolve_tests(paths=['**/**.js', 'accessible/**'])) + self.assertEqual(len(tests), 7) + for t in tests: + path = t['file_relpath'] + self.assertTrue(path.startswith('accessible') or path.endswith('.js')) + + +if __name__ == '__main__': + main() diff --git a/python/mozbuild/mozbuild/test/test_util.py b/python/mozbuild/mozbuild/test/test_util.py new file mode 100644 index 000000000..6c3b39b1e --- /dev/null +++ b/python/mozbuild/mozbuild/test/test_util.py @@ -0,0 +1,924 @@ +# coding: utf-8 +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, You can obtain one at http://mozilla.org/MPL/2.0/. + +from __future__ import unicode_literals + +import itertools +import hashlib +import os +import unittest +import shutil +import string +import sys +import tempfile +import textwrap + +from mozfile.mozfile import NamedTemporaryFile +from mozunit import ( + main, + MockedOpen, +) + +from mozbuild.util import ( + expand_variables, + FileAvoidWrite, + group_unified_files, + hash_file, + indented_repr, + memoize, + memoized_property, + pair, + resolve_target_to_make, + MozbuildDeletionError, + HierarchicalStringList, + EnumString, + EnumStringComparisonError, + ListWithAction, + StrictOrderingOnAppendList, + StrictOrderingOnAppendListWithFlagsFactory, + TypedList, + TypedNamedTuple, + UnsortedError, +) + +if sys.version_info[0] == 3: + str_type = 'str' +else: + str_type = 'unicode' + +data_path = os.path.abspath(os.path.dirname(__file__)) +data_path = os.path.join(data_path, 'data') + + +class TestHashing(unittest.TestCase): + def test_hash_file_known_hash(self): + """Ensure a known hash value is recreated.""" + data = b'The quick brown fox jumps over the lazy cog' + expected = 'de9f2c7fd25e1b3afad3e85a0bd17d9b100db4b3' + + temp = NamedTemporaryFile() + temp.write(data) + temp.flush() + + actual = hash_file(temp.name) + + self.assertEqual(actual, expected) + + def test_hash_file_large(self): + """Ensure that hash_file seems to work with a large file.""" + data = b'x' * 1048576 + + hasher = hashlib.sha1() + hasher.update(data) + expected = hasher.hexdigest() + + temp = NamedTemporaryFile() + temp.write(data) + temp.flush() + + actual = hash_file(temp.name) + + self.assertEqual(actual, expected) + + +class TestFileAvoidWrite(unittest.TestCase): + def test_file_avoid_write(self): + with MockedOpen({'file': 'content'}): + # Overwriting an existing file replaces its content + faw = FileAvoidWrite('file') + faw.write('bazqux') + self.assertEqual(faw.close(), (True, True)) + self.assertEqual(open('file', 'r').read(), 'bazqux') + + # Creating a new file (obviously) stores its content + faw = FileAvoidWrite('file2') + faw.write('content') + self.assertEqual(faw.close(), (False, True)) + self.assertEqual(open('file2').read(), 'content') + + with MockedOpen({'file': 'content'}): + with FileAvoidWrite('file') as file: + file.write('foobar') + + self.assertEqual(open('file', 'r').read(), 'foobar') + + class MyMockedOpen(MockedOpen): + '''MockedOpen extension to raise an exception if something + attempts to write in an opened file. + ''' + def __call__(self, name, mode): + if 'w' in mode: + raise Exception, 'Unexpected open with write mode' + return MockedOpen.__call__(self, name, mode) + + with MyMockedOpen({'file': 'content'}): + # Validate that MyMockedOpen works as intended + file = FileAvoidWrite('file') + file.write('foobar') + self.assertRaises(Exception, file.close) + + # Check that no write actually happens when writing the + # same content as what already is in the file + faw = FileAvoidWrite('file') + faw.write('content') + self.assertEqual(faw.close(), (True, False)) + + def test_diff_not_default(self): + """Diffs are not produced by default.""" + + with MockedOpen({'file': 'old'}): + faw = FileAvoidWrite('file') + faw.write('dummy') + faw.close() + self.assertIsNone(faw.diff) + + def test_diff_update(self): + """Diffs are produced on file update.""" + + with MockedOpen({'file': 'old'}): + faw = FileAvoidWrite('file', capture_diff=True) + faw.write('new') + faw.close() + + diff = '\n'.join(faw.diff) + self.assertIn('-old', diff) + self.assertIn('+new', diff) + + def test_diff_create(self): + """Diffs are produced when files are created.""" + + tmpdir = tempfile.mkdtemp() + try: + path = os.path.join(tmpdir, 'file') + faw = FileAvoidWrite(path, capture_diff=True) + faw.write('new') + faw.close() + + diff = '\n'.join(faw.diff) + self.assertIn('+new', diff) + finally: + shutil.rmtree(tmpdir) + +class TestResolveTargetToMake(unittest.TestCase): + def setUp(self): + self.topobjdir = data_path + + def assertResolve(self, path, expected): + # Handle Windows path separators. + (reldir, target) = resolve_target_to_make(self.topobjdir, path) + if reldir is not None: + reldir = reldir.replace(os.sep, '/') + if target is not None: + target = target.replace(os.sep, '/') + self.assertEqual((reldir, target), expected) + + def test_root_path(self): + self.assertResolve('/test-dir', ('test-dir', None)) + self.assertResolve('/test-dir/with', ('test-dir/with', None)) + self.assertResolve('/test-dir/without', ('test-dir', None)) + self.assertResolve('/test-dir/without/with', ('test-dir/without/with', None)) + + def test_dir(self): + self.assertResolve('test-dir', ('test-dir', None)) + self.assertResolve('test-dir/with', ('test-dir/with', None)) + self.assertResolve('test-dir/with', ('test-dir/with', None)) + self.assertResolve('test-dir/without', ('test-dir', None)) + self.assertResolve('test-dir/without/with', ('test-dir/without/with', None)) + + def test_top_level(self): + self.assertResolve('package', (None, 'package')) + # Makefile handling shouldn't affect top-level targets. + self.assertResolve('Makefile', (None, 'Makefile')) + + def test_regular_file(self): + self.assertResolve('test-dir/with/file', ('test-dir/with', 'file')) + self.assertResolve('test-dir/with/without/file', ('test-dir/with', 'without/file')) + self.assertResolve('test-dir/with/without/with/file', ('test-dir/with/without/with', 'file')) + + self.assertResolve('test-dir/without/file', ('test-dir', 'without/file')) + self.assertResolve('test-dir/without/with/file', ('test-dir/without/with', 'file')) + self.assertResolve('test-dir/without/with/without/file', ('test-dir/without/with', 'without/file')) + + def test_Makefile(self): + self.assertResolve('test-dir/with/Makefile', ('test-dir', 'with/Makefile')) + self.assertResolve('test-dir/with/without/Makefile', ('test-dir/with', 'without/Makefile')) + self.assertResolve('test-dir/with/without/with/Makefile', ('test-dir/with', 'without/with/Makefile')) + + self.assertResolve('test-dir/without/Makefile', ('test-dir', 'without/Makefile')) + self.assertResolve('test-dir/without/with/Makefile', ('test-dir', 'without/with/Makefile')) + self.assertResolve('test-dir/without/with/without/Makefile', ('test-dir/without/with', 'without/Makefile')) + +class TestHierarchicalStringList(unittest.TestCase): + def setUp(self): + self.EXPORTS = HierarchicalStringList() + + def test_exports_append(self): + self.assertEqual(self.EXPORTS._strings, []) + self.EXPORTS += ["foo.h"] + self.assertEqual(self.EXPORTS._strings, ["foo.h"]) + self.EXPORTS += ["bar.h"] + self.assertEqual(self.EXPORTS._strings, ["foo.h", "bar.h"]) + + def test_exports_subdir(self): + self.assertEqual(self.EXPORTS._children, {}) + self.EXPORTS.foo += ["foo.h"] + self.assertItemsEqual(self.EXPORTS._children, {"foo" : True}) + self.assertEqual(self.EXPORTS.foo._strings, ["foo.h"]) + self.EXPORTS.bar += ["bar.h"] + self.assertItemsEqual(self.EXPORTS._children, + {"foo" : True, "bar" : True}) + self.assertEqual(self.EXPORTS.foo._strings, ["foo.h"]) + self.assertEqual(self.EXPORTS.bar._strings, ["bar.h"]) + + def test_exports_multiple_subdir(self): + self.EXPORTS.foo.bar = ["foobar.h"] + self.assertItemsEqual(self.EXPORTS._children, {"foo" : True}) + self.assertItemsEqual(self.EXPORTS.foo._children, {"bar" : True}) + self.assertItemsEqual(self.EXPORTS.foo.bar._children, {}) + self.assertEqual(self.EXPORTS._strings, []) + self.assertEqual(self.EXPORTS.foo._strings, []) + self.assertEqual(self.EXPORTS.foo.bar._strings, ["foobar.h"]) + + def test_invalid_exports_append(self): + with self.assertRaises(ValueError) as ve: + self.EXPORTS += "foo.h" + self.assertEqual(str(ve.exception), + "Expected a list of strings, not " % str_type) + + def test_invalid_exports_set(self): + with self.assertRaises(ValueError) as ve: + self.EXPORTS.foo = "foo.h" + + self.assertEqual(str(ve.exception), + "Expected a list of strings, not " % str_type) + + def test_invalid_exports_append_base(self): + with self.assertRaises(ValueError) as ve: + self.EXPORTS += "foo.h" + + self.assertEqual(str(ve.exception), + "Expected a list of strings, not " % str_type) + + def test_invalid_exports_bool(self): + with self.assertRaises(ValueError) as ve: + self.EXPORTS += [True] + + self.assertEqual(str(ve.exception), + "Expected a list of strings, not an element of " + "") + + def test_del_exports(self): + with self.assertRaises(MozbuildDeletionError) as mde: + self.EXPORTS.foo += ['bar.h'] + del self.EXPORTS.foo + + def test_unsorted(self): + with self.assertRaises(UnsortedError) as ee: + self.EXPORTS += ['foo.h', 'bar.h'] + + with self.assertRaises(UnsortedError) as ee: + self.EXPORTS.foo = ['foo.h', 'bar.h'] + + with self.assertRaises(UnsortedError) as ee: + self.EXPORTS.foo += ['foo.h', 'bar.h'] + + def test_reassign(self): + self.EXPORTS.foo = ['foo.h'] + + with self.assertRaises(KeyError) as ee: + self.EXPORTS.foo = ['bar.h'] + + def test_walk(self): + l = HierarchicalStringList() + l += ['root1', 'root2', 'root3'] + l.child1 += ['child11', 'child12', 'child13'] + l.child1.grandchild1 += ['grandchild111', 'grandchild112'] + l.child1.grandchild2 += ['grandchild121', 'grandchild122'] + l.child2.grandchild1 += ['grandchild211', 'grandchild212'] + l.child2.grandchild1 += ['grandchild213', 'grandchild214'] + + els = list((path, list(seq)) for path, seq in l.walk()) + self.assertEqual(els, [ + ('', ['root1', 'root2', 'root3']), + ('child1', ['child11', 'child12', 'child13']), + ('child1/grandchild1', ['grandchild111', 'grandchild112']), + ('child1/grandchild2', ['grandchild121', 'grandchild122']), + ('child2/grandchild1', ['grandchild211', 'grandchild212', + 'grandchild213', 'grandchild214']), + ]) + + def test_merge(self): + l1 = HierarchicalStringList() + l1 += ['root1', 'root2', 'root3'] + l1.child1 += ['child11', 'child12', 'child13'] + l1.child1.grandchild1 += ['grandchild111', 'grandchild112'] + l1.child1.grandchild2 += ['grandchild121', 'grandchild122'] + l1.child2.grandchild1 += ['grandchild211', 'grandchild212'] + l1.child2.grandchild1 += ['grandchild213', 'grandchild214'] + l2 = HierarchicalStringList() + l2.child1 += ['child14', 'child15'] + l2.child1.grandchild2 += ['grandchild123'] + l2.child3 += ['child31', 'child32'] + + l1 += l2 + els = list((path, list(seq)) for path, seq in l1.walk()) + self.assertEqual(els, [ + ('', ['root1', 'root2', 'root3']), + ('child1', ['child11', 'child12', 'child13', 'child14', + 'child15']), + ('child1/grandchild1', ['grandchild111', 'grandchild112']), + ('child1/grandchild2', ['grandchild121', 'grandchild122', + 'grandchild123']), + ('child2/grandchild1', ['grandchild211', 'grandchild212', + 'grandchild213', 'grandchild214']), + ('child3', ['child31', 'child32']), + ]) + + +class TestStrictOrderingOnAppendList(unittest.TestCase): + def test_init(self): + l = StrictOrderingOnAppendList() + self.assertEqual(len(l), 0) + + l = StrictOrderingOnAppendList(['a', 'b', 'c']) + self.assertEqual(len(l), 3) + + with self.assertRaises(UnsortedError): + StrictOrderingOnAppendList(['c', 'b', 'a']) + + self.assertEqual(len(l), 3) + + def test_extend(self): + l = StrictOrderingOnAppendList() + l.extend(['a', 'b']) + self.assertEqual(len(l), 2) + self.assertIsInstance(l, StrictOrderingOnAppendList) + + with self.assertRaises(UnsortedError): + l.extend(['d', 'c']) + + self.assertEqual(len(l), 2) + + def test_slicing(self): + l = StrictOrderingOnAppendList() + l[:] = ['a', 'b'] + self.assertEqual(len(l), 2) + self.assertIsInstance(l, StrictOrderingOnAppendList) + + with self.assertRaises(UnsortedError): + l[:] = ['b', 'a'] + + self.assertEqual(len(l), 2) + + def test_add(self): + l = StrictOrderingOnAppendList() + l2 = l + ['a', 'b'] + self.assertEqual(len(l), 0) + self.assertEqual(len(l2), 2) + self.assertIsInstance(l2, StrictOrderingOnAppendList) + + with self.assertRaises(UnsortedError): + l2 = l + ['b', 'a'] + + self.assertEqual(len(l), 0) + + def test_iadd(self): + l = StrictOrderingOnAppendList() + l += ['a', 'b'] + self.assertEqual(len(l), 2) + self.assertIsInstance(l, StrictOrderingOnAppendList) + + with self.assertRaises(UnsortedError): + l += ['b', 'a'] + + self.assertEqual(len(l), 2) + + def test_add_after_iadd(self): + l = StrictOrderingOnAppendList(['b']) + l += ['a'] + l2 = l + ['c', 'd'] + self.assertEqual(len(l), 2) + self.assertEqual(len(l2), 4) + self.assertIsInstance(l2, StrictOrderingOnAppendList) + with self.assertRaises(UnsortedError): + l2 = l + ['d', 'c'] + + self.assertEqual(len(l), 2) + + def test_add_StrictOrderingOnAppendList(self): + l = StrictOrderingOnAppendList() + l += ['c', 'd'] + l += ['a', 'b'] + l2 = StrictOrderingOnAppendList() + with self.assertRaises(UnsortedError): + l2 += list(l) + # Adding a StrictOrderingOnAppendList to another shouldn't throw + l2 += l + + +class TestListWithAction(unittest.TestCase): + def setUp(self): + self.action = lambda a: (a, id(a)) + + def assertSameList(self, expected, actual): + self.assertEqual(len(expected), len(actual)) + for idx, item in enumerate(actual): + self.assertEqual(item, expected[idx]) + + def test_init(self): + l = ListWithAction(action=self.action) + self.assertEqual(len(l), 0) + original = ['a', 'b', 'c'] + l = ListWithAction(['a', 'b', 'c'], action=self.action) + expected = map(self.action, original) + self.assertSameList(expected, l) + + with self.assertRaises(ValueError): + ListWithAction('abc', action=self.action) + + with self.assertRaises(ValueError): + ListWithAction() + + def test_extend(self): + l = ListWithAction(action=self.action) + original = ['a', 'b'] + l.extend(original) + expected = map(self.action, original) + self.assertSameList(expected, l) + + with self.assertRaises(ValueError): + l.extend('ab') + + def test_slicing(self): + l = ListWithAction(action=self.action) + original = ['a', 'b'] + l[:] = original + expected = map(self.action, original) + self.assertSameList(expected, l) + + with self.assertRaises(ValueError): + l[:] = 'ab' + + def test_add(self): + l = ListWithAction(action=self.action) + original = ['a', 'b'] + l2 = l + original + expected = map(self.action, original) + self.assertSameList(expected, l2) + + with self.assertRaises(ValueError): + l + 'abc' + + def test_iadd(self): + l = ListWithAction(action=self.action) + original = ['a', 'b'] + l += original + expected = map(self.action, original) + self.assertSameList(expected, l) + + with self.assertRaises(ValueError): + l += 'abc' + + +class TestStrictOrderingOnAppendListWithFlagsFactory(unittest.TestCase): + def test_strict_ordering_on_append_list_with_flags_factory(self): + cls = StrictOrderingOnAppendListWithFlagsFactory({ + 'foo': bool, + 'bar': int, + }) + + l = cls() + l += ['a', 'b'] + + with self.assertRaises(Exception): + l['a'] = 'foo' + + with self.assertRaises(Exception): + c = l['c'] + + self.assertEqual(l['a'].foo, False) + l['a'].foo = True + self.assertEqual(l['a'].foo, True) + + with self.assertRaises(TypeError): + l['a'].bar = 'bar' + + self.assertEqual(l['a'].bar, 0) + l['a'].bar = 42 + self.assertEqual(l['a'].bar, 42) + + l['b'].foo = True + self.assertEqual(l['b'].foo, True) + + with self.assertRaises(AttributeError): + l['b'].baz = False + + l['b'].update(foo=False, bar=12) + self.assertEqual(l['b'].foo, False) + self.assertEqual(l['b'].bar, 12) + + with self.assertRaises(AttributeError): + l['b'].update(xyz=1) + + def test_strict_ordering_on_append_list_with_flags_factory_extend(self): + FooList = StrictOrderingOnAppendListWithFlagsFactory({ + 'foo': bool, 'bar': unicode + }) + foo = FooList(['a', 'b', 'c']) + foo['a'].foo = True + foo['b'].bar = 'bar' + + # Don't allow extending lists with different flag definitions. + BarList = StrictOrderingOnAppendListWithFlagsFactory({ + 'foo': unicode, 'baz': bool + }) + bar = BarList(['d', 'e', 'f']) + bar['d'].foo = 'foo' + bar['e'].baz = True + with self.assertRaises(ValueError): + foo + bar + with self.assertRaises(ValueError): + bar + foo + + # It's not obvious what to do with duplicate list items with possibly + # different flag values, so don't allow that case. + with self.assertRaises(ValueError): + foo + foo + + def assertExtended(l): + self.assertEqual(len(l), 6) + self.assertEqual(l['a'].foo, True) + self.assertEqual(l['b'].bar, 'bar') + self.assertTrue('c' in l) + self.assertEqual(l['d'].foo, True) + self.assertEqual(l['e'].bar, 'bar') + self.assertTrue('f' in l) + + # Test extend. + zot = FooList(['d', 'e', 'f']) + zot['d'].foo = True + zot['e'].bar = 'bar' + zot.extend(foo) + assertExtended(zot) + + # Test __add__. + zot = FooList(['d', 'e', 'f']) + zot['d'].foo = True + zot['e'].bar = 'bar' + assertExtended(foo + zot) + assertExtended(zot + foo) + + # Test __iadd__. + foo += zot + assertExtended(foo) + + # Test __setslice__. + foo[3:] = [] + self.assertEqual(len(foo), 3) + foo[3:] = zot + assertExtended(foo) + + +class TestMemoize(unittest.TestCase): + def test_memoize(self): + self._count = 0 + @memoize + def wrapped(a, b): + self._count += 1 + return a + b + + self.assertEqual(self._count, 0) + self.assertEqual(wrapped(1, 1), 2) + self.assertEqual(self._count, 1) + self.assertEqual(wrapped(1, 1), 2) + self.assertEqual(self._count, 1) + self.assertEqual(wrapped(2, 1), 3) + self.assertEqual(self._count, 2) + self.assertEqual(wrapped(1, 2), 3) + self.assertEqual(self._count, 3) + self.assertEqual(wrapped(1, 2), 3) + self.assertEqual(self._count, 3) + self.assertEqual(wrapped(1, 1), 2) + self.assertEqual(self._count, 3) + + def test_memoize_method(self): + class foo(object): + def __init__(self): + self._count = 0 + + @memoize + def wrapped(self, a, b): + self._count += 1 + return a + b + + instance = foo() + refcount = sys.getrefcount(instance) + self.assertEqual(instance._count, 0) + self.assertEqual(instance.wrapped(1, 1), 2) + self.assertEqual(instance._count, 1) + self.assertEqual(instance.wrapped(1, 1), 2) + self.assertEqual(instance._count, 1) + self.assertEqual(instance.wrapped(2, 1), 3) + self.assertEqual(instance._count, 2) + self.assertEqual(instance.wrapped(1, 2), 3) + self.assertEqual(instance._count, 3) + self.assertEqual(instance.wrapped(1, 2), 3) + self.assertEqual(instance._count, 3) + self.assertEqual(instance.wrapped(1, 1), 2) + self.assertEqual(instance._count, 3) + + # Memoization of methods is expected to not keep references to + # instances, so the refcount shouldn't have changed after executing the + # memoized method. + self.assertEqual(refcount, sys.getrefcount(instance)) + + def test_memoized_property(self): + class foo(object): + def __init__(self): + self._count = 0 + + @memoized_property + def wrapped(self): + self._count += 1 + return 42 + + instance = foo() + self.assertEqual(instance._count, 0) + self.assertEqual(instance.wrapped, 42) + self.assertEqual(instance._count, 1) + self.assertEqual(instance.wrapped, 42) + self.assertEqual(instance._count, 1) + + +class TestTypedList(unittest.TestCase): + def test_init(self): + cls = TypedList(int) + l = cls() + self.assertEqual(len(l), 0) + + l = cls([1, 2, 3]) + self.assertEqual(len(l), 3) + + with self.assertRaises(ValueError): + cls([1, 2, 'c']) + + def test_extend(self): + cls = TypedList(int) + l = cls() + l.extend([1, 2]) + self.assertEqual(len(l), 2) + self.assertIsInstance(l, cls) + + with self.assertRaises(ValueError): + l.extend([3, 'c']) + + self.assertEqual(len(l), 2) + + def test_slicing(self): + cls = TypedList(int) + l = cls() + l[:] = [1, 2] + self.assertEqual(len(l), 2) + self.assertIsInstance(l, cls) + + with self.assertRaises(ValueError): + l[:] = [3, 'c'] + + self.assertEqual(len(l), 2) + + def test_add(self): + cls = TypedList(int) + l = cls() + l2 = l + [1, 2] + self.assertEqual(len(l), 0) + self.assertEqual(len(l2), 2) + self.assertIsInstance(l2, cls) + + with self.assertRaises(ValueError): + l2 = l + [3, 'c'] + + self.assertEqual(len(l), 0) + + def test_iadd(self): + cls = TypedList(int) + l = cls() + l += [1, 2] + self.assertEqual(len(l), 2) + self.assertIsInstance(l, cls) + + with self.assertRaises(ValueError): + l += [3, 'c'] + + self.assertEqual(len(l), 2) + + def test_add_coercion(self): + objs = [] + + class Foo(object): + def __init__(self, obj): + objs.append(obj) + + cls = TypedList(Foo) + l = cls() + l += [1, 2] + self.assertEqual(len(objs), 2) + self.assertEqual(type(l[0]), Foo) + self.assertEqual(type(l[1]), Foo) + + # Adding a TypedList to a TypedList shouldn't trigger coercion again + l2 = cls() + l2 += l + self.assertEqual(len(objs), 2) + self.assertEqual(type(l2[0]), Foo) + self.assertEqual(type(l2[1]), Foo) + + # Adding a TypedList to a TypedList shouldn't even trigger the code + # that does coercion at all. + l2 = cls() + list.__setslice__(l, 0, -1, [1, 2]) + l2 += l + self.assertEqual(len(objs), 2) + self.assertEqual(type(l2[0]), int) + self.assertEqual(type(l2[1]), int) + + def test_memoized(self): + cls = TypedList(int) + cls2 = TypedList(str) + self.assertEqual(TypedList(int), cls) + self.assertNotEqual(cls, cls2) + + +class TypedTestStrictOrderingOnAppendList(unittest.TestCase): + def test_init(self): + class Unicode(unicode): + def __init__(self, other): + if not isinstance(other, unicode): + raise ValueError() + super(Unicode, self).__init__(other) + + cls = TypedList(Unicode, StrictOrderingOnAppendList) + l = cls() + self.assertEqual(len(l), 0) + + l = cls(['a', 'b', 'c']) + self.assertEqual(len(l), 3) + + with self.assertRaises(UnsortedError): + cls(['c', 'b', 'a']) + + with self.assertRaises(ValueError): + cls(['a', 'b', 3]) + + self.assertEqual(len(l), 3) + + +class TestTypedNamedTuple(unittest.TestCase): + def test_simple(self): + FooBar = TypedNamedTuple('FooBar', [('foo', unicode), ('bar', int)]) + + t = FooBar(foo='foo', bar=2) + self.assertEquals(type(t), FooBar) + self.assertEquals(t.foo, 'foo') + self.assertEquals(t.bar, 2) + self.assertEquals(t[0], 'foo') + self.assertEquals(t[1], 2) + + FooBar('foo', 2) + + with self.assertRaises(TypeError): + FooBar('foo', 'not integer') + with self.assertRaises(TypeError): + FooBar(2, 4) + + # Passing a tuple as the first argument is the same as passing multiple + # arguments. + t1 = ('foo', 3) + t2 = FooBar(t1) + self.assertEquals(type(t2), FooBar) + self.assertEqual(FooBar(t1), FooBar('foo', 3)) + + +class TestGroupUnifiedFiles(unittest.TestCase): + FILES = ['%s.cpp' % letter for letter in string.ascii_lowercase] + + def test_multiple_files(self): + mapping = list(group_unified_files(self.FILES, 'Unified', 'cpp', 5)) + + def check_mapping(index, expected_num_source_files): + (unified_file, source_files) = mapping[index] + + self.assertEqual(unified_file, 'Unified%d.cpp' % index) + self.assertEqual(len(source_files), expected_num_source_files) + + all_files = list(itertools.chain(*[files for (_, files) in mapping])) + self.assertEqual(len(all_files), len(self.FILES)) + self.assertEqual(set(all_files), set(self.FILES)) + + expected_amounts = [5, 5, 5, 5, 5, 1] + for i, amount in enumerate(expected_amounts): + check_mapping(i, amount) + + def test_unsorted_files(self): + unsorted_files = ['a%d.cpp' % i for i in range(11)] + sorted_files = sorted(unsorted_files) + mapping = list(group_unified_files(unsorted_files, 'Unified', 'cpp', 5)) + + self.assertEqual(mapping[0][1], sorted_files[0:5]) + self.assertEqual(mapping[1][1], sorted_files[5:10]) + self.assertEqual(mapping[2][1], sorted_files[10:]) + + +class TestMisc(unittest.TestCase): + def test_pair(self): + self.assertEqual( + list(pair([1, 2, 3, 4, 5, 6])), + [(1, 2), (3, 4), (5, 6)] + ) + + self.assertEqual( + list(pair([1, 2, 3, 4, 5, 6, 7])), + [(1, 2), (3, 4), (5, 6), (7, None)] + ) + + def test_expand_variables(self): + self.assertEqual( + expand_variables('$(var)', {'var': 'value'}), + 'value' + ) + + self.assertEqual( + expand_variables('$(a) and $(b)', {'a': '1', 'b': '2'}), + '1 and 2' + ) + + self.assertEqual( + expand_variables('$(a) and $(undefined)', {'a': '1', 'b': '2'}), + '1 and ' + ) + + self.assertEqual( + expand_variables('before $(string) between $(list) after', { + 'string': 'abc', + 'list': ['a', 'b', 'c'] + }), + 'before abc between a b c after' + ) + +class TestEnumString(unittest.TestCase): + def test_string(self): + CompilerType = EnumString.subclass('msvc', 'gcc', 'clang', 'clang-cl') + + type = CompilerType('msvc') + self.assertEquals(type, 'msvc') + self.assertNotEquals(type, 'gcc') + self.assertNotEquals(type, 'clang') + self.assertNotEquals(type, 'clang-cl') + self.assertIn(type, ('msvc', 'clang-cl')) + self.assertNotIn(type, ('gcc', 'clang')) + + with self.assertRaises(EnumStringComparisonError): + self.assertEquals(type, 'foo') + + with self.assertRaises(EnumStringComparisonError): + self.assertNotEquals(type, 'foo') + + with self.assertRaises(EnumStringComparisonError): + self.assertIn(type, ('foo', 'gcc')) + + with self.assertRaises(ValueError): + type = CompilerType('foo') + + +class TestIndentedRepr(unittest.TestCase): + def test_indented_repr(self): + data = textwrap.dedent(r''' + { + 'a': 1, + 'b': b'abc', + b'c': 'xyz', + 'd': False, + 'e': { + 'a': 1, + 'b': b'2', + 'c': '3', + }, + 'f': [ + 1, + b'2', + '3', + ], + 'pile_of_bytes': b'\xf0\x9f\x92\xa9', + 'pile_of_poo': '💩', + 'special_chars': '\\\'"\x08\n\t', + 'with_accents': 'éàñ', + }''').lstrip() + + obj = eval(data) + + self.assertEqual(indented_repr(obj), data) + + +if __name__ == '__main__': + main() diff --git a/python/mozbuild/mozbuild/testing.py b/python/mozbuild/mozbuild/testing.py new file mode 100644 index 000000000..b327cd74f --- /dev/null +++ b/python/mozbuild/mozbuild/testing.py @@ -0,0 +1,535 @@ +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, You can obtain one at http://mozilla.org/MPL/2.0/. + +from __future__ import absolute_import, unicode_literals + +import cPickle as pickle +import os +import sys + +import mozpack.path as mozpath + +from mozpack.copier import FileCopier +from mozpack.manifests import InstallManifest + +from .base import MozbuildObject +from .util import OrderedDefaultDict +from collections import defaultdict + +import manifestparser + +def rewrite_test_base(test, new_base, honor_install_to_subdir=False): + """Rewrite paths in a test to be under a new base path. + + This is useful for running tests from a separate location from where they + were defined. + + honor_install_to_subdir and the underlying install-to-subdir field are a + giant hack intended to work around the restriction where the mochitest + runner can't handle single test files with multiple configurations. This + argument should be removed once the mochitest runner talks manifests + (bug 984670). + """ + test['here'] = mozpath.join(new_base, test['dir_relpath']) + + if honor_install_to_subdir and test.get('install-to-subdir'): + manifest_relpath = mozpath.relpath(test['path'], + mozpath.dirname(test['manifest'])) + test['path'] = mozpath.join(new_base, test['dir_relpath'], + test['install-to-subdir'], manifest_relpath) + else: + test['path'] = mozpath.join(new_base, test['file_relpath']) + + return test + + +class TestMetadata(object): + """Holds information about tests. + + This class provides an API to query tests active in the build + configuration. + """ + + def __init__(self, all_tests, test_defaults=None): + self._tests_by_path = OrderedDefaultDict(list) + self._tests_by_flavor = defaultdict(set) + self._test_dirs = set() + + with open(all_tests, 'rb') as fh: + test_data = pickle.load(fh) + defaults = None + if test_defaults: + with open(test_defaults, 'rb') as fh: + defaults = pickle.load(fh) + for path, tests in test_data.items(): + for metadata in tests: + if defaults: + manifest = metadata['manifest'] + manifest_defaults = defaults.get(manifest) + if manifest_defaults: + metadata = manifestparser.combine_fields(manifest_defaults, + metadata) + self._tests_by_path[path].append(metadata) + self._test_dirs.add(os.path.dirname(path)) + flavor = metadata.get('flavor') + self._tests_by_flavor[flavor].add(path) + + def tests_with_flavor(self, flavor): + """Obtain all tests having the specified flavor. + + This is a generator of dicts describing each test. + """ + + for path in sorted(self._tests_by_flavor.get(flavor, [])): + yield self._tests_by_path[path] + + def resolve_tests(self, paths=None, flavor=None, subsuite=None, under_path=None, + tags=None): + """Resolve tests from an identifier. + + This is a generator of dicts describing each test. + + ``paths`` can be an iterable of values to use to identify tests to run. + If an entry is a known test file, tests associated with that file are + returned (there may be multiple configurations for a single file). If + an entry is a directory, or a prefix of a directory containing tests, + all tests in that directory are returned. If the string appears in a + known test file, that test file is considered. If the path contains + a wildcard pattern, tests matching that pattern are returned. + + If ``under_path`` is a string, it will be used to filter out tests that + aren't in the specified path prefix relative to topsrcdir or the + test's installed dir. + + If ``flavor`` is a string, it will be used to filter returned tests + to only be the flavor specified. A flavor is something like + ``xpcshell``. + + If ``subsuite`` is a string, it will be used to filter returned tests + to only be in the subsuite specified. + + If ``tags`` are specified, they will be used to filter returned tests + to only those with a matching tag. + """ + if tags: + tags = set(tags) + + def fltr(tests): + for test in tests: + if flavor: + if (flavor == 'devtools' and test.get('flavor') != 'browser-chrome') or \ + (flavor != 'devtools' and test.get('flavor') != flavor): + continue + + if subsuite and test.get('subsuite') != subsuite: + continue + + if tags and not (tags & set(test.get('tags', '').split())): + continue + + if under_path \ + and not test['file_relpath'].startswith(under_path): + continue + + # Make a copy so modifications don't change the source. + yield dict(test) + + paths = paths or [] + paths = [mozpath.normpath(p) for p in paths] + if not paths: + paths = [None] + + candidate_paths = set() + + for path in sorted(paths): + if path is None: + candidate_paths |= set(self._tests_by_path.keys()) + continue + + if '*' in path: + candidate_paths |= {p for p in self._tests_by_path + if mozpath.match(p, path)} + continue + + # If the path is a directory, or the path is a prefix of a directory + # containing tests, pull in all tests in that directory. + if (path in self._test_dirs or + any(p.startswith(path) for p in self._tests_by_path)): + candidate_paths |= {p for p in self._tests_by_path + if p.startswith(path)} + continue + + # If it's a test file, add just that file. + candidate_paths |= {p for p in self._tests_by_path if path in p} + + for p in sorted(candidate_paths): + tests = self._tests_by_path[p] + + for test in fltr(tests): + yield test + + +class TestResolver(MozbuildObject): + """Helper to resolve tests from the current environment to test files.""" + + def __init__(self, *args, **kwargs): + MozbuildObject.__init__(self, *args, **kwargs) + + # If installing tests is going to result in re-generating the build + # backend, we need to do this here, so that the updated contents of + # all-tests.pkl make it to the set of tests to run. + self._run_make(target='run-tests-deps', pass_thru=True, + print_directory=False) + + self._tests = TestMetadata(os.path.join(self.topobjdir, + 'all-tests.pkl'), + test_defaults=os.path.join(self.topobjdir, + 'test-defaults.pkl')) + + self._test_rewrites = { + 'a11y': os.path.join(self.topobjdir, '_tests', 'testing', + 'mochitest', 'a11y'), + 'browser-chrome': os.path.join(self.topobjdir, '_tests', 'testing', + 'mochitest', 'browser'), + 'jetpack-package': os.path.join(self.topobjdir, '_tests', 'testing', + 'mochitest', 'jetpack-package'), + 'jetpack-addon': os.path.join(self.topobjdir, '_tests', 'testing', + 'mochitest', 'jetpack-addon'), + 'chrome': os.path.join(self.topobjdir, '_tests', 'testing', + 'mochitest', 'chrome'), + 'mochitest': os.path.join(self.topobjdir, '_tests', 'testing', + 'mochitest', 'tests'), + 'web-platform-tests': os.path.join(self.topobjdir, '_tests', 'testing', + 'web-platform'), + 'xpcshell': os.path.join(self.topobjdir, '_tests', 'xpcshell'), + } + + def resolve_tests(self, cwd=None, **kwargs): + """Resolve tests in the context of the current environment. + + This is a more intelligent version of TestMetadata.resolve_tests(). + + This function provides additional massaging and filtering of low-level + results. + + Paths in returned tests are automatically translated to the paths in + the _tests directory under the object directory. + + If cwd is defined, we will limit our results to tests under the + directory specified. The directory should be defined as an absolute + path under topsrcdir or topobjdir for it to work properly. + """ + rewrite_base = None + + if cwd: + norm_cwd = mozpath.normpath(cwd) + norm_srcdir = mozpath.normpath(self.topsrcdir) + norm_objdir = mozpath.normpath(self.topobjdir) + + reldir = None + + if norm_cwd.startswith(norm_objdir): + reldir = norm_cwd[len(norm_objdir)+1:] + elif norm_cwd.startswith(norm_srcdir): + reldir = norm_cwd[len(norm_srcdir)+1:] + + result = self._tests.resolve_tests(under_path=reldir, + **kwargs) + + else: + result = self._tests.resolve_tests(**kwargs) + + for test in result: + rewrite_base = self._test_rewrites.get(test['flavor'], None) + + if rewrite_base: + yield rewrite_test_base(test, rewrite_base, + honor_install_to_subdir=True) + else: + yield test + +# These definitions provide a single source of truth for modules attempting +# to get a view of all tests for a build. Used by the emitter to figure out +# how to read/install manifests and by test dependency annotations in Files() +# entries to enumerate test flavors. + +# While there are multiple test manifests, the behavior is very similar +# across them. We enforce this by having common handling of all +# manifests and outputting a single class type with the differences +# described inside the instance. +# +# Keys are variable prefixes and values are tuples describing how these +# manifests should be handled: +# +# (flavor, install_root, install_subdir, package_tests) +# +# flavor identifies the flavor of this test. +# install_root is the path prefix to install the files starting from the root +# directory and not as specified by the manifest location. (bug 972168) +# install_subdir is the path of where to install the files in +# the tests directory. +# package_tests indicates whether to package test files into the test +# package; suites that compile the test files should not install +# them into the test package. +# +TEST_MANIFESTS = dict( + A11Y=('a11y', 'testing/mochitest', 'a11y', True), + BROWSER_CHROME=('browser-chrome', 'testing/mochitest', 'browser', True), + ANDROID_INSTRUMENTATION=('instrumentation', 'instrumentation', '.', False), + JETPACK_PACKAGE=('jetpack-package', 'testing/mochitest', 'jetpack-package', True), + JETPACK_ADDON=('jetpack-addon', 'testing/mochitest', 'jetpack-addon', False), + FIREFOX_UI_FUNCTIONAL=('firefox-ui-functional', 'firefox-ui', '.', False), + FIREFOX_UI_UPDATE=('firefox-ui-update', 'firefox-ui', '.', False), + PUPPETEER_FIREFOX=('firefox-ui-functional', 'firefox-ui', '.', False), + + # marionette tests are run from the srcdir + # TODO(ato): make packaging work as for other test suites + MARIONETTE=('marionette', 'marionette', '.', False), + MARIONETTE_UNIT=('marionette', 'marionette', '.', False), + MARIONETTE_WEBAPI=('marionette', 'marionette', '.', False), + + METRO_CHROME=('metro-chrome', 'testing/mochitest', 'metro', True), + MOCHITEST=('mochitest', 'testing/mochitest', 'tests', True), + MOCHITEST_CHROME=('chrome', 'testing/mochitest', 'chrome', True), + WEBRTC_SIGNALLING_TEST=('steeplechase', 'steeplechase', '.', True), + XPCSHELL_TESTS=('xpcshell', 'xpcshell', '.', True), +) + +# Reftests have their own manifest format and are processed separately. +REFTEST_FLAVORS = ('crashtest', 'reftest') + +# Web platform tests have their own manifest format and are processed separately. +WEB_PLATFORM_TESTS_FLAVORS = ('web-platform-tests',) + +def all_test_flavors(): + return ([v[0] for v in TEST_MANIFESTS.values()] + + list(REFTEST_FLAVORS) + + list(WEB_PLATFORM_TESTS_FLAVORS) + + ['python']) + +class TestInstallInfo(object): + def __init__(self): + self.seen = set() + self.pattern_installs = [] + self.installs = [] + self.external_installs = set() + self.deferred_installs = set() + + def __ior__(self, other): + self.pattern_installs.extend(other.pattern_installs) + self.installs.extend(other.installs) + self.external_installs |= other.external_installs + self.deferred_installs |= other.deferred_installs + return self + +class SupportFilesConverter(object): + """Processes a "support-files" entry from a test object, either from + a parsed object from a test manifests or its representation in + moz.build and returns the installs to perform for this test object. + + Processing the same support files multiple times will not have any further + effect, and the structure of the parsed objects from manifests will have a + lot of repeated entries, so this class takes care of memoizing. + """ + def __init__(self): + self._fields = (('head', set()), + ('tail', set()), + ('support-files', set()), + ('generated-files', set())) + + def convert_support_files(self, test, install_root, manifest_dir, out_dir): + # Arguments: + # test - The test object to process. + # install_root - The directory under $objdir/_tests that will contain + # the tests for this harness (examples are "testing/mochitest", + # "xpcshell"). + # manifest_dir - Absoulute path to the (srcdir) directory containing the + # manifest that included this test + # out_dir - The path relative to $objdir/_tests used as the destination for the + # test, based on the relative path to the manifest in the srcdir, + # the install_root, and 'install-to-subdir', if present in the manifest. + info = TestInstallInfo() + for field, seen in self._fields: + value = test.get(field, '') + for pattern in value.split(): + + # We track uniqueness locally (per test) where duplicates are forbidden, + # and globally, where they are permitted. If a support file appears multiple + # times for a single test, there are unnecessary entries in the manifest. But + # many entries will be shared across tests that share defaults. + # We need to memoize on the basis of both the path and the output + # directory for the benefit of tests specifying 'install-to-subdir'. + key = field, pattern, out_dir + if key in info.seen: + raise ValueError("%s appears multiple times in a test manifest under a %s field," + " please omit the duplicate entry." % (pattern, field)) + info.seen.add(key) + if key in seen: + continue + seen.add(key) + + if field == 'generated-files': + info.external_installs.add(mozpath.normpath(mozpath.join(out_dir, pattern))) + # '!' indicates our syntax for inter-directory support file + # dependencies. These receive special handling in the backend. + elif pattern[0] == '!': + info.deferred_installs.add(pattern) + # We only support globbing on support-files because + # the harness doesn't support * for head and tail. + elif '*' in pattern and field == 'support-files': + info.pattern_installs.append((manifest_dir, pattern, out_dir)) + # "absolute" paths identify files that are to be + # placed in the install_root directory (no globs) + elif pattern[0] == '/': + full = mozpath.normpath(mozpath.join(manifest_dir, + mozpath.basename(pattern))) + info.installs.append((full, mozpath.join(install_root, pattern[1:]))) + else: + full = mozpath.normpath(mozpath.join(manifest_dir, pattern)) + dest_path = mozpath.join(out_dir, pattern) + + # If the path resolves to a different directory + # tree, we take special behavior depending on the + # entry type. + if not full.startswith(manifest_dir): + # If it's a support file, we install the file + # into the current destination directory. + # This implementation makes installing things + # with custom prefixes impossible. If this is + # needed, we can add support for that via a + # special syntax later. + if field == 'support-files': + dest_path = mozpath.join(out_dir, + os.path.basename(pattern)) + # If it's not a support file, we ignore it. + # This preserves old behavior so things like + # head files doesn't get installed multiple + # times. + else: + continue + info.installs.append((full, mozpath.normpath(dest_path))) + return info + +def _resolve_installs(paths, topobjdir, manifest): + """Using the given paths as keys, find any unresolved installs noted + by the build backend corresponding to those keys, and add them + to the given manifest. + """ + filename = os.path.join(topobjdir, 'test-installs.pkl') + with open(filename, 'rb') as fh: + resolved_installs = pickle.load(fh) + + for path in paths: + path = path[2:] + if path not in resolved_installs: + raise Exception('A cross-directory support file path noted in a ' + 'test manifest does not appear in any other manifest.\n "%s" ' + 'must appear in another test manifest to specify an install ' + 'for "!/%s".' % (path, path)) + installs = resolved_installs[path] + for install_info in installs: + try: + if len(install_info) == 3: + manifest.add_pattern_symlink(*install_info) + if len(install_info) == 2: + manifest.add_symlink(*install_info) + except ValueError: + # A duplicate value here is pretty likely when running + # multiple directories at once, and harmless. + pass + +def install_test_files(topsrcdir, topobjdir, tests_root, test_objs): + """Installs the requested test files to the objdir. This is invoked by + test runners to avoid installing tens of thousands of test files when + only a few tests need to be run. + """ + flavor_info = {flavor: (root, prefix, install) + for (flavor, root, prefix, install) in TEST_MANIFESTS.values()} + objdir_dest = mozpath.join(topobjdir, tests_root) + + converter = SupportFilesConverter() + install_info = TestInstallInfo() + for o in test_objs: + flavor = o['flavor'] + if flavor not in flavor_info: + # This is a test flavor that isn't installed by the build system. + continue + root, prefix, install = flavor_info[flavor] + if not install: + # This flavor isn't installed to the objdir. + continue + + manifest_path = o['manifest'] + manifest_dir = mozpath.dirname(manifest_path) + + out_dir = mozpath.join(root, prefix, manifest_dir[len(topsrcdir) + 1:]) + file_relpath = o['file_relpath'] + source = mozpath.join(topsrcdir, file_relpath) + dest = mozpath.join(root, prefix, file_relpath) + if 'install-to-subdir' in o: + out_dir = mozpath.join(out_dir, o['install-to-subdir']) + manifest_relpath = mozpath.relpath(source, mozpath.dirname(manifest_path)) + dest = mozpath.join(out_dir, manifest_relpath) + + install_info.installs.append((source, dest)) + install_info |= converter.convert_support_files(o, root, + manifest_dir, + out_dir) + + manifest = InstallManifest() + + for source, dest in set(install_info.installs): + if dest in install_info.external_installs: + continue + manifest.add_symlink(source, dest) + for base, pattern, dest in install_info.pattern_installs: + manifest.add_pattern_symlink(base, pattern, dest) + + _resolve_installs(install_info.deferred_installs, topobjdir, manifest) + + # Harness files are treated as a monolith and installed each time we run tests. + # Fortunately there are not very many. + manifest |= InstallManifest(mozpath.join(topobjdir, + '_build_manifests', + 'install', tests_root)) + copier = FileCopier() + manifest.populate_registry(copier) + copier.copy(objdir_dest, + remove_unaccounted=False) + + +# Convenience methods for test manifest reading. +def read_manifestparser_manifest(context, manifest_path): + path = mozpath.normpath(mozpath.join(context.srcdir, manifest_path)) + return manifestparser.TestManifest(manifests=[path], strict=True, + rootdir=context.config.topsrcdir, + finder=context._finder, + handle_defaults=False) + +def read_reftest_manifest(context, manifest_path): + import reftest + path = mozpath.normpath(mozpath.join(context.srcdir, manifest_path)) + manifest = reftest.ReftestManifest(finder=context._finder) + manifest.load(path) + return manifest + +def read_wpt_manifest(context, paths): + manifest_path, tests_root = paths + full_path = mozpath.normpath(mozpath.join(context.srcdir, manifest_path)) + old_path = sys.path[:] + try: + # Setup sys.path to include all the dependencies required to import + # the web-platform-tests manifest parser. web-platform-tests provides + # a the localpaths.py to do the path manipulation, which we load, + # providing the __file__ variable so it can resolve the relative + # paths correctly. + paths_file = os.path.join(context.config.topsrcdir, "testing", + "web-platform", "tests", "tools", "localpaths.py") + _globals = {"__file__": paths_file} + execfile(paths_file, _globals) + import manifest as wptmanifest + finally: + sys.path = old_path + f = context._finder.get(full_path) + return wptmanifest.manifest.load(tests_root, f) diff --git a/python/mozbuild/mozbuild/util.py b/python/mozbuild/mozbuild/util.py new file mode 100644 index 000000000..58dd9daf0 --- /dev/null +++ b/python/mozbuild/mozbuild/util.py @@ -0,0 +1,1264 @@ +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this file, +# You can obtain one at http://mozilla.org/MPL/2.0/. + +# This file contains miscellaneous utility functions that don't belong anywhere +# in particular. + +from __future__ import absolute_import, unicode_literals, print_function + +import argparse +import collections +import ctypes +import difflib +import errno +import functools +import hashlib +import itertools +import os +import re +import stat +import sys +import time +import types + +from collections import ( + defaultdict, + Iterable, + OrderedDict, +) +from io import ( + StringIO, + BytesIO, +) + + +if sys.version_info[0] == 3: + str_type = str +else: + str_type = basestring + +if sys.platform == 'win32': + _kernel32 = ctypes.windll.kernel32 + _FILE_ATTRIBUTE_NOT_CONTENT_INDEXED = 0x2000 + + +def exec_(object, globals=None, locals=None): + """Wrapper around the exec statement to avoid bogus errors like: + + SyntaxError: unqualified exec is not allowed in function ... + it is a nested function. + + or + + SyntaxError: unqualified exec is not allowed in function ... + it contains a nested function with free variable + + which happen with older versions of python 2.7. + """ + exec(object, globals, locals) + + +def hash_file(path, hasher=None): + """Hashes a file specified by the path given and returns the hex digest.""" + + # If the default hashing function changes, this may invalidate + # lots of cached data. Don't change it lightly. + h = hasher or hashlib.sha1() + + with open(path, 'rb') as fh: + while True: + data = fh.read(8192) + + if not len(data): + break + + h.update(data) + + return h.hexdigest() + + +class EmptyValue(unicode): + """A dummy type that behaves like an empty string and sequence. + + This type exists in order to support + :py:class:`mozbuild.frontend.reader.EmptyConfig`. It should likely not be + used elsewhere. + """ + def __init__(self): + super(EmptyValue, self).__init__() + + +class ReadOnlyNamespace(object): + """A class for objects with immutable attributes set at initialization.""" + def __init__(self, **kwargs): + for k, v in kwargs.iteritems(): + super(ReadOnlyNamespace, self).__setattr__(k, v) + + def __delattr__(self, key): + raise Exception('Object does not support deletion.') + + def __setattr__(self, key, value): + raise Exception('Object does not support assignment.') + + def __ne__(self, other): + return not (self == other) + + def __eq__(self, other): + return self is other or ( + hasattr(other, '__dict__') and self.__dict__ == other.__dict__) + + def __repr__(self): + return '<%s %r>' % (self.__class__.__name__, self.__dict__) + + +class ReadOnlyDict(dict): + """A read-only dictionary.""" + def __init__(self, *args, **kwargs): + dict.__init__(self, *args, **kwargs) + + def __delitem__(self, key): + raise Exception('Object does not support deletion.') + + def __setitem__(self, key, value): + raise Exception('Object does not support assignment.') + + def update(self, *args, **kwargs): + raise Exception('Object does not support update.') + + +class undefined_default(object): + """Represents an undefined argument value that isn't None.""" + + +undefined = undefined_default() + + +class ReadOnlyDefaultDict(ReadOnlyDict): + """A read-only dictionary that supports default values on retrieval.""" + def __init__(self, default_factory, *args, **kwargs): + ReadOnlyDict.__init__(self, *args, **kwargs) + self._default_factory = default_factory + + def __missing__(self, key): + value = self._default_factory() + dict.__setitem__(self, key, value) + return value + + +def ensureParentDir(path): + """Ensures the directory parent to the given file exists.""" + d = os.path.dirname(path) + if d and not os.path.exists(path): + try: + os.makedirs(d) + except OSError, error: + if error.errno != errno.EEXIST: + raise + + +def mkdir(path, not_indexed=False): + """Ensure a directory exists. + + If ``not_indexed`` is True, an attribute is set that disables content + indexing on the directory. + """ + try: + os.makedirs(path) + except OSError as e: + if e.errno != errno.EEXIST: + raise + + if not_indexed: + if sys.platform == 'win32': + if isinstance(path, str_type): + fn = _kernel32.SetFileAttributesW + else: + fn = _kernel32.SetFileAttributesA + + fn(path, _FILE_ATTRIBUTE_NOT_CONTENT_INDEXED) + elif sys.platform == 'darwin': + with open(os.path.join(path, '.metadata_never_index'), 'a'): + pass + + +def simple_diff(filename, old_lines, new_lines): + """Returns the diff between old_lines and new_lines, in unified diff form, + as a list of lines. + + old_lines and new_lines are lists of non-newline terminated lines to + compare. + old_lines can be None, indicating a file creation. + new_lines can be None, indicating a file deletion. + """ + + old_name = '/dev/null' if old_lines is None else filename + new_name = '/dev/null' if new_lines is None else filename + + return difflib.unified_diff(old_lines or [], new_lines or [], + old_name, new_name, n=4, lineterm='') + + +class FileAvoidWrite(BytesIO): + """File-like object that buffers output and only writes if content changed. + + We create an instance from an existing filename. New content is written to + it. When we close the file object, if the content in the in-memory buffer + differs from what is on disk, then we write out the new content. Otherwise, + the original file is untouched. + + Instances can optionally capture diffs of file changes. This feature is not + enabled by default because it a) doesn't make sense for binary files b) + could add unwanted overhead to calls. + + Additionally, there is dry run mode where the file is not actually written + out, but reports whether the file was existing and would have been updated + still occur, as well as diff capture if requested. + """ + def __init__(self, filename, capture_diff=False, dry_run=False, mode='rU'): + BytesIO.__init__(self) + self.name = filename + self._capture_diff = capture_diff + self._dry_run = dry_run + self.diff = None + self.mode = mode + + def write(self, buf): + if isinstance(buf, unicode): + buf = buf.encode('utf-8') + BytesIO.write(self, buf) + + def close(self): + """Stop accepting writes, compare file contents, and rewrite if needed. + + Returns a tuple of bools indicating what action was performed: + + (file existed, file updated) + + If ``capture_diff`` was specified at construction time and the + underlying file was changed, ``.diff`` will be populated with the diff + of the result. + """ + buf = self.getvalue() + BytesIO.close(self) + existed = False + old_content = None + + try: + existing = open(self.name, self.mode) + existed = True + except IOError: + pass + else: + try: + old_content = existing.read() + if old_content == buf: + return True, False + except IOError: + pass + finally: + existing.close() + + if not self._dry_run: + ensureParentDir(self.name) + # Maintain 'b' if specified. 'U' only applies to modes starting with + # 'r', so it is dropped. + writemode = 'w' + if 'b' in self.mode: + writemode += 'b' + with open(self.name, writemode) as file: + file.write(buf) + + if self._capture_diff: + try: + old_lines = old_content.splitlines() if existed else None + new_lines = buf.splitlines() + + self.diff = simple_diff(self.name, old_lines, new_lines) + # FileAvoidWrite isn't unicode/bytes safe. So, files with non-ascii + # content or opened and written in different modes may involve + # implicit conversion and this will make Python unhappy. Since + # diffing isn't a critical feature, we just ignore the failure. + # This can go away once FileAvoidWrite uses io.BytesIO and + # io.StringIO. But that will require a lot of work. + except (UnicodeDecodeError, UnicodeEncodeError): + self.diff = ['Binary or non-ascii file changed: %s' % + self.name] + + return existed, True + + def __enter__(self): + return self + def __exit__(self, type, value, traceback): + if not self.closed: + self.close() + + +def resolve_target_to_make(topobjdir, target): + r''' + Resolve `target` (a target, directory, or file) to a make target. + + `topobjdir` is the object directory; all make targets will be + rooted at or below the top-level Makefile in this directory. + + Returns a pair `(reldir, target)` where `reldir` is a directory + relative to `topobjdir` containing a Makefile and `target` is a + make target (possibly `None`). + + A directory resolves to the nearest directory at or above + containing a Makefile, and target `None`. + + A regular (non-Makefile) file resolves to the nearest directory at + or above the file containing a Makefile, and an appropriate + target. + + A Makefile resolves to the nearest parent strictly above the + Makefile containing a different Makefile, and an appropriate + target. + ''' + + target = target.replace(os.sep, '/').lstrip('/') + abs_target = os.path.join(topobjdir, target) + + # For directories, run |make -C dir|. If the directory does not + # contain a Makefile, check parents until we find one. At worst, + # this will terminate at the root. + if os.path.isdir(abs_target): + current = abs_target + + while True: + make_path = os.path.join(current, 'Makefile') + if os.path.exists(make_path): + return (current[len(topobjdir) + 1:], None) + + current = os.path.dirname(current) + + # If it's not in a directory, this is probably a top-level make + # target. Treat it as such. + if '/' not in target: + return (None, target) + + # We have a relative path within the tree. We look for a Makefile + # as far into the path as possible. Then, we compute the make + # target as relative to that directory. + reldir = os.path.dirname(target) + target = os.path.basename(target) + + while True: + make_path = os.path.join(topobjdir, reldir, 'Makefile') + + # We append to target every iteration, so the check below + # happens exactly once. + if target != 'Makefile' and os.path.exists(make_path): + return (reldir, target) + + target = os.path.join(os.path.basename(reldir), target) + reldir = os.path.dirname(reldir) + + +class ListMixin(object): + def __init__(self, iterable=None, **kwargs): + if iterable is None: + iterable = [] + if not isinstance(iterable, list): + raise ValueError('List can only be created from other list instances.') + + self._kwargs = kwargs + return super(ListMixin, self).__init__(iterable, **kwargs) + + def extend(self, l): + if not isinstance(l, list): + raise ValueError('List can only be extended with other list instances.') + + return super(ListMixin, self).extend(l) + + def __setslice__(self, i, j, sequence): + if not isinstance(sequence, list): + raise ValueError('List can only be sliced with other list instances.') + + return super(ListMixin, self).__setslice__(i, j, sequence) + + def __add__(self, other): + # Allow None and EmptyValue is a special case because it makes undefined + # variable references in moz.build behave better. + other = [] if isinstance(other, (types.NoneType, EmptyValue)) else other + if not isinstance(other, list): + raise ValueError('Only lists can be appended to lists.') + + new_list = self.__class__(self, **self._kwargs) + new_list.extend(other) + return new_list + + def __iadd__(self, other): + other = [] if isinstance(other, (types.NoneType, EmptyValue)) else other + if not isinstance(other, list): + raise ValueError('Only lists can be appended to lists.') + + return super(ListMixin, self).__iadd__(other) + + +class List(ListMixin, list): + """A list specialized for moz.build environments. + + We overload the assignment and append operations to require that the + appended thing is a list. This avoids bad surprises coming from appending + a string to a list, which would just add each letter of the string. + """ + + +class UnsortedError(Exception): + def __init__(self, srtd, original): + assert len(srtd) == len(original) + + self.sorted = srtd + self.original = original + + for i, orig in enumerate(original): + s = srtd[i] + + if orig != s: + self.i = i + break + + def __str__(self): + s = StringIO() + + s.write('An attempt was made to add an unsorted sequence to a list. ') + s.write('The incoming list is unsorted starting at element %d. ' % + self.i) + s.write('We expected "%s" but got "%s"' % ( + self.sorted[self.i], self.original[self.i])) + + return s.getvalue() + + +class StrictOrderingOnAppendListMixin(object): + @staticmethod + def ensure_sorted(l): + if isinstance(l, StrictOrderingOnAppendList): + return + + def _first_element(e): + # If the list entry is a tuple, we sort based on the first element + # in the tuple. + return e[0] if isinstance(e, tuple) else e + srtd = sorted(l, key=lambda x: _first_element(x).lower()) + + if srtd != l: + raise UnsortedError(srtd, l) + + def __init__(self, iterable=None, **kwargs): + if iterable is None: + iterable = [] + + StrictOrderingOnAppendListMixin.ensure_sorted(iterable) + + super(StrictOrderingOnAppendListMixin, self).__init__(iterable, **kwargs) + + def extend(self, l): + StrictOrderingOnAppendListMixin.ensure_sorted(l) + + return super(StrictOrderingOnAppendListMixin, self).extend(l) + + def __setslice__(self, i, j, sequence): + StrictOrderingOnAppendListMixin.ensure_sorted(sequence) + + return super(StrictOrderingOnAppendListMixin, self).__setslice__(i, j, + sequence) + + def __add__(self, other): + StrictOrderingOnAppendListMixin.ensure_sorted(other) + + return super(StrictOrderingOnAppendListMixin, self).__add__(other) + + def __iadd__(self, other): + StrictOrderingOnAppendListMixin.ensure_sorted(other) + + return super(StrictOrderingOnAppendListMixin, self).__iadd__(other) + + +class StrictOrderingOnAppendList(ListMixin, StrictOrderingOnAppendListMixin, + list): + """A list specialized for moz.build environments. + + We overload the assignment and append operations to require that incoming + elements be ordered. This enforces cleaner style in moz.build files. + """ + +class ListWithActionMixin(object): + """Mixin to create lists with pre-processing. See ListWithAction.""" + def __init__(self, iterable=None, action=None): + if iterable is None: + iterable = [] + if not callable(action): + raise ValueError('A callabe action is required to construct ' + 'a ListWithAction') + + self._action = action + iterable = [self._action(i) for i in iterable] + super(ListWithActionMixin, self).__init__(iterable) + + def extend(self, l): + l = [self._action(i) for i in l] + return super(ListWithActionMixin, self).extend(l) + + def __setslice__(self, i, j, sequence): + sequence = [self._action(item) for item in sequence] + return super(ListWithActionMixin, self).__setslice__(i, j, sequence) + + def __iadd__(self, other): + other = [self._action(i) for i in other] + return super(ListWithActionMixin, self).__iadd__(other) + +class StrictOrderingOnAppendListWithAction(StrictOrderingOnAppendListMixin, + ListMixin, ListWithActionMixin, list): + """An ordered list that accepts a callable to be applied to each item. + + A callable (action) passed to the constructor is run on each item of input. + The result of running the callable on each item will be stored in place of + the original input, but the original item must be used to enforce sortedness. + Note that the order of superclasses is therefore significant. + """ + +class ListWithAction(ListMixin, ListWithActionMixin, list): + """A list that accepts a callable to be applied to each item. + + A callable (action) may optionally be passed to the constructor to run on + each item of input. The result of calling the callable on each item will be + stored in place of the original input. + """ + +class MozbuildDeletionError(Exception): + pass + + +def FlagsFactory(flags): + """Returns a class which holds optional flags for an item in a list. + + The flags are defined in the dict given as argument, where keys are + the flag names, and values the type used for the value of that flag. + + The resulting class is used by the various WithFlagsFactory + functions below. + """ + assert isinstance(flags, dict) + assert all(isinstance(v, type) for v in flags.values()) + + class Flags(object): + __slots__ = flags.keys() + _flags = flags + + def update(self, **kwargs): + for k, v in kwargs.iteritems(): + setattr(self, k, v) + + def __getattr__(self, name): + if name not in self.__slots__: + raise AttributeError("'%s' object has no attribute '%s'" % + (self.__class__.__name__, name)) + try: + return object.__getattr__(self, name) + except AttributeError: + value = self._flags[name]() + self.__setattr__(name, value) + return value + + def __setattr__(self, name, value): + if name not in self.__slots__: + raise AttributeError("'%s' object has no attribute '%s'" % + (self.__class__.__name__, name)) + if not isinstance(value, self._flags[name]): + raise TypeError("'%s' attribute of class '%s' must be '%s'" % + (name, self.__class__.__name__, + self._flags[name].__name__)) + return object.__setattr__(self, name, value) + + def __delattr__(self, name): + raise MozbuildDeletionError('Unable to delete attributes for this object') + + return Flags + + +class StrictOrderingOnAppendListWithFlags(StrictOrderingOnAppendList): + """A list with flags specialized for moz.build environments. + + Each subclass has a set of typed flags; this class lets us use `isinstance` + for natural testing. + """ + + +def StrictOrderingOnAppendListWithFlagsFactory(flags): + """Returns a StrictOrderingOnAppendList-like object, with optional + flags on each item. + + The flags are defined in the dict given as argument, where keys are + the flag names, and values the type used for the value of that flag. + + Example: + FooList = StrictOrderingOnAppendListWithFlagsFactory({ + 'foo': bool, 'bar': unicode + }) + foo = FooList(['a', 'b', 'c']) + foo['a'].foo = True + foo['b'].bar = 'bar' + """ + class StrictOrderingOnAppendListWithFlagsSpecialization(StrictOrderingOnAppendListWithFlags): + def __init__(self, iterable=None): + if iterable is None: + iterable = [] + StrictOrderingOnAppendListWithFlags.__init__(self, iterable) + self._flags_type = FlagsFactory(flags) + self._flags = dict() + + def __getitem__(self, name): + if name not in self._flags: + if name not in self: + raise KeyError("'%s'" % name) + self._flags[name] = self._flags_type() + return self._flags[name] + + def __setitem__(self, name, value): + raise TypeError("'%s' object does not support item assignment" % + self.__class__.__name__) + + def _update_flags(self, other): + if self._flags_type._flags != other._flags_type._flags: + raise ValueError('Expected a list of strings with flags like %s, not like %s' % + (self._flags_type._flags, other._flags_type._flags)) + intersection = set(self._flags.keys()) & set(other._flags.keys()) + if intersection: + raise ValueError('Cannot update flags: both lists of strings with flags configure %s' % + intersection) + self._flags.update(other._flags) + + def extend(self, l): + result = super(StrictOrderingOnAppendList, self).extend(l) + if isinstance(l, StrictOrderingOnAppendListWithFlags): + self._update_flags(l) + return result + + def __setslice__(self, i, j, sequence): + result = super(StrictOrderingOnAppendList, self).__setslice__(i, j, sequence) + # We may have removed items. + for name in set(self._flags.keys()) - set(self): + del self._flags[name] + if isinstance(sequence, StrictOrderingOnAppendListWithFlags): + self._update_flags(sequence) + return result + + def __add__(self, other): + result = super(StrictOrderingOnAppendList, self).__add__(other) + if isinstance(other, StrictOrderingOnAppendListWithFlags): + # Result has flags from other but not from self, since + # internally we duplicate self and then extend with other, and + # only extend knows about flags. Since we don't allow updating + # when the set of flag keys intersect, which we instance we pass + # to _update_flags here matters. This needs to be correct but + # is an implementation detail. + result._update_flags(self) + return result + + def __iadd__(self, other): + result = super(StrictOrderingOnAppendList, self).__iadd__(other) + if isinstance(other, StrictOrderingOnAppendListWithFlags): + self._update_flags(other) + return result + + return StrictOrderingOnAppendListWithFlagsSpecialization + + +class HierarchicalStringList(object): + """A hierarchy of lists of strings. + + Each instance of this object contains a list of strings, which can be set or + appended to. A sub-level of the hierarchy is also an instance of this class, + can be added by appending to an attribute instead. + + For example, the moz.build variable EXPORTS is an instance of this class. We + can do: + + EXPORTS += ['foo.h'] + EXPORTS.mozilla.dom += ['bar.h'] + + In this case, we have 3 instances (EXPORTS, EXPORTS.mozilla, and + EXPORTS.mozilla.dom), and the first and last each have one element in their + list. + """ + __slots__ = ('_strings', '_children') + + def __init__(self): + # Please change ContextDerivedTypedHierarchicalStringList in context.py + # if you make changes here. + self._strings = StrictOrderingOnAppendList() + self._children = {} + + class StringListAdaptor(collections.Sequence): + def __init__(self, hsl): + self._hsl = hsl + + def __getitem__(self, index): + return self._hsl._strings[index] + + def __len__(self): + return len(self._hsl._strings) + + def walk(self): + """Walk over all HierarchicalStringLists in the hierarchy. + + This is a generator of (path, sequence). + + The path is '' for the root level and '/'-delimited strings for + any descendants. The sequence is a read-only sequence of the + strings contained at that level. + """ + + if self._strings: + path_to_here = '' + yield path_to_here, self.StringListAdaptor(self) + + for k, l in sorted(self._children.items()): + for p, v in l.walk(): + path_to_there = '%s/%s' % (k, p) + yield path_to_there.strip('/'), v + + def __setattr__(self, name, value): + if name in self.__slots__: + return object.__setattr__(self, name, value) + + # __setattr__ can be called with a list when a simple assignment is + # used: + # + # EXPORTS.foo = ['file.h'] + # + # In this case, we need to overwrite foo's current list of strings. + # + # However, __setattr__ is also called with a HierarchicalStringList + # to try to actually set the attribute. We want to ignore this case, + # since we don't actually create an attribute called 'foo', but just add + # it to our list of children (using _get_exportvariable()). + self._set_exportvariable(name, value) + + def __getattr__(self, name): + if name.startswith('__'): + return object.__getattr__(self, name) + return self._get_exportvariable(name) + + def __delattr__(self, name): + raise MozbuildDeletionError('Unable to delete attributes for this object') + + def __iadd__(self, other): + if isinstance(other, HierarchicalStringList): + self._strings += other._strings + for c in other._children: + self[c] += other[c] + else: + self._check_list(other) + self._strings += other + return self + + def __getitem__(self, name): + return self._get_exportvariable(name) + + def __setitem__(self, name, value): + self._set_exportvariable(name, value) + + def _get_exportvariable(self, name): + # Please change ContextDerivedTypedHierarchicalStringList in context.py + # if you make changes here. + child = self._children.get(name) + if not child: + child = self._children[name] = HierarchicalStringList() + return child + + def _set_exportvariable(self, name, value): + if name in self._children: + if value is self._get_exportvariable(name): + return + raise KeyError('global_ns', 'reassign', + '.%s' % name) + + exports = self._get_exportvariable(name) + exports._check_list(value) + exports._strings += value + + def _check_list(self, value): + if not isinstance(value, list): + raise ValueError('Expected a list of strings, not %s' % type(value)) + for v in value: + if not isinstance(v, str_type): + raise ValueError( + 'Expected a list of strings, not an element of %s' % type(v)) + + +class LockFile(object): + """LockFile is used by the lock_file method to hold the lock. + + This object should not be used directly, but only through + the lock_file method below. + """ + + def __init__(self, lockfile): + self.lockfile = lockfile + + def __del__(self): + while True: + try: + os.remove(self.lockfile) + break + except OSError as e: + if e.errno == errno.EACCES: + # Another process probably has the file open, we'll retry. + # Just a short sleep since we want to drop the lock ASAP + # (but we need to let some other process close the file + # first). + time.sleep(0.1) + else: + # Re-raise unknown errors + raise + + +def lock_file(lockfile, max_wait = 600): + """Create and hold a lockfile of the given name, with the given timeout. + + To release the lock, delete the returned object. + """ + + # FUTURE This function and object could be written as a context manager. + + while True: + try: + fd = os.open(lockfile, os.O_EXCL | os.O_RDWR | os.O_CREAT) + # We created the lockfile, so we're the owner + break + except OSError as e: + if (e.errno == errno.EEXIST or + (sys.platform == "win32" and e.errno == errno.EACCES)): + pass + else: + # Should not occur + raise + + try: + # The lock file exists, try to stat it to get its age + # and read its contents to report the owner PID + f = open(lockfile, 'r') + s = os.stat(lockfile) + except EnvironmentError as e: + if e.errno == errno.ENOENT or e.errno == errno.EACCES: + # We didn't create the lockfile, so it did exist, but it's + # gone now. Just try again + continue + + raise Exception('{0} exists but stat() failed: {1}'.format( + lockfile, e.strerror)) + + # We didn't create the lockfile and it's still there, check + # its age + now = int(time.time()) + if now - s[stat.ST_MTIME] > max_wait: + pid = f.readline().rstrip() + raise Exception('{0} has been locked for more than ' + '{1} seconds (PID {2})'.format(lockfile, max_wait, pid)) + + # It's not been locked too long, wait a while and retry + f.close() + time.sleep(1) + + # if we get here. we have the lockfile. Convert the os.open file + # descriptor into a Python file object and record our PID in it + f = os.fdopen(fd, 'w') + f.write('{0}\n'.format(os.getpid())) + f.close() + + return LockFile(lockfile) + + +class OrderedDefaultDict(OrderedDict): + '''A combination of OrderedDict and defaultdict.''' + def __init__(self, default_factory, *args, **kwargs): + OrderedDict.__init__(self, *args, **kwargs) + self._default_factory = default_factory + + def __missing__(self, key): + value = self[key] = self._default_factory() + return value + + +class KeyedDefaultDict(dict): + '''Like a defaultdict, but the default_factory function takes the key as + argument''' + def __init__(self, default_factory, *args, **kwargs): + dict.__init__(self, *args, **kwargs) + self._default_factory = default_factory + + def __missing__(self, key): + value = self._default_factory(key) + dict.__setitem__(self, key, value) + return value + + +class ReadOnlyKeyedDefaultDict(KeyedDefaultDict, ReadOnlyDict): + '''Like KeyedDefaultDict, but read-only.''' + + +class memoize(dict): + '''A decorator to memoize the results of function calls depending + on its arguments. + Both functions and instance methods are handled, although in the + instance method case, the results are cache in the instance itself. + ''' + def __init__(self, func): + self.func = func + functools.update_wrapper(self, func) + + def __call__(self, *args): + if args not in self: + self[args] = self.func(*args) + return self[args] + + def method_call(self, instance, *args): + name = '_%s' % self.func.__name__ + if not hasattr(instance, name): + setattr(instance, name, {}) + cache = getattr(instance, name) + if args not in cache: + cache[args] = self.func(instance, *args) + return cache[args] + + def __get__(self, instance, cls): + return functools.update_wrapper( + functools.partial(self.method_call, instance), self.func) + + +class memoized_property(object): + '''A specialized version of the memoize decorator that works for + class instance properties. + ''' + def __init__(self, func): + self.func = func + + def __get__(self, instance, cls): + name = '_%s' % self.func.__name__ + if not hasattr(instance, name): + setattr(instance, name, self.func(instance)) + return getattr(instance, name) + + +def TypedNamedTuple(name, fields): + """Factory for named tuple types with strong typing. + + Arguments are an iterable of 2-tuples. The first member is the + the field name. The second member is a type the field will be validated + to be. + + Construction of instances varies from ``collections.namedtuple``. + + First, if a single tuple argument is given to the constructor, this is + treated as the equivalent of passing each tuple value as a separate + argument into __init__. e.g.:: + + t = (1, 2) + TypedTuple(t) == TypedTuple(1, 2) + + This behavior is meant for moz.build files, so vanilla tuples are + automatically cast to typed tuple instances. + + Second, fields in the tuple are validated to be instances of the specified + type. This is done via an ``isinstance()`` check. To allow multiple types, + pass a tuple as the allowed types field. + """ + cls = collections.namedtuple(name, (name for name, typ in fields)) + + class TypedTuple(cls): + __slots__ = () + + def __new__(klass, *args, **kwargs): + if len(args) == 1 and not kwargs and isinstance(args[0], tuple): + args = args[0] + + return super(TypedTuple, klass).__new__(klass, *args, **kwargs) + + def __init__(self, *args, **kwargs): + for i, (fname, ftype) in enumerate(self._fields): + value = self[i] + + if not isinstance(value, ftype): + raise TypeError('field in tuple not of proper type: %s; ' + 'got %s, expected %s' % (fname, + type(value), ftype)) + + super(TypedTuple, self).__init__(*args, **kwargs) + + TypedTuple._fields = fields + + return TypedTuple + + +class TypedListMixin(object): + '''Mixin for a list with type coercion. See TypedList.''' + + def _ensure_type(self, l): + if isinstance(l, self.__class__): + return l + + return [self.normalize(e) for e in l] + + def __init__(self, iterable=None, **kwargs): + if iterable is None: + iterable = [] + iterable = self._ensure_type(iterable) + + super(TypedListMixin, self).__init__(iterable, **kwargs) + + def extend(self, l): + l = self._ensure_type(l) + + return super(TypedListMixin, self).extend(l) + + def __setslice__(self, i, j, sequence): + sequence = self._ensure_type(sequence) + + return super(TypedListMixin, self).__setslice__(i, j, + sequence) + + def __add__(self, other): + other = self._ensure_type(other) + + return super(TypedListMixin, self).__add__(other) + + def __iadd__(self, other): + other = self._ensure_type(other) + + return super(TypedListMixin, self).__iadd__(other) + + def append(self, other): + self += [other] + + +@memoize +def TypedList(type, base_class=List): + '''A list with type coercion. + + The given ``type`` is what list elements are being coerced to. It may do + strict validation, throwing ValueError exceptions. + + A ``base_class`` type can be given for more specific uses than a List. For + example, a Typed StrictOrderingOnAppendList can be created with: + + TypedList(unicode, StrictOrderingOnAppendList) + ''' + class _TypedList(TypedListMixin, base_class): + @staticmethod + def normalize(e): + if not isinstance(e, type): + e = type(e) + return e + + return _TypedList + +def group_unified_files(files, unified_prefix, unified_suffix, + files_per_unified_file): + """Return an iterator of (unified_filename, source_filenames) tuples. + + We compile most C and C++ files in "unified mode"; instead of compiling + ``a.cpp``, ``b.cpp``, and ``c.cpp`` separately, we compile a single file + that looks approximately like:: + + #include "a.cpp" + #include "b.cpp" + #include "c.cpp" + + This function handles the details of generating names for the unified + files, and determining which original source files go in which unified + file.""" + + # Make sure the input list is sorted. If it's not, bad things could happen! + files = sorted(files) + + # Our last returned list of source filenames may be short, and we + # don't want the fill value inserted by izip_longest to be an + # issue. So we do a little dance to filter it out ourselves. + dummy_fill_value = ("dummy",) + def filter_out_dummy(iterable): + return itertools.ifilter(lambda x: x != dummy_fill_value, + iterable) + + # From the itertools documentation, slightly modified: + def grouper(n, iterable): + "grouper(3, 'ABCDEFG', 'x') --> ABC DEF Gxx" + args = [iter(iterable)] * n + return itertools.izip_longest(fillvalue=dummy_fill_value, *args) + + for i, unified_group in enumerate(grouper(files_per_unified_file, + files)): + just_the_filenames = list(filter_out_dummy(unified_group)) + yield '%s%d.%s' % (unified_prefix, i, unified_suffix), just_the_filenames + + +def pair(iterable): + '''Given an iterable, returns an iterable pairing its items. + + For example, + list(pair([1,2,3,4,5,6])) + returns + [(1,2), (3,4), (5,6)] + ''' + i = iter(iterable) + return itertools.izip_longest(i, i) + + +VARIABLES_RE = re.compile('\$\((\w+)\)') + + +def expand_variables(s, variables): + '''Given a string with $(var) variable references, replace those references + with the corresponding entries from the given `variables` dict. + + If a variable value is not a string, it is iterated and its items are + joined with a whitespace.''' + result = '' + for s, name in pair(VARIABLES_RE.split(s)): + result += s + value = variables.get(name) + if not value: + continue + if not isinstance(value, types.StringTypes): + value = ' '.join(value) + result += value + return result + + +class DefinesAction(argparse.Action): + '''An ArgumentParser action to handle -Dvar[=value] type of arguments.''' + def __call__(self, parser, namespace, values, option_string): + defines = getattr(namespace, self.dest) + if defines is None: + defines = {} + values = values.split('=', 1) + if len(values) == 1: + name, value = values[0], 1 + else: + name, value = values + if value.isdigit(): + value = int(value) + defines[name] = value + setattr(namespace, self.dest, defines) + + +class EnumStringComparisonError(Exception): + pass + + +class EnumString(unicode): + '''A string type that only can have a limited set of values, similarly to + an Enum, and can only be compared against that set of values. + + The class is meant to be subclassed, where the subclass defines + POSSIBLE_VALUES. The `subclass` method is a helper to create such + subclasses. + ''' + POSSIBLE_VALUES = () + def __init__(self, value): + if value not in self.POSSIBLE_VALUES: + raise ValueError("'%s' is not a valid value for %s" + % (value, self.__class__.__name__)) + + def __eq__(self, other): + if other not in self.POSSIBLE_VALUES: + raise EnumStringComparisonError( + 'Can only compare with %s' + % ', '.join("'%s'" % v for v in self.POSSIBLE_VALUES)) + return super(EnumString, self).__eq__(other) + + def __ne__(self, other): + return not (self == other) + + @staticmethod + def subclass(*possible_values): + class EnumStringSubclass(EnumString): + POSSIBLE_VALUES = possible_values + return EnumStringSubclass + + +def _escape_char(c): + # str.encode('unicode_espace') doesn't escape quotes, presumably because + # quoting could be done with either ' or ". + if c == "'": + return "\\'" + return unicode(c.encode('unicode_escape')) + +# Mapping table between raw characters below \x80 and their escaped +# counterpart, when they differ +_INDENTED_REPR_TABLE = { + c: e + for c, e in map(lambda x: (x, _escape_char(x)), + map(unichr, range(128))) + if c != e +} +# Regexp matching all characters to escape. +_INDENTED_REPR_RE = re.compile( + '([' + ''.join(_INDENTED_REPR_TABLE.values()) + ']+)') + + +def indented_repr(o, indent=4): + '''Similar to repr(), but returns an indented representation of the object + + One notable difference with repr is that the returned representation + assumes `from __future__ import unicode_literals`. + ''' + one_indent = ' ' * indent + def recurse_indented_repr(o, level): + if isinstance(o, dict): + yield '{\n' + for k, v in sorted(o.items()): + yield one_indent * (level + 1) + for d in recurse_indented_repr(k, level + 1): + yield d + yield ': ' + for d in recurse_indented_repr(v, level + 1): + yield d + yield ',\n' + yield one_indent * level + yield '}' + elif isinstance(o, bytes): + yield 'b' + yield repr(o) + elif isinstance(o, unicode): + yield "'" + # We want a readable string (non escaped unicode), but some + # special characters need escaping (e.g. \n, \t, etc.) + for i, s in enumerate(_INDENTED_REPR_RE.split(o)): + if i % 2: + for c in s: + yield _INDENTED_REPR_TABLE[c] + else: + yield s + yield "'" + elif hasattr(o, '__iter__'): + yield '[\n' + for i in o: + yield one_indent * (level + 1) + for d in recurse_indented_repr(i, level + 1): + yield d + yield ',\n' + yield one_indent * level + yield ']' + else: + yield repr(o) + return ''.join(recurse_indented_repr(o, 0)) + + +def encode(obj, encoding='utf-8'): + '''Recursively encode unicode strings with the given encoding.''' + if isinstance(obj, dict): + return { + encode(k, encoding): encode(v, encoding) + for k, v in obj.iteritems() + } + if isinstance(obj, bytes): + return obj + if isinstance(obj, unicode): + return obj.encode(encoding) + if isinstance(obj, Iterable): + return [encode(i, encoding) for i in obj] + return obj diff --git a/python/mozbuild/mozbuild/vendor_rust.py b/python/mozbuild/mozbuild/vendor_rust.py new file mode 100644 index 000000000..92103e1cb --- /dev/null +++ b/python/mozbuild/mozbuild/vendor_rust.py @@ -0,0 +1,86 @@ +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, # You can obtain one at http://mozilla.org/MPL/2.0/. + +from __future__ import absolute_import, print_function, unicode_literals + +from distutils.version import LooseVersion +import logging +from mozbuild.base import ( + BuildEnvironmentNotFoundException, + MozbuildObject, +) +import mozfile +import mozpack.path as mozpath +import os +import subprocess +import sys + +class VendorRust(MozbuildObject): + def get_cargo_path(self): + try: + # If the build isn't --enable-rust then CARGO won't be set. + return self.substs['CARGO'] + except (BuildEnvironmentNotFoundException, KeyError): + # Default if this tree isn't configured. + import which + return which.which('cargo') + + def check_cargo_version(self, cargo): + ''' + Ensure that cargo is new enough. cargo 0.12 added support + for source replacement, which is required for vendoring to work. + ''' + out = subprocess.check_output([cargo, '--version']).splitlines()[0] + if not out.startswith('cargo'): + return False + return LooseVersion(out.split()[1]) >= b'0.13' + + def check_modified_files(self): + ''' + Ensure that there aren't any uncommitted changes to files + in the working copy, since we're going to change some state + on the user. Allow changes to Cargo.{toml,lock} since that's + likely to be a common use case. + ''' + modified = [f for f in self.repository.get_modified_files() if os.path.basename(f) not in ('Cargo.toml', 'Cargo.lock')] + if modified: + self.log(logging.ERROR, 'modified_files', {}, + '''You have uncommitted changes to the following files: + +{files} + +Please commit or stash these changes before vendoring, or re-run with `--ignore-modified`. +'''.format(files='\n'.join(sorted(modified)))) + sys.exit(1) + + def vendor(self, ignore_modified=False): + self.populate_logger() + self.log_manager.enable_unstructured() + if not ignore_modified: + self.check_modified_files() + cargo = self.get_cargo_path() + if not self.check_cargo_version(cargo): + self.log(logging.ERROR, 'cargo_version', {}, 'Cargo >= 0.13 required (install Rust 1.12 or newer)') + return + else: + self.log(logging.DEBUG, 'cargo_version', {}, 'cargo is new enough') + have_vendor = any(l.strip() == 'vendor' for l in subprocess.check_output([cargo, '--list']).splitlines()) + if not have_vendor: + self.log(logging.INFO, 'installing', {}, 'Installing cargo-vendor') + self.run_process(args=[cargo, 'install', 'cargo-vendor']) + else: + self.log(logging.DEBUG, 'cargo_vendor', {}, 'cargo-vendor already intalled') + vendor_dir = mozpath.join(self.topsrcdir, 'third_party/rust') + self.log(logging.INFO, 'rm_vendor_dir', {}, 'rm -rf %s' % vendor_dir) + mozfile.remove(vendor_dir) + # Once we require a new enough cargo to switch to workspaces, we can + # just do this once on the workspace root crate. + for crate_root in ('toolkit/library/rust/', + 'toolkit/library/gtest/rust'): + path = mozpath.join(self.topsrcdir, crate_root) + self._run_command_in_srcdir(args=[cargo, 'generate-lockfile', '--manifest-path', mozpath.join(path, 'Cargo.toml')]) + self._run_command_in_srcdir(args=[cargo, 'vendor', '--sync', mozpath.join(path, 'Cargo.lock'), vendor_dir]) + #TODO: print stats on size of files added/removed, warn or error + # when adding very large files (bug 1306078) + self.repository.add_remove_files(vendor_dir) diff --git a/python/mozbuild/mozbuild/virtualenv.py b/python/mozbuild/mozbuild/virtualenv.py new file mode 100644 index 000000000..05d30424b --- /dev/null +++ b/python/mozbuild/mozbuild/virtualenv.py @@ -0,0 +1,568 @@ +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, You can obtain one at http://mozilla.org/MPL/2.0/. + +# This file contains code for populating the virtualenv environment for +# Mozilla's build system. It is typically called as part of configure. + +from __future__ import absolute_import, print_function, unicode_literals + +import distutils.sysconfig +import os +import shutil +import subprocess +import sys +import warnings + +from distutils.version import LooseVersion + +IS_NATIVE_WIN = (sys.platform == 'win32' and os.sep == '\\') +IS_MSYS2 = (sys.platform == 'win32' and os.sep == '/') +IS_CYGWIN = (sys.platform == 'cygwin') + +# Minimum version of Python required to build. +MINIMUM_PYTHON_VERSION = LooseVersion('2.7.3') +MINIMUM_PYTHON_MAJOR = 2 + + +UPGRADE_WINDOWS = ''' +Please upgrade to the latest MozillaBuild development environment. See +https://developer.mozilla.org/en-US/docs/Developer_Guide/Build_Instructions/Windows_Prerequisites +'''.lstrip() + +UPGRADE_OTHER = ''' +Run |mach bootstrap| to ensure your system is up to date. + +If you still receive this error, your shell environment is likely detecting +another Python version. Ensure a modern Python can be found in the paths +defined by the $PATH environment variable and try again. +'''.lstrip() + + +class VirtualenvManager(object): + """Contains logic for managing virtualenvs for building the tree.""" + + def __init__(self, topsrcdir, topobjdir, virtualenv_path, log_handle, + manifest_path): + """Create a new manager. + + Each manager is associated with a source directory, a path where you + want the virtualenv to be created, and a handle to write output to. + """ + assert os.path.isabs(manifest_path), "manifest_path must be an absolute path: %s" % (manifest_path) + self.topsrcdir = topsrcdir + self.topobjdir = topobjdir + self.virtualenv_root = virtualenv_path + + # Record the Python executable that was used to create the Virtualenv + # so we can check this against sys.executable when verifying the + # integrity of the virtualenv. + self.exe_info_path = os.path.join(self.virtualenv_root, + 'python_exe.txt') + + self.log_handle = log_handle + self.manifest_path = manifest_path + + @property + def virtualenv_script_path(self): + """Path to virtualenv's own populator script.""" + return os.path.join(self.topsrcdir, 'python', 'virtualenv', + 'virtualenv.py') + + @property + def bin_path(self): + # virtualenv.py provides a similar API via path_locations(). However, + # we have a bit of a chicken-and-egg problem and can't reliably + # import virtualenv. The functionality is trivial, so just implement + # it here. + if IS_CYGWIN or IS_NATIVE_WIN: + return os.path.join(self.virtualenv_root, 'Scripts') + + return os.path.join(self.virtualenv_root, 'bin') + + @property + def python_path(self): + binary = 'python' + if sys.platform in ('win32', 'cygwin'): + binary += '.exe' + + return os.path.join(self.bin_path, binary) + + @property + def activate_path(self): + return os.path.join(self.bin_path, 'activate_this.py') + + def get_exe_info(self): + """Returns the version and file size of the python executable that was in + use when this virutalenv was created. + """ + with open(self.exe_info_path, 'r') as fh: + version, size = fh.read().splitlines() + return int(version), int(size) + + def write_exe_info(self, python): + """Records the the version of the python executable that was in use when + this virutalenv was created. We record this explicitly because + on OS X our python path may end up being a different or modified + executable. + """ + ver = subprocess.check_output([python, '-c', 'import sys; print(sys.hexversion)']).rstrip() + with open(self.exe_info_path, 'w') as fh: + fh.write("%s\n" % ver) + fh.write("%s\n" % os.path.getsize(python)) + + def up_to_date(self, python=sys.executable): + """Returns whether the virtualenv is present and up to date.""" + + deps = [self.manifest_path, __file__] + + # check if virtualenv exists + if not os.path.exists(self.virtualenv_root) or \ + not os.path.exists(self.activate_path): + + return False + + # check modification times + activate_mtime = os.path.getmtime(self.activate_path) + dep_mtime = max(os.path.getmtime(p) for p in deps) + if dep_mtime > activate_mtime: + return False + + # Verify that the Python we're checking here is either the virutalenv + # python, or we have the Python version that was used to create the + # virtualenv. If this fails, it is likely system Python has been + # upgraded, and our virtualenv would not be usable. + python_size = os.path.getsize(python) + if ((python, python_size) != (self.python_path, os.path.getsize(self.python_path)) and + (sys.hexversion, python_size) != self.get_exe_info()): + return False + + # recursively check sub packages.txt files + submanifests = [i[1] for i in self.packages() + if i[0] == 'packages.txt'] + for submanifest in submanifests: + submanifest = os.path.join(self.topsrcdir, submanifest) + submanager = VirtualenvManager(self.topsrcdir, + self.topobjdir, + self.virtualenv_root, + self.log_handle, + submanifest) + if not submanager.up_to_date(python): + return False + + return True + + def ensure(self, python=sys.executable): + """Ensure the virtualenv is present and up to date. + + If the virtualenv is up to date, this does nothing. Otherwise, it + creates and populates the virtualenv as necessary. + + This should be the main API used from this class as it is the + highest-level. + """ + if self.up_to_date(python): + return self.virtualenv_root + return self.build(python) + + def _log_process_output(self, *args, **kwargs): + if hasattr(self.log_handle, 'fileno'): + return subprocess.call(*args, stdout=self.log_handle, + stderr=subprocess.STDOUT, **kwargs) + + proc = subprocess.Popen(*args, stdout=subprocess.PIPE, + stderr=subprocess.STDOUT, **kwargs) + + for line in proc.stdout: + self.log_handle.write(line) + + return proc.wait() + + def create(self, python=sys.executable): + """Create a new, empty virtualenv. + + Receives the path to virtualenv's virtualenv.py script (which will be + called out to), the path to create the virtualenv in, and a handle to + write output to. + """ + env = dict(os.environ) + env.pop('PYTHONDONTWRITEBYTECODE', None) + + args = [python, self.virtualenv_script_path, + # Without this, virtualenv.py may attempt to contact the outside + # world and search for or download a newer version of pip, + # setuptools, or wheel. This is bad for security, reproducibility, + # and speed. + '--no-download', + self.virtualenv_root] + + result = self._log_process_output(args, env=env) + + if result: + raise Exception( + 'Failed to create virtualenv: %s' % self.virtualenv_root) + + self.write_exe_info(python) + + return self.virtualenv_root + + def packages(self): + with file(self.manifest_path, 'rU') as fh: + packages = [line.rstrip().split(':') + for line in fh] + return packages + + def populate(self): + """Populate the virtualenv. + + The manifest file consists of colon-delimited fields. The first field + specifies the action. The remaining fields are arguments to that + action. The following actions are supported: + + setup.py -- Invoke setup.py for a package. Expects the arguments: + 1. relative path directory containing setup.py. + 2. argument(s) to setup.py. e.g. "develop". Each program argument + is delimited by a colon. Arguments with colons are not yet + supported. + + filename.pth -- Adds the path given as argument to filename.pth under + the virtualenv site packages directory. + + optional -- This denotes the action as optional. The requested action + is attempted. If it fails, we issue a warning and go on. The + initial "optional" field is stripped then the remaining line is + processed like normal. e.g. + "optional:setup.py:python/foo:built_ext:-i" + + copy -- Copies the given file in the virtualenv site packages + directory. + + packages.txt -- Denotes that the specified path is a child manifest. It + will be read and processed as if its contents were concatenated + into the manifest being read. + + objdir -- Denotes a relative path in the object directory to add to the + search path. e.g. "objdir:build" will add $topobjdir/build to the + search path. + + Note that the Python interpreter running this function should be the + one from the virtualenv. If it is the system Python or if the + environment is not configured properly, packages could be installed + into the wrong place. This is how virtualenv's work. + """ + + packages = self.packages() + python_lib = distutils.sysconfig.get_python_lib() + + def handle_package(package): + if package[0] == 'setup.py': + assert len(package) >= 2 + + self.call_setup(os.path.join(self.topsrcdir, package[1]), + package[2:]) + + return True + + if package[0] == 'copy': + assert len(package) == 2 + + src = os.path.join(self.topsrcdir, package[1]) + dst = os.path.join(python_lib, os.path.basename(package[1])) + + shutil.copy(src, dst) + + return True + + if package[0] == 'packages.txt': + assert len(package) == 2 + + src = os.path.join(self.topsrcdir, package[1]) + assert os.path.isfile(src), "'%s' does not exist" % src + submanager = VirtualenvManager(self.topsrcdir, + self.topobjdir, + self.virtualenv_root, + self.log_handle, + src) + submanager.populate() + + return True + + if package[0].endswith('.pth'): + assert len(package) == 2 + + path = os.path.join(self.topsrcdir, package[1]) + + with open(os.path.join(python_lib, package[0]), 'a') as f: + # This path is relative to the .pth file. Using a + # relative path allows the srcdir/objdir combination + # to be moved around (as long as the paths relative to + # each other remain the same). + try: + f.write("%s\n" % os.path.relpath(path, python_lib)) + except ValueError: + # When objdir is on a separate drive, relpath throws + f.write("%s\n" % os.path.join(python_lib, path)) + + return True + + if package[0] == 'optional': + try: + handle_package(package[1:]) + return True + except: + print('Error processing command. Ignoring', \ + 'because optional. (%s)' % ':'.join(package), + file=self.log_handle) + return False + + if package[0] == 'objdir': + assert len(package) == 2 + path = os.path.join(self.topobjdir, package[1]) + + with open(os.path.join(python_lib, 'objdir.pth'), 'a') as f: + f.write('%s\n' % path) + + return True + + raise Exception('Unknown action: %s' % package[0]) + + # We always target the OS X deployment target that Python itself was + # built with, regardless of what's in the current environment. If we + # don't do # this, we may run into a Python bug. See + # http://bugs.python.org/issue9516 and bug 659881. + # + # Note that this assumes that nothing compiled in the virtualenv is + # shipped as part of a distribution. If we do ship anything, the + # deployment target here may be different from what's targeted by the + # shipping binaries and # virtualenv-produced binaries may fail to + # work. + # + # We also ignore environment variables that may have been altered by + # configure or a mozconfig activated in the current shell. We trust + # Python is smart enough to find a proper compiler and to use the + # proper compiler flags. If it isn't your Python is likely broken. + IGNORE_ENV_VARIABLES = ('CC', 'CXX', 'CFLAGS', 'CXXFLAGS', 'LDFLAGS', + 'PYTHONDONTWRITEBYTECODE') + + try: + old_target = os.environ.get('MACOSX_DEPLOYMENT_TARGET', None) + sysconfig_target = \ + distutils.sysconfig.get_config_var('MACOSX_DEPLOYMENT_TARGET') + + if sysconfig_target is not None: + os.environ['MACOSX_DEPLOYMENT_TARGET'] = sysconfig_target + + old_env_variables = {} + for k in IGNORE_ENV_VARIABLES: + if k not in os.environ: + continue + + old_env_variables[k] = os.environ[k] + del os.environ[k] + + # HACK ALERT. + # + # The following adjustment to the VSNNCOMNTOOLS environment + # variables are wrong. This is done as a hack to facilitate the + # building of binary Python packages - notably psutil - on Windows + # machines that don't have the Visual Studio 2008 binaries + # installed. This hack assumes the Python on that system was built + # with Visual Studio 2008. The hack is wrong for the reasons + # explained at + # http://stackoverflow.com/questions/3047542/building-lxml-for-python-2-7-on-windows/5122521#5122521. + if sys.platform in ('win32', 'cygwin') and \ + 'VS90COMNTOOLS' not in os.environ: + + warnings.warn('Hacking environment to allow binary Python ' + 'extensions to build. You can make this warning go away ' + 'by installing Visual Studio 2008. You can download the ' + 'Express Edition installer from ' + 'http://go.microsoft.com/?linkid=7729279') + + # We list in order from oldest to newest to prefer the closest + # to 2008 so differences are minimized. + for ver in ('100', '110', '120'): + var = 'VS%sCOMNTOOLS' % ver + if var in os.environ: + os.environ['VS90COMNTOOLS'] = os.environ[var] + break + + for package in packages: + handle_package(package) + + sitecustomize = os.path.join( + os.path.dirname(os.__file__), 'sitecustomize.py') + with open(sitecustomize, 'w') as f: + f.write( + '# Importing mach_bootstrap has the side effect of\n' + '# installing an import hook\n' + 'import mach_bootstrap\n' + ) + + finally: + os.environ.pop('MACOSX_DEPLOYMENT_TARGET', None) + + if old_target is not None: + os.environ['MACOSX_DEPLOYMENT_TARGET'] = old_target + + os.environ.update(old_env_variables) + + def call_setup(self, directory, arguments): + """Calls setup.py in a directory.""" + setup = os.path.join(directory, 'setup.py') + + program = [self.python_path, setup] + program.extend(arguments) + + # We probably could call the contents of this file inside the context + # of this interpreter using execfile() or similar. However, if global + # variables like sys.path are adjusted, this could cause all kinds of + # havoc. While this may work, invoking a new process is safer. + + try: + output = subprocess.check_output(program, cwd=directory, stderr=subprocess.STDOUT) + print(output) + except subprocess.CalledProcessError as e: + if 'Python.h: No such file or directory' in e.output: + print('WARNING: Python.h not found. Install Python development headers.') + else: + print(e.output) + + raise Exception('Error installing package: %s' % directory) + + def build(self, python=sys.executable): + """Build a virtualenv per tree conventions. + + This returns the path of the created virtualenv. + """ + + self.create(python) + + # We need to populate the virtualenv using the Python executable in + # the virtualenv for paths to be proper. + + args = [self.python_path, __file__, 'populate', self.topsrcdir, + self.topobjdir, self.virtualenv_root, self.manifest_path] + + result = self._log_process_output(args, cwd=self.topsrcdir) + + if result != 0: + raise Exception('Error populating virtualenv.') + + os.utime(self.activate_path, None) + + return self.virtualenv_root + + def activate(self): + """Activate the virtualenv in this Python context. + + If you run a random Python script and wish to "activate" the + virtualenv, you can simply instantiate an instance of this class + and call .ensure() and .activate() to make the virtualenv active. + """ + + execfile(self.activate_path, dict(__file__=self.activate_path)) + if isinstance(os.environ['PATH'], unicode): + os.environ['PATH'] = os.environ['PATH'].encode('utf-8') + + def install_pip_package(self, package): + """Install a package via pip. + + The supplied package is specified using a pip requirement specifier. + e.g. 'foo' or 'foo==1.0'. + + If the package is already installed, this is a no-op. + """ + from pip.req import InstallRequirement + + req = InstallRequirement.from_line(package) + if req.check_if_exists(): + return + + args = [ + 'install', + '--use-wheel', + package, + ] + + return self._run_pip(args) + + def install_pip_requirements(self, path, require_hashes=True): + """Install a pip requirements.txt file. + + The supplied path is a text file containing pip requirement + specifiers. + + If require_hashes is True, each specifier must contain the + expected hash of the downloaded package. See: + https://pip.pypa.io/en/stable/reference/pip_install/#hash-checking-mode + """ + + if not os.path.isabs(path): + path = os.path.join(self.topsrcdir, path) + + args = [ + 'install', + '--requirement', + path, + ] + + if require_hashes: + args.append('--require-hashes') + + return self._run_pip(args) + + def _run_pip(self, args): + # It's tempting to call pip natively via pip.main(). However, + # the current Python interpreter may not be the virtualenv python. + # This will confuse pip and cause the package to attempt to install + # against the executing interpreter. By creating a new process, we + # force the virtualenv's interpreter to be used and all is well. + # It /might/ be possible to cheat and set sys.executable to + # self.python_path. However, this seems more risk than it's worth. + subprocess.check_call([os.path.join(self.bin_path, 'pip')] + args, + stderr=subprocess.STDOUT) + + +def verify_python_version(log_handle): + """Ensure the current version of Python is sufficient.""" + major, minor, micro = sys.version_info[:3] + + our = LooseVersion('%d.%d.%d' % (major, minor, micro)) + + if major != MINIMUM_PYTHON_MAJOR or our < MINIMUM_PYTHON_VERSION: + log_handle.write('Python %s or greater (but not Python 3) is ' + 'required to build. ' % MINIMUM_PYTHON_VERSION) + log_handle.write('You are running Python %s.\n' % our) + + if os.name in ('nt', 'ce'): + log_handle.write(UPGRADE_WINDOWS) + else: + log_handle.write(UPGRADE_OTHER) + + sys.exit(1) + + +if __name__ == '__main__': + if len(sys.argv) < 5: + print('Usage: populate_virtualenv.py /path/to/topsrcdir /path/to/topobjdir /path/to/virtualenv /path/to/virtualenv_manifest') + sys.exit(1) + + verify_python_version(sys.stdout) + + topsrcdir, topobjdir, virtualenv_path, manifest_path = sys.argv[1:5] + populate = False + + # This should only be called internally. + if sys.argv[1] == 'populate': + populate = True + topsrcdir, topobjdir, virtualenv_path, manifest_path = sys.argv[2:] + + manager = VirtualenvManager(topsrcdir, topobjdir, virtualenv_path, + sys.stdout, manifest_path) + + if populate: + manager.populate() + else: + manager.ensure() + diff --git a/python/mozbuild/mozpack/__init__.py b/python/mozbuild/mozpack/__init__.py new file mode 100644 index 000000000..e69de29bb diff --git a/python/mozbuild/mozpack/archive.py b/python/mozbuild/mozpack/archive.py new file mode 100644 index 000000000..f3015ff21 --- /dev/null +++ b/python/mozbuild/mozpack/archive.py @@ -0,0 +1,107 @@ +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, You can obtain one at http://mozilla.org/MPL/2.0/. + +from __future__ import absolute_import + +import bz2 +import gzip +import stat +import tarfile + + +# 2016-01-01T00:00:00+0000 +DEFAULT_MTIME = 1451606400 + + +def create_tar_from_files(fp, files): + """Create a tar file deterministically. + + Receives a dict mapping names of files in the archive to local filesystem + paths. + + The files will be archived and written to the passed file handle opened + for writing. + + Only regular files can be written. + + FUTURE accept mozpack.files classes for writing + FUTURE accept a filename argument (or create APIs to write files) + """ + with tarfile.open(name='', mode='w', fileobj=fp, dereference=True) as tf: + for archive_path, fs_path in sorted(files.items()): + ti = tf.gettarinfo(fs_path, archive_path) + + if not ti.isreg(): + raise ValueError('not a regular file: %s' % fs_path) + + # Disallow setuid and setgid bits. This is an arbitrary restriction. + # However, since we set uid/gid to root:root, setuid and setgid + # would be a glaring security hole if the archive were + # uncompressed as root. + if ti.mode & (stat.S_ISUID | stat.S_ISGID): + raise ValueError('cannot add file with setuid or setgid set: ' + '%s' % fs_path) + + # Set uid, gid, username, and group as deterministic values. + ti.uid = 0 + ti.gid = 0 + ti.uname = '' + ti.gname = '' + + # Set mtime to a constant value. + ti.mtime = DEFAULT_MTIME + + with open(fs_path, 'rb') as fh: + tf.addfile(ti, fh) + + +def create_tar_gz_from_files(fp, files, filename=None, compresslevel=9): + """Create a tar.gz file deterministically from files. + + This is a glorified wrapper around ``create_tar_from_files`` that + adds gzip compression. + + The passed file handle should be opened for writing in binary mode. + When the function returns, all data has been written to the handle. + """ + # Offset 3-7 in the gzip header contains an mtime. Pin it to a known + # value so output is deterministic. + gf = gzip.GzipFile(filename=filename or '', mode='wb', fileobj=fp, + compresslevel=compresslevel, mtime=DEFAULT_MTIME) + with gf: + create_tar_from_files(gf, files) + + +class _BZ2Proxy(object): + """File object that proxies writes to a bz2 compressor.""" + def __init__(self, fp, compresslevel=9): + self.fp = fp + self.compressor = bz2.BZ2Compressor(compresslevel=compresslevel) + self.pos = 0 + + def tell(self): + return self.pos + + def write(self, data): + data = self.compressor.compress(data) + self.pos += len(data) + self.fp.write(data) + + def close(self): + data = self.compressor.flush() + self.pos += len(data) + self.fp.write(data) + + +def create_tar_bz2_from_files(fp, files, compresslevel=9): + """Create a tar.bz2 file deterministically from files. + + This is a glorified wrapper around ``create_tar_from_files`` that + adds bzip2 compression. + + This function is similar to ``create_tar_gzip_from_files()``. + """ + proxy = _BZ2Proxy(fp, compresslevel=compresslevel) + create_tar_from_files(proxy, files) + proxy.close() diff --git a/python/mozbuild/mozpack/chrome/__init__.py b/python/mozbuild/mozpack/chrome/__init__.py new file mode 100644 index 000000000..e69de29bb diff --git a/python/mozbuild/mozpack/chrome/flags.py b/python/mozbuild/mozpack/chrome/flags.py new file mode 100644 index 000000000..8c5c9a54c --- /dev/null +++ b/python/mozbuild/mozpack/chrome/flags.py @@ -0,0 +1,258 @@ +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, You can obtain one at http://mozilla.org/MPL/2.0/. + +from __future__ import absolute_import + +import re +from distutils.version import LooseVersion +from mozpack.errors import errors +from collections import OrderedDict + + +class Flag(object): + ''' + Class for flags in manifest entries in the form: + "flag" (same as "flag=true") + "flag=yes|true|1" + "flag=no|false|0" + ''' + def __init__(self, name): + ''' + Initialize a Flag with the given name. + ''' + self.name = name + self.value = None + + def add_definition(self, definition): + ''' + Add a flag value definition. Replaces any previously set value. + ''' + if definition == self.name: + self.value = True + return + assert(definition.startswith(self.name)) + if definition[len(self.name)] != '=': + return errors.fatal('Malformed flag: %s' % definition) + value = definition[len(self.name) + 1:] + if value in ('yes', 'true', '1', 'no', 'false', '0'): + self.value = value + else: + return errors.fatal('Unknown value in: %s' % definition) + + def matches(self, value): + ''' + Return whether the flag value matches the given value. The values + are canonicalized for comparison. + ''' + if value in ('yes', 'true', '1', True): + return self.value in ('yes', 'true', '1', True) + if value in ('no', 'false', '0', False): + return self.value in ('no', 'false', '0', False, None) + raise RuntimeError('Invalid value: %s' % value) + + def __str__(self): + ''' + Serialize the flag value in the same form given to the last + add_definition() call. + ''' + if self.value is None: + return '' + if self.value is True: + return self.name + return '%s=%s' % (self.name, self.value) + + +class StringFlag(object): + ''' + Class for string flags in manifest entries in the form: + "flag=string" + "flag!=string" + ''' + def __init__(self, name): + ''' + Initialize a StringFlag with the given name. + ''' + self.name = name + self.values = [] + + def add_definition(self, definition): + ''' + Add a string flag definition. + ''' + assert(definition.startswith(self.name)) + value = definition[len(self.name):] + if value.startswith('='): + self.values.append(('==', value[1:])) + elif value.startswith('!='): + self.values.append(('!=', value[2:])) + else: + return errors.fatal('Malformed flag: %s' % definition) + + def matches(self, value): + ''' + Return whether one of the string flag definitions matches the given + value. + For example, + flag = StringFlag('foo') + flag.add_definition('foo!=bar') + flag.matches('bar') returns False + flag.matches('qux') returns True + flag = StringFlag('foo') + flag.add_definition('foo=bar') + flag.add_definition('foo=baz') + flag.matches('bar') returns True + flag.matches('baz') returns True + flag.matches('qux') returns False + ''' + if not self.values: + return True + for comparison, val in self.values: + if eval('value %s val' % comparison): + return True + return False + + def __str__(self): + ''' + Serialize the flag definitions in the same form given to each + add_definition() call. + ''' + res = [] + for comparison, val in self.values: + if comparison == '==': + res.append('%s=%s' % (self.name, val)) + else: + res.append('%s!=%s' % (self.name, val)) + return ' '.join(res) + + +class VersionFlag(object): + ''' + Class for version flags in manifest entries in the form: + "flag=version" + "flag<=version" + "flag=version" + "flag>version" + ''' + def __init__(self, name): + ''' + Initialize a VersionFlag with the given name. + ''' + self.name = name + self.values = [] + + def add_definition(self, definition): + ''' + Add a version flag definition. + ''' + assert(definition.startswith(self.name)) + value = definition[len(self.name):] + if value.startswith('='): + self.values.append(('==', LooseVersion(value[1:]))) + elif len(value) > 1 and value[0] in ['<', '>']: + if value[1] == '=': + if len(value) < 3: + return errors.fatal('Malformed flag: %s' % definition) + self.values.append((value[0:2], LooseVersion(value[2:]))) + else: + self.values.append((value[0], LooseVersion(value[1:]))) + else: + return errors.fatal('Malformed flag: %s' % definition) + + def matches(self, value): + ''' + Return whether one of the version flag definitions matches the given + value. + For example, + flag = VersionFlag('foo') + flag.add_definition('foo>=1.0') + flag.matches('1.0') returns True + flag.matches('1.1') returns True + flag.matches('0.9') returns False + flag = VersionFlag('foo') + flag.add_definition('foo>=1.0') + flag.add_definition('foo<0.5') + flag.matches('0.4') returns True + flag.matches('1.0') returns True + flag.matches('0.6') returns False + ''' + value = LooseVersion(value) + if not self.values: + return True + for comparison, val in self.values: + if eval('value %s val' % comparison): + return True + return False + + def __str__(self): + ''' + Serialize the flag definitions in the same form given to each + add_definition() call. + ''' + res = [] + for comparison, val in self.values: + if comparison == '==': + res.append('%s=%s' % (self.name, val)) + else: + res.append('%s%s%s' % (self.name, comparison, val)) + return ' '.join(res) + + +class Flags(OrderedDict): + ''' + Class to handle a set of flags definitions given on a single manifest + entry. + ''' + FLAGS = { + 'application': StringFlag, + 'appversion': VersionFlag, + 'platformversion': VersionFlag, + 'contentaccessible': Flag, + 'os': StringFlag, + 'osversion': VersionFlag, + 'abi': StringFlag, + 'platform': Flag, + 'xpcnativewrappers': Flag, + 'tablet': Flag, + 'process': StringFlag, + } + RE = re.compile(r'([!<>=]+)') + + def __init__(self, *flags): + ''' + Initialize a set of flags given in string form. + flags = Flags('contentaccessible=yes', 'appversion>=3.5') + ''' + OrderedDict.__init__(self) + for f in flags: + name = self.RE.split(f) + name = name[0] + if not name in self.FLAGS: + errors.fatal('Unknown flag: %s' % name) + continue + if not name in self: + self[name] = self.FLAGS[name](name) + self[name].add_definition(f) + + def __str__(self): + ''' + Serialize the set of flags. + ''' + return ' '.join(str(self[k]) for k in self) + + def match(self, **filter): + ''' + Return whether the set of flags match the set of given filters. + flags = Flags('contentaccessible=yes', 'appversion>=3.5', + 'application=foo') + flags.match(application='foo') returns True + flags.match(application='foo', appversion='3.5') returns True + flags.match(application='foo', appversion='3.0') returns False + ''' + for name, value in filter.iteritems(): + if not name in self: + continue + if not self[name].matches(value): + return False + return True diff --git a/python/mozbuild/mozpack/chrome/manifest.py b/python/mozbuild/mozpack/chrome/manifest.py new file mode 100644 index 000000000..71241764d --- /dev/null +++ b/python/mozbuild/mozpack/chrome/manifest.py @@ -0,0 +1,368 @@ +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, You can obtain one at http://mozilla.org/MPL/2.0/. + +from __future__ import absolute_import + +import re +import os +from urlparse import urlparse +import mozpack.path as mozpath +from mozpack.chrome.flags import Flags +from mozpack.errors import errors + + +class ManifestEntry(object): + ''' + Base class for all manifest entry types. + Subclasses may define the following class or member variables: + - localized: indicates whether the manifest entry is used for localized + data. + - type: the manifest entry type (e.g. 'content' in + 'content global content/global/') + - allowed_flags: a set of flags allowed to be defined for the given + manifest entry type. + + A manifest entry is attached to a base path, defining where the manifest + entry is bound to, and that is used to find relative paths defined in + entries. + ''' + localized = False + type = None + allowed_flags = [ + 'application', + 'platformversion', + 'os', + 'osversion', + 'abi', + 'xpcnativewrappers', + 'tablet', + 'process', + ] + + def __init__(self, base, *flags): + ''' + Initialize a manifest entry with the given base path and flags. + ''' + self.base = base + self.flags = Flags(*flags) + if not all(f in self.allowed_flags for f in self.flags): + errors.fatal('%s unsupported for %s manifest entries' % + (','.join(f for f in self.flags + if not f in self.allowed_flags), self.type)) + + def serialize(self, *args): + ''' + Serialize the manifest entry. + ''' + entry = [self.type] + list(args) + flags = str(self.flags) + if flags: + entry.append(flags) + return ' '.join(entry) + + def __eq__(self, other): + return self.base == other.base and str(self) == str(other) + + def __ne__(self, other): + return not self.__eq__(other) + + def __repr__(self): + return '<%s@%s>' % (str(self), self.base) + + def move(self, base): + ''' + Return a new manifest entry with a different base path. + ''' + return parse_manifest_line(base, str(self)) + + def rebase(self, base): + ''' + Return a new manifest entry with all relative paths defined in the + entry relative to a new base directory. + The base class doesn't define relative paths, so it is equivalent to + move(). + ''' + return self.move(base) + + +class ManifestEntryWithRelPath(ManifestEntry): + ''' + Abstract manifest entry type with a relative path definition. + ''' + def __init__(self, base, relpath, *flags): + ManifestEntry.__init__(self, base, *flags) + self.relpath = relpath + + def __str__(self): + return self.serialize(self.relpath) + + def rebase(self, base): + ''' + Return a new manifest entry with all relative paths defined in the + entry relative to a new base directory. + ''' + clone = ManifestEntry.rebase(self, base) + clone.relpath = mozpath.rebase(self.base, base, self.relpath) + return clone + + @property + def path(self): + return mozpath.normpath(mozpath.join(self.base, + self.relpath)) + + +class Manifest(ManifestEntryWithRelPath): + ''' + Class for 'manifest' entries. + manifest some/path/to/another.manifest + ''' + type = 'manifest' + + +class ManifestChrome(ManifestEntryWithRelPath): + ''' + Abstract class for chrome entries. + ''' + def __init__(self, base, name, relpath, *flags): + ManifestEntryWithRelPath.__init__(self, base, relpath, *flags) + self.name = name + + @property + def location(self): + return mozpath.join(self.base, self.relpath) + + +class ManifestContent(ManifestChrome): + ''' + Class for 'content' entries. + content global content/global/ + ''' + type = 'content' + allowed_flags = ManifestChrome.allowed_flags + [ + 'contentaccessible', + 'platform', + ] + + def __str__(self): + return self.serialize(self.name, self.relpath) + + +class ManifestMultiContent(ManifestChrome): + ''' + Abstract class for chrome entries with multiple definitions. + Used for locale and skin entries. + ''' + type = None + + def __init__(self, base, name, id, relpath, *flags): + ManifestChrome.__init__(self, base, name, relpath, *flags) + self.id = id + + def __str__(self): + return self.serialize(self.name, self.id, self.relpath) + + +class ManifestLocale(ManifestMultiContent): + ''' + Class for 'locale' entries. + locale global en-US content/en-US/ + locale global fr content/fr/ + ''' + localized = True + type = 'locale' + + +class ManifestSkin(ManifestMultiContent): + ''' + Class for 'skin' entries. + skin global classic/1.0 content/skin/classic/ + ''' + type = 'skin' + + +class ManifestOverload(ManifestEntry): + ''' + Abstract class for chrome entries defining some kind of overloading. + Used for overlay, override or style entries. + ''' + type = None + + def __init__(self, base, overloaded, overload, *flags): + ManifestEntry.__init__(self, base, *flags) + self.overloaded = overloaded + self.overload = overload + + def __str__(self): + return self.serialize(self.overloaded, self.overload) + + @property + def localized(self): + u = urlparse(self.overload) + return u.scheme == 'chrome' and \ + u.path.split('/')[0:2] == ['', 'locale'] + + +class ManifestOverlay(ManifestOverload): + ''' + Class for 'overlay' entries. + overlay chrome://global/content/viewSource.xul \ + chrome://browser/content/viewSourceOverlay.xul + ''' + type = 'overlay' + + +class ManifestStyle(ManifestOverload): + ''' + Class for 'style' entries. + style chrome://global/content/customizeToolbar.xul \ + chrome://browser/skin/ + ''' + type = 'style' + + +class ManifestOverride(ManifestOverload): + ''' + Class for 'override' entries. + override chrome://global/locale/netError.dtd \ + chrome://browser/locale/netError.dtd + ''' + type = 'override' + + +class ManifestResource(ManifestEntry): + ''' + Class for 'resource' entries. + resource gre-resources toolkit/res/ + resource services-sync resource://gre/modules/services-sync/ + + The target may be a relative path or a resource or chrome url. + ''' + type = 'resource' + + def __init__(self, base, name, target, *flags): + ManifestEntry.__init__(self, base, *flags) + self.name = name + self.target = target + + def __str__(self): + return self.serialize(self.name, self.target) + + def rebase(self, base): + u = urlparse(self.target) + if u.scheme and u.scheme != 'jar': + return ManifestEntry.rebase(self, base) + clone = ManifestEntry.rebase(self, base) + clone.target = mozpath.rebase(self.base, base, self.target) + return clone + + +class ManifestBinaryComponent(ManifestEntryWithRelPath): + ''' + Class for 'binary-component' entries. + binary-component some/path/to/a/component.dll + ''' + type = 'binary-component' + + +class ManifestComponent(ManifestEntryWithRelPath): + ''' + Class for 'component' entries. + component {b2bba4df-057d-41ea-b6b1-94a10a8ede68} foo.js + ''' + type = 'component' + + def __init__(self, base, cid, file, *flags): + ManifestEntryWithRelPath.__init__(self, base, file, *flags) + self.cid = cid + + def __str__(self): + return self.serialize(self.cid, self.relpath) + + +class ManifestInterfaces(ManifestEntryWithRelPath): + ''' + Class for 'interfaces' entries. + interfaces foo.xpt + ''' + type = 'interfaces' + + +class ManifestCategory(ManifestEntry): + ''' + Class for 'category' entries. + category command-line-handler m-browser @mozilla.org/browser/clh; + ''' + type = 'category' + + def __init__(self, base, category, name, value, *flags): + ManifestEntry.__init__(self, base, *flags) + self.category = category + self.name = name + self.value = value + + def __str__(self): + return self.serialize(self.category, self.name, self.value) + + +class ManifestContract(ManifestEntry): + ''' + Class for 'contract' entries. + contract @mozilla.org/foo;1 {b2bba4df-057d-41ea-b6b1-94a10a8ede68} + ''' + type = 'contract' + + def __init__(self, base, contractID, cid, *flags): + ManifestEntry.__init__(self, base, *flags) + self.contractID = contractID + self.cid = cid + + def __str__(self): + return self.serialize(self.contractID, self.cid) + +# All manifest classes by their type name. +MANIFESTS_TYPES = dict([(c.type, c) for c in globals().values() + if type(c) == type and issubclass(c, ManifestEntry) + and hasattr(c, 'type') and c.type]) + +MANIFEST_RE = re.compile(r'^#.*$') + + +def parse_manifest_line(base, line): + ''' + Parse a line from a manifest file with the given base directory and + return the corresponding ManifestEntry instance. + ''' + # Remove comments + cmd = MANIFEST_RE.sub('', line).strip().split() + if not cmd: + return None + if not cmd[0] in MANIFESTS_TYPES: + return errors.fatal('Unknown manifest directive: %s' % cmd[0]) + return MANIFESTS_TYPES[cmd[0]](base, *cmd[1:]) + + +def parse_manifest(root, path, fileobj=None): + ''' + Parse a manifest file. + ''' + base = mozpath.dirname(path) + if root: + path = os.path.normpath(os.path.abspath(os.path.join(root, path))) + if not fileobj: + fileobj = open(path) + linenum = 0 + for line in fileobj: + linenum += 1 + with errors.context(path, linenum): + e = parse_manifest_line(base, line) + if e: + yield e + + +def is_manifest(path): + ''' + Return whether the given path is that of a manifest file. + ''' + return path.endswith('.manifest') and not path.endswith('.CRT.manifest') \ + and not path.endswith('.exe.manifest') diff --git a/python/mozbuild/mozpack/copier.py b/python/mozbuild/mozpack/copier.py new file mode 100644 index 000000000..386930fe7 --- /dev/null +++ b/python/mozbuild/mozpack/copier.py @@ -0,0 +1,568 @@ +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, You can obtain one at http://mozilla.org/MPL/2.0/. + +from __future__ import absolute_import + +import os +import stat +import sys + +from mozpack.errors import errors +from mozpack.files import ( + BaseFile, + Dest, +) +import mozpack.path as mozpath +import errno +from collections import ( + Counter, + OrderedDict, +) +import concurrent.futures as futures + + +class FileRegistry(object): + ''' + Generic container to keep track of a set of BaseFile instances. It + preserves the order under which the files are added, but doesn't keep + track of empty directories (directories are not stored at all). + The paths associated with the BaseFile instances are relative to an + unspecified (virtual) root directory. + + registry = FileRegistry() + registry.add('foo/bar', file_instance) + ''' + + def __init__(self): + self._files = OrderedDict() + self._required_directories = Counter() + self._partial_paths_cache = {} + + def _partial_paths(self, path): + ''' + Turn "foo/bar/baz/zot" into ["foo/bar/baz", "foo/bar", "foo"]. + ''' + dir_name = path.rpartition('/')[0] + if not dir_name: + return [] + + partial_paths = self._partial_paths_cache.get(dir_name) + if partial_paths: + return partial_paths + + partial_paths = [dir_name] + self._partial_paths(dir_name) + + self._partial_paths_cache[dir_name] = partial_paths + return partial_paths + + def add(self, path, content): + ''' + Add a BaseFile instance to the container, under the given path. + ''' + assert isinstance(content, BaseFile) + if path in self._files: + return errors.error("%s already added" % path) + if self._required_directories[path] > 0: + return errors.error("Can't add %s: it is a required directory" % + path) + # Check whether any parent of the given path is already stored + partial_paths = self._partial_paths(path) + for partial_path in partial_paths: + if partial_path in self._files: + return errors.error("Can't add %s: %s is a file" % + (path, partial_path)) + self._files[path] = content + self._required_directories.update(partial_paths) + + def match(self, pattern): + ''' + Return the list of paths, stored in the container, matching the + given pattern. See the mozpack.path.match documentation for a + description of the handled patterns. + ''' + if '*' in pattern: + return [p for p in self.paths() + if mozpath.match(p, pattern)] + if pattern == '': + return self.paths() + if pattern in self._files: + return [pattern] + return [p for p in self.paths() + if mozpath.basedir(p, [pattern]) == pattern] + + def remove(self, pattern): + ''' + Remove paths matching the given pattern from the container. See the + mozpack.path.match documentation for a description of the handled + patterns. + ''' + items = self.match(pattern) + if not items: + return errors.error("Can't remove %s: %s" % (pattern, + "not matching anything previously added")) + for i in items: + del self._files[i] + self._required_directories.subtract(self._partial_paths(i)) + + def paths(self): + ''' + Return all paths stored in the container, in the order they were added. + ''' + return self._files.keys() + + def __len__(self): + ''' + Return number of paths stored in the container. + ''' + return len(self._files) + + def __contains__(self, pattern): + raise RuntimeError("'in' operator forbidden for %s. Use contains()." % + self.__class__.__name__) + + def contains(self, pattern): + ''' + Return whether the container contains paths matching the given + pattern. See the mozpack.path.match documentation for a description of + the handled patterns. + ''' + return len(self.match(pattern)) > 0 + + def __getitem__(self, path): + ''' + Return the BaseFile instance stored in the container for the given + path. + ''' + return self._files[path] + + def __iter__(self): + ''' + Iterate over all (path, BaseFile instance) pairs from the container. + for path, file in registry: + (...) + ''' + return self._files.iteritems() + + def required_directories(self): + ''' + Return the set of directories required by the paths in the container, + in no particular order. The returned directories are relative to an + unspecified (virtual) root directory (and do not include said root + directory). + ''' + return set(k for k, v in self._required_directories.items() if v > 0) + + +class FileRegistrySubtree(object): + '''A proxy class to give access to a subtree of an existing FileRegistry. + + Note this doesn't implement the whole FileRegistry interface.''' + def __new__(cls, base, registry): + if not base: + return registry + return object.__new__(cls) + + def __init__(self, base, registry): + self._base = base + self._registry = registry + + def _get_path(self, path): + # mozpath.join will return a trailing slash if path is empty, and we + # don't want that. + return mozpath.join(self._base, path) if path else self._base + + def add(self, path, content): + return self._registry.add(self._get_path(path), content) + + def match(self, pattern): + return [mozpath.relpath(p, self._base) + for p in self._registry.match(self._get_path(pattern))] + + def remove(self, pattern): + return self._registry.remove(self._get_path(pattern)) + + def paths(self): + return [p for p, f in self] + + def __len__(self): + return len(self.paths()) + + def contains(self, pattern): + return self._registry.contains(self._get_path(pattern)) + + def __getitem__(self, path): + return self._registry[self._get_path(path)] + + def __iter__(self): + for p, f in self._registry: + if mozpath.basedir(p, [self._base]): + yield mozpath.relpath(p, self._base), f + + +class FileCopyResult(object): + """Represents results of a FileCopier.copy operation.""" + + def __init__(self): + self.updated_files = set() + self.existing_files = set() + self.removed_files = set() + self.removed_directories = set() + + @property + def updated_files_count(self): + return len(self.updated_files) + + @property + def existing_files_count(self): + return len(self.existing_files) + + @property + def removed_files_count(self): + return len(self.removed_files) + + @property + def removed_directories_count(self): + return len(self.removed_directories) + + +class FileCopier(FileRegistry): + ''' + FileRegistry with the ability to copy the registered files to a separate + directory. + ''' + def copy(self, destination, skip_if_older=True, + remove_unaccounted=True, + remove_all_directory_symlinks=True, + remove_empty_directories=True): + ''' + Copy all registered files to the given destination path. The given + destination can be an existing directory, or not exist at all. It + can't be e.g. a file. + The copy process acts a bit like rsync: files are not copied when they + don't need to (see mozpack.files for details on file.copy). + + By default, files in the destination directory that aren't + registered are removed and empty directories are deleted. In + addition, all directory symlinks in the destination directory + are deleted: this is a conservative approach to ensure that we + never accidently write files into a directory that is not the + destination directory. In the worst case, we might have a + directory symlink in the object directory to the source + directory. + + To disable removing of unregistered files, pass + remove_unaccounted=False. To disable removing empty + directories, pass remove_empty_directories=False. In rare + cases, you might want to maintain directory symlinks in the + destination directory (at least those that are not required to + be regular directories): pass + remove_all_directory_symlinks=False. Exercise caution with + this flag: you almost certainly do not want to preserve + directory symlinks. + + Returns a FileCopyResult that details what changed. + ''' + assert isinstance(destination, basestring) + assert not os.path.exists(destination) or os.path.isdir(destination) + + result = FileCopyResult() + have_symlinks = hasattr(os, 'symlink') + destination = os.path.normpath(destination) + + # We create the destination directory specially. We can't do this as + # part of the loop doing mkdir() below because that loop munges + # symlinks and permissions and parent directories of the destination + # directory may have their own weird schema. The contract is we only + # manage children of destination, not its parents. + try: + os.makedirs(destination) + except OSError as e: + if e.errno != errno.EEXIST: + raise + + # Because we could be handling thousands of files, code in this + # function is optimized to minimize system calls. We prefer CPU time + # in Python over possibly I/O bound filesystem calls to stat() and + # friends. + + required_dirs = set([destination]) + required_dirs |= set(os.path.normpath(os.path.join(destination, d)) + for d in self.required_directories()) + + # Ensure destination directories are in place and proper. + # + # The "proper" bit is important. We need to ensure that directories + # have appropriate permissions or we will be unable to discover + # and write files. Furthermore, we need to verify directories aren't + # symlinks. + # + # Symlinked directories (a symlink whose target is a directory) are + # incompatible with us because our manifest talks in terms of files, + # not directories. If we leave symlinked directories unchecked, we + # would blindly follow symlinks and this might confuse file + # installation. For example, if an existing directory is a symlink + # to directory X and we attempt to install a symlink in this directory + # to a file in directory X, we may create a recursive symlink! + for d in sorted(required_dirs, key=len): + try: + os.mkdir(d) + except OSError as error: + if error.errno != errno.EEXIST: + raise + + # We allow the destination to be a symlink because the caller + # is responsible for managing the destination and we assume + # they know what they are doing. + if have_symlinks and d != destination: + st = os.lstat(d) + if stat.S_ISLNK(st.st_mode): + # While we have remove_unaccounted, it doesn't apply + # to directory symlinks because if it did, our behavior + # could be very wrong. + os.remove(d) + os.mkdir(d) + + if not os.access(d, os.W_OK): + umask = os.umask(0o077) + os.umask(umask) + os.chmod(d, 0o777 & ~umask) + + if isinstance(remove_unaccounted, FileRegistry): + existing_files = set(os.path.normpath(os.path.join(destination, p)) + for p in remove_unaccounted.paths()) + existing_dirs = set(os.path.normpath(os.path.join(destination, p)) + for p in remove_unaccounted + .required_directories()) + existing_dirs |= {os.path.normpath(destination)} + else: + # While we have remove_unaccounted, it doesn't apply to empty + # directories because it wouldn't make sense: an empty directory + # is empty, so removing it should have no effect. + existing_dirs = set() + existing_files = set() + for root, dirs, files in os.walk(destination): + # We need to perform the same symlink detection as above. + # os.walk() doesn't follow symlinks into directories by + # default, so we need to check dirs (we can't wait for root). + if have_symlinks: + filtered = [] + for d in dirs: + full = os.path.join(root, d) + st = os.lstat(full) + if stat.S_ISLNK(st.st_mode): + # This directory symlink is not a required + # directory: any such symlink would have been + # removed and a directory created above. + if remove_all_directory_symlinks: + os.remove(full) + result.removed_files.add( + os.path.normpath(full)) + else: + existing_files.add(os.path.normpath(full)) + else: + filtered.append(d) + + dirs[:] = filtered + + existing_dirs.add(os.path.normpath(root)) + + for d in dirs: + existing_dirs.add(os.path.normpath(os.path.join(root, d))) + + for f in files: + existing_files.add(os.path.normpath(os.path.join(root, f))) + + # Now we reconcile the state of the world against what we want. + dest_files = set() + + # Install files. + # + # Creating/appending new files on Windows/NTFS is slow. So we use a + # thread pool to speed it up significantly. The performance of this + # loop is so critical to common build operations on Linux that the + # overhead of the thread pool is worth avoiding, so we have 2 code + # paths. We also employ a low water mark to prevent thread pool + # creation if number of files is too small to benefit. + copy_results = [] + if sys.platform == 'win32' and len(self) > 100: + with futures.ThreadPoolExecutor(4) as e: + fs = [] + for p, f in self: + destfile = os.path.normpath(os.path.join(destination, p)) + fs.append((destfile, e.submit(f.copy, destfile, skip_if_older))) + + copy_results = [(destfile, f.result) for destfile, f in fs] + else: + for p, f in self: + destfile = os.path.normpath(os.path.join(destination, p)) + copy_results.append((destfile, f.copy(destfile, skip_if_older))) + + for destfile, copy_result in copy_results: + dest_files.add(destfile) + if copy_result: + result.updated_files.add(destfile) + else: + result.existing_files.add(destfile) + + # Remove files no longer accounted for. + if remove_unaccounted: + for f in existing_files - dest_files: + # Windows requires write access to remove files. + if os.name == 'nt' and not os.access(f, os.W_OK): + # It doesn't matter what we set permissions to since we + # will remove this file shortly. + os.chmod(f, 0o600) + + os.remove(f) + result.removed_files.add(f) + + if not remove_empty_directories: + return result + + # Figure out which directories can be removed. This is complicated + # by the fact we optionally remove existing files. This would be easy + # if we walked the directory tree after installing files. But, we're + # trying to minimize system calls. + + # Start with the ideal set. + remove_dirs = existing_dirs - required_dirs + + # Then don't remove directories if we didn't remove unaccounted files + # and one of those files exists. + if not remove_unaccounted: + parents = set() + pathsep = os.path.sep + for f in existing_files: + path = f + while True: + # All the paths are normalized and relative by this point, + # so os.path.dirname would only do extra work. + dirname = path.rpartition(pathsep)[0] + if dirname in parents: + break + parents.add(dirname) + path = dirname + remove_dirs -= parents + + # Remove empty directories that aren't required. + for d in sorted(remove_dirs, key=len, reverse=True): + try: + try: + os.rmdir(d) + except OSError as e: + if e.errno in (errno.EPERM, errno.EACCES): + # Permissions may not allow deletion. So ensure write + # access is in place before attempting to rmdir again. + os.chmod(d, 0o700) + os.rmdir(d) + else: + raise + except OSError as e: + # If remove_unaccounted is a # FileRegistry, then we have a + # list of directories that may not be empty, so ignore rmdir + # ENOTEMPTY errors for them. + if (isinstance(remove_unaccounted, FileRegistry) and + e.errno == errno.ENOTEMPTY): + continue + raise + result.removed_directories.add(d) + + return result + + +class Jarrer(FileRegistry, BaseFile): + ''' + FileRegistry with the ability to copy and pack the registered files as a + jar file. Also acts as a BaseFile instance, to be copied with a FileCopier. + ''' + def __init__(self, compress=True, optimize=True): + ''' + Create a Jarrer instance. See mozpack.mozjar.JarWriter documentation + for details on the compress and optimize arguments. + ''' + self.compress = compress + self.optimize = optimize + self._preload = [] + self._compress_options = {} # Map path to compress boolean option. + FileRegistry.__init__(self) + + def add(self, path, content, compress=None): + FileRegistry.add(self, path, content) + if compress is not None: + self._compress_options[path] = compress + + def copy(self, dest, skip_if_older=True): + ''' + Pack all registered files in the given destination jar. The given + destination jar may be a path to jar file, or a Dest instance for + a jar file. + If the destination jar file exists, its (compressed) contents are used + instead of the registered BaseFile instances when appropriate. + ''' + class DeflaterDest(Dest): + ''' + Dest-like class, reading from a file-like object initially, but + switching to a Deflater object if written to. + + dest = DeflaterDest(original_file) + dest.read() # Reads original_file + dest.write(data) # Creates a Deflater and write data there + dest.read() # Re-opens the Deflater and reads from it + ''' + def __init__(self, orig=None, compress=True): + self.mode = None + self.deflater = orig + self.compress = compress + + def read(self, length=-1): + if self.mode != 'r': + assert self.mode is None + self.mode = 'r' + return self.deflater.read(length) + + def write(self, data): + if self.mode != 'w': + from mozpack.mozjar import Deflater + self.deflater = Deflater(self.compress) + self.mode = 'w' + self.deflater.write(data) + + def exists(self): + return self.deflater is not None + + if isinstance(dest, basestring): + dest = Dest(dest) + assert isinstance(dest, Dest) + + from mozpack.mozjar import JarWriter, JarReader + try: + old_jar = JarReader(fileobj=dest) + except Exception: + old_jar = [] + + old_contents = dict([(f.filename, f) for f in old_jar]) + + with JarWriter(fileobj=dest, compress=self.compress, + optimize=self.optimize) as jar: + for path, file in self: + compress = self._compress_options.get(path, self.compress) + + if path in old_contents: + deflater = DeflaterDest(old_contents[path], compress) + else: + deflater = DeflaterDest(compress=compress) + file.copy(deflater, skip_if_older) + jar.add(path, deflater.deflater, mode=file.mode, compress=compress) + if self._preload: + jar.preload(self._preload) + + def open(self): + raise RuntimeError('unsupported') + + def preload(self, paths): + ''' + Add the given set of paths to the list of preloaded files. See + mozpack.mozjar.JarWriter documentation for details on jar preloading. + ''' + self._preload.extend(paths) diff --git a/python/mozbuild/mozpack/dmg.py b/python/mozbuild/mozpack/dmg.py new file mode 100644 index 000000000..036302214 --- /dev/null +++ b/python/mozbuild/mozpack/dmg.py @@ -0,0 +1,121 @@ +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, You can obtain one at http://mozilla.org/MPL/2.0/. + +import errno +import mozfile +import os +import platform +import shutil +import subprocess + +is_linux = platform.system() == 'Linux' + +def mkdir(dir): + if not os.path.isdir(dir): + try: + os.makedirs(dir) + except OSError as e: + if e.errno != errno.EEXIST: + raise + + +def chmod(dir): + 'Set permissions of DMG contents correctly' + subprocess.check_call(['chmod', '-R', 'a+rX,a-st,u+w,go-w', dir]) + + +def rsync(source, dest): + 'rsync the contents of directory source into directory dest' + # Ensure a trailing slash so rsync copies the *contents* of source. + if not source.endswith('/'): + source += '/' + subprocess.check_call(['rsync', '-a', '--copy-unsafe-links', + source, dest]) + + +def set_folder_icon(dir): + 'Set HFS attributes of dir to use a custom icon' + if not is_linux: + #TODO: bug 1197325 - figure out how to support this on Linux + subprocess.check_call(['SetFile', '-a', 'C', dir]) + + +def create_dmg_from_staged(stagedir, output_dmg, tmpdir, volume_name): + 'Given a prepared directory stagedir, produce a DMG at output_dmg.' + if not is_linux: + # Running on OS X + hybrid = os.path.join(tmpdir, 'hybrid.dmg') + subprocess.check_call(['hdiutil', 'makehybrid', '-hfs', + '-hfs-volume-name', volume_name, + '-hfs-openfolder', stagedir, + '-ov', stagedir, + '-o', hybrid]) + subprocess.check_call(['hdiutil', 'convert', '-format', 'UDBZ', + '-imagekey', 'bzip2-level=9', + '-ov', hybrid, '-o', output_dmg]) + else: + import buildconfig + uncompressed = os.path.join(tmpdir, 'uncompressed.dmg') + subprocess.check_call([ + buildconfig.substs['GENISOIMAGE'], + '-V', volume_name, + '-D', '-R', '-apple', '-no-pad', + '-o', uncompressed, + stagedir + ]) + subprocess.check_call([ + buildconfig.substs['DMG_TOOL'], + 'dmg', + uncompressed, + output_dmg + ], + # dmg is seriously chatty + stdout=open(os.devnull, 'wb')) + +def check_tools(*tools): + ''' + Check that each tool named in tools exists in SUBSTS and is executable. + ''' + import buildconfig + for tool in tools: + path = buildconfig.substs[tool] + if not path: + raise Exception('Required tool "%s" not found' % tool) + if not os.path.isfile(path): + raise Exception('Required tool "%s" not found at path "%s"' % (tool, path)) + if not os.access(path, os.X_OK): + raise Exception('Required tool "%s" at path "%s" is not executable' % (tool, path)) + + +def create_dmg(source_directory, output_dmg, volume_name, extra_files): + ''' + Create a DMG disk image at the path output_dmg from source_directory. + + Use volume_name as the disk image volume name, and + use extra_files as a list of tuples of (filename, relative path) to copy + into the disk image. + ''' + if platform.system() not in ('Darwin', 'Linux'): + raise Exception("Don't know how to build a DMG on '%s'" % platform.system()) + + if is_linux: + check_tools('DMG_TOOL', 'GENISOIMAGE') + with mozfile.TemporaryDirectory() as tmpdir: + stagedir = os.path.join(tmpdir, 'stage') + os.mkdir(stagedir) + # Copy the app bundle over using rsync + rsync(source_directory, stagedir) + # Copy extra files + for source, target in extra_files: + full_target = os.path.join(stagedir, target) + mkdir(os.path.dirname(full_target)) + shutil.copyfile(source, full_target) + # Make a symlink to /Applications. The symlink name is a space + # so we don't have to localize it. The Applications folder icon + # will be shown in Finder, which should be clear enough for users. + os.symlink('/Applications', os.path.join(stagedir, ' ')) + # Set the folder attributes to use a custom icon + set_folder_icon(stagedir) + chmod(stagedir) + create_dmg_from_staged(stagedir, output_dmg, tmpdir, volume_name) diff --git a/python/mozbuild/mozpack/errors.py b/python/mozbuild/mozpack/errors.py new file mode 100644 index 000000000..8b4b80072 --- /dev/null +++ b/python/mozbuild/mozpack/errors.py @@ -0,0 +1,139 @@ +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, You can obtain one at http://mozilla.org/MPL/2.0/. + +from __future__ import absolute_import + +import sys +from contextlib import contextmanager + + +class ErrorMessage(Exception): + '''Exception type raised from errors.error() and errors.fatal()''' + + +class AccumulatedErrors(Exception): + '''Exception type raised from errors.accumulate()''' + + +class ErrorCollector(object): + ''' + Error handling/logging class. A global instance, errors, is provided for + convenience. + + Warnings, errors and fatal errors may be logged by calls to the following + functions: + errors.warn(message) + errors.error(message) + errors.fatal(message) + + Warnings only send the message on the logging output, while errors and + fatal errors send the message and throw an ErrorMessage exception. The + exception, however, may be deferred. See further below. + + Errors may be ignored by calling: + errors.ignore_errors() + + After calling that function, only fatal errors throw an exception. + + The warnings, errors or fatal errors messages may be augmented with context + information when a context is provided. Context is defined by a pair + (filename, linenumber), and may be set with errors.context() used as a + context manager: + with errors.context(filename, linenumber): + errors.warn(message) + + Arbitrary nesting is supported, both for errors.context calls: + with errors.context(filename1, linenumber1): + errors.warn(message) + with errors.context(filename2, linenumber2): + errors.warn(message) + + as well as for function calls: + def func(): + errors.warn(message) + with errors.context(filename, linenumber): + func() + + Errors and fatal errors can have their exception thrown at a later time, + allowing for several different errors to be reported at once before + throwing. This is achieved with errors.accumulate() as a context manager: + with errors.accumulate(): + if test1: + errors.error(message1) + if test2: + errors.error(message2) + + In such cases, a single AccumulatedErrors exception is thrown, but doesn't + contain information about the exceptions. The logged messages do. + ''' + out = sys.stderr + WARN = 1 + ERROR = 2 + FATAL = 3 + _level = ERROR + _context = [] + _count = None + + def ignore_errors(self, ignore=True): + if ignore: + self._level = self.FATAL + else: + self._level = self.ERROR + + def _full_message(self, level, msg): + if level >= self._level: + level = 'Error' + else: + level = 'Warning' + if self._context: + file, line = self._context[-1] + return "%s: %s:%d: %s" % (level, file, line, msg) + return "%s: %s" % (level, msg) + + def _handle(self, level, msg): + msg = self._full_message(level, msg) + if level >= self._level: + if self._count is None: + raise ErrorMessage(msg) + self._count += 1 + print >>self.out, msg + + def fatal(self, msg): + self._handle(self.FATAL, msg) + + def error(self, msg): + self._handle(self.ERROR, msg) + + def warn(self, msg): + self._handle(self.WARN, msg) + + def get_context(self): + if self._context: + return self._context[-1] + + @contextmanager + def context(self, file, line): + if file and line: + self._context.append((file, line)) + yield + if file and line: + self._context.pop() + + @contextmanager + def accumulate(self): + assert self._count is None + self._count = 0 + yield + count = self._count + self._count = None + if count: + raise AccumulatedErrors() + + @property + def count(self): + # _count can be None. + return self._count if self._count else 0 + + +errors = ErrorCollector() diff --git a/python/mozbuild/mozpack/executables.py b/python/mozbuild/mozpack/executables.py new file mode 100644 index 000000000..c943564fa --- /dev/null +++ b/python/mozbuild/mozpack/executables.py @@ -0,0 +1,124 @@ +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, You can obtain one at http://mozilla.org/MPL/2.0/. + +from __future__ import absolute_import + +import os +import struct +import subprocess +from mozpack.errors import errors + +MACHO_SIGNATURES = [ + 0xfeedface, # mach-o 32-bits big endian + 0xcefaedfe, # mach-o 32-bits little endian + 0xfeedfacf, # mach-o 64-bits big endian + 0xcffaedfe, # mach-o 64-bits little endian +] + +FAT_SIGNATURE = 0xcafebabe # mach-o FAT binary + +ELF_SIGNATURE = 0x7f454c46 # Elf binary + +UNKNOWN = 0 +MACHO = 1 +ELF = 2 + +def get_type(path): + ''' + Check the signature of the give file and returns what kind of executable + matches. + ''' + with open(path, 'rb') as f: + signature = f.read(4) + if len(signature) < 4: + return UNKNOWN + signature = struct.unpack('>L', signature)[0] + if signature == ELF_SIGNATURE: + return ELF + if signature in MACHO_SIGNATURES: + return MACHO + if signature != FAT_SIGNATURE: + return UNKNOWN + # We have to sanity check the second four bytes, because Java class + # files use the same magic number as Mach-O fat binaries. + # This logic is adapted from file(1), which says that Mach-O uses + # these bytes to count the number of architectures within, while + # Java uses it for a version number. Conveniently, there are only + # 18 labelled Mach-O architectures, and Java's first released + # class format used the version 43.0. + num = f.read(4) + if len(num) < 4: + return UNKNOWN + num = struct.unpack('>L', num)[0] + if num < 20: + return MACHO + return UNKNOWN + + +def is_executable(path): + ''' + Return whether a given file path points to an executable or a library, + where an executable or library is identified by: + - the file extension on OS/2 and WINNT + - the file signature on OS/X and ELF systems (GNU/Linux, Android, BSD, + Solaris) + + As this function is intended for use to choose between the ExecutableFile + and File classes in FileFinder, and choosing ExecutableFile only matters + on OS/2, OS/X, ELF and WINNT (in GCC build) systems, we don't bother + detecting other kind of executables. + ''' + from buildconfig import substs + if not os.path.exists(path): + return False + + if substs['OS_ARCH'] == 'WINNT': + return path.lower().endswith((substs['DLL_SUFFIX'], + substs['BIN_SUFFIX'])) + + return get_type(path) != UNKNOWN + + +def may_strip(path): + ''' + Return whether strip() should be called + ''' + from buildconfig import substs + return not substs['PKG_SKIP_STRIP'] + + +def strip(path): + ''' + Execute the STRIP command with STRIP_FLAGS on the given path. + ''' + from buildconfig import substs + strip = substs['STRIP'] + flags = substs['STRIP_FLAGS'].split() if 'STRIP_FLAGS' in substs else [] + cmd = [strip] + flags + [path] + if subprocess.call(cmd) != 0: + errors.fatal('Error executing ' + ' '.join(cmd)) + + +def may_elfhack(path): + ''' + Return whether elfhack() should be called + ''' + # elfhack only supports libraries. We should check the ELF header for + # the right flag, but checking the file extension works too. + from buildconfig import substs + return ('USE_ELF_HACK' in substs and substs['USE_ELF_HACK'] and + path.endswith(substs['DLL_SUFFIX']) and + 'COMPILE_ENVIRONMENT' in substs and substs['COMPILE_ENVIRONMENT']) + + +def elfhack(path): + ''' + Execute the elfhack command on the given path. + ''' + from buildconfig import topobjdir + cmd = [os.path.join(topobjdir, 'build/unix/elfhack/elfhack'), path] + if 'ELF_HACK_FLAGS' in os.environ: + cmd[1:0] = os.environ['ELF_HACK_FLAGS'].split() + if subprocess.call(cmd) != 0: + errors.fatal('Error executing ' + ' '.join(cmd)) diff --git a/python/mozbuild/mozpack/files.py b/python/mozbuild/mozpack/files.py new file mode 100644 index 000000000..64902e195 --- /dev/null +++ b/python/mozbuild/mozpack/files.py @@ -0,0 +1,1106 @@ +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, You can obtain one at http://mozilla.org/MPL/2.0/. + +from __future__ import absolute_import + +import errno +import os +import platform +import shutil +import stat +import subprocess +import uuid +import mozbuild.makeutil as makeutil +from mozbuild.preprocessor import Preprocessor +from mozbuild.util import FileAvoidWrite +from mozpack.executables import ( + is_executable, + may_strip, + strip, + may_elfhack, + elfhack, +) +from mozpack.chrome.manifest import ManifestEntry +from io import BytesIO +from mozpack.errors import ( + ErrorMessage, + errors, +) +from mozpack.mozjar import JarReader +import mozpack.path as mozpath +from collections import OrderedDict +from jsmin import JavascriptMinify +from tempfile import ( + mkstemp, + NamedTemporaryFile, +) +from tarfile import ( + TarFile, + TarInfo, +) +try: + import hglib +except ImportError: + hglib = None + + +# For clean builds, copying files on win32 using CopyFile through ctypes is +# ~2x as fast as using shutil.copyfile. +if platform.system() != 'Windows': + _copyfile = shutil.copyfile +else: + import ctypes + _kernel32 = ctypes.windll.kernel32 + _CopyFileA = _kernel32.CopyFileA + _CopyFileW = _kernel32.CopyFileW + + def _copyfile(src, dest): + # False indicates `dest` should be overwritten if it exists already. + if isinstance(src, unicode) and isinstance(dest, unicode): + _CopyFileW(src, dest, False) + elif isinstance(src, str) and isinstance(dest, str): + _CopyFileA(src, dest, False) + else: + raise TypeError('mismatched path types!') + +class Dest(object): + ''' + Helper interface for BaseFile.copy. The interface works as follows: + - read() and write() can be used to sequentially read/write from the + underlying file. + - a call to read() after a write() will re-open the underlying file and + read from it. + - a call to write() after a read() will re-open the underlying file, + emptying it, and write to it. + ''' + def __init__(self, path): + self.path = path + self.mode = None + + @property + def name(self): + return self.path + + def read(self, length=-1): + if self.mode != 'r': + self.file = open(self.path, 'rb') + self.mode = 'r' + return self.file.read(length) + + def write(self, data): + if self.mode != 'w': + self.file = open(self.path, 'wb') + self.mode = 'w' + return self.file.write(data) + + def exists(self): + return os.path.exists(self.path) + + def close(self): + if self.mode: + self.mode = None + self.file.close() + + +class BaseFile(object): + ''' + Base interface and helper for file copying. Derived class may implement + their own copy function, or rely on BaseFile.copy using the open() member + function and/or the path property. + ''' + @staticmethod + def is_older(first, second): + ''' + Compares the modification time of two files, and returns whether the + ``first`` file is older than the ``second`` file. + ''' + # os.path.getmtime returns a result in seconds with precision up to + # the microsecond. But microsecond is too precise because + # shutil.copystat only copies milliseconds, and seconds is not + # enough precision. + return int(os.path.getmtime(first) * 1000) \ + <= int(os.path.getmtime(second) * 1000) + + @staticmethod + def any_newer(dest, inputs): + ''' + Compares the modification time of ``dest`` to multiple input files, and + returns whether any of the ``inputs`` is newer (has a later mtime) than + ``dest``. + ''' + # os.path.getmtime returns a result in seconds with precision up to + # the microsecond. But microsecond is too precise because + # shutil.copystat only copies milliseconds, and seconds is not + # enough precision. + dest_mtime = int(os.path.getmtime(dest) * 1000) + for input in inputs: + if dest_mtime < int(os.path.getmtime(input) * 1000): + return True + return False + + @staticmethod + def normalize_mode(mode): + # Normalize file mode: + # - keep file type (e.g. S_IFREG) + ret = stat.S_IFMT(mode) + # - expand user read and execute permissions to everyone + if mode & 0400: + ret |= 0444 + if mode & 0100: + ret |= 0111 + # - keep user write permissions + if mode & 0200: + ret |= 0200 + # - leave away sticky bit, setuid, setgid + return ret + + def copy(self, dest, skip_if_older=True): + ''' + Copy the BaseFile content to the destination given as a string or a + Dest instance. Avoids replacing existing files if the BaseFile content + matches that of the destination, or in case of plain files, if the + destination is newer than the original file. This latter behaviour is + disabled when skip_if_older is False. + Returns whether a copy was actually performed (True) or not (False). + ''' + if isinstance(dest, basestring): + dest = Dest(dest) + else: + assert isinstance(dest, Dest) + + can_skip_content_check = False + if not dest.exists(): + can_skip_content_check = True + elif getattr(self, 'path', None) and getattr(dest, 'path', None): + if skip_if_older and BaseFile.is_older(self.path, dest.path): + return False + elif os.path.getsize(self.path) != os.path.getsize(dest.path): + can_skip_content_check = True + + if can_skip_content_check: + if getattr(self, 'path', None) and getattr(dest, 'path', None): + _copyfile(self.path, dest.path) + shutil.copystat(self.path, dest.path) + else: + # Ensure the file is always created + if not dest.exists(): + dest.write('') + shutil.copyfileobj(self.open(), dest) + return True + + src = self.open() + copy_content = '' + while True: + dest_content = dest.read(32768) + src_content = src.read(32768) + copy_content += src_content + if len(dest_content) == len(src_content) == 0: + break + # If the read content differs between origin and destination, + # write what was read up to now, and copy the remainder. + if dest_content != src_content: + dest.write(copy_content) + shutil.copyfileobj(src, dest) + break + if hasattr(self, 'path') and hasattr(dest, 'path'): + shutil.copystat(self.path, dest.path) + return True + + def open(self): + ''' + Return a file-like object allowing to read() the content of the + associated file. This is meant to be overloaded in subclasses to return + a custom file-like object. + ''' + assert self.path is not None + return open(self.path, 'rb') + + def read(self): + raise NotImplementedError('BaseFile.read() not implemented. Bug 1170329.') + + @property + def mode(self): + ''' + Return the file's unix mode, or None if it has no meaning. + ''' + return None + + +class File(BaseFile): + ''' + File class for plain files. + ''' + def __init__(self, path): + self.path = path + + @property + def mode(self): + ''' + Return the file's unix mode, as returned by os.stat().st_mode. + ''' + if platform.system() == 'Windows': + return None + assert self.path is not None + mode = os.stat(self.path).st_mode + return self.normalize_mode(mode) + + def read(self): + '''Return the contents of the file.''' + with open(self.path, 'rb') as fh: + return fh.read() + + +class ExecutableFile(File): + ''' + File class for executable and library files on OS/2, OS/X and ELF systems. + (see mozpack.executables.is_executable documentation). + ''' + def copy(self, dest, skip_if_older=True): + real_dest = dest + if not isinstance(dest, basestring): + fd, dest = mkstemp() + os.close(fd) + os.remove(dest) + assert isinstance(dest, basestring) + # If File.copy didn't actually copy because dest is newer, check the + # file sizes. If dest is smaller, it means it is already stripped and + # elfhacked, so we can skip. + if not File.copy(self, dest, skip_if_older) and \ + os.path.getsize(self.path) > os.path.getsize(dest): + return False + try: + if may_strip(dest): + strip(dest) + if may_elfhack(dest): + elfhack(dest) + except ErrorMessage: + os.remove(dest) + raise + + if real_dest != dest: + f = File(dest) + ret = f.copy(real_dest, skip_if_older) + os.remove(dest) + return ret + return True + + +class AbsoluteSymlinkFile(File): + '''File class that is copied by symlinking (if available). + + This class only works if the target path is absolute. + ''' + + def __init__(self, path): + if not os.path.isabs(path): + raise ValueError('Symlink target not absolute: %s' % path) + + File.__init__(self, path) + + def copy(self, dest, skip_if_older=True): + assert isinstance(dest, basestring) + + # The logic in this function is complicated by the fact that symlinks + # aren't universally supported. So, where symlinks aren't supported, we + # fall back to file copying. Keep in mind that symlink support is + # per-filesystem, not per-OS. + + # Handle the simple case where symlinks are definitely not supported by + # falling back to file copy. + if not hasattr(os, 'symlink'): + return File.copy(self, dest, skip_if_older=skip_if_older) + + # Always verify the symlink target path exists. + if not os.path.exists(self.path): + raise ErrorMessage('Symlink target path does not exist: %s' % self.path) + + st = None + + try: + st = os.lstat(dest) + except OSError as ose: + if ose.errno != errno.ENOENT: + raise + + # If the dest is a symlink pointing to us, we have nothing to do. + # If it's the wrong symlink, the filesystem must support symlinks, + # so we replace with a proper symlink. + if st and stat.S_ISLNK(st.st_mode): + link = os.readlink(dest) + if link == self.path: + return False + + os.remove(dest) + os.symlink(self.path, dest) + return True + + # If the destination doesn't exist, we try to create a symlink. If that + # fails, we fall back to copy code. + if not st: + try: + os.symlink(self.path, dest) + return True + except OSError: + return File.copy(self, dest, skip_if_older=skip_if_older) + + # Now the complicated part. If the destination exists, we could be + # replacing a file with a symlink. Or, the filesystem may not support + # symlinks. We want to minimize I/O overhead for performance reasons, + # so we keep the existing destination file around as long as possible. + # A lot of the system calls would be eliminated if we cached whether + # symlinks are supported. However, even if we performed a single + # up-front test of whether the root of the destination directory + # supports symlinks, there's no guarantee that all operations for that + # dest (or source) would be on the same filesystem and would support + # symlinks. + # + # Our strategy is to attempt to create a new symlink with a random + # name. If that fails, we fall back to copy mode. If that works, we + # remove the old destination and move the newly-created symlink into + # its place. + + temp_dest = os.path.join(os.path.dirname(dest), str(uuid.uuid4())) + try: + os.symlink(self.path, temp_dest) + # TODO Figure out exactly how symlink creation fails and only trap + # that. + except EnvironmentError: + return File.copy(self, dest, skip_if_older=skip_if_older) + + # If removing the original file fails, don't forget to clean up the + # temporary symlink. + try: + os.remove(dest) + except EnvironmentError: + os.remove(temp_dest) + raise + + os.rename(temp_dest, dest) + return True + + +class ExistingFile(BaseFile): + ''' + File class that represents a file that may exist but whose content comes + from elsewhere. + + This purpose of this class is to account for files that are installed via + external means. It is typically only used in manifests or in registries to + account for files. + + When asked to copy, this class does nothing because nothing is known about + the source file/data. + + Instances of this class come in two flavors: required and optional. If an + existing file is required, it must exist during copy() or an error is + raised. + ''' + def __init__(self, required): + self.required = required + + def copy(self, dest, skip_if_older=True): + if isinstance(dest, basestring): + dest = Dest(dest) + else: + assert isinstance(dest, Dest) + + if not self.required: + return + + if not dest.exists(): + errors.fatal("Required existing file doesn't exist: %s" % + dest.path) + + +class PreprocessedFile(BaseFile): + ''' + File class for a file that is preprocessed. PreprocessedFile.copy() runs + the preprocessor on the file to create the output. + ''' + def __init__(self, path, depfile_path, marker, defines, extra_depends=None, + silence_missing_directive_warnings=False): + self.path = path + self.depfile = depfile_path + self.marker = marker + self.defines = defines + self.extra_depends = list(extra_depends or []) + self.silence_missing_directive_warnings = \ + silence_missing_directive_warnings + + def copy(self, dest, skip_if_older=True): + ''' + Invokes the preprocessor to create the destination file. + ''' + if isinstance(dest, basestring): + dest = Dest(dest) + else: + assert isinstance(dest, Dest) + + # We have to account for the case where the destination exists and is a + # symlink to something. Since we know the preprocessor is certainly not + # going to create a symlink, we can just remove the existing one. If the + # destination is not a symlink, we leave it alone, since we're going to + # overwrite its contents anyway. + # If symlinks aren't supported at all, we can skip this step. + if hasattr(os, 'symlink'): + if os.path.islink(dest.path): + os.remove(dest.path) + + pp_deps = set(self.extra_depends) + + # If a dependency file was specified, and it exists, add any + # dependencies from that file to our list. + if self.depfile and os.path.exists(self.depfile): + target = mozpath.normpath(dest.name) + with open(self.depfile, 'rb') as fileobj: + for rule in makeutil.read_dep_makefile(fileobj): + if target in rule.targets(): + pp_deps.update(rule.dependencies()) + + skip = False + if dest.exists() and skip_if_older: + # If a dependency file was specified, and it doesn't exist, + # assume that the preprocessor needs to be rerun. That will + # regenerate the dependency file. + if self.depfile and not os.path.exists(self.depfile): + skip = False + else: + skip = not BaseFile.any_newer(dest.path, pp_deps) + + if skip: + return False + + deps_out = None + if self.depfile: + deps_out = FileAvoidWrite(self.depfile) + pp = Preprocessor(defines=self.defines, marker=self.marker) + pp.setSilenceDirectiveWarnings(self.silence_missing_directive_warnings) + + with open(self.path, 'rU') as input: + pp.processFile(input=input, output=dest, depfile=deps_out) + + dest.close() + if self.depfile: + deps_out.close() + + return True + + +class GeneratedFile(BaseFile): + ''' + File class for content with no previous existence on the filesystem. + ''' + def __init__(self, content): + self.content = content + + def open(self): + return BytesIO(self.content) + + +class DeflatedFile(BaseFile): + ''' + File class for members of a jar archive. DeflatedFile.copy() effectively + extracts the file from the jar archive. + ''' + def __init__(self, file): + from mozpack.mozjar import JarFileReader + assert isinstance(file, JarFileReader) + self.file = file + + def open(self): + self.file.seek(0) + return self.file + +class ExtractedTarFile(GeneratedFile): + ''' + File class for members of a tar archive. Contents of the underlying file + are extracted immediately and stored in memory. + ''' + def __init__(self, tar, info): + assert isinstance(info, TarInfo) + assert isinstance(tar, TarFile) + GeneratedFile.__init__(self, tar.extractfile(info).read()) + self._mode = self.normalize_mode(info.mode) + + @property + def mode(self): + return self._mode + + def read(self): + return self.content + +class XPTFile(GeneratedFile): + ''' + File class for a linked XPT file. It takes several XPT files as input + (using the add() and remove() member functions), and links them at copy() + time. + ''' + def __init__(self): + self._files = set() + + def add(self, xpt): + ''' + Add the given XPT file (as a BaseFile instance) to the list of XPTs + to link. + ''' + assert isinstance(xpt, BaseFile) + self._files.add(xpt) + + def remove(self, xpt): + ''' + Remove the given XPT file (as a BaseFile instance) from the list of + XPTs to link. + ''' + assert isinstance(xpt, BaseFile) + self._files.remove(xpt) + + def copy(self, dest, skip_if_older=True): + ''' + Link the registered XPTs and place the resulting linked XPT at the + destination given as a string or a Dest instance. Avoids an expensive + XPT linking if the interfaces in an existing destination match those of + the individual XPTs to link. + skip_if_older is ignored. + ''' + if isinstance(dest, basestring): + dest = Dest(dest) + assert isinstance(dest, Dest) + + from xpt import xpt_link, Typelib, Interface + all_typelibs = [Typelib.read(f.open()) for f in self._files] + if dest.exists(): + # Typelib.read() needs to seek(), so use a BytesIO for dest + # content. + dest_interfaces = \ + dict((i.name, i) + for i in Typelib.read(BytesIO(dest.read())).interfaces + if i.iid != Interface.UNRESOLVED_IID) + identical = True + for f in self._files: + typelib = Typelib.read(f.open()) + for i in typelib.interfaces: + if i.iid != Interface.UNRESOLVED_IID and \ + not (i.name in dest_interfaces and + i == dest_interfaces[i.name]): + identical = False + break + if identical: + return False + s = BytesIO() + xpt_link(all_typelibs).write(s) + dest.write(s.getvalue()) + return True + + def open(self): + raise RuntimeError("Unsupported") + + def isempty(self): + ''' + Return whether there are XPT files to link. + ''' + return len(self._files) == 0 + + +class ManifestFile(BaseFile): + ''' + File class for a manifest file. It takes individual manifest entries (using + the add() and remove() member functions), and adjusts them to be relative + to the base path for the manifest, given at creation. + Example: + There is a manifest entry "content foobar foobar/content/" relative + to "foobar/chrome". When packaging, the entry will be stored in + jar:foobar/omni.ja!/chrome/chrome.manifest, which means the entry + will have to be relative to "chrome" instead of "foobar/chrome". This + doesn't really matter when serializing the entry, since this base path + is not written out, but it matters when moving the entry at the same + time, e.g. to jar:foobar/omni.ja!/chrome.manifest, which we don't do + currently but could in the future. + ''' + def __init__(self, base, entries=None): + self._entries = entries if entries else [] + self._base = base + + def add(self, entry): + ''' + Add the given entry to the manifest. Entries are rebased at open() time + instead of add() time so that they can be more easily remove()d. + ''' + assert isinstance(entry, ManifestEntry) + self._entries.append(entry) + + def remove(self, entry): + ''' + Remove the given entry from the manifest. + ''' + assert isinstance(entry, ManifestEntry) + self._entries.remove(entry) + + def open(self): + ''' + Return a file-like object allowing to read() the serialized content of + the manifest. + ''' + return BytesIO(''.join('%s\n' % e.rebase(self._base) + for e in self._entries)) + + def __iter__(self): + ''' + Iterate over entries in the manifest file. + ''' + return iter(self._entries) + + def isempty(self): + ''' + Return whether there are manifest entries to write + ''' + return len(self._entries) == 0 + + +class MinifiedProperties(BaseFile): + ''' + File class for minified properties. This wraps around a BaseFile instance, + and removes lines starting with a # from its content. + ''' + def __init__(self, file): + assert isinstance(file, BaseFile) + self._file = file + + def open(self): + ''' + Return a file-like object allowing to read() the minified content of + the properties file. + ''' + return BytesIO(''.join(l for l in self._file.open().readlines() + if not l.startswith('#'))) + + +class MinifiedJavaScript(BaseFile): + ''' + File class for minifying JavaScript files. + ''' + def __init__(self, file, verify_command=None): + assert isinstance(file, BaseFile) + self._file = file + self._verify_command = verify_command + + def open(self): + output = BytesIO() + minify = JavascriptMinify(self._file.open(), output, quote_chars="'\"`") + minify.minify() + output.seek(0) + + if not self._verify_command: + return output + + input_source = self._file.open().read() + output_source = output.getvalue() + + with NamedTemporaryFile() as fh1, NamedTemporaryFile() as fh2: + fh1.write(input_source) + fh2.write(output_source) + fh1.flush() + fh2.flush() + + try: + args = list(self._verify_command) + args.extend([fh1.name, fh2.name]) + subprocess.check_output(args, stderr=subprocess.STDOUT) + except subprocess.CalledProcessError as e: + errors.warn('JS minification verification failed for %s:' % + (getattr(self._file, 'path', ''))) + # Prefix each line with "Warning:" so mozharness doesn't + # think these error messages are real errors. + for line in e.output.splitlines(): + errors.warn(line) + + return self._file.open() + + return output + + +class BaseFinder(object): + def __init__(self, base, minify=False, minify_js=False, + minify_js_verify_command=None): + ''' + Initializes the instance with a reference base directory. + + The optional minify argument specifies whether minification of code + should occur. minify_js is an additional option to control minification + of JavaScript. It requires minify to be True. + + minify_js_verify_command can be used to optionally verify the results + of JavaScript minification. If defined, it is expected to be an iterable + that will constitute the first arguments to a called process which will + receive the filenames of the original and minified JavaScript files. + The invoked process can then verify the results. If minification is + rejected, the process exits with a non-0 exit code and the original + JavaScript source is used. An example value for this argument is + ('/path/to/js', '/path/to/verify/script.js'). + ''' + if minify_js and not minify: + raise ValueError('minify_js requires minify.') + + self.base = base + self._minify = minify + self._minify_js = minify_js + self._minify_js_verify_command = minify_js_verify_command + + def find(self, pattern): + ''' + Yield path, BaseFile_instance pairs for all files under the base + directory and its subdirectories that match the given pattern. See the + mozpack.path.match documentation for a description of the handled + patterns. + ''' + while pattern.startswith('/'): + pattern = pattern[1:] + for p, f in self._find(pattern): + yield p, self._minify_file(p, f) + + def get(self, path): + """Obtain a single file. + + Where ``find`` is tailored towards matching multiple files, this method + is used for retrieving a single file. Use this method when performance + is critical. + + Returns a ``BaseFile`` if at most one file exists or ``None`` otherwise. + """ + files = list(self.find(path)) + if len(files) != 1: + return None + return files[0][1] + + def __iter__(self): + ''' + Iterates over all files under the base directory (excluding files + starting with a '.' and files at any level under a directory starting + with a '.'). + for path, file in finder: + ... + ''' + return self.find('') + + def __contains__(self, pattern): + raise RuntimeError("'in' operator forbidden for %s. Use contains()." % + self.__class__.__name__) + + def contains(self, pattern): + ''' + Return whether some files under the base directory match the given + pattern. See the mozpack.path.match documentation for a description of + the handled patterns. + ''' + return any(self.find(pattern)) + + def _minify_file(self, path, file): + ''' + Return an appropriate MinifiedSomething wrapper for the given BaseFile + instance (file), according to the file type (determined by the given + path), if the FileFinder was created with minification enabled. + Otherwise, just return the given BaseFile instance. + ''' + if not self._minify or isinstance(file, ExecutableFile): + return file + + if path.endswith('.properties'): + return MinifiedProperties(file) + + if self._minify_js and path.endswith(('.js', '.jsm')): + return MinifiedJavaScript(file, self._minify_js_verify_command) + + return file + + def _find_helper(self, pattern, files, file_getter): + """Generic implementation of _find. + + A few *Finder implementations share logic for returning results. + This function implements the custom logic. + + The ``file_getter`` argument is a callable that receives a path + that is known to exist. The callable should return a ``BaseFile`` + instance. + """ + if '*' in pattern: + for p in files: + if mozpath.match(p, pattern): + yield p, file_getter(p) + elif pattern == '': + for p in files: + yield p, file_getter(p) + elif pattern in files: + yield pattern, file_getter(pattern) + else: + for p in files: + if mozpath.basedir(p, [pattern]) == pattern: + yield p, file_getter(p) + + +class FileFinder(BaseFinder): + ''' + Helper to get appropriate BaseFile instances from the file system. + ''' + def __init__(self, base, find_executables=True, ignore=(), + find_dotfiles=False, **kargs): + ''' + Create a FileFinder for files under the given base directory. + + The find_executables argument determines whether the finder needs to + try to guess whether files are executables. Disabling this guessing + when not necessary can speed up the finder significantly. + + ``ignore`` accepts an iterable of patterns to ignore. Entries are + strings that match paths relative to ``base`` using + ``mozpath.match()``. This means if an entry corresponds + to a directory, all files under that directory will be ignored. If + an entry corresponds to a file, that particular file will be ignored. + ''' + BaseFinder.__init__(self, base, **kargs) + self.find_dotfiles = find_dotfiles + self.find_executables = find_executables + self.ignore = ignore + + def _find(self, pattern): + ''' + Actual implementation of FileFinder.find(), dispatching to specialized + member functions depending on what kind of pattern was given. + Note all files with a name starting with a '.' are ignored when + scanning directories, but are not ignored when explicitely requested. + ''' + if '*' in pattern: + return self._find_glob('', mozpath.split(pattern)) + elif os.path.isdir(os.path.join(self.base, pattern)): + return self._find_dir(pattern) + else: + f = self.get(pattern) + return ((pattern, f),) if f else () + + def _find_dir(self, path): + ''' + Actual implementation of FileFinder.find() when the given pattern + corresponds to an existing directory under the base directory. + Ignores file names starting with a '.' under the given path. If the + path itself has leafs starting with a '.', they are not ignored. + ''' + for p in self.ignore: + if mozpath.match(path, p): + return + + # The sorted makes the output idempotent. Otherwise, we are + # likely dependent on filesystem implementation details, such as + # inode ordering. + for p in sorted(os.listdir(os.path.join(self.base, path))): + if p.startswith('.'): + if p in ('.', '..'): + continue + if not self.find_dotfiles: + continue + for p_, f in self._find(mozpath.join(path, p)): + yield p_, f + + def get(self, path): + srcpath = os.path.join(self.base, path) + if not os.path.exists(srcpath): + return None + + for p in self.ignore: + if mozpath.match(path, p): + return None + + if self.find_executables and is_executable(srcpath): + return ExecutableFile(srcpath) + else: + return File(srcpath) + + def _find_glob(self, base, pattern): + ''' + Actual implementation of FileFinder.find() when the given pattern + contains globbing patterns ('*' or '**'). This is meant to be an + equivalent of: + for p, f in self: + if mozpath.match(p, pattern): + yield p, f + but avoids scanning the entire tree. + ''' + if not pattern: + for p, f in self._find(base): + yield p, f + elif pattern[0] == '**': + for p, f in self._find(base): + if mozpath.match(p, mozpath.join(*pattern)): + yield p, f + elif '*' in pattern[0]: + if not os.path.exists(os.path.join(self.base, base)): + return + + for p in self.ignore: + if mozpath.match(base, p): + return + + # See above comment w.r.t. sorted() and idempotent behavior. + for p in sorted(os.listdir(os.path.join(self.base, base))): + if p.startswith('.') and not pattern[0].startswith('.'): + continue + if mozpath.match(p, pattern[0]): + for p_, f in self._find_glob(mozpath.join(base, p), + pattern[1:]): + yield p_, f + else: + for p, f in self._find_glob(mozpath.join(base, pattern[0]), + pattern[1:]): + yield p, f + + +class JarFinder(BaseFinder): + ''' + Helper to get appropriate DeflatedFile instances from a JarReader. + ''' + def __init__(self, base, reader, **kargs): + ''' + Create a JarFinder for files in the given JarReader. The base argument + is used as an indication of the Jar file location. + ''' + assert isinstance(reader, JarReader) + BaseFinder.__init__(self, base, **kargs) + self._files = OrderedDict((f.filename, f) for f in reader) + + def _find(self, pattern): + ''' + Actual implementation of JarFinder.find(), dispatching to specialized + member functions depending on what kind of pattern was given. + ''' + return self._find_helper(pattern, self._files, + lambda x: DeflatedFile(self._files[x])) + + +class TarFinder(BaseFinder): + ''' + Helper to get files from a TarFile. + ''' + def __init__(self, base, tar, **kargs): + ''' + Create a TarFinder for files in the given TarFile. The base argument + is used as an indication of the Tar file location. + ''' + assert isinstance(tar, TarFile) + self._tar = tar + BaseFinder.__init__(self, base, **kargs) + self._files = OrderedDict((f.name, f) for f in tar if f.isfile()) + + def _find(self, pattern): + ''' + Actual implementation of TarFinder.find(), dispatching to specialized + member functions depending on what kind of pattern was given. + ''' + return self._find_helper(pattern, self._files, + lambda x: ExtractedTarFile(self._tar, + self._files[x])) + + +class ComposedFinder(BaseFinder): + ''' + Composes multiple File Finders in some sort of virtual file system. + + A ComposedFinder is initialized from a dictionary associating paths to + *Finder instances. + + Note this could be optimized to be smarter than getting all the files + in advance. + ''' + def __init__(self, finders): + # Can't import globally, because of the dependency of mozpack.copier + # on this module. + from mozpack.copier import FileRegistry + self.files = FileRegistry() + + for base, finder in sorted(finders.iteritems()): + if self.files.contains(base): + self.files.remove(base) + for p, f in finder.find(''): + self.files.add(mozpath.join(base, p), f) + + def find(self, pattern): + for p in self.files.match(pattern): + yield p, self.files[p] + + +class MercurialFile(BaseFile): + """File class for holding data from Mercurial.""" + def __init__(self, client, rev, path): + self._content = client.cat([path], rev=rev) + + def read(self): + return self._content + + +class MercurialRevisionFinder(BaseFinder): + """A finder that operates on a specific Mercurial revision.""" + + def __init__(self, repo, rev='.', recognize_repo_paths=False, **kwargs): + """Create a finder attached to a specific revision in a repository. + + If no revision is given, open the parent of the working directory. + + ``recognize_repo_paths`` will enable a mode where ``.get()`` will + recognize full paths that include the repo's path. Typically Finder + instances are "bound" to a base directory and paths are relative to + that directory. This mode changes that. When this mode is activated, + ``.find()`` will not work! This mode exists to support the moz.build + reader, which uses absolute paths instead of relative paths. The reader + should eventually be rewritten to use relative paths and this hack + should be removed (TODO bug 1171069). + """ + if not hglib: + raise Exception('hglib package not found') + + super(MercurialRevisionFinder, self).__init__(base=repo, **kwargs) + + self._root = mozpath.normpath(repo).rstrip('/') + self._recognize_repo_paths = recognize_repo_paths + + # We change directories here otherwise we have to deal with relative + # paths. + oldcwd = os.getcwd() + os.chdir(self._root) + try: + self._client = hglib.open(path=repo, encoding=b'utf-8') + finally: + os.chdir(oldcwd) + self._rev = rev if rev is not None else b'.' + self._files = OrderedDict() + + # Immediately populate the list of files in the repo since nearly every + # operation requires this list. + out = self._client.rawcommand([b'files', b'--rev', str(self._rev)]) + for relpath in out.splitlines(): + self._files[relpath] = None + + def _find(self, pattern): + if self._recognize_repo_paths: + raise NotImplementedError('cannot use find with recognize_repo_path') + + return self._find_helper(pattern, self._files, self._get) + + def get(self, path): + if self._recognize_repo_paths: + if not path.startswith(self._root): + raise ValueError('lookups in recognize_repo_paths mode must be ' + 'prefixed with repo path: %s' % path) + path = path[len(self._root) + 1:] + + try: + return self._get(path) + except KeyError: + return None + + def _get(self, path): + # We lazy populate self._files because potentially creating tens of + # thousands of MercurialFile instances for every file in the repo is + # inefficient. + f = self._files[path] + if not f: + f = MercurialFile(self._client, self._rev, path) + self._files[path] = f + + return f diff --git a/python/mozbuild/mozpack/hg.py b/python/mozbuild/mozpack/hg.py new file mode 100644 index 000000000..79876061f --- /dev/null +++ b/python/mozbuild/mozpack/hg.py @@ -0,0 +1,95 @@ +# Copyright (C) 2015 Mozilla Contributors +# +# This program is free software; you can redistribute it and/or +# modify it under the terms of the GNU General Public License +# as published by the Free Software Foundation; either version 2 +# of the License, or (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program; if not, write to the Free Software +# Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. +# +# As a special exception, the copyright holders of this code give you +# permission to combine this code with the software known as 'mozbuild', +# and to distribute those combinations without any restriction +# coming from the use of this file. (The General Public License +# restrictions do apply in other respects; for example, they cover +# modification of the file, and distribution when not combined with +# mozbuild.) +# +# If you modify this code, you may extend this exception to your +# version of the code, but you are not obliged to do so. If you +# do not wish to do so, delete this exception statement from your +# version. + +from __future__ import absolute_import + +import mercurial.error as error +import mercurial.hg as hg +import mercurial.ui as hgui + +from .files import ( + BaseFinder, + MercurialFile, +) +import mozpack.path as mozpath + + +# This isn't a complete implementation of BaseFile. But it is complete +# enough for moz.build reading. +class MercurialNativeFile(MercurialFile): + def __init__(self, data): + self.data = data + + def read(self): + return self.data + + +class MercurialNativeRevisionFinder(BaseFinder): + def __init__(self, repo, rev='.', recognize_repo_paths=False): + """Create a finder attached to a specific changeset. + + Accepts a Mercurial localrepo and changectx instance. + """ + if isinstance(repo, (str, unicode)): + path = repo + repo = hg.repository(hgui.ui(), repo) + else: + path = repo.root + + super(MercurialNativeRevisionFinder, self).__init__(base=repo.root) + + self._repo = repo + self._rev = rev + self._root = mozpath.normpath(path) + self._recognize_repo_paths = recognize_repo_paths + + def _find(self, pattern): + if self._recognize_repo_paths: + raise NotImplementedError('cannot use find with recognize_repo_path') + + return self._find_helper(pattern, self._repo[self._rev], self._get) + + def get(self, path): + if self._recognize_repo_paths: + if not path.startswith(self._root): + raise ValueError('lookups in recognize_repo_paths mode must be ' + 'prefixed with repo path: %s' % path) + path = path[len(self._root) + 1:] + + return self._get(path) + + def _get(self, path): + if isinstance(path, unicode): + path = path.encode('utf-8', 'replace') + + try: + fctx = self._repo.filectx(path, self._rev) + return MercurialNativeFile(fctx.data()) + except error.LookupError: + return None diff --git a/python/mozbuild/mozpack/manifests.py b/python/mozbuild/mozpack/manifests.py new file mode 100644 index 000000000..93bd6c2ca --- /dev/null +++ b/python/mozbuild/mozpack/manifests.py @@ -0,0 +1,419 @@ +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, You can obtain one at http://mozilla.org/MPL/2.0/. + +from __future__ import absolute_import, unicode_literals + +from contextlib import contextmanager +import json + +from .files import ( + AbsoluteSymlinkFile, + ExistingFile, + File, + FileFinder, + GeneratedFile, + PreprocessedFile, +) +import mozpack.path as mozpath + + +# This probably belongs in a more generic module. Where? +@contextmanager +def _auto_fileobj(path, fileobj, mode='r'): + if path and fileobj: + raise AssertionError('Only 1 of path or fileobj may be defined.') + + if not path and not fileobj: + raise AssertionError('Must specified 1 of path or fileobj.') + + if path: + fileobj = open(path, mode) + + try: + yield fileobj + finally: + if path: + fileobj.close() + + +class UnreadableInstallManifest(Exception): + """Raised when an invalid install manifest is parsed.""" + + +class InstallManifest(object): + """Describes actions to be used with a copier.FileCopier instance. + + This class facilitates serialization and deserialization of data used to + construct a copier.FileCopier and to perform copy operations. + + The manifest defines source paths, destination paths, and a mechanism by + which the destination file should come into existence. + + Entries in the manifest correspond to the following types: + + copy -- The file specified as the source path will be copied to the + destination path. + + symlink -- The destination path will be a symlink to the source path. + If symlinks are not supported, a copy will be performed. + + exists -- The destination path is accounted for and won't be deleted by + the FileCopier. If the destination path doesn't exist, an error is + raised. + + optional -- The destination path is accounted for and won't be deleted by + the FileCopier. No error is raised if the destination path does not + exist. + + patternsymlink -- Paths matched by the expression in the source path + will be symlinked to the destination directory. + + patterncopy -- Similar to patternsymlink except files are copied, not + symlinked. + + preprocess -- The file specified at the source path will be run through + the preprocessor, and the output will be written to the destination + path. + + content -- The destination file will be created with the given content. + + Version 1 of the manifest was the initial version. + Version 2 added optional path support + Version 3 added support for pattern entries. + Version 4 added preprocessed file support. + Version 5 added content support. + """ + + CURRENT_VERSION = 5 + + FIELD_SEPARATOR = '\x1f' + + # Negative values are reserved for non-actionable items, that is, metadata + # that doesn't describe files in the destination. + SYMLINK = 1 + COPY = 2 + REQUIRED_EXISTS = 3 + OPTIONAL_EXISTS = 4 + PATTERN_SYMLINK = 5 + PATTERN_COPY = 6 + PREPROCESS = 7 + CONTENT = 8 + + def __init__(self, path=None, fileobj=None): + """Create a new InstallManifest entry. + + If path is defined, the manifest will be populated with data from the + file path. + + If fileobj is defined, the manifest will be populated with data read + from the specified file object. + + Both path and fileobj cannot be defined. + """ + self._dests = {} + self._source_files = set() + + if path or fileobj: + with _auto_fileobj(path, fileobj, 'rb') as fh: + self._source_files.add(fh.name) + self._load_from_fileobj(fh) + + def _load_from_fileobj(self, fileobj): + version = fileobj.readline().rstrip() + if version not in ('1', '2', '3', '4', '5'): + raise UnreadableInstallManifest('Unknown manifest version: %s' % + version) + + for line in fileobj: + line = line.rstrip() + + fields = line.split(self.FIELD_SEPARATOR) + + record_type = int(fields[0]) + + if record_type == self.SYMLINK: + dest, source = fields[1:] + self.add_symlink(source, dest) + continue + + if record_type == self.COPY: + dest, source = fields[1:] + self.add_copy(source, dest) + continue + + if record_type == self.REQUIRED_EXISTS: + _, path = fields + self.add_required_exists(path) + continue + + if record_type == self.OPTIONAL_EXISTS: + _, path = fields + self.add_optional_exists(path) + continue + + if record_type == self.PATTERN_SYMLINK: + _, base, pattern, dest = fields[1:] + self.add_pattern_symlink(base, pattern, dest) + continue + + if record_type == self.PATTERN_COPY: + _, base, pattern, dest = fields[1:] + self.add_pattern_copy(base, pattern, dest) + continue + + if record_type == self.PREPROCESS: + dest, source, deps, marker, defines, warnings = fields[1:] + + self.add_preprocess(source, dest, deps, marker, + self._decode_field_entry(defines), + silence_missing_directive_warnings=bool(int(warnings))) + continue + + if record_type == self.CONTENT: + dest, content = fields[1:] + + self.add_content( + self._decode_field_entry(content).encode('utf-8'), dest) + continue + + # Don't fail for non-actionable items, allowing + # forward-compatibility with those we will add in the future. + if record_type >= 0: + raise UnreadableInstallManifest('Unknown record type: %d' % + record_type) + + def __len__(self): + return len(self._dests) + + def __contains__(self, item): + return item in self._dests + + def __eq__(self, other): + return isinstance(other, InstallManifest) and self._dests == other._dests + + def __neq__(self, other): + return not self.__eq__(other) + + def __ior__(self, other): + if not isinstance(other, InstallManifest): + raise ValueError('Can only | with another instance of InstallManifest.') + + # We must copy source files to ourselves so extra dependencies from + # the preprocessor are taken into account. Ideally, we would track + # which source file each entry came from. However, this is more + # complicated and not yet implemented. The current implementation + # will result in over invalidation, possibly leading to performance + # loss. + self._source_files |= other._source_files + + for dest in sorted(other._dests): + self._add_entry(dest, other._dests[dest]) + + return self + + def _encode_field_entry(self, data): + """Converts an object into a format that can be stored in the manifest file. + + Complex data types, such as ``dict``, need to be converted into a text + representation before they can be written to a file. + """ + return json.dumps(data, sort_keys=True) + + def _decode_field_entry(self, data): + """Restores an object from a format that can be stored in the manifest file. + + Complex data types, such as ``dict``, need to be converted into a text + representation before they can be written to a file. + """ + return json.loads(data) + + def write(self, path=None, fileobj=None): + """Serialize this manifest to a file or file object. + + If path is specified, that file will be written to. If fileobj is specified, + the serialized content will be written to that file object. + + It is an error if both are specified. + """ + with _auto_fileobj(path, fileobj, 'wb') as fh: + fh.write('%d\n' % self.CURRENT_VERSION) + + for dest in sorted(self._dests): + entry = self._dests[dest] + + parts = ['%d' % entry[0], dest] + parts.extend(entry[1:]) + fh.write('%s\n' % self.FIELD_SEPARATOR.join( + p.encode('utf-8') for p in parts)) + + def add_symlink(self, source, dest): + """Add a symlink to this manifest. + + dest will be a symlink to source. + """ + self._add_entry(dest, (self.SYMLINK, source)) + + def add_copy(self, source, dest): + """Add a copy to this manifest. + + source will be copied to dest. + """ + self._add_entry(dest, (self.COPY, source)) + + def add_required_exists(self, dest): + """Record that a destination file must exist. + + This effectively prevents the listed file from being deleted. + """ + self._add_entry(dest, (self.REQUIRED_EXISTS,)) + + def add_optional_exists(self, dest): + """Record that a destination file may exist. + + This effectively prevents the listed file from being deleted. Unlike a + "required exists" file, files of this type do not raise errors if the + destination file does not exist. + """ + self._add_entry(dest, (self.OPTIONAL_EXISTS,)) + + def add_pattern_symlink(self, base, pattern, dest): + """Add a pattern match that results in symlinks being created. + + A ``FileFinder`` will be created with its base set to ``base`` + and ``FileFinder.find()`` will be called with ``pattern`` to discover + source files. Each source file will be symlinked under ``dest``. + + Filenames under ``dest`` are constructed by taking the path fragment + after ``base`` and concatenating it with ``dest``. e.g. + + /foo/bar.h -> /foo/bar.h + """ + self._add_entry(mozpath.join(base, pattern, dest), + (self.PATTERN_SYMLINK, base, pattern, dest)) + + def add_pattern_copy(self, base, pattern, dest): + """Add a pattern match that results in copies. + + See ``add_pattern_symlink()`` for usage. + """ + self._add_entry(mozpath.join(base, pattern, dest), + (self.PATTERN_COPY, base, pattern, dest)) + + def add_preprocess(self, source, dest, deps, marker='#', defines={}, + silence_missing_directive_warnings=False): + """Add a preprocessed file to this manifest. + + ``source`` will be passed through preprocessor.py, and the output will be + written to ``dest``. + """ + self._add_entry(dest, ( + self.PREPROCESS, + source, + deps, + marker, + self._encode_field_entry(defines), + '1' if silence_missing_directive_warnings else '0', + )) + + def add_content(self, content, dest): + """Add a file with the given content.""" + self._add_entry(dest, ( + self.CONTENT, + self._encode_field_entry(content), + )) + + def _add_entry(self, dest, entry): + if dest in self._dests: + raise ValueError('Item already in manifest: %s' % dest) + + self._dests[dest] = entry + + def populate_registry(self, registry, defines_override={}): + """Populate a mozpack.copier.FileRegistry instance with data from us. + + The caller supplied a FileRegistry instance (or at least something that + conforms to its interface) and that instance is populated with data + from this manifest. + + Defines can be given to override the ones in the manifest for + preprocessing. + """ + for dest in sorted(self._dests): + entry = self._dests[dest] + install_type = entry[0] + + if install_type == self.SYMLINK: + registry.add(dest, AbsoluteSymlinkFile(entry[1])) + continue + + if install_type == self.COPY: + registry.add(dest, File(entry[1])) + continue + + if install_type == self.REQUIRED_EXISTS: + registry.add(dest, ExistingFile(required=True)) + continue + + if install_type == self.OPTIONAL_EXISTS: + registry.add(dest, ExistingFile(required=False)) + continue + + if install_type in (self.PATTERN_SYMLINK, self.PATTERN_COPY): + _, base, pattern, dest = entry + finder = FileFinder(base, find_executables=False) + paths = [f[0] for f in finder.find(pattern)] + + if install_type == self.PATTERN_SYMLINK: + cls = AbsoluteSymlinkFile + else: + cls = File + + for path in paths: + source = mozpath.join(base, path) + registry.add(mozpath.join(dest, path), cls(source)) + + continue + + if install_type == self.PREPROCESS: + defines = self._decode_field_entry(entry[4]) + if defines_override: + defines.update(defines_override) + registry.add(dest, PreprocessedFile(entry[1], + depfile_path=entry[2], + marker=entry[3], + defines=defines, + extra_depends=self._source_files, + silence_missing_directive_warnings=bool(int(entry[5])))) + + continue + + if install_type == self.CONTENT: + # GeneratedFile expect the buffer interface, which the unicode + # type doesn't have, so encode to a str. + content = self._decode_field_entry(entry[1]).encode('utf-8') + registry.add(dest, GeneratedFile(content)) + continue + + raise Exception('Unknown install type defined in manifest: %d' % + install_type) + + +class InstallManifestNoSymlinks(InstallManifest): + """Like InstallManifest, but files are never installed as symbolic links. + Instead, they are always copied. + """ + + def add_symlink(self, source, dest): + """A wrapper that accept symlink entries and install file copies. + + source will be copied to dest. + """ + self.add_copy(source, dest) + + def add_pattern_symlink(self, base, pattern, dest): + """A wrapper that accepts symlink patterns and installs file copies. + + Files discovered with ``pattern`` will be copied to ``dest``. + """ + self.add_pattern_copy(base, pattern, dest) diff --git a/python/mozbuild/mozpack/mozjar.py b/python/mozbuild/mozpack/mozjar.py new file mode 100644 index 000000000..a1ada8594 --- /dev/null +++ b/python/mozbuild/mozpack/mozjar.py @@ -0,0 +1,816 @@ +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, You can obtain one at http://mozilla.org/MPL/2.0/. + +from __future__ import absolute_import + +from io import BytesIO +import struct +import zlib +import os +from zipfile import ( + ZIP_STORED, + ZIP_DEFLATED, +) +from collections import OrderedDict +from urlparse import urlparse, ParseResult +import mozpack.path as mozpath + +JAR_STORED = ZIP_STORED +JAR_DEFLATED = ZIP_DEFLATED +MAX_WBITS = 15 + + +class JarReaderError(Exception): + '''Error type for Jar reader errors.''' + + +class JarWriterError(Exception): + '''Error type for Jar writer errors.''' + + +class JarStruct(object): + ''' + Helper used to define ZIP archive raw data structures. Data structures + handled by this helper all start with a magic number, defined in + subclasses MAGIC field as a 32-bits unsigned integer, followed by data + structured as described in subclasses STRUCT field. + + The STRUCT field contains a list of (name, type) pairs where name is a + field name, and the type can be one of 'uint32', 'uint16' or one of the + field names. In the latter case, the field is considered to be a string + buffer with a length given in that field. + For example, + STRUCT = [ + ('version', 'uint32'), + ('filename_size', 'uint16'), + ('filename', 'filename_size') + ] + describes a structure with a 'version' 32-bits unsigned integer field, + followed by a 'filename_size' 16-bits unsigned integer field, followed by a + filename_size-long string buffer 'filename'. + + Fields that are used as other fields size are not stored in objects. In the + above example, an instance of such subclass would only have two attributes: + obj['version'] + obj['filename'] + filename_size would be obtained with len(obj['filename']). + + JarStruct subclasses instances can be either initialized from existing data + (deserialized), or with empty fields. + ''' + + TYPE_MAPPING = {'uint32': ('I', 4), 'uint16': ('H', 2)} + + def __init__(self, data=None): + ''' + Create an instance from the given data. Data may be omitted to create + an instance with empty fields. + ''' + assert self.MAGIC and isinstance(self.STRUCT, OrderedDict) + self.size_fields = set(t for t in self.STRUCT.itervalues() + if not t in JarStruct.TYPE_MAPPING) + self._values = {} + if data: + self._init_data(data) + else: + self._init_empty() + + def _init_data(self, data): + ''' + Initialize an instance from data, following the data structure + described in self.STRUCT. The self.MAGIC signature is expected at + data[:4]. + ''' + assert data is not None + self.signature, size = JarStruct.get_data('uint32', data) + if self.signature != self.MAGIC: + raise JarReaderError('Bad magic') + offset = size + # For all fields used as other fields sizes, keep track of their value + # separately. + sizes = dict((t, 0) for t in self.size_fields) + for name, t in self.STRUCT.iteritems(): + if t in JarStruct.TYPE_MAPPING: + value, size = JarStruct.get_data(t, data[offset:]) + else: + size = sizes[t] + value = data[offset:offset + size] + if isinstance(value, memoryview): + value = value.tobytes() + if not name in sizes: + self._values[name] = value + else: + sizes[name] = value + offset += size + + def _init_empty(self): + ''' + Initialize an instance with empty fields. + ''' + self.signature = self.MAGIC + for name, t in self.STRUCT.iteritems(): + if name in self.size_fields: + continue + self._values[name] = 0 if t in JarStruct.TYPE_MAPPING else '' + + @staticmethod + def get_data(type, data): + ''' + Deserialize a single field of given type (must be one of + JarStruct.TYPE_MAPPING) at the given offset in the given data. + ''' + assert type in JarStruct.TYPE_MAPPING + assert data is not None + format, size = JarStruct.TYPE_MAPPING[type] + data = data[:size] + if isinstance(data, memoryview): + data = data.tobytes() + return struct.unpack('<' + format, data)[0], size + + def serialize(self): + ''' + Serialize the data structure according to the data structure definition + from self.STRUCT. + ''' + serialized = struct.pack('" % (self.__class__.__name__, + ' '.join('%s=%s' % (n, v) for n, v in self)) + + +class JarCdirEnd(JarStruct): + ''' + End of central directory record. + ''' + MAGIC = 0x06054b50 + STRUCT = OrderedDict([ + ('disk_num', 'uint16'), + ('cdir_disk', 'uint16'), + ('disk_entries', 'uint16'), + ('cdir_entries', 'uint16'), + ('cdir_size', 'uint32'), + ('cdir_offset', 'uint32'), + ('comment_size', 'uint16'), + ('comment', 'comment_size'), + ]) + +CDIR_END_SIZE = JarCdirEnd().size + + +class JarCdirEntry(JarStruct): + ''' + Central directory file header + ''' + MAGIC = 0x02014b50 + STRUCT = OrderedDict([ + ('creator_version', 'uint16'), + ('min_version', 'uint16'), + ('general_flag', 'uint16'), + ('compression', 'uint16'), + ('lastmod_time', 'uint16'), + ('lastmod_date', 'uint16'), + ('crc32', 'uint32'), + ('compressed_size', 'uint32'), + ('uncompressed_size', 'uint32'), + ('filename_size', 'uint16'), + ('extrafield_size', 'uint16'), + ('filecomment_size', 'uint16'), + ('disknum', 'uint16'), + ('internal_attr', 'uint16'), + ('external_attr', 'uint32'), + ('offset', 'uint32'), + ('filename', 'filename_size'), + ('extrafield', 'extrafield_size'), + ('filecomment', 'filecomment_size'), + ]) + + +class JarLocalFileHeader(JarStruct): + ''' + Local file header + ''' + MAGIC = 0x04034b50 + STRUCT = OrderedDict([ + ('min_version', 'uint16'), + ('general_flag', 'uint16'), + ('compression', 'uint16'), + ('lastmod_time', 'uint16'), + ('lastmod_date', 'uint16'), + ('crc32', 'uint32'), + ('compressed_size', 'uint32'), + ('uncompressed_size', 'uint32'), + ('filename_size', 'uint16'), + ('extra_field_size', 'uint16'), + ('filename', 'filename_size'), + ('extra_field', 'extra_field_size'), + ]) + + +class JarFileReader(object): + ''' + File-like class for use by JarReader to give access to individual files + within a Jar archive. + ''' + def __init__(self, header, data): + ''' + Initialize a JarFileReader. header is the local file header + corresponding to the file in the jar archive, data a buffer containing + the file data. + ''' + assert header['compression'] in [JAR_DEFLATED, JAR_STORED] + self._data = data + # Copy some local file header fields. + for name in ['filename', 'compressed_size', + 'uncompressed_size', 'crc32']: + setattr(self, name, header[name]) + self.compressed = header['compression'] == JAR_DEFLATED + + def read(self, length=-1): + ''' + Read some amount of uncompressed data. + ''' + return self.uncompressed_data.read(length) + + def readlines(self): + ''' + Return a list containing all the lines of data in the uncompressed + data. + ''' + return self.read().splitlines(True) + + def __iter__(self): + ''' + Iterator, to support the "for line in fileobj" constructs. + ''' + return iter(self.readlines()) + + def seek(self, pos, whence=os.SEEK_SET): + ''' + Change the current position in the uncompressed data. Subsequent reads + will start from there. + ''' + return self.uncompressed_data.seek(pos, whence) + + def close(self): + ''' + Free the uncompressed data buffer. + ''' + self.uncompressed_data.close() + + @property + def compressed_data(self): + ''' + Return the raw compressed data. + ''' + return self._data[:self.compressed_size] + + @property + def uncompressed_data(self): + ''' + Return the uncompressed data. + ''' + if hasattr(self, '_uncompressed_data'): + return self._uncompressed_data + data = self.compressed_data + if self.compressed: + data = zlib.decompress(data.tobytes(), -MAX_WBITS) + else: + data = data.tobytes() + if len(data) != self.uncompressed_size: + raise JarReaderError('Corrupted file? %s' % self.filename) + self._uncompressed_data = BytesIO(data) + return self._uncompressed_data + + +class JarReader(object): + ''' + Class with methods to read Jar files. Can open standard jar files as well + as Mozilla jar files (see further details in the JarWriter documentation). + ''' + def __init__(self, file=None, fileobj=None, data=None): + ''' + Opens the given file as a Jar archive. Use the given file-like object + if one is given instead of opening the given file name. + ''' + if fileobj: + data = fileobj.read() + elif file: + data = open(file, 'rb').read() + self._data = memoryview(data) + # The End of Central Directory Record has a variable size because of + # comments it may contain, so scan for it from the end of the file. + offset = -CDIR_END_SIZE + while True: + signature = JarStruct.get_data('uint32', self._data[offset:])[0] + if signature == JarCdirEnd.MAGIC: + break + if offset == -len(self._data): + raise JarReaderError('Not a jar?') + offset -= 1 + self._cdir_end = JarCdirEnd(self._data[offset:]) + + def close(self): + ''' + Free some resources associated with the Jar. + ''' + del self._data + + @property + def entries(self): + ''' + Return an ordered dict of central directory entries, indexed by + filename, in the order they appear in the Jar archive central + directory. Directory entries are skipped. + ''' + if hasattr(self, '_entries'): + return self._entries + preload = 0 + if self.is_optimized: + preload = JarStruct.get_data('uint32', self._data)[0] + entries = OrderedDict() + offset = self._cdir_end['cdir_offset'] + for e in xrange(self._cdir_end['cdir_entries']): + entry = JarCdirEntry(self._data[offset:]) + offset += entry.size + # Creator host system. 0 is MSDOS, 3 is Unix + host = entry['creator_version'] >> 8 + # External attributes values depend on host above. On Unix the + # higher bits are the stat.st_mode value. On MSDOS, the lower bits + # are the FAT attributes. + xattr = entry['external_attr'] + # Skip directories + if (host == 0 and xattr & 0x10) or (host == 3 and + xattr & (040000 << 16)): + continue + entries[entry['filename']] = entry + if entry['offset'] < preload: + self._last_preloaded = entry['filename'] + self._entries = entries + return entries + + @property + def is_optimized(self): + ''' + Return whether the jar archive is optimized. + ''' + # In optimized jars, the central directory is at the beginning of the + # file, after a single 32-bits value, which is the length of data + # preloaded. + return self._cdir_end['cdir_offset'] == \ + JarStruct.TYPE_MAPPING['uint32'][1] + + @property + def last_preloaded(self): + ''' + Return the name of the last file that is set to be preloaded. + See JarWriter documentation for more details on preloading. + ''' + if hasattr(self, '_last_preloaded'): + return self._last_preloaded + self._last_preloaded = None + self.entries + return self._last_preloaded + + def _getreader(self, entry): + ''' + Helper to create a JarFileReader corresponding to the given central + directory entry. + ''' + header = JarLocalFileHeader(self._data[entry['offset']:]) + for key, value in entry: + if key in header and header[key] != value: + raise JarReaderError('Central directory and file header ' + + 'mismatch. Corrupted archive?') + return JarFileReader(header, + self._data[entry['offset'] + header.size:]) + + def __iter__(self): + ''' + Iterate over all files in the Jar archive, in the form of + JarFileReaders. + for file in jarReader: + ... + ''' + for entry in self.entries.itervalues(): + yield self._getreader(entry) + + def __getitem__(self, name): + ''' + Get a JarFileReader for the given file name. + ''' + return self._getreader(self.entries[name]) + + def __contains__(self, name): + ''' + Return whether the given file name appears in the Jar archive. + ''' + return name in self.entries + + +class JarWriter(object): + ''' + Class with methods to write Jar files. Can write more-or-less standard jar + archives as well as jar archives optimized for Gecko. See the documentation + for the close() member function for a description of both layouts. + ''' + def __init__(self, file=None, fileobj=None, compress=True, optimize=True, + compress_level=9): + ''' + Initialize a Jar archive in the given file. Use the given file-like + object if one is given instead of opening the given file name. + The compress option determines the default behavior for storing data + in the jar archive. The optimize options determines whether the jar + archive should be optimized for Gecko or not. ``compress_level`` + defines the zlib compression level. It must be a value between 0 and 9 + and defaults to 9, the highest and slowest level of compression. + ''' + if fileobj: + self._data = fileobj + else: + self._data = open(file, 'wb') + self._compress = compress + self._compress_level = compress_level + self._contents = OrderedDict() + self._last_preloaded = None + self._optimize = optimize + + def __enter__(self): + ''' + Context manager __enter__ method for JarWriter. + ''' + return self + + def __exit__(self, type, value, tb): + ''' + Context manager __exit__ method for JarWriter. + ''' + self.finish() + + def finish(self): + ''' + Flush and close the Jar archive. + + Standard jar archives are laid out like the following: + - Local file header 1 + - File data 1 + - Local file header 2 + - File data 2 + - (...) + - Central directory entry pointing at Local file header 1 + - Central directory entry pointing at Local file header 2 + - (...) + - End of central directory, pointing at first central directory + entry. + + Jar archives optimized for Gecko are laid out like the following: + - 32-bits unsigned integer giving the amount of data to preload. + - Central directory entry pointing at Local file header 1 + - Central directory entry pointing at Local file header 2 + - (...) + - End of central directory, pointing at first central directory + entry. + - Local file header 1 + - File data 1 + - Local file header 2 + - File data 2 + - (...) + - End of central directory, pointing at first central directory + entry. + The duplication of the End of central directory is to accomodate some + Zip reading tools that want an end of central directory structure to + follow the central directory entries. + ''' + offset = 0 + headers = {} + preload_size = 0 + # Prepare central directory entries + for entry, content in self._contents.itervalues(): + header = JarLocalFileHeader() + for name in entry.STRUCT: + if name in header: + header[name] = entry[name] + entry['offset'] = offset + offset += len(content) + header.size + if entry['filename'] == self._last_preloaded: + preload_size = offset + headers[entry] = header + # Prepare end of central directory + end = JarCdirEnd() + end['disk_entries'] = len(self._contents) + end['cdir_entries'] = end['disk_entries'] + end['cdir_size'] = reduce(lambda x, y: x + y[0].size, + self._contents.values(), 0) + # On optimized archives, store the preloaded size and the central + # directory entries, followed by the first end of central directory. + if self._optimize: + end['cdir_offset'] = 4 + offset = end['cdir_size'] + end['cdir_offset'] + end.size + if preload_size: + preload_size += offset + self._data.write(struct.pack(' 0: + errors.fatal('Malformed manifest: space in component name "%s"' + % component) + self._name = name + self._destdir = destdir + + def __repr__(self): + s = self.name + if self.destdir: + s += ' destdir="%s"' % self.destdir + return s + + @property + def name(self): + return self._name + + @property + def destdir(self): + return self._destdir + + @staticmethod + def _triples(lst): + ''' + Split [1, 2, 3, 4, 5, 6, 7] into [(1, 2, 3), (4, 5, 6)]. + ''' + return zip(*[iter(lst)] * 3) + + KEY_VALUE_RE = re.compile(r''' + \s* # optional whitespace. + ([a-zA-Z0-9_]+) # key. + \s*=\s* # optional space around =. + "([^"]*)" # value without surrounding quotes. + (?:\s+|$) + ''', re.VERBOSE) + + @staticmethod + def _split_options(string): + ''' + Split 'key1="value1" key2="value2"' into + {'key1':'value1', 'key2':'value2'}. + + Returned keys and values are all strings. + + Throws ValueError if the input is malformed. + ''' + options = {} + splits = Component.KEY_VALUE_RE.split(string) + if len(splits) % 3 != 1: + # This should never happen -- we expect to always split + # into ['', ('key', 'val', '')*]. + raise ValueError("Bad input") + if splits[0]: + raise ValueError('Unrecognized input ' + splits[0]) + for key, val, no_match in Component._triples(splits[1:]): + if no_match: + raise ValueError('Unrecognized input ' + no_match) + options[key] = val + return options + + @staticmethod + def _split_component_and_options(string): + ''' + Split 'name key1="value1" key2="value2"' into + ('name', {'key1':'value1', 'key2':'value2'}). + + Returned name, keys and values are all strings. + + Raises ValueError if the input is malformed. + ''' + splits = string.strip().split(None, 1) + if not splits: + raise ValueError('No component found') + component = splits[0].strip() + if not component: + raise ValueError('No component found') + if not re.match('[a-zA-Z0-9_\-]+$', component): + raise ValueError('Bad component name ' + component) + options = Component._split_options(splits[1]) if len(splits) > 1 else {} + return component, options + + @staticmethod + def from_string(string): + ''' + Create a component from a string. + ''' + try: + name, options = Component._split_component_and_options(string) + except ValueError as e: + errors.fatal('Malformed manifest: %s' % e) + return + destdir = options.pop('destdir', '') + if options: + errors.fatal('Malformed manifest: options %s not recognized' + % options.keys()) + return Component(name, destdir=destdir) + + +class PackageManifestParser(object): + ''' + Class for parsing of a package manifest, after preprocessing. + + A package manifest is a list of file paths, with some syntaxic sugar: + [] designates a toplevel component. Example: [xpcom] + - in front of a file specifies it to be removed + * wildcard support + ** expands to all files and zero or more directories + ; file comment + + The parser takes input from the preprocessor line by line, and pushes + parsed information to a sink object. + + The add and remove methods of the sink object are called with the + current Component instance and a path. + ''' + def __init__(self, sink): + ''' + Initialize the package manifest parser with the given sink. + ''' + self._component = Component('') + self._sink = sink + + def handle_line(self, str): + ''' + Handle a line of input and push the parsed information to the sink + object. + ''' + # Remove comments. + str = str.strip() + if not str or str.startswith(';'): + return + if str.startswith('[') and str.endswith(']'): + self._component = Component.from_string(str[1:-1]) + elif str.startswith('-'): + str = str[1:] + self._sink.remove(self._component, str) + elif ',' in str: + errors.fatal('Incompatible syntax') + else: + self._sink.add(self._component, str) + + +class PreprocessorOutputWrapper(object): + ''' + File-like helper to handle the preprocessor output and send it to a parser. + The parser's handle_line method is called in the relevant errors.context. + ''' + def __init__(self, preprocessor, parser): + self._parser = parser + self._pp = preprocessor + + def write(self, str): + file = os.path.normpath(os.path.abspath(self._pp.context['FILE'])) + with errors.context(file, self._pp.context['LINE']): + self._parser.handle_line(str) + + +def preprocess(input, parser, defines={}): + ''' + Preprocess the file-like input with the given defines, and send the + preprocessed output line by line to the given parser. + ''' + pp = Preprocessor() + pp.context.update(defines) + pp.do_filter('substitution') + pp.out = PreprocessorOutputWrapper(pp, parser) + pp.do_include(input) + + +def preprocess_manifest(sink, manifest, defines={}): + ''' + Preprocess the given file-like manifest with the given defines, and push + the parsed information to a sink. See PackageManifestParser documentation + for more details on the sink. + ''' + preprocess(manifest, PackageManifestParser(sink), defines) + + +class CallDeque(deque): + ''' + Queue of function calls to make. + ''' + def append(self, function, *args): + deque.append(self, (errors.get_context(), function, args)) + + def execute(self): + while True: + try: + context, function, args = self.popleft() + except IndexError: + return + if context: + with errors.context(context[0], context[1]): + function(*args) + else: + function(*args) + + +class SimplePackager(object): + ''' + Helper used to translate and buffer instructions from the + SimpleManifestSink to a formatter. Formatters expect some information to be + given first that the simple manifest contents can't guarantee before the + end of the input. + ''' + def __init__(self, formatter): + self.formatter = formatter + # Queue for formatter.add_interfaces()/add_manifest() calls. + self._queue = CallDeque() + # Queue for formatter.add_manifest() calls for ManifestChrome. + self._chrome_queue = CallDeque() + # Queue for formatter.add() calls. + self._file_queue = CallDeque() + # All paths containing addons. (key is path, value is whether it + # should be packed or unpacked) + self._addons = {} + # All manifest paths imported. + self._manifests = set() + # All manifest paths included from some other manifest. + self._included_manifests = {} + self._closed = False + + # Parsing RDF is complex, and would require an external library to do + # properly. Just go with some hackish but probably sufficient regexp + UNPACK_ADDON_RE = re.compile(r'''(?: + true + |em:unpack=(?P["']?)true(?P=quote) + )''', re.VERBOSE) + + def add(self, path, file): + ''' + Add the given BaseFile instance with the given path. + ''' + assert not self._closed + if is_manifest(path): + self._add_manifest_file(path, file) + elif path.endswith('.xpt'): + self._queue.append(self.formatter.add_interfaces, path, file) + else: + self._file_queue.append(self.formatter.add, path, file) + if mozpath.basename(path) == 'install.rdf': + addon = True + install_rdf = file.open().read() + if self.UNPACK_ADDON_RE.search(install_rdf): + addon = 'unpacked' + self._addons[mozpath.dirname(path)] = addon + + def _add_manifest_file(self, path, file): + ''' + Add the given BaseFile with manifest file contents with the given path. + ''' + self._manifests.add(path) + base = '' + if hasattr(file, 'path'): + # Find the directory the given path is relative to. + b = mozpath.normsep(file.path) + if b.endswith('/' + path) or b == path: + base = os.path.normpath(b[:-len(path)]) + for e in parse_manifest(base, path, file.open()): + # ManifestResources need to be given after ManifestChrome, so just + # put all ManifestChrome in a separate queue to make them first. + if isinstance(e, ManifestChrome): + # e.move(e.base) just returns a clone of the entry. + self._chrome_queue.append(self.formatter.add_manifest, + e.move(e.base)) + elif not isinstance(e, (Manifest, ManifestInterfaces)): + self._queue.append(self.formatter.add_manifest, e.move(e.base)) + # If a binary component is added to an addon, prevent the addon + # from being packed. + if isinstance(e, ManifestBinaryComponent): + addon = mozpath.basedir(e.base, self._addons) + if addon: + self._addons[addon] = 'unpacked' + if isinstance(e, Manifest): + if e.flags: + errors.fatal('Flags are not supported on ' + + '"manifest" entries') + self._included_manifests[e.path] = path + + def get_bases(self, addons=True): + ''' + Return all paths under which root manifests have been found. Root + manifests are manifests that are included in no other manifest. + `addons` indicates whether to include addon bases as well. + ''' + all_bases = set(mozpath.dirname(m) + for m in self._manifests + - set(self._included_manifests)) + if not addons: + all_bases -= set(self._addons) + else: + # If for some reason some detected addon doesn't have a + # non-included manifest. + all_bases |= set(self._addons) + return all_bases + + def close(self): + ''' + Push all instructions to the formatter. + ''' + self._closed = True + + bases = self.get_bases() + broken_bases = sorted( + m for m, includer in self._included_manifests.iteritems() + if mozpath.basedir(m, bases) != mozpath.basedir(includer, bases)) + for m in broken_bases: + errors.fatal('"%s" is included from "%s", which is outside "%s"' % + (m, self._included_manifests[m], + mozpath.basedir(m, bases))) + for base in sorted(bases): + self.formatter.add_base(base, self._addons.get(base, False)) + self._chrome_queue.execute() + self._queue.execute() + self._file_queue.execute() + + +class SimpleManifestSink(object): + ''' + Parser sink for "simple" package manifests. Simple package manifests use + the format described in the PackageManifestParser documentation, but don't + support file removals, and require manifests, interfaces and chrome data to + be explicitely listed. + Entries starting with bin/ are searched under bin/ in the FileFinder, but + are packaged without the bin/ prefix. + ''' + def __init__(self, finder, formatter): + ''' + Initialize the SimpleManifestSink. The given FileFinder is used to + get files matching the patterns given in the manifest. The given + formatter does the packaging job. + ''' + self._finder = finder + self.packager = SimplePackager(formatter) + self._closed = False + self._manifests = set() + + @staticmethod + def normalize_path(path): + ''' + Remove any bin/ prefix. + ''' + if mozpath.basedir(path, ['bin']) == 'bin': + return mozpath.relpath(path, 'bin') + return path + + def add(self, component, pattern): + ''' + Add files with the given pattern in the given component. + ''' + assert not self._closed + added = False + for p, f in self._finder.find(pattern): + added = True + if is_manifest(p): + self._manifests.add(p) + dest = mozpath.join(component.destdir, SimpleManifestSink.normalize_path(p)) + self.packager.add(dest, f) + if not added: + errors.error('Missing file(s): %s' % pattern) + + def remove(self, component, pattern): + ''' + Remove files with the given pattern in the given component. + ''' + assert not self._closed + errors.fatal('Removal is unsupported') + + def close(self, auto_root_manifest=True): + ''' + Add possibly missing bits and push all instructions to the formatter. + ''' + if auto_root_manifest: + # Simple package manifests don't contain the root manifests, so + # find and add them. + paths = [mozpath.dirname(m) for m in self._manifests] + path = mozpath.dirname(mozpath.commonprefix(paths)) + for p, f in self._finder.find(mozpath.join(path, + 'chrome.manifest')): + if not p in self._manifests: + self.packager.add(SimpleManifestSink.normalize_path(p), f) + self.packager.close() diff --git a/python/mozbuild/mozpack/packager/formats.py b/python/mozbuild/mozpack/packager/formats.py new file mode 100644 index 000000000..c4adabab0 --- /dev/null +++ b/python/mozbuild/mozpack/packager/formats.py @@ -0,0 +1,324 @@ +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, You can obtain one at http://mozilla.org/MPL/2.0/. + +from __future__ import absolute_import + +from mozpack.chrome.manifest import ( + Manifest, + ManifestInterfaces, + ManifestChrome, + ManifestBinaryComponent, + ManifestResource, +) +from urlparse import urlparse +import mozpack.path as mozpath +from mozpack.files import ( + ManifestFile, + XPTFile, +) +from mozpack.copier import ( + FileRegistry, + FileRegistrySubtree, + Jarrer, +) + +STARTUP_CACHE_PATHS = [ + 'jsloader', + 'jssubloader', +] + +''' +Formatters are classes receiving packaging instructions and creating the +appropriate package layout. + +There are three distinct formatters, each handling one of the different chrome +formats: + - flat: essentially, copies files from the source with the same file system + layout. Manifests entries are grouped in a single manifest per directory, + as well as XPT interfaces. + - jar: chrome content is packaged in jar files. + - omni: chrome content, modules, non-binary components, and many other + elements are packaged in an omnijar file for each base directory. + +The base interface provides the following methods: + - add_base(path [, addon]) + Register a base directory for an application or GRE, or an addon. + Base directories usually contain a root manifest (manifests not + included in any other manifest) named chrome.manifest. + The optional addon argument tells whether the base directory + is that of a packed addon (True), unpacked addon ('unpacked') or + otherwise (False). + - add(path, content) + Add the given content (BaseFile instance) at the given virtual path + - add_interfaces(path, content) + Add the given content (BaseFile instance) and link it to other + interfaces in the parent directory of the given virtual path. + - add_manifest(entry) + Add a ManifestEntry. + - contains(path) + Returns whether the given virtual path is known of the formatter. + +The virtual paths mentioned above are paths as they would be with a flat +chrome. + +Formatters all take a FileCopier instance they will fill with the packaged +data. +''' + + +class PiecemealFormatter(object): + ''' + Generic formatter that dispatches across different sub-formatters + according to paths. + ''' + def __init__(self, copier): + assert isinstance(copier, (FileRegistry, FileRegistrySubtree)) + self.copier = copier + self._sub_formatter = {} + self._frozen_bases = False + + def add_base(self, base, addon=False): + # Only allow to add a base directory before calls to _get_base() + assert not self._frozen_bases + assert base not in self._sub_formatter + self._add_base(base, addon) + + def _get_base(self, path): + ''' + Return the deepest base directory containing the given path. + ''' + self._frozen_bases = True + base = mozpath.basedir(path, self._sub_formatter.keys()) + relpath = mozpath.relpath(path, base) if base else path + return base, relpath + + def add(self, path, content): + base, relpath = self._get_base(path) + if base is None: + return self.copier.add(relpath, content) + return self._sub_formatter[base].add(relpath, content) + + def add_manifest(self, entry): + base, relpath = self._get_base(entry.base) + assert base is not None + return self._sub_formatter[base].add_manifest(entry.move(relpath)) + + def add_interfaces(self, path, content): + base, relpath = self._get_base(path) + assert base is not None + return self._sub_formatter[base].add_interfaces(relpath, content) + + def contains(self, path): + assert '*' not in path + base, relpath = self._get_base(path) + if base is None: + return self.copier.contains(relpath) + return self._sub_formatter[base].contains(relpath) + + +class FlatFormatter(PiecemealFormatter): + ''' + Formatter for the flat package format. + ''' + def _add_base(self, base, addon=False): + self._sub_formatter[base] = FlatSubFormatter( + FileRegistrySubtree(base, self.copier)) + + +class FlatSubFormatter(object): + ''' + Sub-formatter for the flat package format. + ''' + def __init__(self, copier): + assert isinstance(copier, (FileRegistry, FileRegistrySubtree)) + self.copier = copier + + def add(self, path, content): + self.copier.add(path, content) + + def add_manifest(self, entry): + # Store manifest entries in a single manifest per directory, named + # after their parent directory, except for root manifests, all named + # chrome.manifest. + if entry.base: + name = mozpath.basename(entry.base) + else: + name = 'chrome' + path = mozpath.normpath(mozpath.join(entry.base, '%s.manifest' % name)) + if not self.copier.contains(path): + # Add a reference to the manifest file in the parent manifest, if + # the manifest file is not a root manifest. + if entry.base: + parent = mozpath.dirname(entry.base) + relbase = mozpath.basename(entry.base) + relpath = mozpath.join(relbase, + mozpath.basename(path)) + self.add_manifest(Manifest(parent, relpath)) + self.copier.add(path, ManifestFile(entry.base)) + self.copier[path].add(entry) + + def add_interfaces(self, path, content): + # Interfaces in the same directory are all linked together in an + # interfaces.xpt file. + interfaces_path = mozpath.join(mozpath.dirname(path), + 'interfaces.xpt') + if not self.copier.contains(interfaces_path): + self.add_manifest(ManifestInterfaces(mozpath.dirname(path), + 'interfaces.xpt')) + self.copier.add(interfaces_path, XPTFile()) + self.copier[interfaces_path].add(content) + + def contains(self, path): + assert '*' not in path + return self.copier.contains(path) + + +class JarFormatter(PiecemealFormatter): + ''' + Formatter for the jar package format. Assumes manifest entries related to + chrome are registered before the chrome data files are added. Also assumes + manifest entries for resources are registered after chrome manifest + entries. + ''' + def __init__(self, copier, compress=True, optimize=True): + PiecemealFormatter.__init__(self, copier) + self._compress=compress + self._optimize=optimize + + def _add_base(self, base, addon=False): + if addon is True: + jarrer = Jarrer(self._compress, self._optimize) + self.copier.add(base + '.xpi', jarrer) + self._sub_formatter[base] = FlatSubFormatter(jarrer) + else: + self._sub_formatter[base] = JarSubFormatter( + FileRegistrySubtree(base, self.copier), + self._compress, self._optimize) + + +class JarSubFormatter(PiecemealFormatter): + ''' + Sub-formatter for the jar package format. It is a PiecemealFormatter that + dispatches between further sub-formatter for each of the jar files it + dispatches the chrome data to, and a FlatSubFormatter for the non-chrome + files. + ''' + def __init__(self, copier, compress=True, optimize=True): + PiecemealFormatter.__init__(self, copier) + self._frozen_chrome = False + self._compress = compress + self._optimize = optimize + self._sub_formatter[''] = FlatSubFormatter(copier) + + def _jarize(self, entry, relpath): + ''' + Transform a manifest entry in one pointing to chrome data in a jar. + Return the corresponding chrome path and the new entry. + ''' + base = entry.base + basepath = mozpath.split(relpath)[0] + chromepath = mozpath.join(base, basepath) + entry = entry.rebase(chromepath) \ + .move(mozpath.join(base, 'jar:%s.jar!' % basepath)) \ + .rebase(base) + return chromepath, entry + + def add_manifest(self, entry): + if isinstance(entry, ManifestChrome) and \ + not urlparse(entry.relpath).scheme: + chromepath, entry = self._jarize(entry, entry.relpath) + assert not self._frozen_chrome + if chromepath not in self._sub_formatter: + jarrer = Jarrer(self._compress, self._optimize) + self.copier.add(chromepath + '.jar', jarrer) + self._sub_formatter[chromepath] = FlatSubFormatter(jarrer) + elif isinstance(entry, ManifestResource) and \ + not urlparse(entry.target).scheme: + chromepath, new_entry = self._jarize(entry, entry.target) + if chromepath in self._sub_formatter: + entry = new_entry + PiecemealFormatter.add_manifest(self, entry) + + +class OmniJarFormatter(JarFormatter): + ''' + Formatter for the omnijar package format. + ''' + def __init__(self, copier, omnijar_name, compress=True, optimize=True, + non_resources=()): + JarFormatter.__init__(self, copier, compress, optimize) + self._omnijar_name = omnijar_name + self._non_resources = non_resources + + def _add_base(self, base, addon=False): + if addon: + JarFormatter._add_base(self, base, addon) + else: + # Initialize a chrome.manifest next to the omnijar file so that + # there's always a chrome.manifest file, even an empty one. + path = mozpath.normpath(mozpath.join(base, 'chrome.manifest')) + if not self.copier.contains(path): + self.copier.add(path, ManifestFile('')) + self._sub_formatter[base] = OmniJarSubFormatter( + FileRegistrySubtree(base, self.copier), self._omnijar_name, + self._compress, self._optimize, self._non_resources) + + +class OmniJarSubFormatter(PiecemealFormatter): + ''' + Sub-formatter for the omnijar package format. It is a PiecemealFormatter + that dispatches between a FlatSubFormatter for the resources data and + another FlatSubFormatter for the other files. + ''' + def __init__(self, copier, omnijar_name, compress=True, optimize=True, + non_resources=()): + PiecemealFormatter.__init__(self, copier) + self._omnijar_name = omnijar_name + self._compress = compress + self._optimize = optimize + self._non_resources = non_resources + self._sub_formatter[''] = FlatSubFormatter(copier) + jarrer = Jarrer(self._compress, self._optimize) + self._sub_formatter[omnijar_name] = FlatSubFormatter(jarrer) + + def _get_base(self, path): + base = self._omnijar_name if self.is_resource(path) else '' + # Only add the omnijar file if something ends up in it. + if base and not self.copier.contains(base): + self.copier.add(base, self._sub_formatter[base].copier) + return base, path + + def add_manifest(self, entry): + base = '' + if not isinstance(entry, ManifestBinaryComponent): + base = self._omnijar_name + formatter = self._sub_formatter[base] + return formatter.add_manifest(entry) + + def is_resource(self, path): + ''' + Return whether the given path corresponds to a resource to be put in an + omnijar archive. + ''' + if any(mozpath.match(path, p.replace('*', '**')) + for p in self._non_resources): + return False + path = mozpath.split(path) + if path[0] == 'chrome': + return len(path) == 1 or path[1] != 'icons' + if path[0] == 'components': + return path[-1].endswith(('.js', '.xpt')) + if path[0] == 'res': + return len(path) == 1 or \ + (path[1] != 'cursors' and path[1] != 'MainMenu.nib') + if path[0] == 'defaults': + return len(path) != 3 or \ + not (path[2] == 'channel-prefs.js' and + path[1] in ['pref', 'preferences']) + return path[0] in [ + 'modules', + 'greprefs.js', + 'hyphenation', + 'update.locale', + ] or path[0] in STARTUP_CACHE_PATHS diff --git a/python/mozbuild/mozpack/packager/l10n.py b/python/mozbuild/mozpack/packager/l10n.py new file mode 100644 index 000000000..758064f59 --- /dev/null +++ b/python/mozbuild/mozpack/packager/l10n.py @@ -0,0 +1,259 @@ +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, You can obtain one at http://mozilla.org/MPL/2.0/. + +from __future__ import absolute_import + +''' +Replace localized parts of a packaged directory with data from a langpack +directory. +''' + +import os +import mozpack.path as mozpath +from mozpack.packager.formats import ( + FlatFormatter, + JarFormatter, + OmniJarFormatter, +) +from mozpack.packager import ( + Component, + SimplePackager, + SimpleManifestSink, +) +from mozpack.files import ( + ComposedFinder, + ManifestFile, +) +from mozpack.copier import ( + FileCopier, + Jarrer, +) +from mozpack.chrome.manifest import ( + ManifestLocale, + ManifestEntryWithRelPath, + is_manifest, + ManifestChrome, + Manifest, +) +from mozpack.errors import errors +from mozpack.packager.unpack import UnpackFinder +from createprecomplete import generate_precomplete + + +class LocaleManifestFinder(object): + def __init__(self, finder): + entries = self.entries = [] + bases = self.bases = [] + + class MockFormatter(object): + def add_interfaces(self, path, content): + pass + + def add(self, path, content): + pass + + def add_manifest(self, entry): + if entry.localized: + entries.append(entry) + + def add_base(self, base, addon=False): + bases.append(base) + + # SimplePackager rejects "manifest foo.manifest" entries with + # additional flags (such as "manifest foo.manifest application=bar"). + # Those type of entries are used by language packs to work as addons, + # but are not necessary for the purpose of l10n repacking. So we wrap + # the finder in order to remove those entries. + class WrapFinder(object): + def __init__(self, finder): + self._finder = finder + + def find(self, pattern): + for p, f in self._finder.find(pattern): + if isinstance(f, ManifestFile): + unwanted = [ + e for e in f._entries + if isinstance(e, Manifest) and e.flags + ] + if unwanted: + f = ManifestFile( + f._base, + [e for e in f._entries if e not in unwanted]) + yield p, f + + sink = SimpleManifestSink(WrapFinder(finder), MockFormatter()) + sink.add(Component(''), '*') + sink.close(False) + + # Find unique locales used in these manifest entries. + self.locales = list(set(e.id for e in self.entries + if isinstance(e, ManifestLocale))) + + +def _repack(app_finder, l10n_finder, copier, formatter, non_chrome=set()): + app = LocaleManifestFinder(app_finder) + l10n = LocaleManifestFinder(l10n_finder) + + # The code further below assumes there's only one locale replaced with + # another one. + if len(app.locales) > 1: + errors.fatal("Multiple app locales aren't supported: " + + ",".join(app.locales)) + if len(l10n.locales) > 1: + errors.fatal("Multiple l10n locales aren't supported: " + + ",".join(l10n.locales)) + locale = app.locales[0] + l10n_locale = l10n.locales[0] + + # For each base directory, store what path a locale chrome package name + # corresponds to. + # e.g., for the following entry under app/chrome: + # locale foo en-US path/to/files + # keep track that the locale path for foo in app is + # app/chrome/path/to/files. + l10n_paths = {} + for e in l10n.entries: + if isinstance(e, ManifestChrome): + base = mozpath.basedir(e.path, app.bases) + l10n_paths.setdefault(base, {}) + l10n_paths[base][e.name] = e.path + + # For chrome and non chrome files or directories, store what langpack path + # corresponds to a package path. + paths = {} + for e in app.entries: + if isinstance(e, ManifestEntryWithRelPath): + base = mozpath.basedir(e.path, app.bases) + if base not in l10n_paths: + errors.fatal("Locale doesn't contain %s/" % base) + # Allow errors to accumulate + continue + if e.name not in l10n_paths[base]: + errors.fatal("Locale doesn't have a manifest entry for '%s'" % + e.name) + # Allow errors to accumulate + continue + paths[e.path] = l10n_paths[base][e.name] + + for pattern in non_chrome: + for base in app.bases: + path = mozpath.join(base, pattern) + left = set(p for p, f in app_finder.find(path)) + right = set(p for p, f in l10n_finder.find(path)) + for p in right: + paths[p] = p + for p in left - right: + paths[p] = None + + # Create a new package, with non localized bits coming from the original + # package, and localized bits coming from the langpack. + packager = SimplePackager(formatter) + for p, f in app_finder: + if is_manifest(p): + # Remove localized manifest entries. + for e in [e for e in f if e.localized]: + f.remove(e) + # If the path is one that needs a locale replacement, use the + # corresponding file from the langpack. + path = None + if p in paths: + path = paths[p] + if not path: + continue + else: + base = mozpath.basedir(p, paths.keys()) + if base: + subpath = mozpath.relpath(p, base) + path = mozpath.normpath(mozpath.join(paths[base], + subpath)) + if path: + files = [f for p, f in l10n_finder.find(path)] + if not len(files): + if base not in non_chrome: + finderBase = "" + if hasattr(l10n_finder, 'base'): + finderBase = l10n_finder.base + errors.error("Missing file: %s" % + os.path.join(finderBase, path)) + else: + packager.add(path, files[0]) + else: + packager.add(p, f) + + # Add localized manifest entries from the langpack. + l10n_manifests = [] + for base in set(e.base for e in l10n.entries): + m = ManifestFile(base, [e for e in l10n.entries if e.base == base]) + path = mozpath.join(base, 'chrome.%s.manifest' % l10n_locale) + l10n_manifests.append((path, m)) + bases = packager.get_bases() + for path, m in l10n_manifests: + base = mozpath.basedir(path, bases) + packager.add(path, m) + # Add a "manifest $path" entry in the top manifest under that base. + m = ManifestFile(base) + m.add(Manifest(base, mozpath.relpath(path, base))) + packager.add(mozpath.join(base, 'chrome.manifest'), m) + + packager.close() + + # Add any remaining non chrome files. + for pattern in non_chrome: + for base in bases: + for p, f in l10n_finder.find(mozpath.join(base, pattern)): + if not formatter.contains(p): + formatter.add(p, f) + + # Transplant jar preloading information. + for path, log in app_finder.jarlogs.iteritems(): + assert isinstance(copier[path], Jarrer) + copier[path].preload([l.replace(locale, l10n_locale) for l in log]) + + +def repack(source, l10n, extra_l10n={}, non_resources=[], non_chrome=set()): + ''' + Replace localized data from the `source` directory with localized data + from `l10n` and `extra_l10n`. + + The `source` argument points to a directory containing a packaged + application (in omnijar, jar or flat form). + The `l10n` argument points to a directory containing the main localized + data (usually in the form of a language pack addon) to use to replace + in the packaged application. + The `extra_l10n` argument contains a dict associating relative paths in + the source to separate directories containing localized data for them. + This can be used to point at different language pack addons for different + parts of the package application. + The `non_resources` argument gives a list of relative paths in the source + that should not be added in an omnijar in case the packaged application + is in that format. + The `non_chrome` argument gives a list of file/directory patterns for + localized files that are not listed in a chrome.manifest. + ''' + app_finder = UnpackFinder(source) + l10n_finder = UnpackFinder(l10n) + if extra_l10n: + finders = { + '': l10n_finder, + } + for base, path in extra_l10n.iteritems(): + finders[base] = UnpackFinder(path) + l10n_finder = ComposedFinder(finders) + copier = FileCopier() + if app_finder.kind == 'flat': + formatter = FlatFormatter(copier) + elif app_finder.kind == 'jar': + formatter = JarFormatter(copier, + optimize=app_finder.optimizedjars, + compress=app_finder.compressed) + elif app_finder.kind == 'omni': + formatter = OmniJarFormatter(copier, app_finder.omnijar, + optimize=app_finder.optimizedjars, + compress=app_finder.compressed, + non_resources=non_resources) + + with errors.accumulate(): + _repack(app_finder, l10n_finder, copier, formatter, non_chrome) + copier.copy(source, skip_if_older=False) + generate_precomplete(source) diff --git a/python/mozbuild/mozpack/packager/unpack.py b/python/mozbuild/mozpack/packager/unpack.py new file mode 100644 index 000000000..fa2b474e7 --- /dev/null +++ b/python/mozbuild/mozpack/packager/unpack.py @@ -0,0 +1,202 @@ +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, You can obtain one at http://mozilla.org/MPL/2.0/. + +from __future__ import absolute_import + +import mozpack.path as mozpath +from mozpack.files import ( + BaseFinder, + FileFinder, + DeflatedFile, + ManifestFile, +) +from mozpack.chrome.manifest import ( + parse_manifest, + ManifestEntryWithRelPath, + ManifestResource, + is_manifest, +) +from mozpack.mozjar import JarReader +from mozpack.copier import ( + FileRegistry, + FileCopier, +) +from mozpack.packager import SimplePackager +from mozpack.packager.formats import ( + FlatFormatter, + STARTUP_CACHE_PATHS, +) +from urlparse import urlparse + + +class UnpackFinder(BaseFinder): + ''' + Special Finder object that treats the source package directory as if it + were in the flat chrome format, whatever chrome format it actually is in. + + This means that for example, paths like chrome/browser/content/... match + files under jar:chrome/browser.jar!/content/... in case of jar chrome + format. + + The only argument to the constructor is a Finder instance or a path. + The UnpackFinder is populated with files from this Finder instance, + or with files from a FileFinder using the given path as its root. + ''' + def __init__(self, source): + if isinstance(source, BaseFinder): + self._finder = source + else: + self._finder = FileFinder(source) + self.base = self._finder.base + self.files = FileRegistry() + self.kind = 'flat' + self.omnijar = None + self.jarlogs = {} + self.optimizedjars = False + self.compressed = True + + jars = set() + + for p, f in self._finder.find('*'): + # Skip the precomplete file, which is generated at packaging time. + if p == 'precomplete': + continue + base = mozpath.dirname(p) + # If the file is a zip/jar that is not a .xpi, and contains a + # chrome.manifest, it is an omnijar. All the files it contains + # go in the directory containing the omnijar. Manifests are merged + # if there is a corresponding manifest in the directory. + if not p.endswith('.xpi') and self._maybe_zip(f) and \ + (mozpath.basename(p) == self.omnijar or + not self.omnijar): + jar = self._open_jar(p, f) + if 'chrome.manifest' in jar: + self.kind = 'omni' + self.omnijar = mozpath.basename(p) + self._fill_with_jar(base, jar) + continue + # If the file is a manifest, scan its entries for some referencing + # jar: urls. If there are some, the files contained in the jar they + # point to, go under a directory named after the jar. + if is_manifest(p): + m = self.files[p] if self.files.contains(p) \ + else ManifestFile(base) + for e in parse_manifest(self.base, p, f.open()): + m.add(self._handle_manifest_entry(e, jars)) + if self.files.contains(p): + continue + f = m + # If the file is a packed addon, unpack it under a directory named + # after the xpi. + if p.endswith('.xpi') and self._maybe_zip(f): + self._fill_with_jar(p[:-4], self._open_jar(p, f)) + continue + if not p in jars: + self.files.add(p, f) + + def _fill_with_jar(self, base, jar): + for j in jar: + path = mozpath.join(base, j.filename) + if is_manifest(j.filename): + m = self.files[path] if self.files.contains(path) \ + else ManifestFile(mozpath.dirname(path)) + for e in parse_manifest(None, path, j): + m.add(e) + if not self.files.contains(path): + self.files.add(path, m) + continue + else: + self.files.add(path, DeflatedFile(j)) + + def _handle_manifest_entry(self, entry, jars): + jarpath = None + if isinstance(entry, ManifestEntryWithRelPath) and \ + urlparse(entry.relpath).scheme == 'jar': + jarpath, entry = self._unjarize(entry, entry.relpath) + elif isinstance(entry, ManifestResource) and \ + urlparse(entry.target).scheme == 'jar': + jarpath, entry = self._unjarize(entry, entry.target) + if jarpath: + # Don't defer unpacking the jar file. If we already saw + # it, take (and remove) it from the registry. If we + # haven't, try to find it now. + if self.files.contains(jarpath): + jar = self.files[jarpath] + self.files.remove(jarpath) + else: + jar = [f for p, f in self._finder.find(jarpath)] + assert len(jar) == 1 + jar = jar[0] + if not jarpath in jars: + base = mozpath.splitext(jarpath)[0] + for j in self._open_jar(jarpath, jar): + self.files.add(mozpath.join(base, + j.filename), + DeflatedFile(j)) + jars.add(jarpath) + self.kind = 'jar' + return entry + + def _open_jar(self, path, file): + ''' + Return a JarReader for the given BaseFile instance, keeping a log of + the preloaded entries it has. + ''' + jar = JarReader(fileobj=file.open()) + if jar.is_optimized: + self.optimizedjars = True + if not any(f.compressed for f in jar): + self.compressed = False + if jar.last_preloaded: + jarlog = jar.entries.keys() + self.jarlogs[path] = jarlog[:jarlog.index(jar.last_preloaded) + 1] + return jar + + def find(self, path): + for p in self.files.match(path): + yield p, self.files[p] + + def _maybe_zip(self, file): + ''' + Return whether the given BaseFile looks like a ZIP/Jar. + ''' + header = file.open().read(8) + return len(header) == 8 and (header[0:2] == 'PK' or + header[4:6] == 'PK') + + def _unjarize(self, entry, relpath): + ''' + Transform a manifest entry pointing to chrome data in a jar in one + pointing to the corresponding unpacked path. Return the jar path and + the new entry. + ''' + base = entry.base + jar, relpath = urlparse(relpath).path.split('!', 1) + entry = entry.rebase(mozpath.join(base, 'jar:%s!' % jar)) \ + .move(mozpath.join(base, mozpath.splitext(jar)[0])) \ + .rebase(base) + return mozpath.join(base, jar), entry + + +def unpack_to_registry(source, registry): + ''' + Transform a jar chrome or omnijar packaged directory into a flat package. + + The given registry is filled with the flat package. + ''' + finder = UnpackFinder(source) + packager = SimplePackager(FlatFormatter(registry)) + for p, f in finder.find('*'): + if mozpath.split(p)[0] not in STARTUP_CACHE_PATHS: + packager.add(p, f) + packager.close() + + +def unpack(source): + ''' + Transform a jar chrome or omnijar packaged directory into a flat package. + ''' + copier = FileCopier() + unpack_to_registry(source, copier) + copier.copy(source, skip_if_older=False) diff --git a/python/mozbuild/mozpack/path.py b/python/mozbuild/mozpack/path.py new file mode 100644 index 000000000..7ea8ea85a --- /dev/null +++ b/python/mozbuild/mozpack/path.py @@ -0,0 +1,136 @@ +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, You can obtain one at http://mozilla.org/MPL/2.0/. + +from __future__ import absolute_import + +import posixpath +import os +import re + +''' +Like os.path, with a reduced set of functions, and with normalized path +separators (always use forward slashes). +Also contains a few additional utilities not found in os.path. +''' + + +def normsep(path): + ''' + Normalize path separators, by using forward slashes instead of whatever + os.sep is. + ''' + if os.sep != '/': + path = path.replace(os.sep, '/') + if os.altsep and os.altsep != '/': + path = path.replace(os.altsep, '/') + return path + + +def relpath(path, start): + rel = normsep(os.path.relpath(path, start)) + return '' if rel == '.' else rel + + +def realpath(path): + return normsep(os.path.realpath(path)) + + +def abspath(path): + return normsep(os.path.abspath(path)) + + +def join(*paths): + return normsep(os.path.join(*paths)) + + +def normpath(path): + return posixpath.normpath(normsep(path)) + + +def dirname(path): + return posixpath.dirname(normsep(path)) + + +def commonprefix(paths): + return posixpath.commonprefix([normsep(path) for path in paths]) + + +def basename(path): + return os.path.basename(path) + + +def splitext(path): + return posixpath.splitext(normsep(path)) + + +def split(path): + ''' + Return the normalized path as a list of its components. + split('foo/bar/baz') returns ['foo', 'bar', 'baz'] + ''' + return normsep(path).split('/') + + +def basedir(path, bases): + ''' + Given a list of directories (bases), return which one contains the given + path. If several matches are found, the deepest base directory is returned. + basedir('foo/bar/baz', ['foo', 'baz', 'foo/bar']) returns 'foo/bar' + ('foo' and 'foo/bar' both match, but 'foo/bar' is the deepest match) + ''' + path = normsep(path) + bases = [normsep(b) for b in bases] + if path in bases: + return path + for b in sorted(bases, reverse=True): + if b == '' or path.startswith(b + '/'): + return b + + +re_cache = {} + +def match(path, pattern): + ''' + Return whether the given path matches the given pattern. + An asterisk can be used to match any string, including the null string, in + one part of the path: + 'foo' matches '*', 'f*' or 'fo*o' + However, an asterisk matching a subdirectory may not match the null string: + 'foo/bar' does *not* match 'foo/*/bar' + If the pattern matches one of the ancestor directories of the path, the + patch is considered matching: + 'foo/bar' matches 'foo' + Two adjacent asterisks can be used to match files and zero or more + directories and subdirectories. + 'foo/bar' matches 'foo/**/bar', or '**/bar' + ''' + if not pattern: + return True + if pattern not in re_cache: + p = re.escape(pattern) + p = re.sub(r'(^|\\\/)\\\*\\\*\\\/', r'\1(?:.+/)?', p) + p = re.sub(r'(^|\\\/)\\\*\\\*$', r'(?:\1.+)?', p) + p = p.replace(r'\*', '[^/]*') + '(?:/.*)?$' + re_cache[pattern] = re.compile(p) + return re_cache[pattern].match(path) is not None + + +def rebase(oldbase, base, relativepath): + ''' + Return relativepath relative to base instead of oldbase. + ''' + if base == oldbase: + return relativepath + if len(base) < len(oldbase): + assert basedir(oldbase, [base]) == base + relbase = relpath(oldbase, base) + result = join(relbase, relativepath) + else: + assert basedir(base, [oldbase]) == oldbase + relbase = relpath(base, oldbase) + result = relpath(relativepath, relbase) + result = normpath(result) + if relativepath.endswith('/') and not result.endswith('/'): + result += '/' + return result diff --git a/python/mozbuild/mozpack/test/__init__.py b/python/mozbuild/mozpack/test/__init__.py new file mode 100644 index 000000000..e69de29bb diff --git a/python/mozbuild/mozpack/test/data/test_data b/python/mozbuild/mozpack/test/data/test_data new file mode 100644 index 000000000..fb7f0c4fc --- /dev/null +++ b/python/mozbuild/mozpack/test/data/test_data @@ -0,0 +1 @@ +test_data \ No newline at end of file diff --git a/python/mozbuild/mozpack/test/support/minify_js_verify.py b/python/mozbuild/mozpack/test/support/minify_js_verify.py new file mode 100644 index 000000000..8e4e8b759 --- /dev/null +++ b/python/mozbuild/mozpack/test/support/minify_js_verify.py @@ -0,0 +1,17 @@ +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, You can obtain one at http://mozilla.org/MPL/2.0/. + +from __future__ import print_function +import sys + + +if len(sys.argv) != 4: + raise Exception('Usage: minify_js_verify ') + +retcode = int(sys.argv[1]) + +if retcode: + print('Error message', file=sys.stderr) + +sys.exit(retcode) diff --git a/python/mozbuild/mozpack/test/test_archive.py b/python/mozbuild/mozpack/test/test_archive.py new file mode 100644 index 000000000..6f61f7eb7 --- /dev/null +++ b/python/mozbuild/mozpack/test/test_archive.py @@ -0,0 +1,190 @@ +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, You can obtain one at http://mozilla.org/MPL/2.0/. + +from __future__ import absolute_import + +import hashlib +import os +import shutil +import stat +import tarfile +import tempfile +import unittest + +from mozpack.archive import ( + DEFAULT_MTIME, + create_tar_from_files, + create_tar_gz_from_files, + create_tar_bz2_from_files, +) + +from mozunit import main + + +MODE_STANDARD = stat.S_IRUSR | stat.S_IWUSR | stat.S_IRGRP | stat.S_IROTH + + +def file_hash(path): + h = hashlib.sha1() + with open(path, 'rb') as fh: + while True: + data = fh.read(8192) + if not data: + break + h.update(data) + + return h.hexdigest() + + +class TestArchive(unittest.TestCase): + def _create_files(self, root): + files = {} + for i in range(10): + p = os.path.join(root, b'file%d' % i) + with open(p, 'wb') as fh: + fh.write(b'file%d' % i) + # Need to set permissions or umask may influence testing. + os.chmod(p, MODE_STANDARD) + files[b'file%d' % i] = p + + return files + + def _verify_basic_tarfile(self, tf): + self.assertEqual(len(tf.getmembers()), 10) + + names = ['file%d' % i for i in range(10)] + self.assertEqual(tf.getnames(), names) + + for ti in tf.getmembers(): + self.assertEqual(ti.uid, 0) + self.assertEqual(ti.gid, 0) + self.assertEqual(ti.uname, '') + self.assertEqual(ti.gname, '') + self.assertEqual(ti.mode, MODE_STANDARD) + self.assertEqual(ti.mtime, DEFAULT_MTIME) + + def test_dirs_refused(self): + d = tempfile.mkdtemp() + try: + tp = os.path.join(d, 'test.tar') + with open(tp, 'wb') as fh: + with self.assertRaisesRegexp(ValueError, 'not a regular'): + create_tar_from_files(fh, {'test': d}) + finally: + shutil.rmtree(d) + + def test_setuid_setgid_refused(self): + d = tempfile.mkdtemp() + try: + uid = os.path.join(d, 'setuid') + gid = os.path.join(d, 'setgid') + with open(uid, 'a'): + pass + with open(gid, 'a'): + pass + + os.chmod(uid, MODE_STANDARD | stat.S_ISUID) + os.chmod(gid, MODE_STANDARD | stat.S_ISGID) + + tp = os.path.join(d, 'test.tar') + with open(tp, 'wb') as fh: + with self.assertRaisesRegexp(ValueError, 'cannot add file with setuid'): + create_tar_from_files(fh, {'test': uid}) + with self.assertRaisesRegexp(ValueError, 'cannot add file with setuid'): + create_tar_from_files(fh, {'test': gid}) + finally: + shutil.rmtree(d) + + def test_create_tar_basic(self): + d = tempfile.mkdtemp() + try: + files = self._create_files(d) + + tp = os.path.join(d, 'test.tar') + with open(tp, 'wb') as fh: + create_tar_from_files(fh, files) + + # Output should be deterministic. + self.assertEqual(file_hash(tp), 'cd16cee6f13391abd94dfa435d2633b61ed727f1') + + with tarfile.open(tp, 'r') as tf: + self._verify_basic_tarfile(tf) + + finally: + shutil.rmtree(d) + + def test_executable_preserved(self): + d = tempfile.mkdtemp() + try: + p = os.path.join(d, 'exec') + with open(p, 'wb') as fh: + fh.write('#!/bin/bash\n') + os.chmod(p, MODE_STANDARD | stat.S_IXUSR) + + tp = os.path.join(d, 'test.tar') + with open(tp, 'wb') as fh: + create_tar_from_files(fh, {'exec': p}) + + self.assertEqual(file_hash(tp), '357e1b81c0b6cfdfa5d2d118d420025c3c76ee93') + + with tarfile.open(tp, 'r') as tf: + m = tf.getmember('exec') + self.assertEqual(m.mode, MODE_STANDARD | stat.S_IXUSR) + + finally: + shutil.rmtree(d) + + def test_create_tar_gz_basic(self): + d = tempfile.mkdtemp() + try: + files = self._create_files(d) + + gp = os.path.join(d, 'test.tar.gz') + with open(gp, 'wb') as fh: + create_tar_gz_from_files(fh, files) + + self.assertEqual(file_hash(gp), 'acb602239c1aeb625da5e69336775609516d60f5') + + with tarfile.open(gp, 'r:gz') as tf: + self._verify_basic_tarfile(tf) + + finally: + shutil.rmtree(d) + + def test_tar_gz_name(self): + d = tempfile.mkdtemp() + try: + files = self._create_files(d) + + gp = os.path.join(d, 'test.tar.gz') + with open(gp, 'wb') as fh: + create_tar_gz_from_files(fh, files, filename='foobar', compresslevel=1) + + self.assertEqual(file_hash(gp), 'fd099f96480cc1100f37baa8e89a6b820dbbcbd3') + + with tarfile.open(gp, 'r:gz') as tf: + self._verify_basic_tarfile(tf) + + finally: + shutil.rmtree(d) + + def test_create_tar_bz2_basic(self): + d = tempfile.mkdtemp() + try: + files = self._create_files(d) + + bp = os.path.join(d, 'test.tar.bz2') + with open(bp, 'wb') as fh: + create_tar_bz2_from_files(fh, files) + + self.assertEqual(file_hash(bp), '1827ad00dfe7acf857b7a1c95ce100361e3f6eea') + + with tarfile.open(bp, 'r:bz2') as tf: + self._verify_basic_tarfile(tf) + finally: + shutil.rmtree(d) + + +if __name__ == '__main__': + main() diff --git a/python/mozbuild/mozpack/test/test_chrome_flags.py b/python/mozbuild/mozpack/test/test_chrome_flags.py new file mode 100644 index 000000000..e6a5257e9 --- /dev/null +++ b/python/mozbuild/mozpack/test/test_chrome_flags.py @@ -0,0 +1,148 @@ +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, You can obtain one at http://mozilla.org/MPL/2.0/. + +import unittest +import mozunit +from mozpack.chrome.flags import ( + Flag, + StringFlag, + VersionFlag, + Flags, +) +from mozpack.errors import ErrorMessage + + +class TestFlag(unittest.TestCase): + def test_flag(self): + flag = Flag('flag') + self.assertEqual(str(flag), '') + self.assertTrue(flag.matches(False)) + self.assertTrue(flag.matches('false')) + self.assertFalse(flag.matches('true')) + self.assertRaises(ErrorMessage, flag.add_definition, 'flag=') + self.assertRaises(ErrorMessage, flag.add_definition, 'flag=42') + self.assertRaises(ErrorMessage, flag.add_definition, 'flag!=false') + + flag.add_definition('flag=1') + self.assertEqual(str(flag), 'flag=1') + self.assertTrue(flag.matches(True)) + self.assertTrue(flag.matches('1')) + self.assertFalse(flag.matches('no')) + + flag.add_definition('flag=true') + self.assertEqual(str(flag), 'flag=true') + self.assertTrue(flag.matches(True)) + self.assertTrue(flag.matches('true')) + self.assertFalse(flag.matches('0')) + + flag.add_definition('flag=no') + self.assertEqual(str(flag), 'flag=no') + self.assertTrue(flag.matches('false')) + self.assertFalse(flag.matches('1')) + + flag.add_definition('flag') + self.assertEqual(str(flag), 'flag') + self.assertFalse(flag.matches('false')) + self.assertTrue(flag.matches('true')) + self.assertFalse(flag.matches(False)) + + def test_string_flag(self): + flag = StringFlag('flag') + self.assertEqual(str(flag), '') + self.assertTrue(flag.matches('foo')) + self.assertRaises(ErrorMessage, flag.add_definition, 'flag>=2') + + flag.add_definition('flag=foo') + self.assertEqual(str(flag), 'flag=foo') + self.assertTrue(flag.matches('foo')) + self.assertFalse(flag.matches('bar')) + + flag.add_definition('flag=bar') + self.assertEqual(str(flag), 'flag=foo flag=bar') + self.assertTrue(flag.matches('foo')) + self.assertTrue(flag.matches('bar')) + self.assertFalse(flag.matches('baz')) + + flag = StringFlag('flag') + flag.add_definition('flag!=bar') + self.assertEqual(str(flag), 'flag!=bar') + self.assertTrue(flag.matches('foo')) + self.assertFalse(flag.matches('bar')) + + def test_version_flag(self): + flag = VersionFlag('flag') + self.assertEqual(str(flag), '') + self.assertTrue(flag.matches('1.0')) + self.assertRaises(ErrorMessage, flag.add_definition, 'flag!=2') + + flag.add_definition('flag=1.0') + self.assertEqual(str(flag), 'flag=1.0') + self.assertTrue(flag.matches('1.0')) + self.assertFalse(flag.matches('2.0')) + + flag.add_definition('flag=2.0') + self.assertEqual(str(flag), 'flag=1.0 flag=2.0') + self.assertTrue(flag.matches('1.0')) + self.assertTrue(flag.matches('2.0')) + self.assertFalse(flag.matches('3.0')) + + flag = VersionFlag('flag') + flag.add_definition('flag>=2.0') + self.assertEqual(str(flag), 'flag>=2.0') + self.assertFalse(flag.matches('1.0')) + self.assertTrue(flag.matches('2.0')) + self.assertTrue(flag.matches('3.0')) + + flag.add_definition('flag<1.10') + self.assertEqual(str(flag), 'flag>=2.0 flag<1.10') + self.assertTrue(flag.matches('1.0')) + self.assertTrue(flag.matches('1.9')) + self.assertFalse(flag.matches('1.10')) + self.assertFalse(flag.matches('1.20')) + self.assertTrue(flag.matches('2.0')) + self.assertTrue(flag.matches('3.0')) + self.assertRaises(Exception, flag.add_definition, 'flag<') + self.assertRaises(Exception, flag.add_definition, 'flag>') + self.assertRaises(Exception, flag.add_definition, 'flag>=') + self.assertRaises(Exception, flag.add_definition, 'flag<=') + self.assertRaises(Exception, flag.add_definition, 'flag!=1.0') + + +class TestFlags(unittest.TestCase): + def setUp(self): + self.flags = Flags('contentaccessible=yes', + 'appversion>=3.5', + 'application=foo', + 'application=bar', + 'appversion<2.0', + 'platform', + 'abi!=Linux_x86-gcc3') + + def test_flags_str(self): + self.assertEqual(str(self.flags), 'contentaccessible=yes ' + + 'appversion>=3.5 appversion<2.0 application=foo ' + + 'application=bar platform abi!=Linux_x86-gcc3') + + def test_flags_match_unset(self): + self.assertTrue(self.flags.match(os='WINNT')) + + def test_flags_match(self): + self.assertTrue(self.flags.match(application='foo')) + self.assertFalse(self.flags.match(application='qux')) + + def test_flags_match_different(self): + self.assertTrue(self.flags.match(abi='WINNT_x86-MSVC')) + self.assertFalse(self.flags.match(abi='Linux_x86-gcc3')) + + def test_flags_match_version(self): + self.assertTrue(self.flags.match(appversion='1.0')) + self.assertTrue(self.flags.match(appversion='1.5')) + self.assertFalse(self.flags.match(appversion='2.0')) + self.assertFalse(self.flags.match(appversion='3.0')) + self.assertTrue(self.flags.match(appversion='3.5')) + self.assertTrue(self.flags.match(appversion='3.10')) + + +if __name__ == '__main__': + mozunit.main() diff --git a/python/mozbuild/mozpack/test/test_chrome_manifest.py b/python/mozbuild/mozpack/test/test_chrome_manifest.py new file mode 100644 index 000000000..690c6acdc --- /dev/null +++ b/python/mozbuild/mozpack/test/test_chrome_manifest.py @@ -0,0 +1,149 @@ +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, You can obtain one at http://mozilla.org/MPL/2.0/. + +import unittest +import mozunit +import os +from mozpack.chrome.manifest import ( + ManifestContent, + ManifestLocale, + ManifestSkin, + Manifest, + ManifestResource, + ManifestOverride, + ManifestComponent, + ManifestContract, + ManifestInterfaces, + ManifestBinaryComponent, + ManifestCategory, + ManifestStyle, + ManifestOverlay, + MANIFESTS_TYPES, + parse_manifest, + parse_manifest_line, +) +from mozpack.errors import errors, AccumulatedErrors +from test_errors import TestErrors + + +class TestManifest(unittest.TestCase): + def test_parse_manifest(self): + manifest = [ + 'content global content/global/', + 'content global content/global/ application=foo application=bar' + + ' platform', + 'locale global en-US content/en-US/', + 'locale global en-US content/en-US/ application=foo', + 'skin global classic/1.0 content/skin/classic/', + 'skin global classic/1.0 content/skin/classic/ application=foo' + + ' os=WINNT', + '', + 'manifest pdfjs/chrome.manifest', + 'resource gre-resources toolkit/res/', + 'override chrome://global/locale/netError.dtd' + + ' chrome://browser/locale/netError.dtd', + '# Comment', + 'component {b2bba4df-057d-41ea-b6b1-94a10a8ede68} foo.js', + 'contract @mozilla.org/foo;1' + + ' {b2bba4df-057d-41ea-b6b1-94a10a8ede68}', + 'interfaces foo.xpt', + 'binary-component bar.so', + 'category command-line-handler m-browser' + + ' @mozilla.org/browser/clh;1' + + ' application={ec8030f7-c20a-464f-9b0e-13a3a9e97384}', + 'style chrome://global/content/customizeToolbar.xul' + + ' chrome://browser/skin/', + 'overlay chrome://global/content/viewSource.xul' + + ' chrome://browser/content/viewSourceOverlay.xul', + ] + other_manifest = [ + 'content global content/global/' + ] + expected_result = [ + ManifestContent('', 'global', 'content/global/'), + ManifestContent('', 'global', 'content/global/', 'application=foo', + 'application=bar', 'platform'), + ManifestLocale('', 'global', 'en-US', 'content/en-US/'), + ManifestLocale('', 'global', 'en-US', 'content/en-US/', + 'application=foo'), + ManifestSkin('', 'global', 'classic/1.0', 'content/skin/classic/'), + ManifestSkin('', 'global', 'classic/1.0', 'content/skin/classic/', + 'application=foo', 'os=WINNT'), + Manifest('', 'pdfjs/chrome.manifest'), + ManifestResource('', 'gre-resources', 'toolkit/res/'), + ManifestOverride('', 'chrome://global/locale/netError.dtd', + 'chrome://browser/locale/netError.dtd'), + ManifestComponent('', '{b2bba4df-057d-41ea-b6b1-94a10a8ede68}', + 'foo.js'), + ManifestContract('', '@mozilla.org/foo;1', + '{b2bba4df-057d-41ea-b6b1-94a10a8ede68}'), + ManifestInterfaces('', 'foo.xpt'), + ManifestBinaryComponent('', 'bar.so'), + ManifestCategory('', 'command-line-handler', 'm-browser', + '@mozilla.org/browser/clh;1', 'application=' + + '{ec8030f7-c20a-464f-9b0e-13a3a9e97384}'), + ManifestStyle('', 'chrome://global/content/customizeToolbar.xul', + 'chrome://browser/skin/'), + ManifestOverlay('', 'chrome://global/content/viewSource.xul', + 'chrome://browser/content/viewSourceOverlay.xul'), + ] + with mozunit.MockedOpen({'manifest': '\n'.join(manifest), + 'other/manifest': '\n'.join(other_manifest)}): + # Ensure we have tests for all types of manifests. + self.assertEqual(set(type(e) for e in expected_result), + set(MANIFESTS_TYPES.values())) + self.assertEqual(list(parse_manifest(os.curdir, 'manifest')), + expected_result) + self.assertEqual(list(parse_manifest(os.curdir, 'other/manifest')), + [ManifestContent('other', 'global', + 'content/global/')]) + + def test_manifest_rebase(self): + m = parse_manifest_line('chrome', 'content global content/global/') + m = m.rebase('') + self.assertEqual(str(m), 'content global chrome/content/global/') + m = m.rebase('chrome') + self.assertEqual(str(m), 'content global content/global/') + + m = parse_manifest_line('chrome/foo', 'content global content/global/') + m = m.rebase('chrome') + self.assertEqual(str(m), 'content global foo/content/global/') + m = m.rebase('chrome/foo') + self.assertEqual(str(m), 'content global content/global/') + + m = parse_manifest_line('modules/foo', 'resource foo ./') + m = m.rebase('modules') + self.assertEqual(str(m), 'resource foo foo/') + m = m.rebase('modules/foo') + self.assertEqual(str(m), 'resource foo ./') + + m = parse_manifest_line('chrome', 'content browser browser/content/') + m = m.rebase('chrome/browser').move('jar:browser.jar!').rebase('') + self.assertEqual(str(m), 'content browser jar:browser.jar!/content/') + + +class TestManifestErrors(TestErrors, unittest.TestCase): + def test_parse_manifest_errors(self): + manifest = [ + 'skin global classic/1.0 content/skin/classic/ platform', + '', + 'binary-component bar.so', + 'unsupported foo', + ] + with mozunit.MockedOpen({'manifest': '\n'.join(manifest)}): + with self.assertRaises(AccumulatedErrors): + with errors.accumulate(): + list(parse_manifest(os.curdir, 'manifest')) + out = self.get_output() + # Expecting 2 errors + self.assertEqual(len(out), 2) + path = os.path.abspath('manifest') + # First on line 1 + self.assertTrue(out[0].startswith('Error: %s:1: ' % path)) + # Second on line 4 + self.assertTrue(out[1].startswith('Error: %s:4: ' % path)) + + +if __name__ == '__main__': + mozunit.main() diff --git a/python/mozbuild/mozpack/test/test_copier.py b/python/mozbuild/mozpack/test/test_copier.py new file mode 100644 index 000000000..6688b3d5e --- /dev/null +++ b/python/mozbuild/mozpack/test/test_copier.py @@ -0,0 +1,529 @@ +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, You can obtain one at http://mozilla.org/MPL/2.0/. + +from mozpack.copier import ( + FileCopier, + FileRegistry, + FileRegistrySubtree, + Jarrer, +) +from mozpack.files import ( + GeneratedFile, + ExistingFile, +) +from mozpack.mozjar import JarReader +import mozpack.path as mozpath +import unittest +import mozunit +import os +import stat +from mozpack.errors import ErrorMessage +from mozpack.test.test_files import ( + MockDest, + MatchTestTemplate, + TestWithTmpDir, +) + + +class BaseTestFileRegistry(MatchTestTemplate): + def add(self, path): + self.registry.add(path, GeneratedFile(path)) + + def do_check(self, pattern, result): + self.checked = True + if result: + self.assertTrue(self.registry.contains(pattern)) + else: + self.assertFalse(self.registry.contains(pattern)) + self.assertEqual(self.registry.match(pattern), result) + + def do_test_file_registry(self, registry): + self.registry = registry + self.registry.add('foo', GeneratedFile('foo')) + bar = GeneratedFile('bar') + self.registry.add('bar', bar) + self.assertEqual(self.registry.paths(), ['foo', 'bar']) + self.assertEqual(self.registry['bar'], bar) + + self.assertRaises(ErrorMessage, self.registry.add, 'foo', + GeneratedFile('foo2')) + + self.assertRaises(ErrorMessage, self.registry.remove, 'qux') + + self.assertRaises(ErrorMessage, self.registry.add, 'foo/bar', + GeneratedFile('foobar')) + self.assertRaises(ErrorMessage, self.registry.add, 'foo/bar/baz', + GeneratedFile('foobar')) + + self.assertEqual(self.registry.paths(), ['foo', 'bar']) + + self.registry.remove('foo') + self.assertEqual(self.registry.paths(), ['bar']) + self.registry.remove('bar') + self.assertEqual(self.registry.paths(), []) + + self.prepare_match_test() + self.do_match_test() + self.assertTrue(self.checked) + self.assertEqual(self.registry.paths(), [ + 'bar', + 'foo/bar', + 'foo/baz', + 'foo/qux/1', + 'foo/qux/bar', + 'foo/qux/2/test', + 'foo/qux/2/test2', + ]) + + self.registry.remove('foo/qux') + self.assertEqual(self.registry.paths(), ['bar', 'foo/bar', 'foo/baz']) + + self.registry.add('foo/qux', GeneratedFile('fooqux')) + self.assertEqual(self.registry.paths(), ['bar', 'foo/bar', 'foo/baz', + 'foo/qux']) + self.registry.remove('foo/b*') + self.assertEqual(self.registry.paths(), ['bar', 'foo/qux']) + + self.assertEqual([f for f, c in self.registry], ['bar', 'foo/qux']) + self.assertEqual(len(self.registry), 2) + + self.add('foo/.foo') + self.assertTrue(self.registry.contains('foo/.foo')) + + def do_test_registry_paths(self, registry): + self.registry = registry + + # Can't add a file if it requires a directory in place of a + # file we also require. + self.registry.add('foo', GeneratedFile('foo')) + self.assertRaises(ErrorMessage, self.registry.add, 'foo/bar', + GeneratedFile('foobar')) + + # Can't add a file if we already have a directory there. + self.registry.add('bar/baz', GeneratedFile('barbaz')) + self.assertRaises(ErrorMessage, self.registry.add, 'bar', + GeneratedFile('bar')) + + # Bump the count of things that require bar/ to 2. + self.registry.add('bar/zot', GeneratedFile('barzot')) + self.assertRaises(ErrorMessage, self.registry.add, 'bar', + GeneratedFile('bar')) + + # Drop the count of things that require bar/ to 1. + self.registry.remove('bar/baz') + self.assertRaises(ErrorMessage, self.registry.add, 'bar', + GeneratedFile('bar')) + + # Drop the count of things that require bar/ to 0. + self.registry.remove('bar/zot') + self.registry.add('bar/zot', GeneratedFile('barzot')) + +class TestFileRegistry(BaseTestFileRegistry, unittest.TestCase): + def test_partial_paths(self): + cases = { + 'foo/bar/baz/zot': ['foo/bar/baz', 'foo/bar', 'foo'], + 'foo/bar': ['foo'], + 'bar': [], + } + reg = FileRegistry() + for path, parts in cases.iteritems(): + self.assertEqual(reg._partial_paths(path), parts) + + def test_file_registry(self): + self.do_test_file_registry(FileRegistry()) + + def test_registry_paths(self): + self.do_test_registry_paths(FileRegistry()) + + def test_required_directories(self): + self.registry = FileRegistry() + + self.registry.add('foo', GeneratedFile('foo')) + self.assertEqual(self.registry.required_directories(), set()) + + self.registry.add('bar/baz', GeneratedFile('barbaz')) + self.assertEqual(self.registry.required_directories(), {'bar'}) + + self.registry.add('bar/zot', GeneratedFile('barzot')) + self.assertEqual(self.registry.required_directories(), {'bar'}) + + self.registry.add('bar/zap/zot', GeneratedFile('barzapzot')) + self.assertEqual(self.registry.required_directories(), {'bar', 'bar/zap'}) + + self.registry.remove('bar/zap/zot') + self.assertEqual(self.registry.required_directories(), {'bar'}) + + self.registry.remove('bar/baz') + self.assertEqual(self.registry.required_directories(), {'bar'}) + + self.registry.remove('bar/zot') + self.assertEqual(self.registry.required_directories(), set()) + + self.registry.add('x/y/z', GeneratedFile('xyz')) + self.assertEqual(self.registry.required_directories(), {'x', 'x/y'}) + + +class TestFileRegistrySubtree(BaseTestFileRegistry, unittest.TestCase): + def test_file_registry_subtree_base(self): + registry = FileRegistry() + self.assertEqual(registry, FileRegistrySubtree('', registry)) + self.assertNotEqual(registry, FileRegistrySubtree('base', registry)) + + def create_registry(self): + registry = FileRegistry() + registry.add('foo/bar', GeneratedFile('foo/bar')) + registry.add('baz/qux', GeneratedFile('baz/qux')) + return FileRegistrySubtree('base/root', registry) + + def test_file_registry_subtree(self): + self.do_test_file_registry(self.create_registry()) + + def test_registry_paths_subtree(self): + registry = FileRegistry() + self.do_test_registry_paths(self.create_registry()) + + +class TestFileCopier(TestWithTmpDir): + def all_dirs(self, base): + all_dirs = set() + for root, dirs, files in os.walk(base): + if not dirs: + all_dirs.add(mozpath.relpath(root, base)) + return all_dirs + + def all_files(self, base): + all_files = set() + for root, dirs, files in os.walk(base): + for f in files: + all_files.add( + mozpath.join(mozpath.relpath(root, base), f)) + return all_files + + def test_file_copier(self): + copier = FileCopier() + copier.add('foo/bar', GeneratedFile('foobar')) + copier.add('foo/qux', GeneratedFile('fooqux')) + copier.add('foo/deep/nested/directory/file', GeneratedFile('fooz')) + copier.add('bar', GeneratedFile('bar')) + copier.add('qux/foo', GeneratedFile('quxfoo')) + copier.add('qux/bar', GeneratedFile('')) + + result = copier.copy(self.tmpdir) + self.assertEqual(self.all_files(self.tmpdir), set(copier.paths())) + self.assertEqual(self.all_dirs(self.tmpdir), + set(['foo/deep/nested/directory', 'qux'])) + + self.assertEqual(result.updated_files, set(self.tmppath(p) for p in + self.all_files(self.tmpdir))) + self.assertEqual(result.existing_files, set()) + self.assertEqual(result.removed_files, set()) + self.assertEqual(result.removed_directories, set()) + + copier.remove('foo') + copier.add('test', GeneratedFile('test')) + result = copier.copy(self.tmpdir) + self.assertEqual(self.all_files(self.tmpdir), set(copier.paths())) + self.assertEqual(self.all_dirs(self.tmpdir), set(['qux'])) + self.assertEqual(result.removed_files, set(self.tmppath(p) for p in + ('foo/bar', 'foo/qux', 'foo/deep/nested/directory/file'))) + + def test_symlink_directory_replaced(self): + """Directory symlinks in destination are replaced if they need to be + real directories.""" + if not self.symlink_supported: + return + + dest = self.tmppath('dest') + + copier = FileCopier() + copier.add('foo/bar/baz', GeneratedFile('foobarbaz')) + + os.makedirs(self.tmppath('dest/foo')) + dummy = self.tmppath('dummy') + os.mkdir(dummy) + link = self.tmppath('dest/foo/bar') + os.symlink(dummy, link) + + result = copier.copy(dest) + + st = os.lstat(link) + self.assertFalse(stat.S_ISLNK(st.st_mode)) + self.assertTrue(stat.S_ISDIR(st.st_mode)) + + self.assertEqual(self.all_files(dest), set(copier.paths())) + + self.assertEqual(result.removed_directories, set()) + self.assertEqual(len(result.updated_files), 1) + + def test_remove_unaccounted_directory_symlinks(self): + """Directory symlinks in destination that are not in the way are + deleted according to remove_unaccounted and + remove_all_directory_symlinks. + """ + if not self.symlink_supported: + return + + dest = self.tmppath('dest') + + copier = FileCopier() + copier.add('foo/bar/baz', GeneratedFile('foobarbaz')) + + os.makedirs(self.tmppath('dest/foo')) + dummy = self.tmppath('dummy') + os.mkdir(dummy) + + os.mkdir(self.tmppath('dest/zot')) + link = self.tmppath('dest/zot/zap') + os.symlink(dummy, link) + + # If not remove_unaccounted but remove_empty_directories, then + # the symlinked directory remains (as does its containing + # directory). + result = copier.copy(dest, remove_unaccounted=False, + remove_empty_directories=True, + remove_all_directory_symlinks=False) + + st = os.lstat(link) + self.assertTrue(stat.S_ISLNK(st.st_mode)) + self.assertFalse(stat.S_ISDIR(st.st_mode)) + + self.assertEqual(self.all_files(dest), set(copier.paths())) + self.assertEqual(self.all_dirs(dest), set(['foo/bar'])) + + self.assertEqual(result.removed_directories, set()) + self.assertEqual(len(result.updated_files), 1) + + # If remove_unaccounted but not remove_empty_directories, then + # only the symlinked directory is removed. + result = copier.copy(dest, remove_unaccounted=True, + remove_empty_directories=False, + remove_all_directory_symlinks=False) + + st = os.lstat(self.tmppath('dest/zot')) + self.assertFalse(stat.S_ISLNK(st.st_mode)) + self.assertTrue(stat.S_ISDIR(st.st_mode)) + + self.assertEqual(result.removed_files, set([link])) + self.assertEqual(result.removed_directories, set()) + + self.assertEqual(self.all_files(dest), set(copier.paths())) + self.assertEqual(self.all_dirs(dest), set(['foo/bar', 'zot'])) + + # If remove_unaccounted and remove_empty_directories, then + # both the symlink and its containing directory are removed. + link = self.tmppath('dest/zot/zap') + os.symlink(dummy, link) + + result = copier.copy(dest, remove_unaccounted=True, + remove_empty_directories=True, + remove_all_directory_symlinks=False) + + self.assertEqual(result.removed_files, set([link])) + self.assertEqual(result.removed_directories, set([self.tmppath('dest/zot')])) + + self.assertEqual(self.all_files(dest), set(copier.paths())) + self.assertEqual(self.all_dirs(dest), set(['foo/bar'])) + + def test_permissions(self): + """Ensure files without write permission can be deleted.""" + with open(self.tmppath('dummy'), 'a'): + pass + + p = self.tmppath('no_perms') + with open(p, 'a'): + pass + + # Make file and directory unwritable. Reminder: making a directory + # unwritable prevents modifications (including deletes) from the list + # of files in that directory. + os.chmod(p, 0o400) + os.chmod(self.tmpdir, 0o400) + + copier = FileCopier() + copier.add('dummy', GeneratedFile('content')) + result = copier.copy(self.tmpdir) + self.assertEqual(result.removed_files_count, 1) + self.assertFalse(os.path.exists(p)) + + def test_no_remove(self): + copier = FileCopier() + copier.add('foo', GeneratedFile('foo')) + + with open(self.tmppath('bar'), 'a'): + pass + + os.mkdir(self.tmppath('emptydir')) + d = self.tmppath('populateddir') + os.mkdir(d) + + with open(self.tmppath('populateddir/foo'), 'a'): + pass + + result = copier.copy(self.tmpdir, remove_unaccounted=False) + + self.assertEqual(self.all_files(self.tmpdir), set(['foo', 'bar', + 'populateddir/foo'])) + self.assertEqual(self.all_dirs(self.tmpdir), set(['populateddir'])) + self.assertEqual(result.removed_files, set()) + self.assertEqual(result.removed_directories, + set([self.tmppath('emptydir')])) + + def test_no_remove_empty_directories(self): + copier = FileCopier() + copier.add('foo', GeneratedFile('foo')) + + with open(self.tmppath('bar'), 'a'): + pass + + os.mkdir(self.tmppath('emptydir')) + d = self.tmppath('populateddir') + os.mkdir(d) + + with open(self.tmppath('populateddir/foo'), 'a'): + pass + + result = copier.copy(self.tmpdir, remove_unaccounted=False, + remove_empty_directories=False) + + self.assertEqual(self.all_files(self.tmpdir), set(['foo', 'bar', + 'populateddir/foo'])) + self.assertEqual(self.all_dirs(self.tmpdir), set(['emptydir', + 'populateddir'])) + self.assertEqual(result.removed_files, set()) + self.assertEqual(result.removed_directories, set()) + + def test_optional_exists_creates_unneeded_directory(self): + """Demonstrate that a directory not strictly required, but specified + as the path to an optional file, will be unnecessarily created. + + This behaviour is wrong; fixing it is tracked by Bug 972432; + and this test exists to guard against unexpected changes in + behaviour. + """ + + dest = self.tmppath('dest') + + copier = FileCopier() + copier.add('foo/bar', ExistingFile(required=False)) + + result = copier.copy(dest) + + st = os.lstat(self.tmppath('dest/foo')) + self.assertFalse(stat.S_ISLNK(st.st_mode)) + self.assertTrue(stat.S_ISDIR(st.st_mode)) + + # What's worse, we have no record that dest was created. + self.assertEquals(len(result.updated_files), 0) + + # But we do have an erroneous record of an optional file + # existing when it does not. + self.assertIn(self.tmppath('dest/foo/bar'), result.existing_files) + + def test_remove_unaccounted_file_registry(self): + """Test FileCopier.copy(remove_unaccounted=FileRegistry())""" + + dest = self.tmppath('dest') + + copier = FileCopier() + copier.add('foo/bar/baz', GeneratedFile('foobarbaz')) + copier.add('foo/bar/qux', GeneratedFile('foobarqux')) + copier.add('foo/hoge/fuga', GeneratedFile('foohogefuga')) + copier.add('foo/toto/tata', GeneratedFile('footototata')) + + os.makedirs(os.path.join(dest, 'bar')) + with open(os.path.join(dest, 'bar', 'bar'), 'w') as fh: + fh.write('barbar'); + os.makedirs(os.path.join(dest, 'foo', 'toto')) + with open(os.path.join(dest, 'foo', 'toto', 'toto'), 'w') as fh: + fh.write('foototototo'); + + result = copier.copy(dest, remove_unaccounted=False) + + self.assertEqual(self.all_files(dest), + set(copier.paths()) | { 'foo/toto/toto', 'bar/bar'}) + self.assertEqual(self.all_dirs(dest), + {'foo/bar', 'foo/hoge', 'foo/toto', 'bar'}) + + copier2 = FileCopier() + copier2.add('foo/hoge/fuga', GeneratedFile('foohogefuga')) + + # We expect only files copied from the first copier to be removed, + # not the extra file that was there beforehand. + result = copier2.copy(dest, remove_unaccounted=copier) + + self.assertEqual(self.all_files(dest), + set(copier2.paths()) | { 'foo/toto/toto', 'bar/bar'}) + self.assertEqual(self.all_dirs(dest), + {'foo/hoge', 'foo/toto', 'bar'}) + self.assertEqual(result.updated_files, + {self.tmppath('dest/foo/hoge/fuga')}) + self.assertEqual(result.existing_files, set()) + self.assertEqual(result.removed_files, {self.tmppath(p) for p in + ('dest/foo/bar/baz', 'dest/foo/bar/qux', 'dest/foo/toto/tata')}) + self.assertEqual(result.removed_directories, + {self.tmppath('dest/foo/bar')}) + + +class TestJarrer(unittest.TestCase): + def check_jar(self, dest, copier): + jar = JarReader(fileobj=dest) + self.assertEqual([f.filename for f in jar], copier.paths()) + for f in jar: + self.assertEqual(f.uncompressed_data.read(), + copier[f.filename].content) + + def test_jarrer(self): + copier = Jarrer() + copier.add('foo/bar', GeneratedFile('foobar')) + copier.add('foo/qux', GeneratedFile('fooqux')) + copier.add('foo/deep/nested/directory/file', GeneratedFile('fooz')) + copier.add('bar', GeneratedFile('bar')) + copier.add('qux/foo', GeneratedFile('quxfoo')) + copier.add('qux/bar', GeneratedFile('')) + + dest = MockDest() + copier.copy(dest) + self.check_jar(dest, copier) + + copier.remove('foo') + copier.add('test', GeneratedFile('test')) + copier.copy(dest) + self.check_jar(dest, copier) + + copier.remove('test') + copier.add('test', GeneratedFile('replaced-content')) + copier.copy(dest) + self.check_jar(dest, copier) + + copier.copy(dest) + self.check_jar(dest, copier) + + preloaded = ['qux/bar', 'bar'] + copier.preload(preloaded) + copier.copy(dest) + + dest.seek(0) + jar = JarReader(fileobj=dest) + self.assertEqual([f.filename for f in jar], preloaded + + [p for p in copier.paths() if not p in preloaded]) + self.assertEqual(jar.last_preloaded, preloaded[-1]) + + + def test_jarrer_compress(self): + copier = Jarrer() + copier.add('foo/bar', GeneratedFile('ffffff')) + copier.add('foo/qux', GeneratedFile('ffffff'), compress=False) + + dest = MockDest() + copier.copy(dest) + self.check_jar(dest, copier) + + dest.seek(0) + jar = JarReader(fileobj=dest) + self.assertTrue(jar['foo/bar'].compressed) + self.assertFalse(jar['foo/qux'].compressed) + + +if __name__ == '__main__': + mozunit.main() diff --git a/python/mozbuild/mozpack/test/test_errors.py b/python/mozbuild/mozpack/test/test_errors.py new file mode 100644 index 000000000..16e2b0496 --- /dev/null +++ b/python/mozbuild/mozpack/test/test_errors.py @@ -0,0 +1,93 @@ +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, You can obtain one at http://mozilla.org/MPL/2.0/. + +from mozpack.errors import ( + errors, + ErrorMessage, + AccumulatedErrors, +) +import unittest +import mozunit +import sys +from cStringIO import StringIO + + +class TestErrors(object): + def setUp(self): + errors.out = StringIO() + errors.ignore_errors(False) + + def tearDown(self): + errors.out = sys.stderr + + def get_output(self): + return [l.strip() for l in errors.out.getvalue().splitlines()] + + +class TestErrorsImpl(TestErrors, unittest.TestCase): + def test_plain_error(self): + errors.warn('foo') + self.assertRaises(ErrorMessage, errors.error, 'foo') + self.assertRaises(ErrorMessage, errors.fatal, 'foo') + self.assertEquals(self.get_output(), ['Warning: foo']) + + def test_ignore_errors(self): + errors.ignore_errors() + errors.warn('foo') + errors.error('bar') + self.assertRaises(ErrorMessage, errors.fatal, 'foo') + self.assertEquals(self.get_output(), ['Warning: foo', 'Warning: bar']) + + def test_no_error(self): + with errors.accumulate(): + errors.warn('1') + + def test_simple_error(self): + with self.assertRaises(AccumulatedErrors): + with errors.accumulate(): + errors.error('1') + self.assertEquals(self.get_output(), ['Error: 1']) + + def test_error_loop(self): + with self.assertRaises(AccumulatedErrors): + with errors.accumulate(): + for i in range(3): + errors.error('%d' % i) + self.assertEquals(self.get_output(), + ['Error: 0', 'Error: 1', 'Error: 2']) + + def test_multiple_errors(self): + with self.assertRaises(AccumulatedErrors): + with errors.accumulate(): + errors.error('foo') + for i in range(3): + if i == 2: + errors.warn('%d' % i) + else: + errors.error('%d' % i) + errors.error('bar') + self.assertEquals(self.get_output(), + ['Error: foo', 'Error: 0', 'Error: 1', + 'Warning: 2', 'Error: bar']) + + def test_errors_context(self): + with self.assertRaises(AccumulatedErrors): + with errors.accumulate(): + self.assertEqual(errors.get_context(), None) + with errors.context('foo', 1): + self.assertEqual(errors.get_context(), ('foo', 1)) + errors.error('a') + with errors.context('bar', 2): + self.assertEqual(errors.get_context(), ('bar', 2)) + errors.error('b') + self.assertEqual(errors.get_context(), ('foo', 1)) + errors.error('c') + self.assertEqual(self.get_output(), [ + 'Error: foo:1: a', + 'Error: bar:2: b', + 'Error: foo:1: c', + ]) + +if __name__ == '__main__': + mozunit.main() diff --git a/python/mozbuild/mozpack/test/test_files.py b/python/mozbuild/mozpack/test/test_files.py new file mode 100644 index 000000000..6fd617828 --- /dev/null +++ b/python/mozbuild/mozpack/test/test_files.py @@ -0,0 +1,1160 @@ +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, You can obtain one at http://mozilla.org/MPL/2.0/. + +from mozbuild.util import ensureParentDir + +from mozpack.errors import ( + ErrorMessage, + errors, +) +from mozpack.files import ( + AbsoluteSymlinkFile, + ComposedFinder, + DeflatedFile, + Dest, + ExistingFile, + ExtractedTarFile, + FileFinder, + File, + GeneratedFile, + JarFinder, + TarFinder, + ManifestFile, + MercurialFile, + MercurialRevisionFinder, + MinifiedJavaScript, + MinifiedProperties, + PreprocessedFile, + XPTFile, +) + +# We don't have hglib installed everywhere. +try: + import hglib +except ImportError: + hglib = None + +try: + from mozpack.hg import MercurialNativeRevisionFinder +except ImportError: + MercurialNativeRevisionFinder = None + +from mozpack.mozjar import ( + JarReader, + JarWriter, +) +from mozpack.chrome.manifest import ( + ManifestContent, + ManifestResource, + ManifestLocale, + ManifestOverride, +) +import unittest +import mozfile +import mozunit +import os +import random +import string +import sys +import tarfile +import mozpack.path as mozpath +from tempfile import mkdtemp +from io import BytesIO +from StringIO import StringIO +from xpt import Typelib + + +class TestWithTmpDir(unittest.TestCase): + def setUp(self): + self.tmpdir = mkdtemp() + + self.symlink_supported = False + + if not hasattr(os, 'symlink'): + return + + dummy_path = self.tmppath('dummy_file') + with open(dummy_path, 'a'): + pass + + try: + os.symlink(dummy_path, self.tmppath('dummy_symlink')) + os.remove(self.tmppath('dummy_symlink')) + except EnvironmentError: + pass + finally: + os.remove(dummy_path) + + self.symlink_supported = True + + + def tearDown(self): + mozfile.rmtree(self.tmpdir) + + def tmppath(self, relpath): + return os.path.normpath(os.path.join(self.tmpdir, relpath)) + + +class MockDest(BytesIO, Dest): + def __init__(self): + BytesIO.__init__(self) + self.mode = None + + def read(self, length=-1): + if self.mode != 'r': + self.seek(0) + self.mode = 'r' + return BytesIO.read(self, length) + + def write(self, data): + if self.mode != 'w': + self.seek(0) + self.truncate(0) + self.mode = 'w' + return BytesIO.write(self, data) + + def exists(self): + return True + + def close(self): + if self.mode: + self.mode = None + + +class DestNoWrite(Dest): + def write(self, data): + raise RuntimeError + + +class TestDest(TestWithTmpDir): + def test_dest(self): + dest = Dest(self.tmppath('dest')) + self.assertFalse(dest.exists()) + dest.write('foo') + self.assertTrue(dest.exists()) + dest.write('foo') + self.assertEqual(dest.read(4), 'foof') + self.assertEqual(dest.read(), 'oo') + self.assertEqual(dest.read(), '') + dest.write('bar') + self.assertEqual(dest.read(4), 'bar') + dest.close() + self.assertEqual(dest.read(), 'bar') + dest.write('foo') + dest.close() + dest.write('qux') + self.assertEqual(dest.read(), 'qux') + +rand = ''.join(random.choice(string.letters) for i in xrange(131597)) +samples = [ + '', + 'test', + 'fooo', + 'same', + 'same', + 'Different and longer', + rand, + rand, + rand[:-1] + '_', + 'test' +] + + +class TestFile(TestWithTmpDir): + def test_file(self): + ''' + Check that File.copy yields the proper content in the destination file + in all situations that trigger different code paths: + - different content + - different content of the same size + - same content + - long content + ''' + src = self.tmppath('src') + dest = self.tmppath('dest') + + for content in samples: + with open(src, 'wb') as tmp: + tmp.write(content) + # Ensure the destination file, when it exists, is older than the + # source + if os.path.exists(dest): + time = os.path.getmtime(src) - 1 + os.utime(dest, (time, time)) + f = File(src) + f.copy(dest) + self.assertEqual(content, open(dest, 'rb').read()) + self.assertEqual(content, f.open().read()) + self.assertEqual(content, f.open().read()) + + def test_file_dest(self): + ''' + Similar to test_file, but for a destination object instead of + a destination file. This ensures the destination object is being + used properly by File.copy, ensuring that other subclasses of Dest + will work. + ''' + src = self.tmppath('src') + dest = MockDest() + + for content in samples: + with open(src, 'wb') as tmp: + tmp.write(content) + f = File(src) + f.copy(dest) + self.assertEqual(content, dest.getvalue()) + + def test_file_open(self): + ''' + Test whether File.open returns an appropriately reset file object. + ''' + src = self.tmppath('src') + content = ''.join(samples) + with open(src, 'wb') as tmp: + tmp.write(content) + + f = File(src) + self.assertEqual(content[:42], f.open().read(42)) + self.assertEqual(content, f.open().read()) + + def test_file_no_write(self): + ''' + Test various conditions where File.copy is expected not to write + in the destination file. + ''' + src = self.tmppath('src') + dest = self.tmppath('dest') + + with open(src, 'wb') as tmp: + tmp.write('test') + + # Initial copy + f = File(src) + f.copy(dest) + + # Ensure subsequent copies won't trigger writes + f.copy(DestNoWrite(dest)) + self.assertEqual('test', open(dest, 'rb').read()) + + # When the source file is newer, but with the same content, no copy + # should occur + time = os.path.getmtime(src) - 1 + os.utime(dest, (time, time)) + f.copy(DestNoWrite(dest)) + self.assertEqual('test', open(dest, 'rb').read()) + + # When the source file is older than the destination file, even with + # different content, no copy should occur. + with open(src, 'wb') as tmp: + tmp.write('fooo') + time = os.path.getmtime(dest) - 1 + os.utime(src, (time, time)) + f.copy(DestNoWrite(dest)) + self.assertEqual('test', open(dest, 'rb').read()) + + # Double check that under conditions where a copy occurs, we would get + # an exception. + time = os.path.getmtime(src) - 1 + os.utime(dest, (time, time)) + self.assertRaises(RuntimeError, f.copy, DestNoWrite(dest)) + + # skip_if_older=False is expected to force a copy in this situation. + f.copy(dest, skip_if_older=False) + self.assertEqual('fooo', open(dest, 'rb').read()) + + +class TestAbsoluteSymlinkFile(TestWithTmpDir): + def test_absolute_relative(self): + AbsoluteSymlinkFile('/foo') + + with self.assertRaisesRegexp(ValueError, 'Symlink target not absolute'): + AbsoluteSymlinkFile('./foo') + + def test_symlink_file(self): + source = self.tmppath('test_path') + with open(source, 'wt') as fh: + fh.write('Hello world') + + s = AbsoluteSymlinkFile(source) + dest = self.tmppath('symlink') + self.assertTrue(s.copy(dest)) + + if self.symlink_supported: + self.assertTrue(os.path.islink(dest)) + link = os.readlink(dest) + self.assertEqual(link, source) + else: + self.assertTrue(os.path.isfile(dest)) + content = open(dest).read() + self.assertEqual(content, 'Hello world') + + def test_replace_file_with_symlink(self): + # If symlinks are supported, an existing file should be replaced by a + # symlink. + source = self.tmppath('test_path') + with open(source, 'wt') as fh: + fh.write('source') + + dest = self.tmppath('dest') + with open(dest, 'a'): + pass + + s = AbsoluteSymlinkFile(source) + s.copy(dest, skip_if_older=False) + + if self.symlink_supported: + self.assertTrue(os.path.islink(dest)) + link = os.readlink(dest) + self.assertEqual(link, source) + else: + self.assertTrue(os.path.isfile(dest)) + content = open(dest).read() + self.assertEqual(content, 'source') + + def test_replace_symlink(self): + if not self.symlink_supported: + return + + source = self.tmppath('source') + with open(source, 'a'): + pass + + dest = self.tmppath('dest') + + os.symlink(self.tmppath('bad'), dest) + self.assertTrue(os.path.islink(dest)) + + s = AbsoluteSymlinkFile(source) + self.assertTrue(s.copy(dest)) + + self.assertTrue(os.path.islink(dest)) + link = os.readlink(dest) + self.assertEqual(link, source) + + def test_noop(self): + if not hasattr(os, 'symlink'): + return + + source = self.tmppath('source') + dest = self.tmppath('dest') + + with open(source, 'a'): + pass + + os.symlink(source, dest) + link = os.readlink(dest) + self.assertEqual(link, source) + + s = AbsoluteSymlinkFile(source) + self.assertFalse(s.copy(dest)) + + link = os.readlink(dest) + self.assertEqual(link, source) + +class TestPreprocessedFile(TestWithTmpDir): + def test_preprocess(self): + ''' + Test that copying the file invokes the preprocessor + ''' + src = self.tmppath('src') + dest = self.tmppath('dest') + + with open(src, 'wb') as tmp: + tmp.write('#ifdef FOO\ntest\n#endif') + + f = PreprocessedFile(src, depfile_path=None, marker='#', defines={'FOO': True}) + self.assertTrue(f.copy(dest)) + + self.assertEqual('test\n', open(dest, 'rb').read()) + + def test_preprocess_file_no_write(self): + ''' + Test various conditions where PreprocessedFile.copy is expected not to + write in the destination file. + ''' + src = self.tmppath('src') + dest = self.tmppath('dest') + depfile = self.tmppath('depfile') + + with open(src, 'wb') as tmp: + tmp.write('#ifdef FOO\ntest\n#endif') + + # Initial copy + f = PreprocessedFile(src, depfile_path=depfile, marker='#', defines={'FOO': True}) + self.assertTrue(f.copy(dest)) + + # Ensure subsequent copies won't trigger writes + self.assertFalse(f.copy(DestNoWrite(dest))) + self.assertEqual('test\n', open(dest, 'rb').read()) + + # When the source file is older than the destination file, even with + # different content, no copy should occur. + with open(src, 'wb') as tmp: + tmp.write('#ifdef FOO\nfooo\n#endif') + time = os.path.getmtime(dest) - 1 + os.utime(src, (time, time)) + self.assertFalse(f.copy(DestNoWrite(dest))) + self.assertEqual('test\n', open(dest, 'rb').read()) + + # skip_if_older=False is expected to force a copy in this situation. + self.assertTrue(f.copy(dest, skip_if_older=False)) + self.assertEqual('fooo\n', open(dest, 'rb').read()) + + def test_preprocess_file_dependencies(self): + ''' + Test that the preprocess runs if the dependencies of the source change + ''' + src = self.tmppath('src') + dest = self.tmppath('dest') + incl = self.tmppath('incl') + deps = self.tmppath('src.pp') + + with open(src, 'wb') as tmp: + tmp.write('#ifdef FOO\ntest\n#endif') + + with open(incl, 'wb') as tmp: + tmp.write('foo bar') + + # Initial copy + f = PreprocessedFile(src, depfile_path=deps, marker='#', defines={'FOO': True}) + self.assertTrue(f.copy(dest)) + + # Update the source so it #includes the include file. + with open(src, 'wb') as tmp: + tmp.write('#include incl\n') + time = os.path.getmtime(dest) + 1 + os.utime(src, (time, time)) + self.assertTrue(f.copy(dest)) + self.assertEqual('foo bar', open(dest, 'rb').read()) + + # If one of the dependencies changes, the file should be updated. The + # mtime of the dependency is set after the destination file, to avoid + # both files having the same time. + with open(incl, 'wb') as tmp: + tmp.write('quux') + time = os.path.getmtime(dest) + 1 + os.utime(incl, (time, time)) + self.assertTrue(f.copy(dest)) + self.assertEqual('quux', open(dest, 'rb').read()) + + # Perform one final copy to confirm that we don't run the preprocessor + # again. We update the mtime of the destination so it's newer than the + # input files. This would "just work" if we weren't changing + time = os.path.getmtime(incl) + 1 + os.utime(dest, (time, time)) + self.assertFalse(f.copy(DestNoWrite(dest))) + + def test_replace_symlink(self): + ''' + Test that if the destination exists, and is a symlink, the target of + the symlink is not overwritten by the preprocessor output. + ''' + if not self.symlink_supported: + return + + source = self.tmppath('source') + dest = self.tmppath('dest') + pp_source = self.tmppath('pp_in') + deps = self.tmppath('deps') + + with open(source, 'a'): + pass + + os.symlink(source, dest) + self.assertTrue(os.path.islink(dest)) + + with open(pp_source, 'wb') as tmp: + tmp.write('#define FOO\nPREPROCESSED') + + f = PreprocessedFile(pp_source, depfile_path=deps, marker='#', + defines={'FOO': True}) + self.assertTrue(f.copy(dest)) + + self.assertEqual('PREPROCESSED', open(dest, 'rb').read()) + self.assertFalse(os.path.islink(dest)) + self.assertEqual('', open(source, 'rb').read()) + +class TestExistingFile(TestWithTmpDir): + def test_required_missing_dest(self): + with self.assertRaisesRegexp(ErrorMessage, 'Required existing file'): + f = ExistingFile(required=True) + f.copy(self.tmppath('dest')) + + def test_required_existing_dest(self): + p = self.tmppath('dest') + with open(p, 'a'): + pass + + f = ExistingFile(required=True) + f.copy(p) + + def test_optional_missing_dest(self): + f = ExistingFile(required=False) + f.copy(self.tmppath('dest')) + + def test_optional_existing_dest(self): + p = self.tmppath('dest') + with open(p, 'a'): + pass + + f = ExistingFile(required=False) + f.copy(p) + + +class TestGeneratedFile(TestWithTmpDir): + def test_generated_file(self): + ''' + Check that GeneratedFile.copy yields the proper content in the + destination file in all situations that trigger different code paths + (see TestFile.test_file) + ''' + dest = self.tmppath('dest') + + for content in samples: + f = GeneratedFile(content) + f.copy(dest) + self.assertEqual(content, open(dest, 'rb').read()) + + def test_generated_file_open(self): + ''' + Test whether GeneratedFile.open returns an appropriately reset file + object. + ''' + content = ''.join(samples) + f = GeneratedFile(content) + self.assertEqual(content[:42], f.open().read(42)) + self.assertEqual(content, f.open().read()) + + def test_generated_file_no_write(self): + ''' + Test various conditions where GeneratedFile.copy is expected not to + write in the destination file. + ''' + dest = self.tmppath('dest') + + # Initial copy + f = GeneratedFile('test') + f.copy(dest) + + # Ensure subsequent copies won't trigger writes + f.copy(DestNoWrite(dest)) + self.assertEqual('test', open(dest, 'rb').read()) + + # When using a new instance with the same content, no copy should occur + f = GeneratedFile('test') + f.copy(DestNoWrite(dest)) + self.assertEqual('test', open(dest, 'rb').read()) + + # Double check that under conditions where a copy occurs, we would get + # an exception. + f = GeneratedFile('fooo') + self.assertRaises(RuntimeError, f.copy, DestNoWrite(dest)) + + +class TestDeflatedFile(TestWithTmpDir): + def test_deflated_file(self): + ''' + Check that DeflatedFile.copy yields the proper content in the + destination file in all situations that trigger different code paths + (see TestFile.test_file) + ''' + src = self.tmppath('src.jar') + dest = self.tmppath('dest') + + contents = {} + with JarWriter(src) as jar: + for content in samples: + name = ''.join(random.choice(string.letters) + for i in xrange(8)) + jar.add(name, content, compress=True) + contents[name] = content + + for j in JarReader(src): + f = DeflatedFile(j) + f.copy(dest) + self.assertEqual(contents[j.filename], open(dest, 'rb').read()) + + def test_deflated_file_open(self): + ''' + Test whether DeflatedFile.open returns an appropriately reset file + object. + ''' + src = self.tmppath('src.jar') + content = ''.join(samples) + with JarWriter(src) as jar: + jar.add('content', content) + + f = DeflatedFile(JarReader(src)['content']) + self.assertEqual(content[:42], f.open().read(42)) + self.assertEqual(content, f.open().read()) + + def test_deflated_file_no_write(self): + ''' + Test various conditions where DeflatedFile.copy is expected not to + write in the destination file. + ''' + src = self.tmppath('src.jar') + dest = self.tmppath('dest') + + with JarWriter(src) as jar: + jar.add('test', 'test') + jar.add('test2', 'test') + jar.add('fooo', 'fooo') + + jar = JarReader(src) + # Initial copy + f = DeflatedFile(jar['test']) + f.copy(dest) + + # Ensure subsequent copies won't trigger writes + f.copy(DestNoWrite(dest)) + self.assertEqual('test', open(dest, 'rb').read()) + + # When using a different file with the same content, no copy should + # occur + f = DeflatedFile(jar['test2']) + f.copy(DestNoWrite(dest)) + self.assertEqual('test', open(dest, 'rb').read()) + + # Double check that under conditions where a copy occurs, we would get + # an exception. + f = DeflatedFile(jar['fooo']) + self.assertRaises(RuntimeError, f.copy, DestNoWrite(dest)) + + +class TestManifestFile(TestWithTmpDir): + def test_manifest_file(self): + f = ManifestFile('chrome') + f.add(ManifestContent('chrome', 'global', 'toolkit/content/global/')) + f.add(ManifestResource('chrome', 'gre-resources', 'toolkit/res/')) + f.add(ManifestResource('chrome/pdfjs', 'pdfjs', './')) + f.add(ManifestContent('chrome/pdfjs', 'pdfjs', 'pdfjs')) + f.add(ManifestLocale('chrome', 'browser', 'en-US', + 'en-US/locale/browser/')) + + f.copy(self.tmppath('chrome.manifest')) + self.assertEqual(open(self.tmppath('chrome.manifest')).readlines(), [ + 'content global toolkit/content/global/\n', + 'resource gre-resources toolkit/res/\n', + 'resource pdfjs pdfjs/\n', + 'content pdfjs pdfjs/pdfjs\n', + 'locale browser en-US en-US/locale/browser/\n', + ]) + + self.assertRaises( + ValueError, + f.remove, + ManifestContent('', 'global', 'toolkit/content/global/') + ) + self.assertRaises( + ValueError, + f.remove, + ManifestOverride('chrome', 'chrome://global/locale/netError.dtd', + 'chrome://browser/locale/netError.dtd') + ) + + f.remove(ManifestContent('chrome', 'global', + 'toolkit/content/global/')) + self.assertRaises( + ValueError, + f.remove, + ManifestContent('chrome', 'global', 'toolkit/content/global/') + ) + + f.copy(self.tmppath('chrome.manifest')) + content = open(self.tmppath('chrome.manifest')).read() + self.assertEqual(content[:42], f.open().read(42)) + self.assertEqual(content, f.open().read()) + +# Compiled typelib for the following IDL: +# interface foo; +# [scriptable, uuid(5f70da76-519c-4858-b71e-e3c92333e2d6)] +# interface bar { +# void bar(in foo f); +# }; +# We need to make this [scriptable] so it doesn't get deleted from the +# typelib. We don't need to make the foo interfaces below [scriptable], +# because they will be automatically included by virtue of being an +# argument to a method of |bar|. +bar_xpt = GeneratedFile( + b'\x58\x50\x43\x4F\x4D\x0A\x54\x79\x70\x65\x4C\x69\x62\x0D\x0A\x1A' + + b'\x01\x02\x00\x02\x00\x00\x00\x7B\x00\x00\x00\x24\x00\x00\x00\x5C' + + b'\x80\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00' + + b'\x00\x00\x00\x00\x00\x00\x01\x00\x00\x00\x00\x00\x00\x00\x00\x5F' + + b'\x70\xDA\x76\x51\x9C\x48\x58\xB7\x1E\xE3\xC9\x23\x33\xE2\xD6\x00' + + b'\x00\x00\x05\x00\x00\x00\x00\x00\x00\x00\x0D\x00\x66\x6F\x6F\x00' + + b'\x62\x61\x72\x00\x62\x61\x72\x00\x00\x00\x00\x01\x00\x00\x00\x00' + + b'\x09\x01\x80\x92\x00\x01\x80\x06\x00\x00\x80' +) + +# Compiled typelib for the following IDL: +# [uuid(3271bebc-927e-4bef-935e-44e0aaf3c1e5)] +# interface foo { +# void foo(); +# }; +foo_xpt = GeneratedFile( + b'\x58\x50\x43\x4F\x4D\x0A\x54\x79\x70\x65\x4C\x69\x62\x0D\x0A\x1A' + + b'\x01\x02\x00\x01\x00\x00\x00\x57\x00\x00\x00\x24\x00\x00\x00\x40' + + b'\x80\x00\x00\x32\x71\xBE\xBC\x92\x7E\x4B\xEF\x93\x5E\x44\xE0\xAA' + + b'\xF3\xC1\xE5\x00\x00\x00\x01\x00\x00\x00\x00\x00\x00\x00\x09\x00' + + b'\x66\x6F\x6F\x00\x66\x6F\x6F\x00\x00\x00\x00\x01\x00\x00\x00\x00' + + b'\x05\x00\x80\x06\x00\x00\x00' +) + +# Compiled typelib for the following IDL: +# [uuid(7057f2aa-fdc2-4559-abde-08d939f7e80d)] +# interface foo { +# void foo(); +# }; +foo2_xpt = GeneratedFile( + b'\x58\x50\x43\x4F\x4D\x0A\x54\x79\x70\x65\x4C\x69\x62\x0D\x0A\x1A' + + b'\x01\x02\x00\x01\x00\x00\x00\x57\x00\x00\x00\x24\x00\x00\x00\x40' + + b'\x80\x00\x00\x70\x57\xF2\xAA\xFD\xC2\x45\x59\xAB\xDE\x08\xD9\x39' + + b'\xF7\xE8\x0D\x00\x00\x00\x01\x00\x00\x00\x00\x00\x00\x00\x09\x00' + + b'\x66\x6F\x6F\x00\x66\x6F\x6F\x00\x00\x00\x00\x01\x00\x00\x00\x00' + + b'\x05\x00\x80\x06\x00\x00\x00' +) + + +def read_interfaces(file): + return dict((i.name, i) for i in Typelib.read(file).interfaces) + + +class TestXPTFile(TestWithTmpDir): + def test_xpt_file(self): + x = XPTFile() + x.add(foo_xpt) + x.add(bar_xpt) + x.copy(self.tmppath('interfaces.xpt')) + + foo = read_interfaces(foo_xpt.open()) + foo2 = read_interfaces(foo2_xpt.open()) + bar = read_interfaces(bar_xpt.open()) + linked = read_interfaces(self.tmppath('interfaces.xpt')) + self.assertEqual(foo['foo'], linked['foo']) + self.assertEqual(bar['bar'], linked['bar']) + + x.remove(foo_xpt) + x.copy(self.tmppath('interfaces2.xpt')) + linked = read_interfaces(self.tmppath('interfaces2.xpt')) + self.assertEqual(bar['foo'], linked['foo']) + self.assertEqual(bar['bar'], linked['bar']) + + x.add(foo_xpt) + x.copy(DestNoWrite(self.tmppath('interfaces.xpt'))) + linked = read_interfaces(self.tmppath('interfaces.xpt')) + self.assertEqual(foo['foo'], linked['foo']) + self.assertEqual(bar['bar'], linked['bar']) + + x = XPTFile() + x.add(foo2_xpt) + x.add(bar_xpt) + x.copy(self.tmppath('interfaces.xpt')) + linked = read_interfaces(self.tmppath('interfaces.xpt')) + self.assertEqual(foo2['foo'], linked['foo']) + self.assertEqual(bar['bar'], linked['bar']) + + x = XPTFile() + x.add(foo_xpt) + x.add(foo2_xpt) + x.add(bar_xpt) + from xpt import DataError + self.assertRaises(DataError, x.copy, self.tmppath('interfaces.xpt')) + + +class TestMinifiedProperties(TestWithTmpDir): + def test_minified_properties(self): + propLines = [ + '# Comments are removed', + 'foo = bar', + '', + '# Another comment', + ] + prop = GeneratedFile('\n'.join(propLines)) + self.assertEqual(MinifiedProperties(prop).open().readlines(), + ['foo = bar\n', '\n']) + open(self.tmppath('prop'), 'wb').write('\n'.join(propLines)) + MinifiedProperties(File(self.tmppath('prop'))) \ + .copy(self.tmppath('prop2')) + self.assertEqual(open(self.tmppath('prop2')).readlines(), + ['foo = bar\n', '\n']) + + +class TestMinifiedJavaScript(TestWithTmpDir): + orig_lines = [ + '// Comment line', + 'let foo = "bar";', + 'var bar = true;', + '', + '// Another comment', + ] + + def test_minified_javascript(self): + orig_f = GeneratedFile('\n'.join(self.orig_lines)) + min_f = MinifiedJavaScript(orig_f) + + mini_lines = min_f.open().readlines() + self.assertTrue(mini_lines) + self.assertTrue(len(mini_lines) < len(self.orig_lines)) + + def _verify_command(self, code): + our_dir = os.path.abspath(os.path.dirname(__file__)) + return [ + sys.executable, + os.path.join(our_dir, 'support', 'minify_js_verify.py'), + code, + ] + + def test_minified_verify_success(self): + orig_f = GeneratedFile('\n'.join(self.orig_lines)) + min_f = MinifiedJavaScript(orig_f, + verify_command=self._verify_command('0')) + + mini_lines = min_f.open().readlines() + self.assertTrue(mini_lines) + self.assertTrue(len(mini_lines) < len(self.orig_lines)) + + def test_minified_verify_failure(self): + orig_f = GeneratedFile('\n'.join(self.orig_lines)) + errors.out = StringIO() + min_f = MinifiedJavaScript(orig_f, + verify_command=self._verify_command('1')) + + mini_lines = min_f.open().readlines() + output = errors.out.getvalue() + errors.out = sys.stderr + self.assertEqual(output, + 'Warning: JS minification verification failed for :\n' + 'Warning: Error message\n') + self.assertEqual(mini_lines, orig_f.open().readlines()) + + +class MatchTestTemplate(object): + def prepare_match_test(self, with_dotfiles=False): + self.add('bar') + self.add('foo/bar') + self.add('foo/baz') + self.add('foo/qux/1') + self.add('foo/qux/bar') + self.add('foo/qux/2/test') + self.add('foo/qux/2/test2') + if with_dotfiles: + self.add('foo/.foo') + self.add('foo/.bar/foo') + + def do_match_test(self): + self.do_check('', [ + 'bar', 'foo/bar', 'foo/baz', 'foo/qux/1', 'foo/qux/bar', + 'foo/qux/2/test', 'foo/qux/2/test2' + ]) + self.do_check('*', [ + 'bar', 'foo/bar', 'foo/baz', 'foo/qux/1', 'foo/qux/bar', + 'foo/qux/2/test', 'foo/qux/2/test2' + ]) + self.do_check('foo/qux', [ + 'foo/qux/1', 'foo/qux/bar', 'foo/qux/2/test', 'foo/qux/2/test2' + ]) + self.do_check('foo/b*', ['foo/bar', 'foo/baz']) + self.do_check('baz', []) + self.do_check('foo/foo', []) + self.do_check('foo/*ar', ['foo/bar']) + self.do_check('*ar', ['bar']) + self.do_check('*/bar', ['foo/bar']) + self.do_check('foo/*ux', [ + 'foo/qux/1', 'foo/qux/bar', 'foo/qux/2/test', 'foo/qux/2/test2' + ]) + self.do_check('foo/q*ux', [ + 'foo/qux/1', 'foo/qux/bar', 'foo/qux/2/test', 'foo/qux/2/test2' + ]) + self.do_check('foo/*/2/test*', ['foo/qux/2/test', 'foo/qux/2/test2']) + self.do_check('**/bar', ['bar', 'foo/bar', 'foo/qux/bar']) + self.do_check('foo/**/test', ['foo/qux/2/test']) + self.do_check('foo', [ + 'foo/bar', 'foo/baz', 'foo/qux/1', 'foo/qux/bar', + 'foo/qux/2/test', 'foo/qux/2/test2' + ]) + self.do_check('foo/**', [ + 'foo/bar', 'foo/baz', 'foo/qux/1', 'foo/qux/bar', + 'foo/qux/2/test', 'foo/qux/2/test2' + ]) + self.do_check('**/2/test*', ['foo/qux/2/test', 'foo/qux/2/test2']) + self.do_check('**/foo', [ + 'foo/bar', 'foo/baz', 'foo/qux/1', 'foo/qux/bar', + 'foo/qux/2/test', 'foo/qux/2/test2' + ]) + self.do_check('**/barbaz', []) + self.do_check('f**/bar', ['foo/bar']) + + def do_finder_test(self, finder): + self.assertTrue(finder.contains('foo/.foo')) + self.assertTrue(finder.contains('foo/.bar')) + self.assertTrue('foo/.foo' in [f for f, c in + finder.find('foo/.foo')]) + self.assertTrue('foo/.bar/foo' in [f for f, c in + finder.find('foo/.bar')]) + self.assertEqual(sorted([f for f, c in finder.find('foo/.*')]), + ['foo/.bar/foo', 'foo/.foo']) + for pattern in ['foo', '**', '**/*', '**/foo', 'foo/*']: + self.assertFalse('foo/.foo' in [f for f, c in + finder.find(pattern)]) + self.assertFalse('foo/.bar/foo' in [f for f, c in + finder.find(pattern)]) + self.assertEqual(sorted([f for f, c in finder.find(pattern)]), + sorted([f for f, c in finder + if mozpath.match(f, pattern)])) + + +def do_check(test, finder, pattern, result): + if result: + test.assertTrue(finder.contains(pattern)) + else: + test.assertFalse(finder.contains(pattern)) + test.assertEqual(sorted(list(f for f, c in finder.find(pattern))), + sorted(result)) + + +class TestFileFinder(MatchTestTemplate, TestWithTmpDir): + def add(self, path): + ensureParentDir(self.tmppath(path)) + open(self.tmppath(path), 'wb').write(path) + + def do_check(self, pattern, result): + do_check(self, self.finder, pattern, result) + + def test_file_finder(self): + self.prepare_match_test(with_dotfiles=True) + self.finder = FileFinder(self.tmpdir) + self.do_match_test() + self.do_finder_test(self.finder) + + def test_get(self): + self.prepare_match_test() + finder = FileFinder(self.tmpdir) + + self.assertIsNone(finder.get('does-not-exist')) + res = finder.get('bar') + self.assertIsInstance(res, File) + self.assertEqual(mozpath.normpath(res.path), + mozpath.join(self.tmpdir, 'bar')) + + def test_ignored_dirs(self): + """Ignored directories should not have results returned.""" + self.prepare_match_test() + self.add('fooz') + + # Present to ensure prefix matching doesn't exclude. + self.add('foo/quxz') + + self.finder = FileFinder(self.tmpdir, ignore=['foo/qux']) + + self.do_check('**', ['bar', 'foo/bar', 'foo/baz', 'foo/quxz', 'fooz']) + self.do_check('foo/*', ['foo/bar', 'foo/baz', 'foo/quxz']) + self.do_check('foo/**', ['foo/bar', 'foo/baz', 'foo/quxz']) + self.do_check('foo/qux/**', []) + self.do_check('foo/qux/*', []) + self.do_check('foo/qux/bar', []) + self.do_check('foo/quxz', ['foo/quxz']) + self.do_check('fooz', ['fooz']) + + def test_ignored_files(self): + """Ignored files should not have results returned.""" + self.prepare_match_test() + + # Be sure prefix match doesn't get ignored. + self.add('barz') + + self.finder = FileFinder(self.tmpdir, ignore=['foo/bar', 'bar']) + self.do_check('**', ['barz', 'foo/baz', 'foo/qux/1', 'foo/qux/2/test', + 'foo/qux/2/test2', 'foo/qux/bar']) + self.do_check('foo/**', ['foo/baz', 'foo/qux/1', 'foo/qux/2/test', + 'foo/qux/2/test2', 'foo/qux/bar']) + + def test_ignored_patterns(self): + """Ignore entries with patterns should be honored.""" + self.prepare_match_test() + + self.add('foo/quxz') + + self.finder = FileFinder(self.tmpdir, ignore=['foo/qux/*']) + self.do_check('**', ['foo/bar', 'foo/baz', 'foo/quxz', 'bar']) + self.do_check('foo/**', ['foo/bar', 'foo/baz', 'foo/quxz']) + + def test_dotfiles(self): + """Finder can find files beginning with . is configured.""" + self.prepare_match_test(with_dotfiles=True) + self.finder = FileFinder(self.tmpdir, find_dotfiles=True) + self.do_check('**', ['bar', 'foo/.foo', 'foo/.bar/foo', + 'foo/bar', 'foo/baz', 'foo/qux/1', 'foo/qux/bar', + 'foo/qux/2/test', 'foo/qux/2/test2']) + + def test_dotfiles_plus_ignore(self): + self.prepare_match_test(with_dotfiles=True) + self.finder = FileFinder(self.tmpdir, find_dotfiles=True, + ignore=['foo/.bar/**']) + self.do_check('foo/**', ['foo/.foo', 'foo/bar', 'foo/baz', + 'foo/qux/1', 'foo/qux/bar', 'foo/qux/2/test', 'foo/qux/2/test2']) + + +class TestJarFinder(MatchTestTemplate, TestWithTmpDir): + def add(self, path): + self.jar.add(path, path, compress=True) + + def do_check(self, pattern, result): + do_check(self, self.finder, pattern, result) + + def test_jar_finder(self): + self.jar = JarWriter(file=self.tmppath('test.jar')) + self.prepare_match_test() + self.jar.finish() + reader = JarReader(file=self.tmppath('test.jar')) + self.finder = JarFinder(self.tmppath('test.jar'), reader) + self.do_match_test() + + self.assertIsNone(self.finder.get('does-not-exist')) + self.assertIsInstance(self.finder.get('bar'), DeflatedFile) + +class TestTarFinder(MatchTestTemplate, TestWithTmpDir): + def add(self, path): + self.tar.addfile(tarfile.TarInfo(name=path)) + + def do_check(self, pattern, result): + do_check(self, self.finder, pattern, result) + + def test_tar_finder(self): + self.tar = tarfile.open(name=self.tmppath('test.tar.bz2'), + mode='w:bz2') + self.prepare_match_test() + self.tar.close() + with tarfile.open(name=self.tmppath('test.tar.bz2'), + mode='r:bz2') as tarreader: + self.finder = TarFinder(self.tmppath('test.tar.bz2'), tarreader) + self.do_match_test() + + self.assertIsNone(self.finder.get('does-not-exist')) + self.assertIsInstance(self.finder.get('bar'), ExtractedTarFile) + + +class TestComposedFinder(MatchTestTemplate, TestWithTmpDir): + def add(self, path, content=None): + # Put foo/qux files under $tmp/b. + if path.startswith('foo/qux/'): + real_path = mozpath.join('b', path[8:]) + else: + real_path = mozpath.join('a', path) + ensureParentDir(self.tmppath(real_path)) + if not content: + content = path + open(self.tmppath(real_path), 'wb').write(content) + + def do_check(self, pattern, result): + if '*' in pattern: + return + do_check(self, self.finder, pattern, result) + + def test_composed_finder(self): + self.prepare_match_test() + # Also add files in $tmp/a/foo/qux because ComposedFinder is + # expected to mask foo/qux entirely with content from $tmp/b. + ensureParentDir(self.tmppath('a/foo/qux/hoge')) + open(self.tmppath('a/foo/qux/hoge'), 'wb').write('hoge') + open(self.tmppath('a/foo/qux/bar'), 'wb').write('not the right content') + self.finder = ComposedFinder({ + '': FileFinder(self.tmppath('a')), + 'foo/qux': FileFinder(self.tmppath('b')), + }) + self.do_match_test() + + self.assertIsNone(self.finder.get('does-not-exist')) + self.assertIsInstance(self.finder.get('bar'), File) + + +@unittest.skipUnless(hglib, 'hglib not available') +class TestMercurialRevisionFinder(MatchTestTemplate, TestWithTmpDir): + def setUp(self): + super(TestMercurialRevisionFinder, self).setUp() + hglib.init(self.tmpdir) + + def add(self, path): + c = hglib.open(self.tmpdir) + ensureParentDir(self.tmppath(path)) + with open(self.tmppath(path), 'wb') as fh: + fh.write(path) + c.add(self.tmppath(path)) + + def do_check(self, pattern, result): + do_check(self, self.finder, pattern, result) + + def _get_finder(self, *args, **kwargs): + return MercurialRevisionFinder(*args, **kwargs) + + def test_default_revision(self): + self.prepare_match_test() + c = hglib.open(self.tmpdir) + c.commit('initial commit') + self.finder = self._get_finder(self.tmpdir) + self.do_match_test() + + self.assertIsNone(self.finder.get('does-not-exist')) + self.assertIsInstance(self.finder.get('bar'), MercurialFile) + + def test_old_revision(self): + c = hglib.open(self.tmpdir) + with open(self.tmppath('foo'), 'wb') as fh: + fh.write('foo initial') + c.add(self.tmppath('foo')) + c.commit('initial') + + with open(self.tmppath('foo'), 'wb') as fh: + fh.write('foo second') + with open(self.tmppath('bar'), 'wb') as fh: + fh.write('bar second') + c.add(self.tmppath('bar')) + c.commit('second') + # This wipes out the working directory, ensuring the finder isn't + # finding anything from the filesystem. + c.rawcommand(['update', 'null']) + + finder = self._get_finder(self.tmpdir, 0) + f = finder.get('foo') + self.assertEqual(f.read(), 'foo initial') + self.assertEqual(f.read(), 'foo initial', 'read again for good measure') + self.assertIsNone(finder.get('bar')) + + finder = MercurialRevisionFinder(self.tmpdir, rev=1) + f = finder.get('foo') + self.assertEqual(f.read(), 'foo second') + f = finder.get('bar') + self.assertEqual(f.read(), 'bar second') + + def test_recognize_repo_paths(self): + c = hglib.open(self.tmpdir) + with open(self.tmppath('foo'), 'wb') as fh: + fh.write('initial') + c.add(self.tmppath('foo')) + c.commit('initial') + c.rawcommand(['update', 'null']) + + finder = self._get_finder(self.tmpdir, 0, + recognize_repo_paths=True) + with self.assertRaises(NotImplementedError): + list(finder.find('')) + + with self.assertRaises(ValueError): + finder.get('foo') + with self.assertRaises(ValueError): + finder.get('') + + f = finder.get(self.tmppath('foo')) + self.assertIsInstance(f, MercurialFile) + self.assertEqual(f.read(), 'initial') + + +@unittest.skipUnless(MercurialNativeRevisionFinder, 'hgnative not available') +class TestMercurialNativeRevisionFinder(TestMercurialRevisionFinder): + def _get_finder(self, *args, **kwargs): + return MercurialNativeRevisionFinder(*args, **kwargs) + + +if __name__ == '__main__': + mozunit.main() diff --git a/python/mozbuild/mozpack/test/test_manifests.py b/python/mozbuild/mozpack/test/test_manifests.py new file mode 100644 index 000000000..b785d014a --- /dev/null +++ b/python/mozbuild/mozpack/test/test_manifests.py @@ -0,0 +1,375 @@ +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, You can obtain one at http://mozilla.org/MPL/2.0/. + +from __future__ import unicode_literals + +import os + +import mozunit + +from mozpack.copier import ( + FileCopier, + FileRegistry, +) +from mozpack.manifests import ( + InstallManifest, + UnreadableInstallManifest, +) +from mozpack.test.test_files import TestWithTmpDir + + +class TestInstallManifest(TestWithTmpDir): + def test_construct(self): + m = InstallManifest() + self.assertEqual(len(m), 0) + + def test_malformed(self): + f = self.tmppath('manifest') + open(f, 'wb').write('junk\n') + with self.assertRaises(UnreadableInstallManifest): + m = InstallManifest(f) + + def test_adds(self): + m = InstallManifest() + m.add_symlink('s_source', 's_dest') + m.add_copy('c_source', 'c_dest') + m.add_required_exists('e_dest') + m.add_optional_exists('o_dest') + m.add_pattern_symlink('ps_base', 'ps/*', 'ps_dest') + m.add_pattern_copy('pc_base', 'pc/**', 'pc_dest') + m.add_preprocess('p_source', 'p_dest', 'p_source.pp') + m.add_content('content', 'content') + + self.assertEqual(len(m), 8) + self.assertIn('s_dest', m) + self.assertIn('c_dest', m) + self.assertIn('p_dest', m) + self.assertIn('e_dest', m) + self.assertIn('o_dest', m) + self.assertIn('content', m) + + with self.assertRaises(ValueError): + m.add_symlink('s_other', 's_dest') + + with self.assertRaises(ValueError): + m.add_copy('c_other', 'c_dest') + + with self.assertRaises(ValueError): + m.add_preprocess('p_other', 'p_dest', 'p_other.pp') + + with self.assertRaises(ValueError): + m.add_required_exists('e_dest') + + with self.assertRaises(ValueError): + m.add_optional_exists('o_dest') + + with self.assertRaises(ValueError): + m.add_pattern_symlink('ps_base', 'ps/*', 'ps_dest') + + with self.assertRaises(ValueError): + m.add_pattern_copy('pc_base', 'pc/**', 'pc_dest') + + with self.assertRaises(ValueError): + m.add_content('content', 'content') + + def _get_test_manifest(self): + m = InstallManifest() + m.add_symlink(self.tmppath('s_source'), 's_dest') + m.add_copy(self.tmppath('c_source'), 'c_dest') + m.add_preprocess(self.tmppath('p_source'), 'p_dest', self.tmppath('p_source.pp'), '#', {'FOO':'BAR', 'BAZ':'QUX'}) + m.add_required_exists('e_dest') + m.add_optional_exists('o_dest') + m.add_pattern_symlink('ps_base', '*', 'ps_dest') + m.add_pattern_copy('pc_base', '**', 'pc_dest') + m.add_content('the content\non\nmultiple lines', 'content') + + return m + + def test_serialization(self): + m = self._get_test_manifest() + + p = self.tmppath('m') + m.write(path=p) + self.assertTrue(os.path.isfile(p)) + + with open(p, 'rb') as fh: + c = fh.read() + + self.assertEqual(c.count('\n'), 9) + + lines = c.splitlines() + self.assertEqual(len(lines), 9) + + self.assertEqual(lines[0], '5') + + m2 = InstallManifest(path=p) + self.assertEqual(m, m2) + p2 = self.tmppath('m2') + m2.write(path=p2) + + with open(p2, 'rb') as fh: + c2 = fh.read() + + self.assertEqual(c, c2) + + def test_populate_registry(self): + m = self._get_test_manifest() + r = FileRegistry() + m.populate_registry(r) + + self.assertEqual(len(r), 6) + self.assertEqual(r.paths(), ['c_dest', 'content', 'e_dest', 'o_dest', + 'p_dest', 's_dest']) + + def test_pattern_expansion(self): + source = self.tmppath('source') + os.mkdir(source) + os.mkdir('%s/base' % source) + os.mkdir('%s/base/foo' % source) + + with open('%s/base/foo/file1' % source, 'a'): + pass + + with open('%s/base/foo/file2' % source, 'a'): + pass + + m = InstallManifest() + m.add_pattern_symlink('%s/base' % source, '**', 'dest') + + c = FileCopier() + m.populate_registry(c) + self.assertEqual(c.paths(), ['dest/foo/file1', 'dest/foo/file2']) + + def test_or(self): + m1 = self._get_test_manifest() + orig_length = len(m1) + m2 = InstallManifest() + m2.add_symlink('s_source2', 's_dest2') + m2.add_copy('c_source2', 'c_dest2') + + m1 |= m2 + + self.assertEqual(len(m2), 2) + self.assertEqual(len(m1), orig_length + 2) + + self.assertIn('s_dest2', m1) + self.assertIn('c_dest2', m1) + + def test_copier_application(self): + dest = self.tmppath('dest') + os.mkdir(dest) + + to_delete = self.tmppath('dest/to_delete') + with open(to_delete, 'a'): + pass + + with open(self.tmppath('s_source'), 'wt') as fh: + fh.write('symlink!') + + with open(self.tmppath('c_source'), 'wt') as fh: + fh.write('copy!') + + with open(self.tmppath('p_source'), 'wt') as fh: + fh.write('#define FOO 1\npreprocess!') + + with open(self.tmppath('dest/e_dest'), 'a'): + pass + + with open(self.tmppath('dest/o_dest'), 'a'): + pass + + m = self._get_test_manifest() + c = FileCopier() + m.populate_registry(c) + result = c.copy(dest) + + self.assertTrue(os.path.exists(self.tmppath('dest/s_dest'))) + self.assertTrue(os.path.exists(self.tmppath('dest/c_dest'))) + self.assertTrue(os.path.exists(self.tmppath('dest/p_dest'))) + self.assertTrue(os.path.exists(self.tmppath('dest/e_dest'))) + self.assertTrue(os.path.exists(self.tmppath('dest/o_dest'))) + self.assertTrue(os.path.exists(self.tmppath('dest/content'))) + self.assertFalse(os.path.exists(to_delete)) + + with open(self.tmppath('dest/s_dest'), 'rt') as fh: + self.assertEqual(fh.read(), 'symlink!') + + with open(self.tmppath('dest/c_dest'), 'rt') as fh: + self.assertEqual(fh.read(), 'copy!') + + with open(self.tmppath('dest/p_dest'), 'rt') as fh: + self.assertEqual(fh.read(), 'preprocess!') + + self.assertEqual(result.updated_files, set(self.tmppath(p) for p in ( + 'dest/s_dest', 'dest/c_dest', 'dest/p_dest', 'dest/content'))) + self.assertEqual(result.existing_files, + set([self.tmppath('dest/e_dest'), self.tmppath('dest/o_dest')])) + self.assertEqual(result.removed_files, {to_delete}) + self.assertEqual(result.removed_directories, set()) + + def test_preprocessor(self): + manifest = self.tmppath('m') + deps = self.tmppath('m.pp') + dest = self.tmppath('dest') + include = self.tmppath('p_incl') + + with open(include, 'wt') as fh: + fh.write('#define INCL\n') + time = os.path.getmtime(include) - 3 + os.utime(include, (time, time)) + + with open(self.tmppath('p_source'), 'wt') as fh: + fh.write('#ifdef FOO\n#if BAZ == QUX\nPASS1\n#endif\n#endif\n') + fh.write('#ifdef DEPTEST\nPASS2\n#endif\n') + fh.write('#include p_incl\n#ifdef INCLTEST\nPASS3\n#endif\n') + time = os.path.getmtime(self.tmppath('p_source')) - 3 + os.utime(self.tmppath('p_source'), (time, time)) + + # Create and write a manifest with the preprocessed file, then apply it. + # This should write out our preprocessed file. + m = InstallManifest() + m.add_preprocess(self.tmppath('p_source'), 'p_dest', deps, '#', {'FOO':'BAR', 'BAZ':'QUX'}) + m.write(path=manifest) + + m = InstallManifest(path=manifest) + c = FileCopier() + m.populate_registry(c) + c.copy(dest) + + self.assertTrue(os.path.exists(self.tmppath('dest/p_dest'))) + + with open(self.tmppath('dest/p_dest'), 'rt') as fh: + self.assertEqual(fh.read(), 'PASS1\n') + + # Create a second manifest with the preprocessed file, then apply it. + # Since this manifest does not exist on the disk, there should not be a + # dependency on it, and the preprocessed file should not be modified. + m2 = InstallManifest() + m2.add_preprocess(self.tmppath('p_source'), 'p_dest', deps, '#', {'DEPTEST':True}) + c = FileCopier() + m2.populate_registry(c) + result = c.copy(dest) + + self.assertFalse(self.tmppath('dest/p_dest') in result.updated_files) + self.assertTrue(self.tmppath('dest/p_dest') in result.existing_files) + + # Write out the second manifest, then load it back in from the disk. + # This should add the dependency on the manifest file, so our + # preprocessed file should be regenerated with the new defines. + # We also set the mtime on the destination file back, so it will be + # older than the manifest file. + m2.write(path=manifest) + time = os.path.getmtime(manifest) - 1 + os.utime(self.tmppath('dest/p_dest'), (time, time)) + m2 = InstallManifest(path=manifest) + c = FileCopier() + m2.populate_registry(c) + self.assertTrue(c.copy(dest)) + + with open(self.tmppath('dest/p_dest'), 'rt') as fh: + self.assertEqual(fh.read(), 'PASS2\n') + + # Set the time on the manifest back, so it won't be picked up as + # modified in the next test + time = os.path.getmtime(manifest) - 1 + os.utime(manifest, (time, time)) + + # Update the contents of a file included by the source file. This should + # cause the destination to be regenerated. + with open(include, 'wt') as fh: + fh.write('#define INCLTEST\n') + + time = os.path.getmtime(include) - 1 + os.utime(self.tmppath('dest/p_dest'), (time, time)) + c = FileCopier() + m2.populate_registry(c) + self.assertTrue(c.copy(dest)) + + with open(self.tmppath('dest/p_dest'), 'rt') as fh: + self.assertEqual(fh.read(), 'PASS2\nPASS3\n') + + def test_preprocessor_dependencies(self): + manifest = self.tmppath('m') + deps = self.tmppath('m.pp') + dest = self.tmppath('dest') + source = self.tmppath('p_source') + destfile = self.tmppath('dest/p_dest') + include = self.tmppath('p_incl') + os.mkdir(dest) + + with open(source, 'wt') as fh: + fh.write('#define SRC\nSOURCE\n') + time = os.path.getmtime(source) - 3 + os.utime(source, (time, time)) + + with open(include, 'wt') as fh: + fh.write('INCLUDE\n') + time = os.path.getmtime(source) - 3 + os.utime(include, (time, time)) + + # Create and write a manifest with the preprocessed file. + m = InstallManifest() + m.add_preprocess(source, 'p_dest', deps, '#', {'FOO':'BAR', 'BAZ':'QUX'}) + m.write(path=manifest) + + time = os.path.getmtime(source) - 5 + os.utime(manifest, (time, time)) + + # Now read the manifest back in, and apply it. This should write out + # our preprocessed file. + m = InstallManifest(path=manifest) + c = FileCopier() + m.populate_registry(c) + self.assertTrue(c.copy(dest)) + + with open(destfile, 'rt') as fh: + self.assertEqual(fh.read(), 'SOURCE\n') + + # Next, modify the source to #INCLUDE another file. + with open(source, 'wt') as fh: + fh.write('SOURCE\n#include p_incl\n') + time = os.path.getmtime(source) - 1 + os.utime(destfile, (time, time)) + + # Apply the manifest, and confirm that it also reads the newly included + # file. + m = InstallManifest(path=manifest) + c = FileCopier() + m.populate_registry(c) + c.copy(dest) + + with open(destfile, 'rt') as fh: + self.assertEqual(fh.read(), 'SOURCE\nINCLUDE\n') + + # Set the time on the source file back, so it won't be picked up as + # modified in the next test. + time = os.path.getmtime(source) - 1 + os.utime(source, (time, time)) + + # Now, modify the include file (but not the original source). + with open(include, 'wt') as fh: + fh.write('INCLUDE MODIFIED\n') + time = os.path.getmtime(include) - 1 + os.utime(destfile, (time, time)) + + # Apply the manifest, and confirm that the change to the include file + # is detected. That should cause the preprocessor to run again. + m = InstallManifest(path=manifest) + c = FileCopier() + m.populate_registry(c) + c.copy(dest) + + with open(destfile, 'rt') as fh: + self.assertEqual(fh.read(), 'SOURCE\nINCLUDE MODIFIED\n') + + # ORing an InstallManifest should copy file dependencies + m = InstallManifest() + m |= InstallManifest(path=manifest) + c = FileCopier() + m.populate_registry(c) + e = c._files['p_dest'] + self.assertEqual(e.extra_depends, [manifest]) + +if __name__ == '__main__': + mozunit.main() diff --git a/python/mozbuild/mozpack/test/test_mozjar.py b/python/mozbuild/mozpack/test/test_mozjar.py new file mode 100644 index 000000000..948403006 --- /dev/null +++ b/python/mozbuild/mozpack/test/test_mozjar.py @@ -0,0 +1,342 @@ +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, You can obtain one at http://mozilla.org/MPL/2.0/. + +from mozpack.files import FileFinder +from mozpack.mozjar import ( + JarReaderError, + JarWriterError, + JarStruct, + JarReader, + JarWriter, + Deflater, + JarLog, +) +from collections import OrderedDict +from mozpack.test.test_files import MockDest +import unittest +import mozunit +from cStringIO import StringIO +from urllib import pathname2url +import mozpack.path as mozpath +import os + + +test_data_path = mozpath.abspath(mozpath.dirname(__file__)) +test_data_path = mozpath.join(test_data_path, 'data') + + +class TestJarStruct(unittest.TestCase): + class Foo(JarStruct): + MAGIC = 0x01020304 + STRUCT = OrderedDict([ + ('foo', 'uint32'), + ('bar', 'uint16'), + ('qux', 'uint16'), + ('length', 'uint16'), + ('length2', 'uint16'), + ('string', 'length'), + ('string2', 'length2'), + ]) + + def test_jar_struct(self): + foo = TestJarStruct.Foo() + self.assertEqual(foo.signature, TestJarStruct.Foo.MAGIC) + self.assertEqual(foo['foo'], 0) + self.assertEqual(foo['bar'], 0) + self.assertEqual(foo['qux'], 0) + self.assertFalse('length' in foo) + self.assertFalse('length2' in foo) + self.assertEqual(foo['string'], '') + self.assertEqual(foo['string2'], '') + + self.assertEqual(foo.size, 16) + + foo['foo'] = 0x42434445 + foo['bar'] = 0xabcd + foo['qux'] = 0xef01 + foo['string'] = 'abcde' + foo['string2'] = 'Arbitrarily long string' + + serialized = b'\x04\x03\x02\x01\x45\x44\x43\x42\xcd\xab\x01\xef' + \ + b'\x05\x00\x17\x00abcdeArbitrarily long string' + self.assertEqual(foo.size, len(serialized)) + foo_serialized = foo.serialize() + self.assertEqual(foo_serialized, serialized) + + def do_test_read_jar_struct(self, data): + self.assertRaises(JarReaderError, TestJarStruct.Foo, data) + self.assertRaises(JarReaderError, TestJarStruct.Foo, data[2:]) + + foo = TestJarStruct.Foo(data[1:]) + self.assertEqual(foo['foo'], 0x45444342) + self.assertEqual(foo['bar'], 0xcdab) + self.assertEqual(foo['qux'], 0x01ef) + self.assertFalse('length' in foo) + self.assertFalse('length2' in foo) + self.assertEqual(foo['string'], '012345') + self.assertEqual(foo['string2'], '67') + + def test_read_jar_struct(self): + data = b'\x00\x04\x03\x02\x01\x42\x43\x44\x45\xab\xcd\xef' + \ + b'\x01\x06\x00\x02\x0001234567890' + self.do_test_read_jar_struct(data) + + def test_read_jar_struct_memoryview(self): + data = b'\x00\x04\x03\x02\x01\x42\x43\x44\x45\xab\xcd\xef' + \ + b'\x01\x06\x00\x02\x0001234567890' + self.do_test_read_jar_struct(memoryview(data)) + + +class TestDeflater(unittest.TestCase): + def wrap(self, data): + return data + + def test_deflater_no_compress(self): + deflater = Deflater(False) + deflater.write(self.wrap('abc')) + self.assertFalse(deflater.compressed) + self.assertEqual(deflater.uncompressed_size, 3) + self.assertEqual(deflater.compressed_size, deflater.uncompressed_size) + self.assertEqual(deflater.compressed_data, 'abc') + self.assertEqual(deflater.crc32, 0x352441c2) + + def test_deflater_compress_no_gain(self): + deflater = Deflater(True) + deflater.write(self.wrap('abc')) + self.assertFalse(deflater.compressed) + self.assertEqual(deflater.uncompressed_size, 3) + self.assertEqual(deflater.compressed_size, deflater.uncompressed_size) + self.assertEqual(deflater.compressed_data, 'abc') + self.assertEqual(deflater.crc32, 0x352441c2) + + def test_deflater_compress(self): + deflater = Deflater(True) + deflater.write(self.wrap('aaaaaaaaaaaaanopqrstuvwxyz')) + self.assertTrue(deflater.compressed) + self.assertEqual(deflater.uncompressed_size, 26) + self.assertNotEqual(deflater.compressed_size, + deflater.uncompressed_size) + self.assertEqual(deflater.crc32, 0xd46b97ed) + # The CRC is the same as when not compressed + deflater = Deflater(False) + self.assertFalse(deflater.compressed) + deflater.write(self.wrap('aaaaaaaaaaaaanopqrstuvwxyz')) + self.assertEqual(deflater.crc32, 0xd46b97ed) + + +class TestDeflaterMemoryView(TestDeflater): + def wrap(self, data): + return memoryview(data) + + +class TestJar(unittest.TestCase): + optimize = False + + def test_jar(self): + s = MockDest() + with JarWriter(fileobj=s, optimize=self.optimize) as jar: + jar.add('foo', 'foo') + self.assertRaises(JarWriterError, jar.add, 'foo', 'bar') + jar.add('bar', 'aaaaaaaaaaaaanopqrstuvwxyz') + jar.add('baz/qux', 'aaaaaaaaaaaaanopqrstuvwxyz', False) + jar.add('baz\\backslash', 'aaaaaaaaaaaaaaa') + + files = [j for j in JarReader(fileobj=s)] + + self.assertEqual(files[0].filename, 'foo') + self.assertFalse(files[0].compressed) + self.assertEqual(files[0].read(), 'foo') + + self.assertEqual(files[1].filename, 'bar') + self.assertTrue(files[1].compressed) + self.assertEqual(files[1].read(), 'aaaaaaaaaaaaanopqrstuvwxyz') + + self.assertEqual(files[2].filename, 'baz/qux') + self.assertFalse(files[2].compressed) + self.assertEqual(files[2].read(), 'aaaaaaaaaaaaanopqrstuvwxyz') + + if os.sep == '\\': + self.assertEqual(files[3].filename, 'baz/backslash', + 'backslashes in filenames on Windows should get normalized') + else: + self.assertEqual(files[3].filename, 'baz\\backslash', + 'backslashes in filenames on POSIX platform are untouched') + + s = MockDest() + with JarWriter(fileobj=s, compress=False, + optimize=self.optimize) as jar: + jar.add('bar', 'aaaaaaaaaaaaanopqrstuvwxyz') + jar.add('foo', 'foo') + jar.add('baz/qux', 'aaaaaaaaaaaaanopqrstuvwxyz', True) + + jar = JarReader(fileobj=s) + files = [j for j in jar] + + self.assertEqual(files[0].filename, 'bar') + self.assertFalse(files[0].compressed) + self.assertEqual(files[0].read(), 'aaaaaaaaaaaaanopqrstuvwxyz') + + self.assertEqual(files[1].filename, 'foo') + self.assertFalse(files[1].compressed) + self.assertEqual(files[1].read(), 'foo') + + self.assertEqual(files[2].filename, 'baz/qux') + self.assertTrue(files[2].compressed) + self.assertEqual(files[2].read(), 'aaaaaaaaaaaaanopqrstuvwxyz') + + self.assertTrue('bar' in jar) + self.assertTrue('foo' in jar) + self.assertFalse('baz' in jar) + self.assertTrue('baz/qux' in jar) + self.assertTrue(jar['bar'], files[1]) + self.assertTrue(jar['foo'], files[0]) + self.assertTrue(jar['baz/qux'], files[2]) + + s.seek(0) + jar = JarReader(fileobj=s) + self.assertTrue('bar' in jar) + self.assertTrue('foo' in jar) + self.assertFalse('baz' in jar) + self.assertTrue('baz/qux' in jar) + + files[0].seek(0) + self.assertEqual(jar['bar'].filename, files[0].filename) + self.assertEqual(jar['bar'].compressed, files[0].compressed) + self.assertEqual(jar['bar'].read(), files[0].read()) + + files[1].seek(0) + self.assertEqual(jar['foo'].filename, files[1].filename) + self.assertEqual(jar['foo'].compressed, files[1].compressed) + self.assertEqual(jar['foo'].read(), files[1].read()) + + files[2].seek(0) + self.assertEqual(jar['baz/qux'].filename, files[2].filename) + self.assertEqual(jar['baz/qux'].compressed, files[2].compressed) + self.assertEqual(jar['baz/qux'].read(), files[2].read()) + + def test_rejar(self): + s = MockDest() + with JarWriter(fileobj=s, optimize=self.optimize) as jar: + jar.add('foo', 'foo') + jar.add('bar', 'aaaaaaaaaaaaanopqrstuvwxyz') + jar.add('baz/qux', 'aaaaaaaaaaaaanopqrstuvwxyz', False) + + new = MockDest() + with JarWriter(fileobj=new, optimize=self.optimize) as jar: + for j in JarReader(fileobj=s): + jar.add(j.filename, j) + + jar = JarReader(fileobj=new) + files = [j for j in jar] + + self.assertEqual(files[0].filename, 'foo') + self.assertFalse(files[0].compressed) + self.assertEqual(files[0].read(), 'foo') + + self.assertEqual(files[1].filename, 'bar') + self.assertTrue(files[1].compressed) + self.assertEqual(files[1].read(), 'aaaaaaaaaaaaanopqrstuvwxyz') + + self.assertEqual(files[2].filename, 'baz/qux') + self.assertTrue(files[2].compressed) + self.assertEqual(files[2].read(), 'aaaaaaaaaaaaanopqrstuvwxyz') + + def test_add_from_finder(self): + s = MockDest() + with JarWriter(fileobj=s, optimize=self.optimize) as jar: + finder = FileFinder(test_data_path) + for p, f in finder.find('test_data'): + jar.add('test_data', f) + + jar = JarReader(fileobj=s) + files = [j for j in jar] + + self.assertEqual(files[0].filename, 'test_data') + self.assertFalse(files[0].compressed) + self.assertEqual(files[0].read(), 'test_data') + + +class TestOptimizeJar(TestJar): + optimize = True + + +class TestPreload(unittest.TestCase): + def test_preload(self): + s = MockDest() + with JarWriter(fileobj=s) as jar: + jar.add('foo', 'foo') + jar.add('bar', 'abcdefghijklmnopqrstuvwxyz') + jar.add('baz/qux', 'aaaaaaaaaaaaanopqrstuvwxyz') + + jar = JarReader(fileobj=s) + self.assertEqual(jar.last_preloaded, None) + + with JarWriter(fileobj=s) as jar: + jar.add('foo', 'foo') + jar.add('bar', 'abcdefghijklmnopqrstuvwxyz') + jar.add('baz/qux', 'aaaaaaaaaaaaanopqrstuvwxyz') + jar.preload(['baz/qux', 'bar']) + + jar = JarReader(fileobj=s) + self.assertEqual(jar.last_preloaded, 'bar') + files = [j for j in jar] + + self.assertEqual(files[0].filename, 'baz/qux') + self.assertEqual(files[1].filename, 'bar') + self.assertEqual(files[2].filename, 'foo') + + +class TestJarLog(unittest.TestCase): + def test_jarlog(self): + base = 'file:' + pathname2url(os.path.abspath(os.curdir)) + s = StringIO('\n'.join([ + base + '/bar/baz.jar first', + base + '/bar/baz.jar second', + base + '/bar/baz.jar third', + base + '/bar/baz.jar second', + base + '/bar/baz.jar second', + 'jar:' + base + '/qux.zip!/omni.ja stuff', + base + '/bar/baz.jar first', + 'jar:' + base + '/qux.zip!/omni.ja other/stuff', + 'jar:' + base + '/qux.zip!/omni.ja stuff', + base + '/bar/baz.jar third', + 'jar:jar:' + base + '/qux.zip!/baz/baz.jar!/omni.ja nested/stuff', + 'jar:jar:jar:' + base + '/qux.zip!/baz/baz.jar!/foo.zip!/omni.ja' + + ' deeply/nested/stuff', + ])) + log = JarLog(fileobj=s) + canonicalize = lambda p: \ + mozpath.normsep(os.path.normcase(os.path.realpath(p))) + baz_jar = canonicalize('bar/baz.jar') + qux_zip = canonicalize('qux.zip') + self.assertEqual(set(log.keys()), set([ + baz_jar, + (qux_zip, 'omni.ja'), + (qux_zip, 'baz/baz.jar', 'omni.ja'), + (qux_zip, 'baz/baz.jar', 'foo.zip', 'omni.ja'), + ])) + self.assertEqual(log[baz_jar], [ + 'first', + 'second', + 'third', + ]) + self.assertEqual(log[(qux_zip, 'omni.ja')], [ + 'stuff', + 'other/stuff', + ]) + self.assertEqual(log[(qux_zip, 'baz/baz.jar', 'omni.ja')], + ['nested/stuff']) + self.assertEqual(log[(qux_zip, 'baz/baz.jar', 'foo.zip', + 'omni.ja')], ['deeply/nested/stuff']) + + # The above tests also indirectly check the value returned by + # JarLog.canonicalize for various jar: and file: urls, but + # JarLog.canonicalize also supports plain paths. + self.assertEqual(JarLog.canonicalize(os.path.abspath('bar/baz.jar')), + baz_jar) + self.assertEqual(JarLog.canonicalize('bar/baz.jar'), baz_jar) + + +if __name__ == '__main__': + mozunit.main() diff --git a/python/mozbuild/mozpack/test/test_packager.py b/python/mozbuild/mozpack/test/test_packager.py new file mode 100644 index 000000000..397f40538 --- /dev/null +++ b/python/mozbuild/mozpack/test/test_packager.py @@ -0,0 +1,490 @@ +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, You can obtain one at http://mozilla.org/MPL/2.0/. + +import unittest +import mozunit +import os +from mozpack.packager import ( + preprocess_manifest, + CallDeque, + Component, + SimplePackager, + SimpleManifestSink, +) +from mozpack.files import GeneratedFile +from mozpack.chrome.manifest import ( + ManifestBinaryComponent, + ManifestContent, + ManifestResource, +) +from mozunit import MockedOpen +from mozbuild.preprocessor import Preprocessor +from mozpack.errors import ( + errors, + ErrorMessage, +) +import mozpack.path as mozpath + +MANIFEST = ''' +bar/* +[foo] +foo/* +-foo/bar +chrome.manifest +[zot destdir="destdir"] +foo/zot +; comment +#ifdef baz +[baz] +baz@SUFFIX@ +#endif +''' + + +class TestPreprocessManifest(unittest.TestCase): + MANIFEST_PATH = os.path.join(os.path.abspath(os.curdir), 'manifest') + + EXPECTED_LOG = [ + ((MANIFEST_PATH, 2), 'add', '', 'bar/*'), + ((MANIFEST_PATH, 4), 'add', 'foo', 'foo/*'), + ((MANIFEST_PATH, 5), 'remove', 'foo', 'foo/bar'), + ((MANIFEST_PATH, 6), 'add', 'foo', 'chrome.manifest'), + ((MANIFEST_PATH, 8), 'add', 'zot destdir="destdir"', 'foo/zot'), + ] + + def setUp(self): + class MockSink(object): + def __init__(self): + self.log = [] + + def add(self, component, path): + self._log(errors.get_context(), 'add', repr(component), path) + + def remove(self, component, path): + self._log(errors.get_context(), 'remove', repr(component), path) + + def _log(self, *args): + self.log.append(args) + + self.sink = MockSink() + + def test_preprocess_manifest(self): + with MockedOpen({'manifest': MANIFEST}): + preprocess_manifest(self.sink, 'manifest') + self.assertEqual(self.sink.log, self.EXPECTED_LOG) + + def test_preprocess_manifest_missing_define(self): + with MockedOpen({'manifest': MANIFEST}): + self.assertRaises( + Preprocessor.Error, + preprocess_manifest, + self.sink, + 'manifest', + {'baz': 1} + ) + + def test_preprocess_manifest_defines(self): + with MockedOpen({'manifest': MANIFEST}): + preprocess_manifest(self.sink, 'manifest', + {'baz': 1, 'SUFFIX': '.exe'}) + self.assertEqual(self.sink.log, self.EXPECTED_LOG + + [((self.MANIFEST_PATH, 12), 'add', 'baz', 'baz.exe')]) + + +class MockFinder(object): + def __init__(self, files): + self.files = files + self.log = [] + + def find(self, path): + self.log.append(path) + for f in sorted(self.files): + if mozpath.match(f, path): + yield f, self.files[f] + + def __iter__(self): + return self.find('') + + +class MockFormatter(object): + def __init__(self): + self.log = [] + + def add_base(self, *args): + self._log(errors.get_context(), 'add_base', *args) + + def add_manifest(self, *args): + self._log(errors.get_context(), 'add_manifest', *args) + + def add_interfaces(self, *args): + self._log(errors.get_context(), 'add_interfaces', *args) + + def add(self, *args): + self._log(errors.get_context(), 'add', *args) + + def _log(self, *args): + self.log.append(args) + + +class TestSimplePackager(unittest.TestCase): + def test_simple_packager(self): + class GeneratedFileWithPath(GeneratedFile): + def __init__(self, path, content): + GeneratedFile.__init__(self, content) + self.path = path + + formatter = MockFormatter() + packager = SimplePackager(formatter) + curdir = os.path.abspath(os.curdir) + file = GeneratedFileWithPath(os.path.join(curdir, 'foo', + 'bar.manifest'), + 'resource bar bar/\ncontent bar bar/') + with errors.context('manifest', 1): + packager.add('foo/bar.manifest', file) + + file = GeneratedFileWithPath(os.path.join(curdir, 'foo', + 'baz.manifest'), + 'resource baz baz/') + with errors.context('manifest', 2): + packager.add('bar/baz.manifest', file) + + with errors.context('manifest', 3): + packager.add('qux/qux.manifest', + GeneratedFile(''.join([ + 'resource qux qux/\n', + 'binary-component qux.so\n', + ]))) + bar_xpt = GeneratedFile('bar.xpt') + qux_xpt = GeneratedFile('qux.xpt') + foo_html = GeneratedFile('foo_html') + bar_html = GeneratedFile('bar_html') + with errors.context('manifest', 4): + packager.add('foo/bar.xpt', bar_xpt) + with errors.context('manifest', 5): + packager.add('foo/bar/foo.html', foo_html) + packager.add('foo/bar/bar.html', bar_html) + + file = GeneratedFileWithPath(os.path.join(curdir, 'foo.manifest'), + ''.join([ + 'manifest foo/bar.manifest\n', + 'manifest bar/baz.manifest\n', + ])) + with errors.context('manifest', 6): + packager.add('foo.manifest', file) + with errors.context('manifest', 7): + packager.add('foo/qux.xpt', qux_xpt) + + file = GeneratedFileWithPath(os.path.join(curdir, 'addon', + 'chrome.manifest'), + 'resource hoge hoge/') + with errors.context('manifest', 8): + packager.add('addon/chrome.manifest', file) + + install_rdf = GeneratedFile('') + with errors.context('manifest', 9): + packager.add('addon/install.rdf', install_rdf) + + with errors.context('manifest', 10): + packager.add('addon2/install.rdf', install_rdf) + packager.add('addon2/chrome.manifest', + GeneratedFile('binary-component addon2.so')) + + with errors.context('manifest', 11): + packager.add('addon3/install.rdf', install_rdf) + packager.add('addon3/chrome.manifest', GeneratedFile( + 'manifest components/components.manifest')) + packager.add('addon3/components/components.manifest', + GeneratedFile('binary-component addon3.so')) + + with errors.context('manifest', 12): + install_rdf_addon4 = GeneratedFile( + '\n<...>\ntrue\n<...>\n') + packager.add('addon4/install.rdf', install_rdf_addon4) + + with errors.context('manifest', 13): + install_rdf_addon5 = GeneratedFile( + '\n<...>\nfalse\n<...>\n') + packager.add('addon5/install.rdf', install_rdf_addon5) + + with errors.context('manifest', 14): + install_rdf_addon6 = GeneratedFile( + '\n<... em:unpack=true>\n<...>\n') + packager.add('addon6/install.rdf', install_rdf_addon6) + + with errors.context('manifest', 15): + install_rdf_addon7 = GeneratedFile( + '\n<... em:unpack=false>\n<...>\n') + packager.add('addon7/install.rdf', install_rdf_addon7) + + with errors.context('manifest', 16): + install_rdf_addon8 = GeneratedFile( + '\n<... em:unpack="true">\n<...>\n') + packager.add('addon8/install.rdf', install_rdf_addon8) + + with errors.context('manifest', 17): + install_rdf_addon9 = GeneratedFile( + '\n<... em:unpack="false">\n<...>\n') + packager.add('addon9/install.rdf', install_rdf_addon9) + + with errors.context('manifest', 18): + install_rdf_addon10 = GeneratedFile( + '\n<... em:unpack=\'true\'>\n<...>\n') + packager.add('addon10/install.rdf', install_rdf_addon10) + + with errors.context('manifest', 19): + install_rdf_addon11 = GeneratedFile( + '\n<... em:unpack=\'false\'>\n<...>\n') + packager.add('addon11/install.rdf', install_rdf_addon11) + + self.assertEqual(formatter.log, []) + + with errors.context('dummy', 1): + packager.close() + self.maxDiff = None + # The formatter is expected to reorder the manifest entries so that + # chrome entries appear before the others. + self.assertEqual(formatter.log, [ + (('dummy', 1), 'add_base', '', False), + (('dummy', 1), 'add_base', 'addon', True), + (('dummy', 1), 'add_base', 'addon10', 'unpacked'), + (('dummy', 1), 'add_base', 'addon11', True), + (('dummy', 1), 'add_base', 'addon2', 'unpacked'), + (('dummy', 1), 'add_base', 'addon3', 'unpacked'), + (('dummy', 1), 'add_base', 'addon4', 'unpacked'), + (('dummy', 1), 'add_base', 'addon5', True), + (('dummy', 1), 'add_base', 'addon6', 'unpacked'), + (('dummy', 1), 'add_base', 'addon7', True), + (('dummy', 1), 'add_base', 'addon8', 'unpacked'), + (('dummy', 1), 'add_base', 'addon9', True), + (('dummy', 1), 'add_base', 'qux', False), + ((os.path.join(curdir, 'foo', 'bar.manifest'), 2), + 'add_manifest', ManifestContent('foo', 'bar', 'bar/')), + ((os.path.join(curdir, 'foo', 'bar.manifest'), 1), + 'add_manifest', ManifestResource('foo', 'bar', 'bar/')), + (('bar/baz.manifest', 1), + 'add_manifest', ManifestResource('bar', 'baz', 'baz/')), + (('qux/qux.manifest', 1), + 'add_manifest', ManifestResource('qux', 'qux', 'qux/')), + (('qux/qux.manifest', 2), + 'add_manifest', ManifestBinaryComponent('qux', 'qux.so')), + (('manifest', 4), 'add_interfaces', 'foo/bar.xpt', bar_xpt), + (('manifest', 7), 'add_interfaces', 'foo/qux.xpt', qux_xpt), + ((os.path.join(curdir, 'addon', 'chrome.manifest'), 1), + 'add_manifest', ManifestResource('addon', 'hoge', 'hoge/')), + (('addon2/chrome.manifest', 1), 'add_manifest', + ManifestBinaryComponent('addon2', 'addon2.so')), + (('addon3/components/components.manifest', 1), 'add_manifest', + ManifestBinaryComponent('addon3/components', 'addon3.so')), + (('manifest', 5), 'add', 'foo/bar/foo.html', foo_html), + (('manifest', 5), 'add', 'foo/bar/bar.html', bar_html), + (('manifest', 9), 'add', 'addon/install.rdf', install_rdf), + (('manifest', 10), 'add', 'addon2/install.rdf', install_rdf), + (('manifest', 11), 'add', 'addon3/install.rdf', install_rdf), + (('manifest', 12), 'add', 'addon4/install.rdf', + install_rdf_addon4), + (('manifest', 13), 'add', 'addon5/install.rdf', + install_rdf_addon5), + (('manifest', 14), 'add', 'addon6/install.rdf', + install_rdf_addon6), + (('manifest', 15), 'add', 'addon7/install.rdf', + install_rdf_addon7), + (('manifest', 16), 'add', 'addon8/install.rdf', + install_rdf_addon8), + (('manifest', 17), 'add', 'addon9/install.rdf', + install_rdf_addon9), + (('manifest', 18), 'add', 'addon10/install.rdf', + install_rdf_addon10), + (('manifest', 19), 'add', 'addon11/install.rdf', + install_rdf_addon11), + ]) + + self.assertEqual(packager.get_bases(), + set(['', 'addon', 'addon2', 'addon3', 'addon4', + 'addon5', 'addon6', 'addon7', 'addon8', + 'addon9', 'addon10', 'addon11', 'qux'])) + self.assertEqual(packager.get_bases(addons=False), set(['', 'qux'])) + + def test_simple_packager_manifest_consistency(self): + formatter = MockFormatter() + # bar/ is detected as an addon because of install.rdf, but top-level + # includes a manifest inside bar/. + packager = SimplePackager(formatter) + packager.add('base.manifest', GeneratedFile( + 'manifest foo/bar.manifest\n' + 'manifest bar/baz.manifest\n' + )) + packager.add('foo/bar.manifest', GeneratedFile('resource bar bar')) + packager.add('bar/baz.manifest', GeneratedFile('resource baz baz')) + packager.add('bar/install.rdf', GeneratedFile('')) + + with self.assertRaises(ErrorMessage) as e: + packager.close() + + self.assertEqual(e.exception.message, + 'Error: "bar/baz.manifest" is included from "base.manifest", ' + 'which is outside "bar"') + + # bar/ is detected as a separate base because of chrome.manifest that + # is included nowhere, but top-level includes another manifest inside + # bar/. + packager = SimplePackager(formatter) + packager.add('base.manifest', GeneratedFile( + 'manifest foo/bar.manifest\n' + 'manifest bar/baz.manifest\n' + )) + packager.add('foo/bar.manifest', GeneratedFile('resource bar bar')) + packager.add('bar/baz.manifest', GeneratedFile('resource baz baz')) + packager.add('bar/chrome.manifest', GeneratedFile('resource baz baz')) + + with self.assertRaises(ErrorMessage) as e: + packager.close() + + self.assertEqual(e.exception.message, + 'Error: "bar/baz.manifest" is included from "base.manifest", ' + 'which is outside "bar"') + + # bar/ is detected as a separate base because of chrome.manifest that + # is included nowhere, but chrome.manifest includes baz.manifest from + # the same directory. This shouldn't error out. + packager = SimplePackager(formatter) + packager.add('base.manifest', GeneratedFile( + 'manifest foo/bar.manifest\n' + )) + packager.add('foo/bar.manifest', GeneratedFile('resource bar bar')) + packager.add('bar/baz.manifest', GeneratedFile('resource baz baz')) + packager.add('bar/chrome.manifest', + GeneratedFile('manifest baz.manifest')) + packager.close() + + +class TestSimpleManifestSink(unittest.TestCase): + def test_simple_manifest_parser(self): + formatter = MockFormatter() + foobar = GeneratedFile('foobar') + foobaz = GeneratedFile('foobaz') + fooqux = GeneratedFile('fooqux') + foozot = GeneratedFile('foozot') + finder = MockFinder({ + 'bin/foo/bar': foobar, + 'bin/foo/baz': foobaz, + 'bin/foo/qux': fooqux, + 'bin/foo/zot': foozot, + 'bin/foo/chrome.manifest': GeneratedFile('resource foo foo/'), + 'bin/chrome.manifest': + GeneratedFile('manifest foo/chrome.manifest'), + }) + parser = SimpleManifestSink(finder, formatter) + component0 = Component('component0') + component1 = Component('component1') + component2 = Component('component2', destdir='destdir') + parser.add(component0, 'bin/foo/b*') + parser.add(component1, 'bin/foo/qux') + parser.add(component1, 'bin/foo/chrome.manifest') + parser.add(component2, 'bin/foo/zot') + self.assertRaises(ErrorMessage, parser.add, 'component1', 'bin/bar') + + self.assertEqual(formatter.log, []) + parser.close() + self.assertEqual(formatter.log, [ + (None, 'add_base', '', False), + (('foo/chrome.manifest', 1), + 'add_manifest', ManifestResource('foo', 'foo', 'foo/')), + (None, 'add', 'foo/bar', foobar), + (None, 'add', 'foo/baz', foobaz), + (None, 'add', 'foo/qux', fooqux), + (None, 'add', 'destdir/foo/zot', foozot), + ]) + + self.assertEqual(finder.log, [ + 'bin/foo/b*', + 'bin/foo/qux', + 'bin/foo/chrome.manifest', + 'bin/foo/zot', + 'bin/bar', + 'bin/chrome.manifest' + ]) + + +class TestCallDeque(unittest.TestCase): + def test_call_deque(self): + class Logger(object): + def __init__(self): + self._log = [] + + def log(self, str): + self._log.append(str) + + @staticmethod + def staticlog(logger, str): + logger.log(str) + + def do_log(logger, str): + logger.log(str) + + logger = Logger() + d = CallDeque() + d.append(logger.log, 'foo') + d.append(logger.log, 'bar') + d.append(logger.staticlog, logger, 'baz') + d.append(do_log, logger, 'qux') + self.assertEqual(logger._log, []) + d.execute() + self.assertEqual(logger._log, ['foo', 'bar', 'baz', 'qux']) + + +class TestComponent(unittest.TestCase): + def do_split(self, string, name, options): + n, o = Component._split_component_and_options(string) + self.assertEqual(name, n) + self.assertEqual(options, o) + + def test_component_split_component_and_options(self): + self.do_split('component', 'component', {}) + self.do_split('trailingspace ', 'trailingspace', {}) + self.do_split(' leadingspace', 'leadingspace', {}) + self.do_split(' trim ', 'trim', {}) + self.do_split(' trim key="value"', 'trim', {'key':'value'}) + self.do_split(' trim empty=""', 'trim', {'empty':''}) + self.do_split(' trim space=" "', 'trim', {'space':' '}) + self.do_split('component key="value" key2="second" ', + 'component', {'key':'value', 'key2':'second'}) + self.do_split( 'trim key=" value with spaces " key2="spaces again"', + 'trim', {'key':' value with spaces ', 'key2': 'spaces again'}) + + def do_split_error(self, string): + self.assertRaises(ValueError, Component._split_component_and_options, string) + + def test_component_split_component_and_options_errors(self): + self.do_split_error('"component') + self.do_split_error('comp"onent') + self.do_split_error('component"') + self.do_split_error('"component"') + self.do_split_error('=component') + self.do_split_error('comp=onent') + self.do_split_error('component=') + self.do_split_error('key="val"') + self.do_split_error('component key=') + self.do_split_error('component key="val') + self.do_split_error('component key=val"') + self.do_split_error('component key="val" x') + self.do_split_error('component x key="val"') + self.do_split_error('component key1="val" x key2="val"') + + def do_from_string(self, string, name, destdir=''): + component = Component.from_string(string) + self.assertEqual(name, component.name) + self.assertEqual(destdir, component.destdir) + + def test_component_from_string(self): + self.do_from_string('component', 'component') + self.do_from_string('component-with-hyphen', 'component-with-hyphen') + self.do_from_string('component destdir="foo/bar"', 'component', 'foo/bar') + self.do_from_string('component destdir="bar spc"', 'component', 'bar spc') + self.assertRaises(ErrorMessage, Component.from_string, '') + self.assertRaises(ErrorMessage, Component.from_string, 'component novalue=') + self.assertRaises(ErrorMessage, Component.from_string, 'component badoption=badvalue') + + +if __name__ == '__main__': + mozunit.main() diff --git a/python/mozbuild/mozpack/test/test_packager_formats.py b/python/mozbuild/mozpack/test/test_packager_formats.py new file mode 100644 index 000000000..1af4336b2 --- /dev/null +++ b/python/mozbuild/mozpack/test/test_packager_formats.py @@ -0,0 +1,428 @@ +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, You can obtain one at http://mozilla.org/MPL/2.0/. + +import mozunit +import unittest +from mozpack.packager.formats import ( + FlatFormatter, + JarFormatter, + OmniJarFormatter, +) +from mozpack.copier import FileRegistry +from mozpack.files import ( + GeneratedFile, + ManifestFile, +) +from mozpack.chrome.manifest import ( + ManifestContent, + ManifestComponent, + ManifestResource, + ManifestBinaryComponent, +) +from mozpack.test.test_files import ( + MockDest, + foo_xpt, + foo2_xpt, + bar_xpt, + read_interfaces, +) +import mozpack.path as mozpath + + +CONTENTS = { + 'bases': { + # base_path: is_addon? + '': False, + 'app': False, + 'addon0': 'unpacked', + 'addon1': True, + }, + 'manifests': [ + ManifestContent('chrome/f', 'oo', 'oo/'), + ManifestContent('chrome/f', 'bar', 'oo/bar/'), + ManifestResource('chrome/f', 'foo', 'resource://bar/'), + ManifestBinaryComponent('components', 'foo.so'), + ManifestContent('app/chrome', 'content', 'foo/'), + ManifestComponent('app/components', '{foo-id}', 'foo.js'), + ManifestContent('addon0/chrome', 'content', 'foo/bar/'), + ManifestContent('addon1/chrome', 'content', 'foo/bar/'), + ], + 'files': { + 'chrome/f/oo/bar/baz': GeneratedFile('foobarbaz'), + 'chrome/f/oo/baz': GeneratedFile('foobaz'), + 'chrome/f/oo/qux': GeneratedFile('fooqux'), + 'components/foo.so': GeneratedFile('foo.so'), + 'components/foo.xpt': foo_xpt, + 'components/bar.xpt': bar_xpt, + 'foo': GeneratedFile('foo'), + 'app/chrome/foo/foo': GeneratedFile('appfoo'), + 'app/components/foo.js': GeneratedFile('foo.js'), + 'addon0/chrome/foo/bar/baz': GeneratedFile('foobarbaz'), + 'addon0/components/foo.xpt': foo2_xpt, + 'addon0/components/bar.xpt': bar_xpt, + 'addon1/chrome/foo/bar/baz': GeneratedFile('foobarbaz'), + 'addon1/components/foo.xpt': foo2_xpt, + 'addon1/components/bar.xpt': bar_xpt, + }, +} + +FILES = CONTENTS['files'] + +RESULT_FLAT = { + 'chrome.manifest': [ + 'manifest chrome/chrome.manifest', + 'manifest components/components.manifest', + ], + 'chrome/chrome.manifest': [ + 'manifest f/f.manifest', + ], + 'chrome/f/f.manifest': [ + 'content oo oo/', + 'content bar oo/bar/', + 'resource foo resource://bar/', + ], + 'chrome/f/oo/bar/baz': FILES['chrome/f/oo/bar/baz'], + 'chrome/f/oo/baz': FILES['chrome/f/oo/baz'], + 'chrome/f/oo/qux': FILES['chrome/f/oo/qux'], + 'components/components.manifest': [ + 'binary-component foo.so', + 'interfaces interfaces.xpt', + ], + 'components/foo.so': FILES['components/foo.so'], + 'components/interfaces.xpt': { + 'foo': read_interfaces(foo_xpt.open())['foo'], + 'bar': read_interfaces(bar_xpt.open())['bar'], + }, + 'foo': FILES['foo'], + 'app/chrome.manifest': [ + 'manifest chrome/chrome.manifest', + 'manifest components/components.manifest', + ], + 'app/chrome/chrome.manifest': [ + 'content content foo/', + ], + 'app/chrome/foo/foo': FILES['app/chrome/foo/foo'], + 'app/components/components.manifest': [ + 'component {foo-id} foo.js', + ], + 'app/components/foo.js': FILES['app/components/foo.js'], +} + +for addon in ('addon0', 'addon1'): + RESULT_FLAT.update({ + mozpath.join(addon, p): f + for p, f in { + 'chrome.manifest': [ + 'manifest chrome/chrome.manifest', + 'manifest components/components.manifest', + ], + 'chrome/chrome.manifest': [ + 'content content foo/bar/', + ], + 'chrome/foo/bar/baz': FILES[mozpath.join(addon, 'chrome/foo/bar/baz')], + 'components/components.manifest': [ + 'interfaces interfaces.xpt', + ], + 'components/interfaces.xpt': { + 'foo': read_interfaces(foo2_xpt.open())['foo'], + 'bar': read_interfaces(bar_xpt.open())['bar'], + }, + }.iteritems() + }) + +RESULT_JAR = { + p: RESULT_FLAT[p] + for p in ( + 'chrome.manifest', + 'chrome/chrome.manifest', + 'components/components.manifest', + 'components/foo.so', + 'components/interfaces.xpt', + 'foo', + 'app/chrome.manifest', + 'app/components/components.manifest', + 'app/components/foo.js', + 'addon0/chrome.manifest', + 'addon0/components/components.manifest', + 'addon0/components/interfaces.xpt', + ) +} + +RESULT_JAR.update({ + 'chrome/f/f.manifest': [ + 'content oo jar:oo.jar!/', + 'content bar jar:oo.jar!/bar/', + 'resource foo resource://bar/', + ], + 'chrome/f/oo.jar': { + 'bar/baz': FILES['chrome/f/oo/bar/baz'], + 'baz': FILES['chrome/f/oo/baz'], + 'qux': FILES['chrome/f/oo/qux'], + }, + 'app/chrome/chrome.manifest': [ + 'content content jar:foo.jar!/', + ], + 'app/chrome/foo.jar': { + 'foo': FILES['app/chrome/foo/foo'], + }, + 'addon0/chrome/chrome.manifest': [ + 'content content jar:foo.jar!/bar/', + ], + 'addon0/chrome/foo.jar': { + 'bar/baz': FILES['addon0/chrome/foo/bar/baz'], + }, + 'addon1.xpi': { + mozpath.relpath(p, 'addon1'): f + for p, f in RESULT_FLAT.iteritems() + if p.startswith('addon1/') + }, +}) + +RESULT_OMNIJAR = { + p: RESULT_FLAT[p] + for p in ( + 'components/foo.so', + 'foo', + ) +} + +RESULT_OMNIJAR.update({ + p: RESULT_JAR[p] + for p in RESULT_JAR + if p.startswith('addon') +}) + +RESULT_OMNIJAR.update({ + 'omni.foo': { + 'components/components.manifest': [ + 'interfaces interfaces.xpt', + ], + }, + 'chrome.manifest': [ + 'manifest components/components.manifest', + ], + 'components/components.manifest': [ + 'binary-component foo.so', + ], + 'app/omni.foo': { + p: RESULT_FLAT['app/' + p] + for p in ( + 'chrome.manifest', + 'chrome/chrome.manifest', + 'chrome/foo/foo', + 'components/components.manifest', + 'components/foo.js', + ) + }, + 'app/chrome.manifest': [], +}) + +RESULT_OMNIJAR['omni.foo'].update({ + p: RESULT_FLAT[p] + for p in ( + 'chrome.manifest', + 'chrome/chrome.manifest', + 'chrome/f/f.manifest', + 'chrome/f/oo/bar/baz', + 'chrome/f/oo/baz', + 'chrome/f/oo/qux', + 'components/interfaces.xpt', + ) +}) + +CONTENTS_WITH_BASE = { + 'bases': { + mozpath.join('base/root', b) if b else 'base/root': a + for b, a in CONTENTS['bases'].iteritems() + }, + 'manifests': [ + m.move(mozpath.join('base/root', m.base)) + for m in CONTENTS['manifests'] + ], + 'files': { + mozpath.join('base/root', p): f + for p, f in CONTENTS['files'].iteritems() + }, +} + +EXTRA_CONTENTS = { + 'extra/file': GeneratedFile('extra file'), +} + +CONTENTS_WITH_BASE['files'].update(EXTRA_CONTENTS) + +def result_with_base(results): + result = { + mozpath.join('base/root', p): v + for p, v in results.iteritems() + } + result.update(EXTRA_CONTENTS) + return result + +RESULT_FLAT_WITH_BASE = result_with_base(RESULT_FLAT) +RESULT_JAR_WITH_BASE = result_with_base(RESULT_JAR) +RESULT_OMNIJAR_WITH_BASE = result_with_base(RESULT_OMNIJAR) + + +class MockDest(MockDest): + def exists(self): + return False + + +def fill_formatter(formatter, contents): + for base, is_addon in contents['bases'].items(): + formatter.add_base(base, is_addon) + + for manifest in contents['manifests']: + formatter.add_manifest(manifest) + + for k, v in contents['files'].iteritems(): + if k.endswith('.xpt'): + formatter.add_interfaces(k, v) + else: + formatter.add(k, v) + + +def get_contents(registry, read_all=False): + result = {} + for k, v in registry: + if k.endswith('.xpt'): + tmpfile = MockDest() + registry[k].copy(tmpfile) + result[k] = read_interfaces(tmpfile) + elif isinstance(v, FileRegistry): + result[k] = get_contents(v) + elif isinstance(v, ManifestFile) or read_all: + result[k] = v.open().read().splitlines() + else: + result[k] = v + return result + + +class TestFormatters(unittest.TestCase): + maxDiff = None + + def test_bases(self): + formatter = FlatFormatter(FileRegistry()) + formatter.add_base('') + formatter.add_base('browser') + formatter.add_base('addon0', addon=True) + self.assertEqual(formatter._get_base('platform.ini'), + ('', 'platform.ini')) + self.assertEqual(formatter._get_base('browser/application.ini'), + ('browser', 'application.ini')) + self.assertEqual(formatter._get_base('addon0/install.rdf'), + ('addon0', 'install.rdf')) + + def do_test_contents(self, formatter, contents): + for f in contents['files']: + # .xpt files are merged, so skip them. + if not f.endswith('.xpt'): + self.assertTrue(formatter.contains(f)) + + def test_flat_formatter(self): + registry = FileRegistry() + formatter = FlatFormatter(registry) + + fill_formatter(formatter, CONTENTS) + self.assertEqual(get_contents(registry), RESULT_FLAT) + self.do_test_contents(formatter, CONTENTS) + + def test_jar_formatter(self): + registry = FileRegistry() + formatter = JarFormatter(registry) + + fill_formatter(formatter, CONTENTS) + self.assertEqual(get_contents(registry), RESULT_JAR) + self.do_test_contents(formatter, CONTENTS) + + def test_omnijar_formatter(self): + registry = FileRegistry() + formatter = OmniJarFormatter(registry, 'omni.foo') + + fill_formatter(formatter, CONTENTS) + self.assertEqual(get_contents(registry), RESULT_OMNIJAR) + self.do_test_contents(formatter, CONTENTS) + + def test_flat_formatter_with_base(self): + registry = FileRegistry() + formatter = FlatFormatter(registry) + + fill_formatter(formatter, CONTENTS_WITH_BASE) + self.assertEqual(get_contents(registry), RESULT_FLAT_WITH_BASE) + self.do_test_contents(formatter, CONTENTS_WITH_BASE) + + def test_jar_formatter_with_base(self): + registry = FileRegistry() + formatter = JarFormatter(registry) + + fill_formatter(formatter, CONTENTS_WITH_BASE) + self.assertEqual(get_contents(registry), RESULT_JAR_WITH_BASE) + self.do_test_contents(formatter, CONTENTS_WITH_BASE) + + def test_omnijar_formatter_with_base(self): + registry = FileRegistry() + formatter = OmniJarFormatter(registry, 'omni.foo') + + fill_formatter(formatter, CONTENTS_WITH_BASE) + self.assertEqual(get_contents(registry), RESULT_OMNIJAR_WITH_BASE) + self.do_test_contents(formatter, CONTENTS_WITH_BASE) + + def test_omnijar_is_resource(self): + def is_resource(base, path): + registry = FileRegistry() + f = OmniJarFormatter(registry, 'omni.foo', non_resources=[ + 'defaults/messenger/mailViews.dat', + 'defaults/foo/*', + '*/dummy', + ]) + f.add_base('') + f.add_base('app') + f.add(mozpath.join(base, path), GeneratedFile('')) + if f.copier.contains(mozpath.join(base, path)): + return False + self.assertTrue(f.copier.contains(mozpath.join(base, 'omni.foo'))) + self.assertTrue(f.copier[mozpath.join(base, 'omni.foo')] + .contains(path)) + return True + + for base in ['', 'app/']: + self.assertTrue(is_resource(base, 'chrome')) + self.assertTrue( + is_resource(base, 'chrome/foo/bar/baz.properties')) + self.assertFalse(is_resource(base, 'chrome/icons/foo.png')) + self.assertTrue(is_resource(base, 'components/foo.js')) + self.assertFalse(is_resource(base, 'components/foo.so')) + self.assertTrue(is_resource(base, 'res/foo.css')) + self.assertFalse(is_resource(base, 'res/cursors/foo.png')) + self.assertFalse(is_resource(base, 'res/MainMenu.nib/foo')) + self.assertTrue(is_resource(base, 'defaults/pref/foo.js')) + self.assertFalse( + is_resource(base, 'defaults/pref/channel-prefs.js')) + self.assertTrue( + is_resource(base, 'defaults/preferences/foo.js')) + self.assertFalse( + is_resource(base, 'defaults/preferences/channel-prefs.js')) + self.assertTrue(is_resource(base, 'modules/foo.jsm')) + self.assertTrue(is_resource(base, 'greprefs.js')) + self.assertTrue(is_resource(base, 'hyphenation/foo')) + self.assertTrue(is_resource(base, 'update.locale')) + self.assertTrue( + is_resource(base, 'jsloader/resource/gre/modules/foo.jsm')) + self.assertFalse(is_resource(base, 'foo')) + self.assertFalse(is_resource(base, 'foo/bar/greprefs.js')) + self.assertTrue(is_resource(base, 'defaults/messenger/foo.dat')) + self.assertFalse( + is_resource(base, 'defaults/messenger/mailViews.dat')) + self.assertTrue(is_resource(base, 'defaults/pref/foo.js')) + self.assertFalse(is_resource(base, 'defaults/foo/bar.dat')) + self.assertFalse(is_resource(base, 'defaults/foo/bar/baz.dat')) + self.assertTrue(is_resource(base, 'chrome/foo/bar/baz/dummy_')) + self.assertFalse(is_resource(base, 'chrome/foo/bar/baz/dummy')) + self.assertTrue(is_resource(base, 'chrome/foo/bar/dummy_')) + self.assertFalse(is_resource(base, 'chrome/foo/bar/dummy')) + + +if __name__ == '__main__': + mozunit.main() diff --git a/python/mozbuild/mozpack/test/test_packager_l10n.py b/python/mozbuild/mozpack/test/test_packager_l10n.py new file mode 100644 index 000000000..c797eadd1 --- /dev/null +++ b/python/mozbuild/mozpack/test/test_packager_l10n.py @@ -0,0 +1,126 @@ +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, You can obtain one at http://mozilla.org/MPL/2.0/. + +import unittest +import mozunit +from test_packager import MockFinder +from mozpack.packager import l10n +from mozpack.files import ( + GeneratedFile, + ManifestFile, +) +from mozpack.chrome.manifest import ( + Manifest, + ManifestLocale, + ManifestContent, +) +from mozpack.copier import FileRegistry +from mozpack.packager.formats import FlatFormatter + + +class TestL10NRepack(unittest.TestCase): + def test_l10n_repack(self): + foo = GeneratedFile('foo') + foobar = GeneratedFile('foobar') + qux = GeneratedFile('qux') + bar = GeneratedFile('bar') + baz = GeneratedFile('baz') + dict_aa = GeneratedFile('dict_aa') + dict_bb = GeneratedFile('dict_bb') + dict_cc = GeneratedFile('dict_cc') + barbaz = GeneratedFile('barbaz') + lst = GeneratedFile('foo\nbar') + app_finder = MockFinder({ + 'bar/foo': foo, + 'chrome/foo/foobar': foobar, + 'chrome/qux/qux.properties': qux, + 'chrome/qux/baz/baz.properties': baz, + 'chrome/chrome.manifest': ManifestFile('chrome', [ + ManifestContent('chrome', 'foo', 'foo/'), + ManifestLocale('chrome', 'qux', 'en-US', 'qux/'), + ]), + 'chrome.manifest': + ManifestFile('', [Manifest('', 'chrome/chrome.manifest')]), + 'dict/aa': dict_aa, + 'app/chrome/bar/barbaz.dtd': barbaz, + 'app/chrome/chrome.manifest': ManifestFile('app/chrome', [ + ManifestLocale('app/chrome', 'bar', 'en-US', 'bar/') + ]), + 'app/chrome.manifest': + ManifestFile('app', [Manifest('app', 'chrome/chrome.manifest')]), + 'app/dict/bb': dict_bb, + 'app/dict/cc': dict_cc, + 'app/chrome/bar/search/foo.xml': foo, + 'app/chrome/bar/search/bar.xml': bar, + 'app/chrome/bar/search/lst.txt': lst, + }) + app_finder.jarlogs = {} + app_finder.base = 'app' + foo_l10n = GeneratedFile('foo_l10n') + qux_l10n = GeneratedFile('qux_l10n') + baz_l10n = GeneratedFile('baz_l10n') + barbaz_l10n = GeneratedFile('barbaz_l10n') + lst_l10n = GeneratedFile('foo\nqux') + l10n_finder = MockFinder({ + 'chrome/qux-l10n/qux.properties': qux_l10n, + 'chrome/qux-l10n/baz/baz.properties': baz_l10n, + 'chrome/chrome.manifest': ManifestFile('chrome', [ + ManifestLocale('chrome', 'qux', 'x-test', 'qux-l10n/'), + ]), + 'chrome.manifest': + ManifestFile('', [Manifest('', 'chrome/chrome.manifest')]), + 'dict/bb': dict_bb, + 'dict/cc': dict_cc, + 'app/chrome/bar-l10n/barbaz.dtd': barbaz_l10n, + 'app/chrome/chrome.manifest': ManifestFile('app/chrome', [ + ManifestLocale('app/chrome', 'bar', 'x-test', 'bar-l10n/') + ]), + 'app/chrome.manifest': + ManifestFile('app', [Manifest('app', 'chrome/chrome.manifest')]), + 'app/dict/aa': dict_aa, + 'app/chrome/bar-l10n/search/foo.xml': foo_l10n, + 'app/chrome/bar-l10n/search/qux.xml': qux_l10n, + 'app/chrome/bar-l10n/search/lst.txt': lst_l10n, + }) + l10n_finder.base = 'l10n' + copier = FileRegistry() + formatter = FlatFormatter(copier) + + l10n._repack(app_finder, l10n_finder, copier, formatter, + ['dict', 'chrome/**/search/*.xml']) + self.maxDiff = None + + repacked = { + 'bar/foo': foo, + 'chrome/foo/foobar': foobar, + 'chrome/qux-l10n/qux.properties': qux_l10n, + 'chrome/qux-l10n/baz/baz.properties': baz_l10n, + 'chrome/chrome.manifest': ManifestFile('chrome', [ + ManifestContent('chrome', 'foo', 'foo/'), + ManifestLocale('chrome', 'qux', 'x-test', 'qux-l10n/'), + ]), + 'chrome.manifest': + ManifestFile('', [Manifest('', 'chrome/chrome.manifest')]), + 'dict/bb': dict_bb, + 'dict/cc': dict_cc, + 'app/chrome/bar-l10n/barbaz.dtd': barbaz_l10n, + 'app/chrome/chrome.manifest': ManifestFile('app/chrome', [ + ManifestLocale('app/chrome', 'bar', 'x-test', 'bar-l10n/') + ]), + 'app/chrome.manifest': + ManifestFile('app', [Manifest('app', 'chrome/chrome.manifest')]), + 'app/dict/aa': dict_aa, + 'app/chrome/bar-l10n/search/foo.xml': foo_l10n, + 'app/chrome/bar-l10n/search/qux.xml': qux_l10n, + 'app/chrome/bar-l10n/search/lst.txt': lst_l10n, + } + + self.assertEqual( + dict((p, f.open().read()) for p, f in copier), + dict((p, f.open().read()) for p, f in repacked.iteritems()) + ) + + +if __name__ == '__main__': + mozunit.main() diff --git a/python/mozbuild/mozpack/test/test_packager_unpack.py b/python/mozbuild/mozpack/test/test_packager_unpack.py new file mode 100644 index 000000000..d201cabf7 --- /dev/null +++ b/python/mozbuild/mozpack/test/test_packager_unpack.py @@ -0,0 +1,65 @@ +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, You can obtain one at http://mozilla.org/MPL/2.0/. + +import mozunit +from mozpack.packager.formats import ( + FlatFormatter, + JarFormatter, + OmniJarFormatter, +) +from mozpack.packager.unpack import unpack_to_registry +from mozpack.copier import ( + FileCopier, + FileRegistry, +) +from mozpack.test.test_packager_formats import ( + CONTENTS, + fill_formatter, + get_contents, +) +from mozpack.test.test_files import TestWithTmpDir + + +class TestUnpack(TestWithTmpDir): + maxDiff = None + + @staticmethod + def _get_copier(cls): + copier = FileCopier() + formatter = cls(copier) + fill_formatter(formatter, CONTENTS) + return copier + + @classmethod + def setUpClass(cls): + cls.contents = get_contents(cls._get_copier(FlatFormatter), + read_all=True) + + def _unpack_test(self, cls): + # Format a package with the given formatter class + copier = self._get_copier(cls) + copier.copy(self.tmpdir) + + # Unpack that package. Its content is expected to match that of a Flat + # formatted package. + registry = FileRegistry() + unpack_to_registry(self.tmpdir, registry) + self.assertEqual(get_contents(registry, read_all=True), self.contents) + + def test_flat_unpack(self): + self._unpack_test(FlatFormatter) + + def test_jar_unpack(self): + self._unpack_test(JarFormatter) + + def test_omnijar_unpack(self): + class OmniFooFormatter(OmniJarFormatter): + def __init__(self, registry): + super(OmniFooFormatter, self).__init__(registry, 'omni.foo') + + self._unpack_test(OmniFooFormatter) + + +if __name__ == '__main__': + mozunit.main() diff --git a/python/mozbuild/mozpack/test/test_path.py b/python/mozbuild/mozpack/test/test_path.py new file mode 100644 index 000000000..ee41e4a69 --- /dev/null +++ b/python/mozbuild/mozpack/test/test_path.py @@ -0,0 +1,143 @@ +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, You can obtain one at http://mozilla.org/MPL/2.0/. + +from mozpack.path import ( + relpath, + join, + normpath, + dirname, + commonprefix, + basename, + split, + splitext, + basedir, + match, + rebase, +) +import unittest +import mozunit +import os + + +class TestPath(unittest.TestCase): + SEP = os.sep + + def test_relpath(self): + self.assertEqual(relpath('foo', 'foo'), '') + self.assertEqual(relpath(self.SEP.join(('foo', 'bar')), 'foo/bar'), '') + self.assertEqual(relpath(self.SEP.join(('foo', 'bar')), 'foo'), 'bar') + self.assertEqual(relpath(self.SEP.join(('foo', 'bar', 'baz')), 'foo'), + 'bar/baz') + self.assertEqual(relpath(self.SEP.join(('foo', 'bar')), 'foo/bar/baz'), + '..') + self.assertEqual(relpath(self.SEP.join(('foo', 'bar')), 'foo/baz'), + '../bar') + self.assertEqual(relpath('foo/', 'foo'), '') + self.assertEqual(relpath('foo/bar/', 'foo'), 'bar') + + def test_join(self): + self.assertEqual(join('foo', 'bar', 'baz'), 'foo/bar/baz') + self.assertEqual(join('foo', '', 'bar'), 'foo/bar') + self.assertEqual(join('', 'foo', 'bar'), 'foo/bar') + self.assertEqual(join('', 'foo', '/bar'), '/bar') + + def test_normpath(self): + self.assertEqual(normpath(self.SEP.join(('foo', 'bar', 'baz', + '..', 'qux'))), 'foo/bar/qux') + + def test_dirname(self): + self.assertEqual(dirname('foo/bar/baz'), 'foo/bar') + self.assertEqual(dirname('foo/bar'), 'foo') + self.assertEqual(dirname('foo'), '') + self.assertEqual(dirname('foo/bar/'), 'foo/bar') + + def test_commonprefix(self): + self.assertEqual(commonprefix([self.SEP.join(('foo', 'bar', 'baz')), + 'foo/qux', 'foo/baz/qux']), 'foo/') + self.assertEqual(commonprefix([self.SEP.join(('foo', 'bar', 'baz')), + 'foo/qux', 'baz/qux']), '') + + def test_basename(self): + self.assertEqual(basename('foo/bar/baz'), 'baz') + self.assertEqual(basename('foo/bar'), 'bar') + self.assertEqual(basename('foo'), 'foo') + self.assertEqual(basename('foo/bar/'), '') + + def test_split(self): + self.assertEqual(split(self.SEP.join(('foo', 'bar', 'baz'))), + ['foo', 'bar', 'baz']) + + def test_splitext(self): + self.assertEqual(splitext(self.SEP.join(('foo', 'bar', 'baz.qux'))), + ('foo/bar/baz', '.qux')) + + def test_basedir(self): + foobarbaz = self.SEP.join(('foo', 'bar', 'baz')) + self.assertEqual(basedir(foobarbaz, ['foo', 'bar', 'baz']), 'foo') + self.assertEqual(basedir(foobarbaz, ['foo', 'foo/bar', 'baz']), + 'foo/bar') + self.assertEqual(basedir(foobarbaz, ['foo/bar', 'foo', 'baz']), + 'foo/bar') + self.assertEqual(basedir(foobarbaz, ['foo', 'bar', '']), 'foo') + self.assertEqual(basedir(foobarbaz, ['bar', 'baz', '']), '') + + def test_match(self): + self.assertTrue(match('foo', '')) + self.assertTrue(match('foo/bar/baz.qux', 'foo/bar')) + self.assertTrue(match('foo/bar/baz.qux', 'foo')) + self.assertTrue(match('foo', '*')) + self.assertTrue(match('foo/bar/baz.qux', 'foo/bar/*')) + self.assertTrue(match('foo/bar/baz.qux', 'foo/bar/*')) + self.assertTrue(match('foo/bar/baz.qux', 'foo/bar/*')) + self.assertTrue(match('foo/bar/baz.qux', 'foo/bar/*')) + self.assertTrue(match('foo/bar/baz.qux', 'foo/*/baz.qux')) + self.assertTrue(match('foo/bar/baz.qux', '*/bar/baz.qux')) + self.assertTrue(match('foo/bar/baz.qux', '*/*/baz.qux')) + self.assertTrue(match('foo/bar/baz.qux', '*/*/*')) + self.assertTrue(match('foo/bar/baz.qux', 'foo/*/*')) + self.assertTrue(match('foo/bar/baz.qux', 'foo/*/*.qux')) + self.assertTrue(match('foo/bar/baz.qux', 'foo/b*/*z.qux')) + self.assertTrue(match('foo/bar/baz.qux', 'foo/b*r/ba*z.qux')) + self.assertFalse(match('foo/bar/baz.qux', 'foo/b*z/ba*r.qux')) + self.assertTrue(match('foo/bar/baz.qux', '**')) + self.assertTrue(match('foo/bar/baz.qux', '**/baz.qux')) + self.assertTrue(match('foo/bar/baz.qux', '**/bar/baz.qux')) + self.assertTrue(match('foo/bar/baz.qux', 'foo/**/baz.qux')) + self.assertTrue(match('foo/bar/baz.qux', 'foo/**/*.qux')) + self.assertTrue(match('foo/bar/baz.qux', '**/foo/bar/baz.qux')) + self.assertTrue(match('foo/bar/baz.qux', 'foo/**/bar/baz.qux')) + self.assertTrue(match('foo/bar/baz.qux', 'foo/**/bar/*.qux')) + self.assertTrue(match('foo/bar/baz.qux', 'foo/**/*.qux')) + self.assertTrue(match('foo/bar/baz.qux', '**/*.qux')) + self.assertFalse(match('foo/bar/baz.qux', '**.qux')) + self.assertFalse(match('foo/bar', 'foo/*/bar')) + self.assertTrue(match('foo/bar/baz.qux', 'foo/**/bar/**')) + self.assertFalse(match('foo/nobar/baz.qux', 'foo/**/bar/**')) + self.assertTrue(match('foo/bar', 'foo/**/bar/**')) + + def test_rebase(self): + self.assertEqual(rebase('foo', 'foo/bar', 'bar/baz'), 'baz') + self.assertEqual(rebase('foo', 'foo', 'bar/baz'), 'bar/baz') + self.assertEqual(rebase('foo/bar', 'foo', 'baz'), 'bar/baz') + + +if os.altsep: + class TestAltPath(TestPath): + SEP = os.altsep + + class TestReverseAltPath(TestPath): + def setUp(self): + sep = os.sep + os.sep = os.altsep + os.altsep = sep + + def tearDown(self): + self.setUp() + + class TestAltReverseAltPath(TestReverseAltPath): + SEP = os.altsep + + +if __name__ == '__main__': + mozunit.main() diff --git a/python/mozbuild/mozpack/test/test_unify.py b/python/mozbuild/mozpack/test/test_unify.py new file mode 100644 index 000000000..a2bbb4470 --- /dev/null +++ b/python/mozbuild/mozpack/test/test_unify.py @@ -0,0 +1,199 @@ +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, You can obtain one at http://mozilla.org/MPL/2.0/. + +from mozbuild.util import ensureParentDir + +from mozpack.unify import ( + UnifiedFinder, + UnifiedBuildFinder, +) +import mozunit +from mozpack.test.test_files import TestWithTmpDir +from mozpack.files import FileFinder +from mozpack.mozjar import JarWriter +from mozpack.test.test_files import MockDest +from cStringIO import StringIO +import os +import sys +from mozpack.errors import ( + ErrorMessage, + AccumulatedErrors, + errors, +) + + +class TestUnified(TestWithTmpDir): + def create_one(self, which, path, content): + file = self.tmppath(os.path.join(which, path)) + ensureParentDir(file) + open(file, 'wb').write(content) + + def create_both(self, path, content): + for p in ['a', 'b']: + self.create_one(p, path, content) + + +class TestUnifiedFinder(TestUnified): + def test_unified_finder(self): + self.create_both('foo/bar', 'foobar') + self.create_both('foo/baz', 'foobaz') + self.create_one('a', 'bar', 'bar') + self.create_one('b', 'baz', 'baz') + self.create_one('a', 'qux', 'foobar') + self.create_one('b', 'qux', 'baz') + self.create_one('a', 'test/foo', 'a\nb\nc\n') + self.create_one('b', 'test/foo', 'b\nc\na\n') + self.create_both('test/bar', 'a\nb\nc\n') + + finder = UnifiedFinder(FileFinder(self.tmppath('a')), + FileFinder(self.tmppath('b')), + sorted=['test']) + self.assertEqual(sorted([(f, c.open().read()) + for f, c in finder.find('foo')]), + [('foo/bar', 'foobar'), ('foo/baz', 'foobaz')]) + self.assertRaises(ErrorMessage, any, finder.find('bar')) + self.assertRaises(ErrorMessage, any, finder.find('baz')) + self.assertRaises(ErrorMessage, any, finder.find('qux')) + self.assertEqual(sorted([(f, c.open().read()) + for f, c in finder.find('test')]), + [('test/bar', 'a\nb\nc\n'), + ('test/foo', 'a\nb\nc\n')]) + + +class TestUnifiedBuildFinder(TestUnified): + def test_unified_build_finder(self): + finder = UnifiedBuildFinder(FileFinder(self.tmppath('a')), + FileFinder(self.tmppath('b'))) + + # Test chrome.manifest unification + self.create_both('chrome.manifest', 'a\nb\nc\n') + self.create_one('a', 'chrome/chrome.manifest', 'a\nb\nc\n') + self.create_one('b', 'chrome/chrome.manifest', 'b\nc\na\n') + self.assertEqual(sorted([(f, c.open().read()) for f, c in + finder.find('**/chrome.manifest')]), + [('chrome.manifest', 'a\nb\nc\n'), + ('chrome/chrome.manifest', 'a\nb\nc\n')]) + + # Test buildconfig.html unification + self.create_one('a', 'chrome/browser/foo/buildconfig.html', + '\n'.join([ + '', + '', + '

about:buildconfig

', + '
foo
', + '', + '', + ])) + self.create_one('b', 'chrome/browser/foo/buildconfig.html', + '\n'.join([ + '', + '', + '

about:buildconfig

', + '
bar
', + '', + '', + ])) + self.assertEqual(sorted([(f, c.open().read()) for f, c in + finder.find('**/buildconfig.html')]), + [('chrome/browser/foo/buildconfig.html', '\n'.join([ + '', + '', + '

about:buildconfig

', + '
foo
', + '
', + '
bar
', + '', + '', + ]))]) + + # Test xpi file unification + xpi = MockDest() + with JarWriter(fileobj=xpi, compress=True) as jar: + jar.add('foo', 'foo') + jar.add('bar', 'bar') + foo_xpi = xpi.read() + self.create_both('foo.xpi', foo_xpi) + + with JarWriter(fileobj=xpi, compress=True) as jar: + jar.add('foo', 'bar') + self.create_one('a', 'bar.xpi', foo_xpi) + self.create_one('b', 'bar.xpi', xpi.read()) + + errors.out = StringIO() + with self.assertRaises(AccumulatedErrors), errors.accumulate(): + self.assertEqual([(f, c.open().read()) for f, c in + finder.find('*.xpi')], + [('foo.xpi', foo_xpi)]) + errors.out = sys.stderr + + # Test install.rdf unification + x86_64 = 'Darwin_x86_64-gcc3' + x86 = 'Darwin_x86-gcc3' + target_tag = '<{em}targetPlatform>{platform}' + target_attr = '{em}targetPlatform="{platform}" ' + + rdf_tag = ''.join([ + '<{RDF}Description {em}bar="bar" {em}qux="qux">', + '<{em}foo>foo', + '{targets}', + '<{em}baz>baz', + '' + ]) + rdf_attr = ''.join([ + '<{RDF}Description {em}bar="bar" {attr}{em}qux="qux">', + '{targets}', + '<{em}foo>foo<{em}baz>baz', + '' + ]) + + for descr_ns, target_ns in (('RDF:', ''), ('', 'em:'), ('RDF:', 'em:')): + # First we need to infuse the above strings with our namespaces and + # platform values. + ns = { 'RDF': descr_ns, 'em': target_ns } + target_tag_x86_64 = target_tag.format(platform=x86_64, **ns) + target_tag_x86 = target_tag.format(platform=x86, **ns) + target_attr_x86_64 = target_attr.format(platform=x86_64, **ns) + target_attr_x86 = target_attr.format(platform=x86, **ns) + + tag_x86_64 = rdf_tag.format(targets=target_tag_x86_64, **ns) + tag_x86 = rdf_tag.format(targets=target_tag_x86, **ns) + tag_merged = rdf_tag.format(targets=target_tag_x86_64 + target_tag_x86, **ns) + tag_empty = rdf_tag.format(targets="", **ns) + + attr_x86_64 = rdf_attr.format(attr=target_attr_x86_64, targets="", **ns) + attr_x86 = rdf_attr.format(attr=target_attr_x86, targets="", **ns) + attr_merged = rdf_attr.format(attr="", targets=target_tag_x86_64 + target_tag_x86, **ns) + + # This table defines the test cases, columns "a" and "b" being the + # contents of the install.rdf of the respective platform and + # "result" the exepected merged content after unification. + testcases = ( + #_____a_____ _____b_____ ___result___# + (tag_x86_64, tag_x86, tag_merged ), + (tag_x86_64, tag_empty, tag_empty ), + (tag_empty, tag_x86, tag_empty ), + (tag_empty, tag_empty, tag_empty ), + + (attr_x86_64, attr_x86, attr_merged ), + (tag_x86_64, attr_x86, tag_merged ), + (attr_x86_64, tag_x86, attr_merged ), + + (attr_x86_64, tag_empty, tag_empty ), + (tag_empty, attr_x86, tag_empty ) + ) + + # Now create the files from the above table and compare + results = [] + for emid, (rdf_a, rdf_b, result) in enumerate(testcases): + filename = 'ext/id{0}/install.rdf'.format(emid) + self.create_one('a', filename, rdf_a) + self.create_one('b', filename, rdf_b) + results.append((filename, result)) + + self.assertEqual(sorted([(f, c.open().read()) for f, c in + finder.find('**/install.rdf')]), results) + + +if __name__ == '__main__': + mozunit.main() diff --git a/python/mozbuild/mozpack/unify.py b/python/mozbuild/mozpack/unify.py new file mode 100644 index 000000000..3c8a8d605 --- /dev/null +++ b/python/mozbuild/mozpack/unify.py @@ -0,0 +1,231 @@ +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, You can obtain one at http://mozilla.org/MPL/2.0/. + +from __future__ import absolute_import + +from mozpack.files import ( + BaseFinder, + JarFinder, + ExecutableFile, + BaseFile, + GeneratedFile, +) +from mozpack.executables import ( + MACHO_SIGNATURES, +) +from mozpack.mozjar import JarReader +from mozpack.errors import errors +from tempfile import mkstemp +import mozpack.path as mozpath +import struct +import os +import re +import subprocess +import buildconfig +from collections import OrderedDict + +# Regular expressions for unifying install.rdf +FIND_TARGET_PLATFORM = re.compile(r""" + <(?P[-._0-9A-Za-z]+:)?targetPlatform> # The targetPlatform tag, with any namespace + (?P[^<]*) # The actual platform value + # The closing tag + """, re.X) +FIND_TARGET_PLATFORM_ATTR = re.compile(r""" + (?P<(?:[-._0-9A-Za-z]+:)?Description) # The opening part of the tag + (?P[^>]*?)\s+ # The initial attributes + (?P[-._0-9A-Za-z]+:)?targetPlatform= # The targetPlatform attribute, with any namespace + [\'"](?P[^\'"]+)[\'"] # The actual platform value + (?P[^>]*?>) # The remaining attributes and closing angle bracket + """, re.X) + +def may_unify_binary(file): + ''' + Return whether the given BaseFile instance is an ExecutableFile that + may be unified. Only non-fat Mach-O binaries are to be unified. + ''' + if isinstance(file, ExecutableFile): + signature = file.open().read(4) + if len(signature) < 4: + return False + signature = struct.unpack('>L', signature)[0] + if signature in MACHO_SIGNATURES: + return True + return False + + +class UnifiedExecutableFile(BaseFile): + ''' + File class for executable and library files that to be unified with 'lipo'. + ''' + def __init__(self, executable1, executable2): + ''' + Initialize a UnifiedExecutableFile with a pair of ExecutableFiles to + be unified. They are expected to be non-fat Mach-O executables. + ''' + assert isinstance(executable1, ExecutableFile) + assert isinstance(executable2, ExecutableFile) + self._executables = (executable1, executable2) + + def copy(self, dest, skip_if_older=True): + ''' + Create a fat executable from the two Mach-O executable given when + creating the instance. + skip_if_older is ignored. + ''' + assert isinstance(dest, basestring) + tmpfiles = [] + try: + for e in self._executables: + fd, f = mkstemp() + os.close(fd) + tmpfiles.append(f) + e.copy(f, skip_if_older=False) + lipo = buildconfig.substs.get('LIPO') or 'lipo' + subprocess.call([lipo, '-create'] + tmpfiles + ['-output', dest]) + finally: + for f in tmpfiles: + os.unlink(f) + + +class UnifiedFinder(BaseFinder): + ''' + Helper to get unified BaseFile instances from two distinct trees on the + file system. + ''' + def __init__(self, finder1, finder2, sorted=[], **kargs): + ''' + Initialize a UnifiedFinder. finder1 and finder2 are BaseFinder + instances from which files are picked. UnifiedFinder.find() will act as + FileFinder.find() but will error out when matches can only be found in + one of the two trees and not the other. It will also error out if + matches can be found on both ends but their contents are not identical. + + The sorted argument gives a list of mozpath.match patterns. File + paths matching one of these patterns will have their contents compared + with their lines sorted. + ''' + assert isinstance(finder1, BaseFinder) + assert isinstance(finder2, BaseFinder) + self._finder1 = finder1 + self._finder2 = finder2 + self._sorted = sorted + BaseFinder.__init__(self, finder1.base, **kargs) + + def _find(self, path): + ''' + UnifiedFinder.find() implementation. + ''' + files1 = OrderedDict() + for p, f in self._finder1.find(path): + files1[p] = f + files2 = set() + for p, f in self._finder2.find(path): + files2.add(p) + if p in files1: + if may_unify_binary(files1[p]) and \ + may_unify_binary(f): + yield p, UnifiedExecutableFile(files1[p], f) + else: + err = errors.count + unified = self.unify_file(p, files1[p], f) + if unified: + yield p, unified + elif err == errors.count: + self._report_difference(p, files1[p], f) + else: + errors.error('File missing in %s: %s' % + (self._finder1.base, p)) + for p in [p for p in files1 if not p in files2]: + errors.error('File missing in %s: %s' % (self._finder2.base, p)) + + def _report_difference(self, path, file1, file2): + ''' + Report differences between files in both trees. + ''' + errors.error("Can't unify %s: file differs between %s and %s" % + (path, self._finder1.base, self._finder2.base)) + if not isinstance(file1, ExecutableFile) and \ + not isinstance(file2, ExecutableFile): + from difflib import unified_diff + for line in unified_diff(file1.open().readlines(), + file2.open().readlines(), + os.path.join(self._finder1.base, path), + os.path.join(self._finder2.base, path)): + errors.out.write(line) + + def unify_file(self, path, file1, file2): + ''' + Given two BaseFiles and the path they were found at, check whether + their content match and return the first BaseFile if they do. + ''' + content1 = file1.open().readlines() + content2 = file2.open().readlines() + if content1 == content2: + return file1 + for pattern in self._sorted: + if mozpath.match(path, pattern): + if sorted(content1) == sorted(content2): + return file1 + break + return None + + +class UnifiedBuildFinder(UnifiedFinder): + ''' + Specialized UnifiedFinder for Mozilla applications packaging. It allows + "*.manifest" files to differ in their order, and unifies "buildconfig.html" + files by merging their content. + ''' + def __init__(self, finder1, finder2, **kargs): + UnifiedFinder.__init__(self, finder1, finder2, + sorted=['**/*.manifest'], **kargs) + + def unify_file(self, path, file1, file2): + ''' + Unify files taking Mozilla application special cases into account. + Otherwise defer to UnifiedFinder.unify_file. + ''' + basename = mozpath.basename(path) + if basename == 'buildconfig.html': + content1 = file1.open().readlines() + content2 = file2.open().readlines() + # Copy everything from the first file up to the end of its , + # insert a
between the two files and copy the second file's + # content beginning after its leading

. + return GeneratedFile(''.join( + content1[:content1.index('\n')] + + ['
\n'] + + content2[content2.index('

about:buildconfig

\n') + 1:] + )) + elif basename == 'install.rdf': + # install.rdf files often have em:targetPlatform (either as + # attribute or as tag) that will differ between platforms. The + # unified install.rdf should contain both em:targetPlatforms if + # they exist, or strip them if only one file has a target platform. + content1, content2 = ( + FIND_TARGET_PLATFORM_ATTR.sub(lambda m: \ + m.group('tag') + m.group('attrs') + m.group('otherattrs') + + '<%stargetPlatform>%s' % \ + (m.group('ns') or "", m.group('platform'), m.group('ns') or ""), + f.open().read() + ) for f in (file1, file2) + ) + + platform2 = FIND_TARGET_PLATFORM.search(content2) + return GeneratedFile(FIND_TARGET_PLATFORM.sub( + lambda m: m.group(0) + platform2.group(0) if platform2 else '', + content1 + )) + elif path.endswith('.xpi'): + finder1 = JarFinder(os.path.join(self._finder1.base, path), + JarReader(fileobj=file1.open())) + finder2 = JarFinder(os.path.join(self._finder2.base, path), + JarReader(fileobj=file2.open())) + unifier = UnifiedFinder(finder1, finder2, sorted=self._sorted) + err = errors.count + all(unifier.find('')) + if err == errors.count: + return file1 + return None + return UnifiedFinder.unify_file(self, path, file1, file2) diff --git a/python/mozbuild/setup.py b/python/mozbuild/setup.py new file mode 100644 index 000000000..448a1362a --- /dev/null +++ b/python/mozbuild/setup.py @@ -0,0 +1,29 @@ +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, You can obtain one at http://mozilla.org/MPL/2.0/. + +from setuptools import setup, find_packages + +VERSION = '0.2' + +setup( + author='Mozilla Foundation', + author_email='dev-builds@lists.mozilla.org', + name='mozbuild', + description='Mozilla build system functionality.', + license='MPL 2.0', + packages=find_packages(), + version=VERSION, + install_requires=[ + 'jsmin', + 'mozfile', + ], + classifiers=[ + 'Development Status :: 3 - Alpha', + 'Topic :: Software Development :: Build Tools', + 'License :: OSI Approved :: Mozilla Public License 2.0 (MPL 2.0)', + 'Programming Language :: Python :: 2.7', + 'Programming Language :: Python :: Implementation :: CPython', + ], + keywords='mozilla build', +) -- cgit v1.2.3