From 5f8de423f190bbb79a62f804151bc24824fa32d8 Mon Sep 17 00:00:00 2001 From: "Matt A. Tobin" Date: Fri, 2 Feb 2018 04:16:08 -0500 Subject: Add m-esr52 at 52.6.0 --- testing/mozbase/README.md | 21 + testing/mozbase/docs/Makefile | 153 ++ testing/mozbase/docs/_static/structured_example.py | 102 + testing/mozbase/docs/conf.py | 258 +++ testing/mozbase/docs/devicemanagement.rst | 11 + testing/mozbase/docs/gettinginfo.rst | 13 + testing/mozbase/docs/index.rst | 57 + testing/mozbase/docs/loggingreporting.rst | 11 + testing/mozbase/docs/make.bat | 190 ++ testing/mozbase/docs/manifestparser.rst | 558 +++++ testing/mozbase/docs/mozcrash.rst | 8 + testing/mozbase/docs/mozdebug.rst | 5 + testing/mozbase/docs/mozdevice.rst | 254 +++ testing/mozbase/docs/mozfile.rst | 10 + testing/mozbase/docs/mozhttpd.rst | 22 + testing/mozbase/docs/mozinfo.rst | 71 + testing/mozbase/docs/mozinstall.rst | 29 + testing/mozbase/docs/mozlog.rst | 486 +++++ testing/mozbase/docs/moznetwork.rst | 9 + testing/mozbase/docs/mozprocess.rst | 324 +++ testing/mozbase/docs/mozprofile.rst | 99 + testing/mozbase/docs/mozrunner.rst | 177 ++ testing/mozbase/docs/mozversion.rst | 112 + testing/mozbase/docs/requirements.txt | 1 + testing/mozbase/docs/setuprunning.rst | 18 + .../manifestparser/manifestparser/__init__.py | 8 + .../mozbase/manifestparser/manifestparser/cli.py | 246 +++ .../manifestparser/manifestparser/expression.py | 324 +++ .../manifestparser/manifestparser/filters.py | 421 ++++ .../mozbase/manifestparser/manifestparser/ini.py | 142 ++ .../manifestparser/manifestparser.py | 804 +++++++ testing/mozbase/manifestparser/setup.py | 27 + .../manifestparser/tests/comment-example.ini | 11 + .../manifestparser/tests/default-skipif.ini | 22 + .../manifestparser/tests/default-suppfiles.ini | 9 + .../manifestparser/tests/filter-example.ini | 11 + testing/mozbase/manifestparser/tests/fleem | 1 + .../manifestparser/tests/include-example.ini | 11 + .../manifestparser/tests/include-invalid.ini | 1 + .../mozbase/manifestparser/tests/include/bar.ini | 4 + .../manifestparser/tests/include/crash-handling | 1 + .../mozbase/manifestparser/tests/include/flowers | 1 + .../mozbase/manifestparser/tests/include/foo.ini | 5 + .../mozbase/manifestparser/tests/just-defaults.ini | 2 + testing/mozbase/manifestparser/tests/manifest.ini | 11 + .../mozbase/manifestparser/tests/missing-path.ini | 2 + .../manifestparser/tests/mozmill-example.ini | 80 + .../tests/mozmill-restart-example.ini | 26 + testing/mozbase/manifestparser/tests/no-tests.ini | 2 + .../tests/parent/include/first/manifest.ini | 3 + .../tests/parent/include/manifest.ini | 8 + .../tests/parent/include/second/manifest.ini | 3 + .../tests/parent/level_1/level_1.ini | 5 + .../tests/parent/level_1/level_1_server-root.ini | 5 + .../tests/parent/level_1/level_2/level_2.ini | 3 + .../parent/level_1/level_2/level_2_server-root.ini | 3 + .../parent/level_1/level_2/level_3/level_3.ini | 3 + .../level_1/level_2/level_3/level_3_default.ini | 6 + .../level_2/level_3/level_3_server-root.ini | 3 + .../tests/parent/level_1/level_2/level_3/test_3 | 1 + .../tests/parent/level_1/level_2/test_2 | 1 + .../manifestparser/tests/parent/level_1/test_1 | 1 + .../mozbase/manifestparser/tests/parent/root/dummy | 0 .../mozbase/manifestparser/tests/path-example.ini | 2 + .../mozbase/manifestparser/tests/relative-path.ini | 5 + testing/mozbase/manifestparser/tests/subsuite.ini | 13 + .../mozbase/manifestparser/tests/test_chunking.py | 302 +++ .../manifestparser/tests/test_convert_directory.py | 181 ++ .../manifestparser/tests/test_convert_symlinks.py | 139 ++ .../manifestparser/tests/test_default_overrides.py | 115 + .../manifestparser/tests/test_expressionparser.py | 152 ++ .../mozbase/manifestparser/tests/test_filters.py | 182 ++ .../manifestparser/tests/test_manifestparser.py | 325 +++ .../mozbase/manifestparser/tests/test_read_ini.py | 70 + .../manifestparser/tests/test_testmanifest.py | 122 ++ .../tests/verifyDirectory/subdir/manifest.ini | 1 + .../tests/verifyDirectory/subdir/test_sub.js | 1 + .../manifestparser/tests/verifyDirectory/test_1.js | 1 + .../manifestparser/tests/verifyDirectory/test_2.js | 1 + .../manifestparser/tests/verifyDirectory/test_3.js | 1 + .../tests/verifyDirectory/verifyDirectory.ini | 4 + .../verifyDirectory/verifyDirectory_incomplete.ini | 3 + .../verifyDirectory_toocomplete.ini | 5 + testing/mozbase/moz.build | 38 + testing/mozbase/mozcrash/mozcrash/__init__.py | 10 + testing/mozbase/mozcrash/mozcrash/mozcrash.py | 557 +++++ testing/mozbase/mozcrash/setup.py | 29 + testing/mozbase/mozcrash/tests/manifest.ini | 1 + testing/mozbase/mozcrash/tests/test.py | 241 +++ testing/mozbase/mozdebug/mozdebug/__init__.py | 31 + testing/mozbase/mozdebug/mozdebug/mozdebug.py | 291 +++ testing/mozbase/mozdebug/setup.py | 27 + .../adb_tests/test_device_running_adb_as_root.py | 48 + .../mozdevice/adb_tests/test_devicemanagerADB.py | 219 ++ testing/mozbase/mozdevice/mozdevice/Zeroconf.py | 1560 ++++++++++++++ testing/mozbase/mozdevice/mozdevice/__init__.py | 15 + testing/mozbase/mozdevice/mozdevice/adb.py | 2271 ++++++++++++++++++++ testing/mozbase/mozdevice/mozdevice/adb_android.py | 493 +++++ testing/mozbase/mozdevice/mozdevice/adb_b2g.py | 122 ++ .../mozbase/mozdevice/mozdevice/devicemanager.py | 674 ++++++ .../mozdevice/mozdevice/devicemanagerADB.py | 893 ++++++++ .../mozdevice/mozdevice/devicemanagerSUT.py | 975 +++++++++ testing/mozbase/mozdevice/mozdevice/dmcli.py | 382 ++++ testing/mozbase/mozdevice/mozdevice/droid.py | 263 +++ testing/mozbase/mozdevice/mozdevice/sutini.py | 126 ++ .../mozbase/mozdevice/mozdevice/version_codes.py | 61 + testing/mozbase/mozdevice/setup.py | 36 + testing/mozbase/mozdevice/sut_tests/README.md | 15 + testing/mozbase/mozdevice/sut_tests/dmunit.py | 55 + testing/mozbase/mozdevice/sut_tests/genfiles.py | 85 + testing/mozbase/mozdevice/sut_tests/runtests.py | 96 + testing/mozbase/mozdevice/sut_tests/setup-tools.sh | 10 + .../mozdevice/sut_tests/test-files/mytext.txt | 177 ++ .../mozdevice/sut_tests/test-files/smalltext.txt | 1 + .../mozdevice/sut_tests/test-files/test_script.sh | 1 + .../mozdevice/sut_tests/test_datachannel.py | 53 + testing/mozbase/mozdevice/sut_tests/test_exec.py | 24 + .../mozbase/mozdevice/sut_tests/test_exec_env.py | 32 + .../mozbase/mozdevice/sut_tests/test_fileExists.py | 37 + testing/mozbase/mozdevice/sut_tests/test_getdir.py | 51 + testing/mozbase/mozdevice/sut_tests/test_info.py | 20 + testing/mozbase/mozdevice/sut_tests/test_prompt.py | 30 + testing/mozbase/mozdevice/sut_tests/test_ps.py | 27 + testing/mozbase/mozdevice/sut_tests/test_pull.py | 34 + testing/mozbase/mozdevice/sut_tests/test_push1.py | 38 + testing/mozbase/mozdevice/sut_tests/test_push2.py | 39 + .../mozbase/mozdevice/sut_tests/test_pushbinary.py | 19 + .../mozdevice/sut_tests/test_pushsmalltext.py | 19 + testing/mozbase/mozdevice/tests/droidsut_launch.py | 36 + testing/mozbase/mozdevice/tests/manifest.ini | 23 + testing/mozbase/mozdevice/tests/sut.py | 89 + testing/mozbase/mozdevice/tests/sut_app.py | 20 + testing/mozbase/mozdevice/tests/sut_basic.py | 73 + testing/mozbase/mozdevice/tests/sut_chmod.py | 22 + testing/mozbase/mozdevice/tests/sut_copytree.py | 67 + testing/mozbase/mozdevice/tests/sut_fileExists.py | 29 + testing/mozbase/mozdevice/tests/sut_fileMethods.py | 72 + testing/mozbase/mozdevice/tests/sut_info.py | 49 + testing/mozbase/mozdevice/tests/sut_ip.py | 37 + testing/mozbase/mozdevice/tests/sut_kill.py | 24 + testing/mozbase/mozdevice/tests/sut_list.py | 22 + testing/mozbase/mozdevice/tests/sut_logcat.py | 52 + testing/mozbase/mozdevice/tests/sut_mkdir.py | 78 + testing/mozbase/mozdevice/tests/sut_movetree.py | 65 + testing/mozbase/mozdevice/tests/sut_ps.py | 50 + testing/mozbase/mozdevice/tests/sut_pull.py | 47 + testing/mozbase/mozdevice/tests/sut_push.py | 88 + testing/mozbase/mozdevice/tests/sut_remove.py | 24 + testing/mozbase/mozdevice/tests/sut_time.py | 18 + testing/mozbase/mozdevice/tests/sut_unpackfile.py | 23 + testing/mozbase/mozfile/mozfile/__init__.py | 8 + testing/mozbase/mozfile/mozfile/mozfile.py | 449 ++++ testing/mozbase/mozfile/setup.py | 25 + .../tests/files/missing_file_attributes.zip | Bin 0 -> 442 bytes testing/mozbase/mozfile/tests/manifest.ini | 6 + testing/mozbase/mozfile/tests/stubs.py | 37 + testing/mozbase/mozfile/tests/test_extract.py | 154 ++ testing/mozbase/mozfile/tests/test_load.py | 62 + testing/mozbase/mozfile/tests/test_move_remove.py | 232 ++ testing/mozbase/mozfile/tests/test_tempdir.py | 42 + testing/mozbase/mozfile/tests/test_tempfile.py | 102 + testing/mozbase/mozfile/tests/test_url.py | 21 + testing/mozbase/mozhttpd/mozhttpd/__init__.py | 48 + testing/mozbase/mozhttpd/mozhttpd/handlers.py | 16 + testing/mozbase/mozhttpd/mozhttpd/mozhttpd.py | 330 +++ testing/mozbase/mozhttpd/setup.py | 29 + testing/mozbase/mozhttpd/tests/api.py | 266 +++ testing/mozbase/mozhttpd/tests/baseurl.py | 19 + testing/mozbase/mozhttpd/tests/basic.py | 46 + testing/mozbase/mozhttpd/tests/filelisting.py | 43 + testing/mozbase/mozhttpd/tests/manifest.ini | 6 + testing/mozbase/mozhttpd/tests/paths.py | 77 + testing/mozbase/mozhttpd/tests/requestlog.py | 41 + testing/mozbase/mozinfo/mozinfo/__init__.py | 60 + testing/mozbase/mozinfo/mozinfo/mozinfo.py | 300 +++ testing/mozbase/mozinfo/mozinfo/string_version.py | 43 + testing/mozbase/mozinfo/setup.py | 31 + testing/mozbase/mozinfo/tests/manifest.ini | 1 + testing/mozbase/mozinfo/tests/test.py | 121 ++ testing/mozbase/mozinstall/mozinstall/__init__.py | 6 + .../mozbase/mozinstall/mozinstall/mozinstall.py | 342 +++ testing/mozbase/mozinstall/setup.py | 53 + .../mozinstall/tests/Installer-Stubs/firefox.dmg | Bin 0 -> 13441 bytes .../tests/Installer-Stubs/firefox.tar.bz2 | Bin 0 -> 2882 bytes .../mozinstall/tests/Installer-Stubs/firefox.zip | Bin 0 -> 8707 bytes testing/mozbase/mozinstall/tests/manifest.ini | 1 + testing/mozbase/mozinstall/tests/test.py | 169 ++ testing/mozbase/mozleak/mozleak/__init__.py | 11 + testing/mozbase/mozleak/mozleak/leaklog.py | 205 ++ testing/mozbase/mozleak/setup.py | 26 + testing/mozbase/mozlog/mozlog/__init__.py | 30 + testing/mozbase/mozlog/mozlog/commandline.py | 282 +++ .../mozbase/mozlog/mozlog/formatters/__init__.py | 23 + testing/mozbase/mozlog/mozlog/formatters/base.py | 20 + .../mozlog/mozlog/formatters/errorsummary.py | 69 + .../mozlog/mozlog/formatters/html/__init__.py | 3 + .../mozbase/mozlog/mozlog/formatters/html/html.py | 236 ++ .../mozbase/mozlog/mozlog/formatters/html/main.js | 172 ++ .../mozlog/mozlog/formatters/html/style.css | 154 ++ .../mozlog/mozlog/formatters/html/xmlgen.py | 283 +++ .../mozlog/mozlog/formatters/machformatter.py | 395 ++++ .../mozbase/mozlog/mozlog/formatters/process.py | 55 + .../mozlog/mozlog/formatters/tbplformatter.py | 244 +++ .../mozbase/mozlog/mozlog/formatters/unittest.py | 60 + testing/mozbase/mozlog/mozlog/formatters/xunit.py | 101 + testing/mozbase/mozlog/mozlog/handlers/__init__.py | 11 + testing/mozbase/mozlog/mozlog/handlers/base.py | 105 + .../mozlog/mozlog/handlers/bufferhandler.py | 83 + .../mozlog/mozlog/handlers/statushandler.py | 52 + .../mozlog/mozlog/handlers/valgrindhandler.py | 140 ++ testing/mozbase/mozlog/mozlog/logtypes.py | 204 ++ testing/mozbase/mozlog/mozlog/proxy.py | 35 + .../mozlog/mozlog/pytest_mozlog/__init__.py | 0 .../mozbase/mozlog/mozlog/pytest_mozlog/plugin.py | 94 + testing/mozbase/mozlog/mozlog/reader.py | 77 + testing/mozbase/mozlog/mozlog/scripts/__init__.py | 32 + testing/mozbase/mozlog/mozlog/scripts/format.py | 42 + testing/mozbase/mozlog/mozlog/scripts/logmerge.py | 82 + testing/mozbase/mozlog/mozlog/scripts/unstable.py | 120 ++ testing/mozbase/mozlog/mozlog/stdadapter.py | 45 + testing/mozbase/mozlog/mozlog/structuredlog.py | 521 +++++ .../mozbase/mozlog/mozlog/unstructured/__init__.py | 8 + .../mozbase/mozlog/mozlog/unstructured/logger.py | 185 ++ .../mozlog/mozlog/unstructured/loggingmixin.py | 44 + .../mozlog/mozlog/unstructured/loglistener.py | 50 + testing/mozbase/mozlog/setup.py | 39 + testing/mozbase/mozlog/tests/manifest.ini | 2 + testing/mozbase/mozlog/tests/test_logger.py | 264 +++ testing/mozbase/mozlog/tests/test_structured.py | 1098 ++++++++++ testing/mozbase/moznetwork/moznetwork/__init__.py | 26 + .../mozbase/moznetwork/moznetwork/moznetwork.py | 172 ++ testing/mozbase/moznetwork/setup.py | 29 + testing/mozbase/moznetwork/tests/manifest.ini | 1 + testing/mozbase/moznetwork/tests/test.py | 85 + testing/mozbase/mozprocess/mozprocess/__init__.py | 8 + .../mozprocess/mozprocess/processhandler.py | 1079 ++++++++++ testing/mozbase/mozprocess/mozprocess/qijo.py | 166 ++ .../mozbase/mozprocess/mozprocess/winprocess.py | 479 +++++ testing/mozbase/mozprocess/setup.py | 33 + testing/mozbase/mozprocess/tests/Makefile | 55 + testing/mozbase/mozprocess/tests/infinite_loop.py | 18 + testing/mozbase/mozprocess/tests/iniparser/AUTHORS | 6 + testing/mozbase/mozprocess/tests/iniparser/INSTALL | 15 + testing/mozbase/mozprocess/tests/iniparser/LICENSE | 21 + .../mozbase/mozprocess/tests/iniparser/Makefile | 85 + testing/mozbase/mozprocess/tests/iniparser/README | 12 + .../mozprocess/tests/iniparser/dictionary.c | 407 ++++ .../mozprocess/tests/iniparser/dictionary.h | 176 ++ .../mozbase/mozprocess/tests/iniparser/iniparser.c | 648 ++++++ .../mozbase/mozprocess/tests/iniparser/iniparser.h | 273 +++ .../mozbase/mozprocess/tests/iniparser/platform.mk | 8 + testing/mozbase/mozprocess/tests/manifest.ini | 18 + testing/mozbase/mozprocess/tests/proccountfive.py | 2 + .../tests/process_normal_broad_python.ini | 30 + .../tests/process_normal_deep_python.ini | 65 + .../mozprocess/tests/process_normal_finish.ini | 11 + .../process_normal_finish_no_process_group.ini | 2 + .../tests/process_normal_finish_python.ini | 17 + .../mozprocess/tests/process_waittimeout.ini | 11 + .../mozprocess/tests/process_waittimeout_10s.ini | 8 + .../tests/process_waittimeout_10s_python.ini | 16 + .../tests/process_waittimeout_python.ini | 16 + testing/mozbase/mozprocess/tests/proclaunch.c | 156 ++ testing/mozbase/mozprocess/tests/proclaunch.py | 199 ++ testing/mozbase/mozprocess/tests/procnonewline.py | 3 + testing/mozbase/mozprocess/tests/proctest.py | 52 + .../mozbase/mozprocess/tests/test_mozprocess.py | 235 ++ .../mozprocess/tests/test_mozprocess_kill.py | 91 + .../tests/test_mozprocess_kill_broad_wait.py | 33 + .../mozprocess/tests/test_mozprocess_misc.py | 41 + .../mozprocess/tests/test_mozprocess_output.py | 57 + .../mozprocess/tests/test_mozprocess_params.py | 84 + .../mozprocess/tests/test_mozprocess_poll.py | 106 + .../mozprocess/tests/test_mozprocess_wait.py | 96 + .../mozprocess/tests/test_process_reader.py | 101 + testing/mozbase/mozprofile/mozprofile/__init__.py | 21 + testing/mozbase/mozprofile/mozprofile/addons.py | 410 ++++ testing/mozbase/mozprofile/mozprofile/cli.py | 131 ++ testing/mozbase/mozprofile/mozprofile/diff.py | 81 + .../mozbase/mozprofile/mozprofile/permissions.py | 415 ++++ testing/mozbase/mozprofile/mozprofile/prefs.py | 232 ++ testing/mozbase/mozprofile/mozprofile/profile.py | 454 ++++ testing/mozbase/mozprofile/mozprofile/view.py | 43 + testing/mozbase/mozprofile/mozprofile/webapps.py | 281 +++ testing/mozbase/mozprofile/setup.py | 45 + testing/mozbase/mozprofile/tests/addon_stubs.py | 78 + testing/mozbase/mozprofile/tests/addonid.py | 184 ++ testing/mozbase/mozprofile/tests/addons/empty.xpi | Bin 0 -> 530 bytes .../mozprofile/tests/addons/empty/install.rdf | 20 + .../mozbase/mozprofile/tests/addons/invalid.xpi | Bin 0 -> 564 bytes testing/mozbase/mozprofile/tests/bug758250.py | 53 + testing/mozbase/mozprofile/tests/bug785146.py | 51 + .../mozprofile/tests/files/not_an_addon.txt | 0 .../mozprofile/tests/files/prefs_with_comments.js | 6 + .../tests/files/prefs_with_interpolation.js | 4 + .../mozbase/mozprofile/tests/files/webapps1.json | 50 + .../mozbase/mozprofile/tests/files/webapps2.json | 37 + .../tests/install_manifests/test_addon_1.rdf | 21 + .../tests/install_manifests/test_addon_2.rdf | 21 + .../tests/install_manifests/test_addon_3.rdf | 22 + .../tests/install_manifests/test_addon_4.rdf | 22 + .../install_manifests/test_addon_invalid_no_id.rdf | 22 + .../test_addon_invalid_not_wellformed.rdf | 23 + .../test_addon_invalid_version.rdf | 23 + .../tests/install_manifests/test_addon_unpack.rdf | 22 + testing/mozbase/mozprofile/tests/manifest.ini | 12 + testing/mozbase/mozprofile/tests/permissions.py | 199 ++ .../mozbase/mozprofile/tests/server_locations.py | 151 ++ testing/mozbase/mozprofile/tests/test_addons.py | 415 ++++ .../mozbase/mozprofile/tests/test_clone_cleanup.py | 63 + testing/mozbase/mozprofile/tests/test_nonce.py | 49 + .../mozbase/mozprofile/tests/test_preferences.py | 378 ++++ testing/mozbase/mozprofile/tests/test_profile.py | 30 + .../mozbase/mozprofile/tests/test_profile_view.py | 75 + testing/mozbase/mozprofile/tests/test_webapps.py | 202 ++ testing/mozbase/mozrunner/mozrunner/__init__.py | 11 + testing/mozbase/mozrunner/mozrunner/application.py | 265 +++ .../mozbase/mozrunner/mozrunner/base/__init__.py | 5 + .../mozbase/mozrunner/mozrunner/base/browser.py | 80 + testing/mozbase/mozrunner/mozrunner/base/device.py | 185 ++ testing/mozbase/mozrunner/mozrunner/base/runner.py | 233 ++ testing/mozbase/mozrunner/mozrunner/cli.py | 152 ++ .../mozrunner/mozrunner/devices/__init__.py | 13 + .../mozrunner/mozrunner/devices/android_device.py | 773 +++++++ .../mozrunner/mozrunner/devices/autophone.py | 651 ++++++ .../mozbase/mozrunner/mozrunner/devices/base.py | 306 +++ .../mozrunner/mozrunner/devices/emulator.py | 288 +++ .../mozrunner/devices/emulator_battery.py | 53 + .../mozrunner/mozrunner/devices/emulator_geo.py | 17 + .../mozrunner/mozrunner/devices/emulator_screen.py | 89 + testing/mozbase/mozrunner/mozrunner/errors.py | 16 + .../mozrunner/resources/metrotestharness.exe | Bin 0 -> 63488 bytes testing/mozbase/mozrunner/mozrunner/runners.py | 211 ++ testing/mozbase/mozrunner/mozrunner/utils.py | 279 +++ testing/mozbase/mozrunner/setup.py | 54 + testing/mozbase/mozrunner/tests/manifest.ini | 7 + testing/mozbase/mozrunner/tests/mozrunnertest.py | 34 + testing/mozbase/mozrunner/tests/test_crash.py | 37 + .../mozbase/mozrunner/tests/test_interactive.py | 53 + testing/mozbase/mozrunner/tests/test_start.py | 45 + testing/mozbase/mozrunner/tests/test_states.py | 18 + testing/mozbase/mozrunner/tests/test_stop.py | 39 + testing/mozbase/mozrunner/tests/test_threads.py | 73 + testing/mozbase/mozrunner/tests/test_wait.py | 29 + .../mozscreenshot/mozscreenshot/__init__.py | 61 + testing/mozbase/mozscreenshot/setup.py | 26 + testing/mozbase/mozsystemmonitor/README.rst | 13 + .../mozsystemmonitor/mozsystemmonitor/__init__.py | 0 .../mozsystemmonitor/resourcemonitor.py | 676 ++++++ .../mozsystemmonitor/test/__init__.py | 0 .../mozsystemmonitor/test/test_resource_monitor.py | 180 ++ testing/mozbase/mozsystemmonitor/setup.py | 29 + testing/mozbase/moztest/moztest/__init__.py | 7 + .../mozbase/moztest/moztest/adapters/__init__.py | 7 + testing/mozbase/moztest/moztest/adapters/unit.py | 225 ++ testing/mozbase/moztest/moztest/output/__init__.py | 0 testing/mozbase/moztest/moztest/output/autolog.py | 73 + testing/mozbase/moztest/moztest/output/base.py | 53 + testing/mozbase/moztest/moztest/output/xunit.py | 93 + testing/mozbase/moztest/moztest/results.py | 323 +++ testing/mozbase/moztest/setup.py | 26 + testing/mozbase/moztest/tests/manifest.ini | 1 + testing/mozbase/moztest/tests/test.py | 55 + testing/mozbase/mozversion/mozversion/__init__.py | 7 + testing/mozbase/mozversion/mozversion/errors.py | 30 + .../mozbase/mozversion/mozversion/mozversion.py | 340 +++ testing/mozbase/mozversion/setup.py | 29 + testing/mozbase/mozversion/tests/manifest.ini | 4 + testing/mozbase/mozversion/tests/test_apk.py | 43 + testing/mozbase/mozversion/tests/test_b2g.py | 75 + testing/mozbase/mozversion/tests/test_binary.py | 177 ++ testing/mozbase/mozversion/tests/test_sources.py | 85 + testing/mozbase/packages.txt | 19 + testing/mozbase/setup_development.py | 273 +++ testing/mozbase/test-manifest.ini | 24 + testing/mozbase/test.py | 104 + testing/mozbase/versioninfo.py | 132 ++ 377 files changed, 43767 insertions(+) create mode 100644 testing/mozbase/README.md create mode 100644 testing/mozbase/docs/Makefile create mode 100644 testing/mozbase/docs/_static/structured_example.py create mode 100644 testing/mozbase/docs/conf.py create mode 100644 testing/mozbase/docs/devicemanagement.rst create mode 100644 testing/mozbase/docs/gettinginfo.rst create mode 100644 testing/mozbase/docs/index.rst create mode 100644 testing/mozbase/docs/loggingreporting.rst create mode 100644 testing/mozbase/docs/make.bat create mode 100644 testing/mozbase/docs/manifestparser.rst create mode 100644 testing/mozbase/docs/mozcrash.rst create mode 100644 testing/mozbase/docs/mozdebug.rst create mode 100644 testing/mozbase/docs/mozdevice.rst create mode 100644 testing/mozbase/docs/mozfile.rst create mode 100644 testing/mozbase/docs/mozhttpd.rst create mode 100644 testing/mozbase/docs/mozinfo.rst create mode 100644 testing/mozbase/docs/mozinstall.rst create mode 100644 testing/mozbase/docs/mozlog.rst create mode 100644 testing/mozbase/docs/moznetwork.rst create mode 100644 testing/mozbase/docs/mozprocess.rst create mode 100644 testing/mozbase/docs/mozprofile.rst create mode 100644 testing/mozbase/docs/mozrunner.rst create mode 100644 testing/mozbase/docs/mozversion.rst create mode 100644 testing/mozbase/docs/requirements.txt create mode 100644 testing/mozbase/docs/setuprunning.rst create mode 100644 testing/mozbase/manifestparser/manifestparser/__init__.py create mode 100644 testing/mozbase/manifestparser/manifestparser/cli.py create mode 100644 testing/mozbase/manifestparser/manifestparser/expression.py create mode 100644 testing/mozbase/manifestparser/manifestparser/filters.py create mode 100644 testing/mozbase/manifestparser/manifestparser/ini.py create mode 100644 testing/mozbase/manifestparser/manifestparser/manifestparser.py create mode 100644 testing/mozbase/manifestparser/setup.py create mode 100644 testing/mozbase/manifestparser/tests/comment-example.ini create mode 100644 testing/mozbase/manifestparser/tests/default-skipif.ini create mode 100644 testing/mozbase/manifestparser/tests/default-suppfiles.ini create mode 100644 testing/mozbase/manifestparser/tests/filter-example.ini create mode 100644 testing/mozbase/manifestparser/tests/fleem create mode 100644 testing/mozbase/manifestparser/tests/include-example.ini create mode 100644 testing/mozbase/manifestparser/tests/include-invalid.ini create mode 100644 testing/mozbase/manifestparser/tests/include/bar.ini create mode 100644 testing/mozbase/manifestparser/tests/include/crash-handling create mode 100644 testing/mozbase/manifestparser/tests/include/flowers create mode 100644 testing/mozbase/manifestparser/tests/include/foo.ini create mode 100644 testing/mozbase/manifestparser/tests/just-defaults.ini create mode 100644 testing/mozbase/manifestparser/tests/manifest.ini create mode 100644 testing/mozbase/manifestparser/tests/missing-path.ini create mode 100644 testing/mozbase/manifestparser/tests/mozmill-example.ini create mode 100644 testing/mozbase/manifestparser/tests/mozmill-restart-example.ini create mode 100644 testing/mozbase/manifestparser/tests/no-tests.ini create mode 100644 testing/mozbase/manifestparser/tests/parent/include/first/manifest.ini create mode 100644 testing/mozbase/manifestparser/tests/parent/include/manifest.ini create mode 100644 testing/mozbase/manifestparser/tests/parent/include/second/manifest.ini create mode 100644 testing/mozbase/manifestparser/tests/parent/level_1/level_1.ini create mode 100644 testing/mozbase/manifestparser/tests/parent/level_1/level_1_server-root.ini create mode 100644 testing/mozbase/manifestparser/tests/parent/level_1/level_2/level_2.ini create mode 100644 testing/mozbase/manifestparser/tests/parent/level_1/level_2/level_2_server-root.ini create mode 100644 testing/mozbase/manifestparser/tests/parent/level_1/level_2/level_3/level_3.ini create mode 100644 testing/mozbase/manifestparser/tests/parent/level_1/level_2/level_3/level_3_default.ini create mode 100644 testing/mozbase/manifestparser/tests/parent/level_1/level_2/level_3/level_3_server-root.ini create mode 100644 testing/mozbase/manifestparser/tests/parent/level_1/level_2/level_3/test_3 create mode 100644 testing/mozbase/manifestparser/tests/parent/level_1/level_2/test_2 create mode 100644 testing/mozbase/manifestparser/tests/parent/level_1/test_1 create mode 100644 testing/mozbase/manifestparser/tests/parent/root/dummy create mode 100644 testing/mozbase/manifestparser/tests/path-example.ini create mode 100644 testing/mozbase/manifestparser/tests/relative-path.ini create mode 100644 testing/mozbase/manifestparser/tests/subsuite.ini create mode 100644 testing/mozbase/manifestparser/tests/test_chunking.py create mode 100755 testing/mozbase/manifestparser/tests/test_convert_directory.py create mode 100755 testing/mozbase/manifestparser/tests/test_convert_symlinks.py create mode 100755 testing/mozbase/manifestparser/tests/test_default_overrides.py create mode 100755 testing/mozbase/manifestparser/tests/test_expressionparser.py create mode 100644 testing/mozbase/manifestparser/tests/test_filters.py create mode 100755 testing/mozbase/manifestparser/tests/test_manifestparser.py create mode 100755 testing/mozbase/manifestparser/tests/test_read_ini.py create mode 100644 testing/mozbase/manifestparser/tests/test_testmanifest.py create mode 100644 testing/mozbase/manifestparser/tests/verifyDirectory/subdir/manifest.ini create mode 100644 testing/mozbase/manifestparser/tests/verifyDirectory/subdir/test_sub.js create mode 100644 testing/mozbase/manifestparser/tests/verifyDirectory/test_1.js create mode 100644 testing/mozbase/manifestparser/tests/verifyDirectory/test_2.js create mode 100644 testing/mozbase/manifestparser/tests/verifyDirectory/test_3.js create mode 100644 testing/mozbase/manifestparser/tests/verifyDirectory/verifyDirectory.ini create mode 100644 testing/mozbase/manifestparser/tests/verifyDirectory/verifyDirectory_incomplete.ini create mode 100644 testing/mozbase/manifestparser/tests/verifyDirectory/verifyDirectory_toocomplete.ini create mode 100644 testing/mozbase/moz.build create mode 100644 testing/mozbase/mozcrash/mozcrash/__init__.py create mode 100644 testing/mozbase/mozcrash/mozcrash/mozcrash.py create mode 100644 testing/mozbase/mozcrash/setup.py create mode 100644 testing/mozbase/mozcrash/tests/manifest.ini create mode 100644 testing/mozbase/mozcrash/tests/test.py create mode 100644 testing/mozbase/mozdebug/mozdebug/__init__.py create mode 100755 testing/mozbase/mozdebug/mozdebug/mozdebug.py create mode 100644 testing/mozbase/mozdebug/setup.py create mode 100644 testing/mozbase/mozdevice/adb_tests/test_device_running_adb_as_root.py create mode 100644 testing/mozbase/mozdevice/adb_tests/test_devicemanagerADB.py create mode 100755 testing/mozbase/mozdevice/mozdevice/Zeroconf.py create mode 100644 testing/mozbase/mozdevice/mozdevice/__init__.py create mode 100644 testing/mozbase/mozdevice/mozdevice/adb.py create mode 100644 testing/mozbase/mozdevice/mozdevice/adb_android.py create mode 100644 testing/mozbase/mozdevice/mozdevice/adb_b2g.py create mode 100644 testing/mozbase/mozdevice/mozdevice/devicemanager.py create mode 100644 testing/mozbase/mozdevice/mozdevice/devicemanagerADB.py create mode 100644 testing/mozbase/mozdevice/mozdevice/devicemanagerSUT.py create mode 100644 testing/mozbase/mozdevice/mozdevice/dmcli.py create mode 100644 testing/mozbase/mozdevice/mozdevice/droid.py create mode 100644 testing/mozbase/mozdevice/mozdevice/sutini.py create mode 100644 testing/mozbase/mozdevice/mozdevice/version_codes.py create mode 100644 testing/mozbase/mozdevice/setup.py create mode 100644 testing/mozbase/mozdevice/sut_tests/README.md create mode 100644 testing/mozbase/mozdevice/sut_tests/dmunit.py create mode 100644 testing/mozbase/mozdevice/sut_tests/genfiles.py create mode 100644 testing/mozbase/mozdevice/sut_tests/runtests.py create mode 100755 testing/mozbase/mozdevice/sut_tests/setup-tools.sh create mode 100644 testing/mozbase/mozdevice/sut_tests/test-files/mytext.txt create mode 100644 testing/mozbase/mozdevice/sut_tests/test-files/smalltext.txt create mode 100644 testing/mozbase/mozdevice/sut_tests/test-files/test_script.sh create mode 100644 testing/mozbase/mozdevice/sut_tests/test_datachannel.py create mode 100644 testing/mozbase/mozdevice/sut_tests/test_exec.py create mode 100644 testing/mozbase/mozdevice/sut_tests/test_exec_env.py create mode 100644 testing/mozbase/mozdevice/sut_tests/test_fileExists.py create mode 100644 testing/mozbase/mozdevice/sut_tests/test_getdir.py create mode 100644 testing/mozbase/mozdevice/sut_tests/test_info.py create mode 100644 testing/mozbase/mozdevice/sut_tests/test_prompt.py create mode 100644 testing/mozbase/mozdevice/sut_tests/test_ps.py create mode 100644 testing/mozbase/mozdevice/sut_tests/test_pull.py create mode 100644 testing/mozbase/mozdevice/sut_tests/test_push1.py create mode 100644 testing/mozbase/mozdevice/sut_tests/test_push2.py create mode 100644 testing/mozbase/mozdevice/sut_tests/test_pushbinary.py create mode 100644 testing/mozbase/mozdevice/sut_tests/test_pushsmalltext.py create mode 100644 testing/mozbase/mozdevice/tests/droidsut_launch.py create mode 100644 testing/mozbase/mozdevice/tests/manifest.ini create mode 100644 testing/mozbase/mozdevice/tests/sut.py create mode 100644 testing/mozbase/mozdevice/tests/sut_app.py create mode 100644 testing/mozbase/mozdevice/tests/sut_basic.py create mode 100644 testing/mozbase/mozdevice/tests/sut_chmod.py create mode 100644 testing/mozbase/mozdevice/tests/sut_copytree.py create mode 100644 testing/mozbase/mozdevice/tests/sut_fileExists.py create mode 100644 testing/mozbase/mozdevice/tests/sut_fileMethods.py create mode 100644 testing/mozbase/mozdevice/tests/sut_info.py create mode 100644 testing/mozbase/mozdevice/tests/sut_ip.py create mode 100644 testing/mozbase/mozdevice/tests/sut_kill.py create mode 100644 testing/mozbase/mozdevice/tests/sut_list.py create mode 100644 testing/mozbase/mozdevice/tests/sut_logcat.py create mode 100644 testing/mozbase/mozdevice/tests/sut_mkdir.py create mode 100644 testing/mozbase/mozdevice/tests/sut_movetree.py create mode 100644 testing/mozbase/mozdevice/tests/sut_ps.py create mode 100644 testing/mozbase/mozdevice/tests/sut_pull.py create mode 100644 testing/mozbase/mozdevice/tests/sut_push.py create mode 100644 testing/mozbase/mozdevice/tests/sut_remove.py create mode 100644 testing/mozbase/mozdevice/tests/sut_time.py create mode 100644 testing/mozbase/mozdevice/tests/sut_unpackfile.py create mode 100644 testing/mozbase/mozfile/mozfile/__init__.py create mode 100644 testing/mozbase/mozfile/mozfile/mozfile.py create mode 100644 testing/mozbase/mozfile/setup.py create mode 100644 testing/mozbase/mozfile/tests/files/missing_file_attributes.zip create mode 100644 testing/mozbase/mozfile/tests/manifest.ini create mode 100644 testing/mozbase/mozfile/tests/stubs.py create mode 100644 testing/mozbase/mozfile/tests/test_extract.py create mode 100755 testing/mozbase/mozfile/tests/test_load.py create mode 100644 testing/mozbase/mozfile/tests/test_move_remove.py create mode 100644 testing/mozbase/mozfile/tests/test_tempdir.py create mode 100644 testing/mozbase/mozfile/tests/test_tempfile.py create mode 100755 testing/mozbase/mozfile/tests/test_url.py create mode 100644 testing/mozbase/mozhttpd/mozhttpd/__init__.py create mode 100644 testing/mozbase/mozhttpd/mozhttpd/handlers.py create mode 100755 testing/mozbase/mozhttpd/mozhttpd/mozhttpd.py create mode 100644 testing/mozbase/mozhttpd/setup.py create mode 100644 testing/mozbase/mozhttpd/tests/api.py create mode 100644 testing/mozbase/mozhttpd/tests/baseurl.py create mode 100644 testing/mozbase/mozhttpd/tests/basic.py create mode 100644 testing/mozbase/mozhttpd/tests/filelisting.py create mode 100644 testing/mozbase/mozhttpd/tests/manifest.ini create mode 100644 testing/mozbase/mozhttpd/tests/paths.py create mode 100644 testing/mozbase/mozhttpd/tests/requestlog.py create mode 100644 testing/mozbase/mozinfo/mozinfo/__init__.py create mode 100755 testing/mozbase/mozinfo/mozinfo/mozinfo.py create mode 100644 testing/mozbase/mozinfo/mozinfo/string_version.py create mode 100644 testing/mozbase/mozinfo/setup.py create mode 100644 testing/mozbase/mozinfo/tests/manifest.ini create mode 100644 testing/mozbase/mozinfo/tests/test.py create mode 100644 testing/mozbase/mozinstall/mozinstall/__init__.py create mode 100755 testing/mozbase/mozinstall/mozinstall/mozinstall.py create mode 100644 testing/mozbase/mozinstall/setup.py create mode 100644 testing/mozbase/mozinstall/tests/Installer-Stubs/firefox.dmg create mode 100644 testing/mozbase/mozinstall/tests/Installer-Stubs/firefox.tar.bz2 create mode 100644 testing/mozbase/mozinstall/tests/Installer-Stubs/firefox.zip create mode 100644 testing/mozbase/mozinstall/tests/manifest.ini create mode 100644 testing/mozbase/mozinstall/tests/test.py create mode 100644 testing/mozbase/mozleak/mozleak/__init__.py create mode 100644 testing/mozbase/mozleak/mozleak/leaklog.py create mode 100644 testing/mozbase/mozleak/setup.py create mode 100644 testing/mozbase/mozlog/mozlog/__init__.py create mode 100644 testing/mozbase/mozlog/mozlog/commandline.py create mode 100644 testing/mozbase/mozlog/mozlog/formatters/__init__.py create mode 100644 testing/mozbase/mozlog/mozlog/formatters/base.py create mode 100644 testing/mozbase/mozlog/mozlog/formatters/errorsummary.py create mode 100644 testing/mozbase/mozlog/mozlog/formatters/html/__init__.py create mode 100755 testing/mozbase/mozlog/mozlog/formatters/html/html.py create mode 100644 testing/mozbase/mozlog/mozlog/formatters/html/main.js create mode 100644 testing/mozbase/mozlog/mozlog/formatters/html/style.css create mode 100644 testing/mozbase/mozlog/mozlog/formatters/html/xmlgen.py create mode 100644 testing/mozbase/mozlog/mozlog/formatters/machformatter.py create mode 100644 testing/mozbase/mozlog/mozlog/formatters/process.py create mode 100644 testing/mozbase/mozlog/mozlog/formatters/tbplformatter.py create mode 100755 testing/mozbase/mozlog/mozlog/formatters/unittest.py create mode 100644 testing/mozbase/mozlog/mozlog/formatters/xunit.py create mode 100644 testing/mozbase/mozlog/mozlog/handlers/__init__.py create mode 100644 testing/mozbase/mozlog/mozlog/handlers/base.py create mode 100644 testing/mozbase/mozlog/mozlog/handlers/bufferhandler.py create mode 100644 testing/mozbase/mozlog/mozlog/handlers/statushandler.py create mode 100644 testing/mozbase/mozlog/mozlog/handlers/valgrindhandler.py create mode 100644 testing/mozbase/mozlog/mozlog/logtypes.py create mode 100644 testing/mozbase/mozlog/mozlog/proxy.py create mode 100644 testing/mozbase/mozlog/mozlog/pytest_mozlog/__init__.py create mode 100644 testing/mozbase/mozlog/mozlog/pytest_mozlog/plugin.py create mode 100644 testing/mozbase/mozlog/mozlog/reader.py create mode 100644 testing/mozbase/mozlog/mozlog/scripts/__init__.py create mode 100644 testing/mozbase/mozlog/mozlog/scripts/format.py create mode 100644 testing/mozbase/mozlog/mozlog/scripts/logmerge.py create mode 100644 testing/mozbase/mozlog/mozlog/scripts/unstable.py create mode 100644 testing/mozbase/mozlog/mozlog/stdadapter.py create mode 100644 testing/mozbase/mozlog/mozlog/structuredlog.py create mode 100644 testing/mozbase/mozlog/mozlog/unstructured/__init__.py create mode 100644 testing/mozbase/mozlog/mozlog/unstructured/logger.py create mode 100644 testing/mozbase/mozlog/mozlog/unstructured/loggingmixin.py create mode 100644 testing/mozbase/mozlog/mozlog/unstructured/loglistener.py create mode 100644 testing/mozbase/mozlog/setup.py create mode 100644 testing/mozbase/mozlog/tests/manifest.ini create mode 100644 testing/mozbase/mozlog/tests/test_logger.py create mode 100644 testing/mozbase/mozlog/tests/test_structured.py create mode 100644 testing/mozbase/moznetwork/moznetwork/__init__.py create mode 100644 testing/mozbase/moznetwork/moznetwork/moznetwork.py create mode 100644 testing/mozbase/moznetwork/setup.py create mode 100644 testing/mozbase/moznetwork/tests/manifest.ini create mode 100644 testing/mozbase/moznetwork/tests/test.py create mode 100644 testing/mozbase/mozprocess/mozprocess/__init__.py create mode 100644 testing/mozbase/mozprocess/mozprocess/processhandler.py create mode 100644 testing/mozbase/mozprocess/mozprocess/qijo.py create mode 100644 testing/mozbase/mozprocess/mozprocess/winprocess.py create mode 100644 testing/mozbase/mozprocess/setup.py create mode 100644 testing/mozbase/mozprocess/tests/Makefile create mode 100644 testing/mozbase/mozprocess/tests/infinite_loop.py create mode 100644 testing/mozbase/mozprocess/tests/iniparser/AUTHORS create mode 100644 testing/mozbase/mozprocess/tests/iniparser/INSTALL create mode 100644 testing/mozbase/mozprocess/tests/iniparser/LICENSE create mode 100644 testing/mozbase/mozprocess/tests/iniparser/Makefile create mode 100644 testing/mozbase/mozprocess/tests/iniparser/README create mode 100644 testing/mozbase/mozprocess/tests/iniparser/dictionary.c create mode 100644 testing/mozbase/mozprocess/tests/iniparser/dictionary.h create mode 100644 testing/mozbase/mozprocess/tests/iniparser/iniparser.c create mode 100644 testing/mozbase/mozprocess/tests/iniparser/iniparser.h create mode 100644 testing/mozbase/mozprocess/tests/iniparser/platform.mk create mode 100644 testing/mozbase/mozprocess/tests/manifest.ini create mode 100644 testing/mozbase/mozprocess/tests/proccountfive.py create mode 100644 testing/mozbase/mozprocess/tests/process_normal_broad_python.ini create mode 100644 testing/mozbase/mozprocess/tests/process_normal_deep_python.ini create mode 100644 testing/mozbase/mozprocess/tests/process_normal_finish.ini create mode 100644 testing/mozbase/mozprocess/tests/process_normal_finish_no_process_group.ini create mode 100644 testing/mozbase/mozprocess/tests/process_normal_finish_python.ini create mode 100644 testing/mozbase/mozprocess/tests/process_waittimeout.ini create mode 100644 testing/mozbase/mozprocess/tests/process_waittimeout_10s.ini create mode 100644 testing/mozbase/mozprocess/tests/process_waittimeout_10s_python.ini create mode 100644 testing/mozbase/mozprocess/tests/process_waittimeout_python.ini create mode 100644 testing/mozbase/mozprocess/tests/proclaunch.c create mode 100644 testing/mozbase/mozprocess/tests/proclaunch.py create mode 100644 testing/mozbase/mozprocess/tests/procnonewline.py create mode 100644 testing/mozbase/mozprocess/tests/proctest.py create mode 100644 testing/mozbase/mozprocess/tests/test_mozprocess.py create mode 100644 testing/mozbase/mozprocess/tests/test_mozprocess_kill.py create mode 100644 testing/mozbase/mozprocess/tests/test_mozprocess_kill_broad_wait.py create mode 100644 testing/mozbase/mozprocess/tests/test_mozprocess_misc.py create mode 100644 testing/mozbase/mozprocess/tests/test_mozprocess_output.py create mode 100644 testing/mozbase/mozprocess/tests/test_mozprocess_params.py create mode 100644 testing/mozbase/mozprocess/tests/test_mozprocess_poll.py create mode 100644 testing/mozbase/mozprocess/tests/test_mozprocess_wait.py create mode 100644 testing/mozbase/mozprocess/tests/test_process_reader.py create mode 100644 testing/mozbase/mozprofile/mozprofile/__init__.py create mode 100644 testing/mozbase/mozprofile/mozprofile/addons.py create mode 100755 testing/mozbase/mozprofile/mozprofile/cli.py create mode 100644 testing/mozbase/mozprofile/mozprofile/diff.py create mode 100644 testing/mozbase/mozprofile/mozprofile/permissions.py create mode 100644 testing/mozbase/mozprofile/mozprofile/prefs.py create mode 100644 testing/mozbase/mozprofile/mozprofile/profile.py create mode 100644 testing/mozbase/mozprofile/mozprofile/view.py create mode 100644 testing/mozbase/mozprofile/mozprofile/webapps.py create mode 100644 testing/mozbase/mozprofile/setup.py create mode 100644 testing/mozbase/mozprofile/tests/addon_stubs.py create mode 100755 testing/mozbase/mozprofile/tests/addonid.py create mode 100644 testing/mozbase/mozprofile/tests/addons/empty.xpi create mode 100644 testing/mozbase/mozprofile/tests/addons/empty/install.rdf create mode 100644 testing/mozbase/mozprofile/tests/addons/invalid.xpi create mode 100755 testing/mozbase/mozprofile/tests/bug758250.py create mode 100755 testing/mozbase/mozprofile/tests/bug785146.py create mode 100644 testing/mozbase/mozprofile/tests/files/not_an_addon.txt create mode 100644 testing/mozbase/mozprofile/tests/files/prefs_with_comments.js create mode 100644 testing/mozbase/mozprofile/tests/files/prefs_with_interpolation.js create mode 100644 testing/mozbase/mozprofile/tests/files/webapps1.json create mode 100644 testing/mozbase/mozprofile/tests/files/webapps2.json create mode 100644 testing/mozbase/mozprofile/tests/install_manifests/test_addon_1.rdf create mode 100644 testing/mozbase/mozprofile/tests/install_manifests/test_addon_2.rdf create mode 100644 testing/mozbase/mozprofile/tests/install_manifests/test_addon_3.rdf create mode 100644 testing/mozbase/mozprofile/tests/install_manifests/test_addon_4.rdf create mode 100644 testing/mozbase/mozprofile/tests/install_manifests/test_addon_invalid_no_id.rdf create mode 100644 testing/mozbase/mozprofile/tests/install_manifests/test_addon_invalid_not_wellformed.rdf create mode 100644 testing/mozbase/mozprofile/tests/install_manifests/test_addon_invalid_version.rdf create mode 100644 testing/mozbase/mozprofile/tests/install_manifests/test_addon_unpack.rdf create mode 100644 testing/mozbase/mozprofile/tests/manifest.ini create mode 100755 testing/mozbase/mozprofile/tests/permissions.py create mode 100644 testing/mozbase/mozprofile/tests/server_locations.py create mode 100644 testing/mozbase/mozprofile/tests/test_addons.py create mode 100644 testing/mozbase/mozprofile/tests/test_clone_cleanup.py create mode 100755 testing/mozbase/mozprofile/tests/test_nonce.py create mode 100755 testing/mozbase/mozprofile/tests/test_preferences.py create mode 100644 testing/mozbase/mozprofile/tests/test_profile.py create mode 100644 testing/mozbase/mozprofile/tests/test_profile_view.py create mode 100755 testing/mozbase/mozprofile/tests/test_webapps.py create mode 100644 testing/mozbase/mozrunner/mozrunner/__init__.py create mode 100644 testing/mozbase/mozrunner/mozrunner/application.py create mode 100644 testing/mozbase/mozrunner/mozrunner/base/__init__.py create mode 100644 testing/mozbase/mozrunner/mozrunner/base/browser.py create mode 100644 testing/mozbase/mozrunner/mozrunner/base/device.py create mode 100644 testing/mozbase/mozrunner/mozrunner/base/runner.py create mode 100644 testing/mozbase/mozrunner/mozrunner/cli.py create mode 100644 testing/mozbase/mozrunner/mozrunner/devices/__init__.py create mode 100644 testing/mozbase/mozrunner/mozrunner/devices/android_device.py create mode 100644 testing/mozbase/mozrunner/mozrunner/devices/autophone.py create mode 100644 testing/mozbase/mozrunner/mozrunner/devices/base.py create mode 100644 testing/mozbase/mozrunner/mozrunner/devices/emulator.py create mode 100644 testing/mozbase/mozrunner/mozrunner/devices/emulator_battery.py create mode 100644 testing/mozbase/mozrunner/mozrunner/devices/emulator_geo.py create mode 100644 testing/mozbase/mozrunner/mozrunner/devices/emulator_screen.py create mode 100644 testing/mozbase/mozrunner/mozrunner/errors.py create mode 100644 testing/mozbase/mozrunner/mozrunner/resources/metrotestharness.exe create mode 100644 testing/mozbase/mozrunner/mozrunner/runners.py create mode 100755 testing/mozbase/mozrunner/mozrunner/utils.py create mode 100644 testing/mozbase/mozrunner/setup.py create mode 100644 testing/mozbase/mozrunner/tests/manifest.ini create mode 100644 testing/mozbase/mozrunner/tests/mozrunnertest.py create mode 100644 testing/mozbase/mozrunner/tests/test_crash.py create mode 100644 testing/mozbase/mozrunner/tests/test_interactive.py create mode 100644 testing/mozbase/mozrunner/tests/test_start.py create mode 100644 testing/mozbase/mozrunner/tests/test_states.py create mode 100644 testing/mozbase/mozrunner/tests/test_stop.py create mode 100644 testing/mozbase/mozrunner/tests/test_threads.py create mode 100644 testing/mozbase/mozrunner/tests/test_wait.py create mode 100644 testing/mozbase/mozscreenshot/mozscreenshot/__init__.py create mode 100644 testing/mozbase/mozscreenshot/setup.py create mode 100644 testing/mozbase/mozsystemmonitor/README.rst create mode 100644 testing/mozbase/mozsystemmonitor/mozsystemmonitor/__init__.py create mode 100644 testing/mozbase/mozsystemmonitor/mozsystemmonitor/resourcemonitor.py create mode 100644 testing/mozbase/mozsystemmonitor/mozsystemmonitor/test/__init__.py create mode 100644 testing/mozbase/mozsystemmonitor/mozsystemmonitor/test/test_resource_monitor.py create mode 100644 testing/mozbase/mozsystemmonitor/setup.py create mode 100644 testing/mozbase/moztest/moztest/__init__.py create mode 100644 testing/mozbase/moztest/moztest/adapters/__init__.py create mode 100644 testing/mozbase/moztest/moztest/adapters/unit.py create mode 100644 testing/mozbase/moztest/moztest/output/__init__.py create mode 100644 testing/mozbase/moztest/moztest/output/autolog.py create mode 100644 testing/mozbase/moztest/moztest/output/base.py create mode 100644 testing/mozbase/moztest/moztest/output/xunit.py create mode 100644 testing/mozbase/moztest/moztest/results.py create mode 100644 testing/mozbase/moztest/setup.py create mode 100644 testing/mozbase/moztest/tests/manifest.ini create mode 100644 testing/mozbase/moztest/tests/test.py create mode 100644 testing/mozbase/mozversion/mozversion/__init__.py create mode 100644 testing/mozbase/mozversion/mozversion/errors.py create mode 100644 testing/mozbase/mozversion/mozversion/mozversion.py create mode 100644 testing/mozbase/mozversion/setup.py create mode 100644 testing/mozbase/mozversion/tests/manifest.ini create mode 100644 testing/mozbase/mozversion/tests/test_apk.py create mode 100644 testing/mozbase/mozversion/tests/test_b2g.py create mode 100644 testing/mozbase/mozversion/tests/test_binary.py create mode 100644 testing/mozbase/mozversion/tests/test_sources.py create mode 100644 testing/mozbase/packages.txt create mode 100755 testing/mozbase/setup_development.py create mode 100644 testing/mozbase/test-manifest.ini create mode 100755 testing/mozbase/test.py create mode 100755 testing/mozbase/versioninfo.py (limited to 'testing/mozbase') diff --git a/testing/mozbase/README.md b/testing/mozbase/README.md new file mode 100644 index 000000000..4e2cabfe5 --- /dev/null +++ b/testing/mozbase/README.md @@ -0,0 +1,21 @@ +# Mozbase + +Mozbase is a set of easy-to-use Python packages forming a supplemental standard +library for Mozilla. It provides consistency and reduces redundancy in +automation and other system-level software. All of Mozilla's test harnesses use +mozbase to some degree, including Talos, mochitest, reftest, Autophone, and +Eideticker. + +Learn more about mozbase at the [project page][]. + +Read [detailed docs][] online, or build them locally by running "make html" in +the docs directory. + +Consult [open][] [bugs][] and feel free to file [new bugs][]. + + +[project page]: https://wiki.mozilla.org/Auto-tools/Projects/Mozbase +[detailed docs]: http://mozbase.readthedocs.org/ +[open]: https://bugzilla.mozilla.org/buglist.cgi?resolution=---&component=Mozbase&product=Testing +[bugs]: https://bugzilla.mozilla.org/buglist.cgi?resolution=---&status_whiteboard_type=allwordssubstr&query_format=advanced&status_whiteboard=mozbase +[new bugs]: https://bugzilla.mozilla.org/enter_bug.cgi?product=Testing&component=Mozbase diff --git a/testing/mozbase/docs/Makefile b/testing/mozbase/docs/Makefile new file mode 100644 index 000000000..386a52db1 --- /dev/null +++ b/testing/mozbase/docs/Makefile @@ -0,0 +1,153 @@ +# Makefile for Sphinx documentation +# + +# You can set these variables from the command line. +SPHINXOPTS = +SPHINXBUILD = sphinx-build +PAPER = +BUILDDIR = _build + +# Internal variables. +PAPEROPT_a4 = -D latex_paper_size=a4 +PAPEROPT_letter = -D latex_paper_size=letter +ALLSPHINXOPTS = -d $(BUILDDIR)/doctrees $(PAPEROPT_$(PAPER)) $(SPHINXOPTS) . +# the i18n builder cannot share the environment and doctrees with the others +I18NSPHINXOPTS = $(PAPEROPT_$(PAPER)) $(SPHINXOPTS) . + +.PHONY: help clean html dirhtml singlehtml pickle json htmlhelp qthelp devhelp epub latex latexpdf text man changes linkcheck doctest gettext + +help: + @echo "Please use \`make ' where is one of" + @echo " html to make standalone HTML files" + @echo " dirhtml to make HTML files named index.html in directories" + @echo " singlehtml to make a single large HTML file" + @echo " pickle to make pickle files" + @echo " json to make JSON files" + @echo " htmlhelp to make HTML files and a HTML help project" + @echo " qthelp to make HTML files and a qthelp project" + @echo " devhelp to make HTML files and a Devhelp project" + @echo " epub to make an epub" + @echo " latex to make LaTeX files, you can set PAPER=a4 or PAPER=letter" + @echo " latexpdf to make LaTeX files and run them through pdflatex" + @echo " text to make text files" + @echo " man to make manual pages" + @echo " texinfo to make Texinfo files" + @echo " info to make Texinfo files and run them through makeinfo" + @echo " gettext to make PO message catalogs" + @echo " changes to make an overview of all changed/added/deprecated items" + @echo " linkcheck to check all external links for integrity" + @echo " doctest to run all doctests embedded in the documentation (if enabled)" + +clean: + -rm -rf $(BUILDDIR)/* + +html: + $(SPHINXBUILD) -b html $(ALLSPHINXOPTS) $(BUILDDIR)/html + @echo + @echo "Build finished. The HTML pages are in $(BUILDDIR)/html." + +dirhtml: + $(SPHINXBUILD) -b dirhtml $(ALLSPHINXOPTS) $(BUILDDIR)/dirhtml + @echo + @echo "Build finished. The HTML pages are in $(BUILDDIR)/dirhtml." + +singlehtml: + $(SPHINXBUILD) -b singlehtml $(ALLSPHINXOPTS) $(BUILDDIR)/singlehtml + @echo + @echo "Build finished. The HTML page is in $(BUILDDIR)/singlehtml." + +pickle: + $(SPHINXBUILD) -b pickle $(ALLSPHINXOPTS) $(BUILDDIR)/pickle + @echo + @echo "Build finished; now you can process the pickle files." + +json: + $(SPHINXBUILD) -b json $(ALLSPHINXOPTS) $(BUILDDIR)/json + @echo + @echo "Build finished; now you can process the JSON files." + +htmlhelp: + $(SPHINXBUILD) -b htmlhelp $(ALLSPHINXOPTS) $(BUILDDIR)/htmlhelp + @echo + @echo "Build finished; now you can run HTML Help Workshop with the" \ + ".hhp project file in $(BUILDDIR)/htmlhelp." + +qthelp: + $(SPHINXBUILD) -b qthelp $(ALLSPHINXOPTS) $(BUILDDIR)/qthelp + @echo + @echo "Build finished; now you can run "qcollectiongenerator" with the" \ + ".qhcp project file in $(BUILDDIR)/qthelp, like this:" + @echo "# qcollectiongenerator $(BUILDDIR)/qthelp/MozBase.qhcp" + @echo "To view the help file:" + @echo "# assistant -collectionFile $(BUILDDIR)/qthelp/MozBase.qhc" + +devhelp: + $(SPHINXBUILD) -b devhelp $(ALLSPHINXOPTS) $(BUILDDIR)/devhelp + @echo + @echo "Build finished." + @echo "To view the help file:" + @echo "# mkdir -p $$HOME/.local/share/devhelp/MozBase" + @echo "# ln -s $(BUILDDIR)/devhelp $$HOME/.local/share/devhelp/MozBase" + @echo "# devhelp" + +epub: + $(SPHINXBUILD) -b epub $(ALLSPHINXOPTS) $(BUILDDIR)/epub + @echo + @echo "Build finished. The epub file is in $(BUILDDIR)/epub." + +latex: + $(SPHINXBUILD) -b latex $(ALLSPHINXOPTS) $(BUILDDIR)/latex + @echo + @echo "Build finished; the LaTeX files are in $(BUILDDIR)/latex." + @echo "Run \`make' in that directory to run these through (pdf)latex" \ + "(use \`make latexpdf' here to do that automatically)." + +latexpdf: + $(SPHINXBUILD) -b latex $(ALLSPHINXOPTS) $(BUILDDIR)/latex + @echo "Running LaTeX files through pdflatex..." + $(MAKE) -C $(BUILDDIR)/latex all-pdf + @echo "pdflatex finished; the PDF files are in $(BUILDDIR)/latex." + +text: + $(SPHINXBUILD) -b text $(ALLSPHINXOPTS) $(BUILDDIR)/text + @echo + @echo "Build finished. The text files are in $(BUILDDIR)/text." + +man: + $(SPHINXBUILD) -b man $(ALLSPHINXOPTS) $(BUILDDIR)/man + @echo + @echo "Build finished. The manual pages are in $(BUILDDIR)/man." + +texinfo: + $(SPHINXBUILD) -b texinfo $(ALLSPHINXOPTS) $(BUILDDIR)/texinfo + @echo + @echo "Build finished. The Texinfo files are in $(BUILDDIR)/texinfo." + @echo "Run \`make' in that directory to run these through makeinfo" \ + "(use \`make info' here to do that automatically)." + +info: + $(SPHINXBUILD) -b texinfo $(ALLSPHINXOPTS) $(BUILDDIR)/texinfo + @echo "Running Texinfo files through makeinfo..." + make -C $(BUILDDIR)/texinfo info + @echo "makeinfo finished; the Info files are in $(BUILDDIR)/texinfo." + +gettext: + $(SPHINXBUILD) -b gettext $(I18NSPHINXOPTS) $(BUILDDIR)/locale + @echo + @echo "Build finished. The message catalogs are in $(BUILDDIR)/locale." + +changes: + $(SPHINXBUILD) -b changes $(ALLSPHINXOPTS) $(BUILDDIR)/changes + @echo + @echo "The overview file is in $(BUILDDIR)/changes." + +linkcheck: + $(SPHINXBUILD) -b linkcheck $(ALLSPHINXOPTS) $(BUILDDIR)/linkcheck + @echo + @echo "Link check complete; look for any errors in the above output " \ + "or in $(BUILDDIR)/linkcheck/output.txt." + +doctest: + $(SPHINXBUILD) -b doctest $(ALLSPHINXOPTS) $(BUILDDIR)/doctest + @echo "Testing of doctests in the sources finished, look at the " \ + "results in $(BUILDDIR)/doctest/output.txt." diff --git a/testing/mozbase/docs/_static/structured_example.py b/testing/mozbase/docs/_static/structured_example.py new file mode 100644 index 000000000..2bbc03810 --- /dev/null +++ b/testing/mozbase/docs/_static/structured_example.py @@ -0,0 +1,102 @@ +import argparse +import sys +import traceback +import types + +from mozlog import commandline, get_default_logger + + +class TestAssertion(Exception): + pass + + +def assert_equals(a, b): + if a != b: + raise TestAssertion("%r not equal to %r" % (a, b)) + + +def expected(status): + def inner(f): + def test_func(): + f() + test_func.__name__ = f.__name__ + test_func._expected = status + return test_func + return inner + + +def test_that_passes(): + assert_equals(1, int("1")) + + +def test_that_fails(): + assert_equals(1, int("2")) + + +def test_that_has_an_error(): + assert_equals(2, 1 + "1") + + +@expected("FAIL") +def test_expected_fail(): + assert_equals(2 + 2, 5) + + +class TestRunner(object): + + def __init__(self): + self.logger = get_default_logger(component='TestRunner') + + def gather_tests(self): + for item in globals().itervalues(): + if isinstance(item, types.FunctionType) and item.__name__.startswith("test_"): + yield item.__name__, item + + def run(self): + tests = list(self.gather_tests()) + + self.logger.suite_start(tests=[name for name, func in tests]) + self.logger.info("Running tests") + for name, func in tests: + self.run_test(name, func) + self.logger.suite_end() + + def run_test(self, name, func): + self.logger.test_start(name) + status = None + message = None + expected = func._expected if hasattr(func, "_expected") else "PASS" + try: + func() + except TestAssertion as e: + status = "FAIL" + message = e.message + except: + status = "ERROR" + message = traceback.format_exc() + else: + status = "PASS" + self.logger.test_end(name, status=status, expected=expected, message=message) + + +def get_parser(): + parser = argparse.ArgumentParser() + return parser + + +def main(): + parser = get_parser() + commandline.add_logging_group(parser) + + args = parser.parse_args() + + logger = commandline.setup_logging("structured-example", args, {"raw": sys.stdout}) + + runner = TestRunner() + try: + runner.run() + except: + logger.critical("Error during test run:\n%s" % traceback.format_exc()) + +if __name__ == "__main__": + main() diff --git a/testing/mozbase/docs/conf.py b/testing/mozbase/docs/conf.py new file mode 100644 index 000000000..95d6de64b --- /dev/null +++ b/testing/mozbase/docs/conf.py @@ -0,0 +1,258 @@ +# -*- coding: utf-8 -*- +# +# MozBase documentation build configuration file, created by +# sphinx-quickstart on Mon Oct 22 14:02:17 2012. +# +# This file is execfile()d with the current directory set to its containing dir. +# +# Note that not all possible configuration values are present in this +# autogenerated file. +# +# All configuration values have a default; values that are commented out +# serve to show the default. + +import sys +import os + +# If extensions (or modules to document with autodoc) are in another directory, +# add these directories to sys.path here. If the directory is relative to the +# documentation root, use os.path.abspath to make it absolute, like shown here. +here = os.path.dirname(os.path.abspath(__file__)) +parent = os.path.dirname(here) +for item in os.listdir(parent): + path = os.path.join(parent, item) + if (not os.path.isdir(path)) or (not os.path.exists(os.path.join(path, 'setup.py'))): + continue + sys.path.insert(0, path) + +# -- General configuration ----------------------------------------------------- + +# If your documentation needs a minimal Sphinx version, state it here. +# needs_sphinx = '1.0' + +# Add any Sphinx extension module names here, as strings. They can be extensions +# coming with Sphinx (named 'sphinx.ext.*') or your custom ones. +extensions = ['sphinx.ext.autodoc', 'sphinx.ext.doctest', 'sphinx.ext.todo', 'sphinx.ext.viewcode'] + +# Add any paths that contain templates here, relative to this directory. +templates_path = ['_templates'] + +# The suffix of source filenames. +source_suffix = '.rst' + +# The encoding of source files. +# source_encoding = 'utf-8-sig' + +# The master toctree document. +master_doc = 'index' + +# General information about the project. +project = u'MozBase' +copyright = u'2012, Mozilla Automation and Tools team' + +# The version info for the project you're documenting, acts as replacement for +# |version| and |release|, also used in various other places throughout the +# built documents. +# +# The short X.Y version. +version = '1' +# The full version, including alpha/beta/rc tags. +release = '1' + +# The language for content autogenerated by Sphinx. Refer to documentation +# for a list of supported languages. +# language = None + +# There are two options for replacing |today|: either, you set today to some +# non-false value, then it is used: +# today = '' +# Else, today_fmt is used as the format for a strftime call. +# today_fmt = '%B %d, %Y' + +# List of patterns, relative to source directory, that match files and +# directories to ignore when looking for source files. +exclude_patterns = ['_build'] + +# The reST default role (used for this markup: `text`) to use for all documents. +# default_role = None + +# If true, '()' will be appended to :func: etc. cross-reference text. +# add_function_parentheses = True + +# If true, the current module name will be prepended to all description +# unit titles (such as .. function::). +# add_module_names = True + +# If true, sectionauthor and moduleauthor directives will be shown in the +# output. They are ignored by default. +# show_authors = False + +# The name of the Pygments (syntax highlighting) style to use. +pygments_style = 'sphinx' + +# A list of ignored prefixes for module index sorting. +# modindex_common_prefix = [] + + +# -- Options for HTML output --------------------------------------------------- + +# The theme to use for HTML and HTML Help pages. See the documentation for +# a list of builtin themes. +html_theme = 'default' +on_rtd = os.environ.get('READTHEDOCS', None) == 'True' + +if not on_rtd: + try: + import sphinx_rtd_theme + html_theme = 'sphinx_rtd_theme' + html_theme_path = [sphinx_rtd_theme.get_html_theme_path()] + except ImportError: + pass + +# Theme options are theme-specific and customize the look and feel of a theme +# further. For a list of options available for each theme, see the +# documentation. +# html_theme_options = {} + +# Add any paths that contain custom themes here, relative to this directory. +# html_theme_path = [] + +# The name for this set of Sphinx documents. If None, it defaults to +# " v documentation". +html_title = "mozbase documentation" + +# A shorter title for the navigation bar. Default is the same as html_title. +# html_short_title = None + +# The name of an image file (relative to this directory) to place at the top +# of the sidebar. +# html_logo = None + +# The name of an image file (within the static path) to use as favicon of the +# docs. This file should be a Windows icon file (.ico) being 16x16 or 32x32 +# pixels large. +# html_favicon = None + +# Add any paths that contain custom static files (such as style sheets) here, +# relative to this directory. They are copied after the builtin static files, +# so a file named "default.css" will overwrite the builtin "default.css". +html_static_path = ['_static'] + +# If not '', a 'Last updated on:' timestamp is inserted at every page bottom, +# using the given strftime format. +# html_last_updated_fmt = '%b %d, %Y' + +# If true, SmartyPants will be used to convert quotes and dashes to +# typographically correct entities. +# html_use_smartypants = True + +# Custom sidebar templates, maps document names to template names. +# html_sidebars = {} + +# Additional templates that should be rendered to pages, maps page names to +# template names. +# html_additional_pages = {} + +# If false, no module index is generated. +# html_domain_indices = True + +# If false, no index is generated. +# html_use_index = True + +# If true, the index is split into individual pages for each letter. +# html_split_index = False + +# If true, links to the reST sources are added to the pages. +# html_show_sourcelink = True + +# If true, "Created using Sphinx" is shown in the HTML footer. Default is True. +# html_show_sphinx = True + +# If true, "(C) Copyright ..." is shown in the HTML footer. Default is True. +# html_show_copyright = True + +# If true, an OpenSearch description file will be output, and all pages will +# contain a tag referring to it. The value of this option must be the +# base URL from which the finished HTML is served. +# html_use_opensearch = '' + +# This is the file name suffix for HTML files (e.g. ".xhtml"). +# html_file_suffix = None + +# Output file base name for HTML help builder. +htmlhelp_basename = 'MozBasedoc' + + +# -- Options for LaTeX output -------------------------------------------------- + +latex_elements = { + # The paper size ('letterpaper' or 'a4paper'). + # 'papersize': 'letterpaper', + + # The font size ('10pt', '11pt' or '12pt'). + # 'pointsize': '10pt', + + # Additional stuff for the LaTeX preamble. + # 'preamble': '', +} + +# Grouping the document tree into LaTeX files. List of tuples +# (source start file, target name, title, author, documentclass [howto/manual]). +latex_documents = [ + ('index', 'MozBase.tex', u'MozBase Documentation', + u'Mozilla Automation and Tools team', 'manual'), +] + +# The name of an image file (relative to this directory) to place at the top of +# the title page. +# latex_logo = None + +# For "manual" documents, if this is true, then toplevel headings are parts, +# not chapters. +# latex_use_parts = False + +# If true, show page references after internal links. +# latex_show_pagerefs = False + +# If true, show URL addresses after external links. +# latex_show_urls = False + +# Documents to append as an appendix to all manuals. +# latex_appendices = [] + +# If false, no module index is generated. +# latex_domain_indices = True + + +# -- Options for manual page output -------------------------------------------- + +# One entry per manual page. List of tuples +# (source start file, name, description, authors, manual section). +man_pages = [ + ('index', 'mozbase', u'MozBase Documentation', + [u'Mozilla Automation and Tools team'], 1) +] + +# If true, show URL addresses after external links. +# man_show_urls = False + + +# -- Options for Texinfo output ------------------------------------------------ + +# Grouping the document tree into Texinfo files. List of tuples +# (source start file, target name, title, author, +# dir menu entry, description, category) +texinfo_documents = [ + ('index', 'MozBase', u'MozBase Documentation', + u'Mozilla Automation and Tools team', 'MozBase', 'One line description of project.', + 'Miscellaneous'), +] + +# Documents to append as an appendix to all manuals. +# texinfo_appendices = [] + +# If false, no module index is generated. +# texinfo_domain_indices = True + +# How to display URL addresses: 'footnote', 'no', or 'inline'. +# texinfo_show_urls = 'footnote' diff --git a/testing/mozbase/docs/devicemanagement.rst b/testing/mozbase/docs/devicemanagement.rst new file mode 100644 index 000000000..80c4af5bd --- /dev/null +++ b/testing/mozbase/docs/devicemanagement.rst @@ -0,0 +1,11 @@ +Device management +----------------- + +Mozbase provides a module called `mozdevice` for the purposes of +running automated tests or scripts on a device (e.g. an Android- or +FirefoxOS-based phone) connected to a workstation. + +.. toctree:: + :maxdepth: 3 + + mozdevice diff --git a/testing/mozbase/docs/gettinginfo.rst b/testing/mozbase/docs/gettinginfo.rst new file mode 100644 index 000000000..35c4c4508 --- /dev/null +++ b/testing/mozbase/docs/gettinginfo.rst @@ -0,0 +1,13 @@ +Getting information on the system under test +============================================ + +It's often necessary to get some information about the system we're +testing, for example to turn on or off some platform specific +behaviour. + +.. toctree:: + :maxdepth: 2 + + mozinfo + moznetwork + mozversion diff --git a/testing/mozbase/docs/index.rst b/testing/mozbase/docs/index.rst new file mode 100644 index 000000000..86da2a14b --- /dev/null +++ b/testing/mozbase/docs/index.rst @@ -0,0 +1,57 @@ +.. MozBase documentation master file, created by + sphinx-quickstart on Mon Oct 22 14:02:17 2012. + You can adapt this file completely to your liking, but it should at least + contain the root `toctree` directive. + +mozbase +======= + +Mozbase is a set of easy-to-use Python packages forming a supplemental standard +library for Mozilla. It provides consistency and reduces redundancy in +automation and other system-level software. All of Mozilla's test harnesses use +mozbase to some degree, including Talos_, mochitest_, reftest_, Autophone_, and +Eideticker_. + +.. _Talos: https://wiki.mozilla.org/Talos + +.. _mochitest: https://developer.mozilla.org/en-US/docs/Mochitest + +.. _reftest: https://developer.mozilla.org/en-US/docs/Creating_reftest-based_unit_tests + +.. _Autophone: https://wiki.mozilla.org/Auto-tools/Projects/AutoPhone + +.. _Eideticker: https://wiki.mozilla.org/Project_Eideticker + +In the course of writing automated tests at Mozilla, we found that +the same tasks came up over and over, regardless of the specific nature of +what we were testing. We figured that consolidating this code into a set of +libraries would save us a good deal of time, and so we spent some effort +factoring out the best-of-breed automation code into something we named +"mozbase" (usually written all in lower case except at the beginning of a +sentence). + +This is the main documentation for users of mozbase. There is also a +project_ wiki page with notes on development practices and administration. + +.. _project: https://wiki.mozilla.org/Auto-tools/Projects/Mozbase + +The documentation is organized by category, then by module. Figure out what you +want to do then dive in! + +.. toctree:: + :maxdepth: 2 + + manifestparser + gettinginfo + setuprunning + mozhttpd + loggingreporting + devicemanagement + +Indices and tables +================== + +* :ref:`genindex` +* :ref:`modindex` +* :ref:`search` + diff --git a/testing/mozbase/docs/loggingreporting.rst b/testing/mozbase/docs/loggingreporting.rst new file mode 100644 index 000000000..a8561a49b --- /dev/null +++ b/testing/mozbase/docs/loggingreporting.rst @@ -0,0 +1,11 @@ +Logging and reporting +===================== + +Ideally output between different types of testing system should be as +uniform as possible, as well as making it easy to make things more or +less verbose. We created some libraries to make doing this easy. + +.. toctree:: + :maxdepth: 2 + + mozlog diff --git a/testing/mozbase/docs/make.bat b/testing/mozbase/docs/make.bat new file mode 100644 index 000000000..d67c86ae9 --- /dev/null +++ b/testing/mozbase/docs/make.bat @@ -0,0 +1,190 @@ +@ECHO OFF + +REM Command file for Sphinx documentation + +if "%SPHINXBUILD%" == "" ( + set SPHINXBUILD=sphinx-build +) +set BUILDDIR=_build +set ALLSPHINXOPTS=-d %BUILDDIR%/doctrees %SPHINXOPTS% . +set I18NSPHINXOPTS=%SPHINXOPTS% . +if NOT "%PAPER%" == "" ( + set ALLSPHINXOPTS=-D latex_paper_size=%PAPER% %ALLSPHINXOPTS% + set I18NSPHINXOPTS=-D latex_paper_size=%PAPER% %I18NSPHINXOPTS% +) + +if "%1" == "" goto help + +if "%1" == "help" ( + :help + echo.Please use `make ^` where ^ is one of + echo. html to make standalone HTML files + echo. dirhtml to make HTML files named index.html in directories + echo. singlehtml to make a single large HTML file + echo. pickle to make pickle files + echo. json to make JSON files + echo. htmlhelp to make HTML files and a HTML help project + echo. qthelp to make HTML files and a qthelp project + echo. devhelp to make HTML files and a Devhelp project + echo. epub to make an epub + echo. latex to make LaTeX files, you can set PAPER=a4 or PAPER=letter + echo. text to make text files + echo. man to make manual pages + echo. texinfo to make Texinfo files + echo. gettext to make PO message catalogs + echo. changes to make an overview over all changed/added/deprecated items + echo. linkcheck to check all external links for integrity + echo. doctest to run all doctests embedded in the documentation if enabled + goto end +) + +if "%1" == "clean" ( + for /d %%i in (%BUILDDIR%\*) do rmdir /q /s %%i + del /q /s %BUILDDIR%\* + goto end +) + +if "%1" == "html" ( + %SPHINXBUILD% -b html %ALLSPHINXOPTS% %BUILDDIR%/html + if errorlevel 1 exit /b 1 + echo. + echo.Build finished. The HTML pages are in %BUILDDIR%/html. + goto end +) + +if "%1" == "dirhtml" ( + %SPHINXBUILD% -b dirhtml %ALLSPHINXOPTS% %BUILDDIR%/dirhtml + if errorlevel 1 exit /b 1 + echo. + echo.Build finished. The HTML pages are in %BUILDDIR%/dirhtml. + goto end +) + +if "%1" == "singlehtml" ( + %SPHINXBUILD% -b singlehtml %ALLSPHINXOPTS% %BUILDDIR%/singlehtml + if errorlevel 1 exit /b 1 + echo. + echo.Build finished. The HTML pages are in %BUILDDIR%/singlehtml. + goto end +) + +if "%1" == "pickle" ( + %SPHINXBUILD% -b pickle %ALLSPHINXOPTS% %BUILDDIR%/pickle + if errorlevel 1 exit /b 1 + echo. + echo.Build finished; now you can process the pickle files. + goto end +) + +if "%1" == "json" ( + %SPHINXBUILD% -b json %ALLSPHINXOPTS% %BUILDDIR%/json + if errorlevel 1 exit /b 1 + echo. + echo.Build finished; now you can process the JSON files. + goto end +) + +if "%1" == "htmlhelp" ( + %SPHINXBUILD% -b htmlhelp %ALLSPHINXOPTS% %BUILDDIR%/htmlhelp + if errorlevel 1 exit /b 1 + echo. + echo.Build finished; now you can run HTML Help Workshop with the ^ +.hhp project file in %BUILDDIR%/htmlhelp. + goto end +) + +if "%1" == "qthelp" ( + %SPHINXBUILD% -b qthelp %ALLSPHINXOPTS% %BUILDDIR%/qthelp + if errorlevel 1 exit /b 1 + echo. + echo.Build finished; now you can run "qcollectiongenerator" with the ^ +.qhcp project file in %BUILDDIR%/qthelp, like this: + echo.^> qcollectiongenerator %BUILDDIR%\qthelp\MozBase.qhcp + echo.To view the help file: + echo.^> assistant -collectionFile %BUILDDIR%\qthelp\MozBase.ghc + goto end +) + +if "%1" == "devhelp" ( + %SPHINXBUILD% -b devhelp %ALLSPHINXOPTS% %BUILDDIR%/devhelp + if errorlevel 1 exit /b 1 + echo. + echo.Build finished. + goto end +) + +if "%1" == "epub" ( + %SPHINXBUILD% -b epub %ALLSPHINXOPTS% %BUILDDIR%/epub + if errorlevel 1 exit /b 1 + echo. + echo.Build finished. The epub file is in %BUILDDIR%/epub. + goto end +) + +if "%1" == "latex" ( + %SPHINXBUILD% -b latex %ALLSPHINXOPTS% %BUILDDIR%/latex + if errorlevel 1 exit /b 1 + echo. + echo.Build finished; the LaTeX files are in %BUILDDIR%/latex. + goto end +) + +if "%1" == "text" ( + %SPHINXBUILD% -b text %ALLSPHINXOPTS% %BUILDDIR%/text + if errorlevel 1 exit /b 1 + echo. + echo.Build finished. The text files are in %BUILDDIR%/text. + goto end +) + +if "%1" == "man" ( + %SPHINXBUILD% -b man %ALLSPHINXOPTS% %BUILDDIR%/man + if errorlevel 1 exit /b 1 + echo. + echo.Build finished. The manual pages are in %BUILDDIR%/man. + goto end +) + +if "%1" == "texinfo" ( + %SPHINXBUILD% -b texinfo %ALLSPHINXOPTS% %BUILDDIR%/texinfo + if errorlevel 1 exit /b 1 + echo. + echo.Build finished. The Texinfo files are in %BUILDDIR%/texinfo. + goto end +) + +if "%1" == "gettext" ( + %SPHINXBUILD% -b gettext %I18NSPHINXOPTS% %BUILDDIR%/locale + if errorlevel 1 exit /b 1 + echo. + echo.Build finished. The message catalogs are in %BUILDDIR%/locale. + goto end +) + +if "%1" == "changes" ( + %SPHINXBUILD% -b changes %ALLSPHINXOPTS% %BUILDDIR%/changes + if errorlevel 1 exit /b 1 + echo. + echo.The overview file is in %BUILDDIR%/changes. + goto end +) + +if "%1" == "linkcheck" ( + %SPHINXBUILD% -b linkcheck %ALLSPHINXOPTS% %BUILDDIR%/linkcheck + if errorlevel 1 exit /b 1 + echo. + echo.Link check complete; look for any errors in the above output ^ +or in %BUILDDIR%/linkcheck/output.txt. + goto end +) + +if "%1" == "doctest" ( + %SPHINXBUILD% -b doctest %ALLSPHINXOPTS% %BUILDDIR%/doctest + if errorlevel 1 exit /b 1 + echo. + echo.Testing of doctests in the sources finished, look at the ^ +results in %BUILDDIR%/doctest/output.txt. + goto end +) + +:end diff --git a/testing/mozbase/docs/manifestparser.rst b/testing/mozbase/docs/manifestparser.rst new file mode 100644 index 000000000..e93317b40 --- /dev/null +++ b/testing/mozbase/docs/manifestparser.rst @@ -0,0 +1,558 @@ +Managing lists of tests +======================= + +.. py:currentmodule:: manifestparser + +We don't always want to run all tests, all the time. Sometimes a test +may be broken, in other cases we only want to run a test on a specific +platform or build of Mozilla. To handle these cases (and more), we +created a python library to create and use test "manifests", which +codify this information. + +:mod:`manifestparser` --- Create and manage test manifests +----------------------------------------------------------- + +manifestparser lets you easily create and use test manifests, to +control which tests are run under what circumstances. + +What manifestparser gives you: + +* manifests are ordered lists of tests +* tests may have an arbitrary number of key, value pairs +* the parser returns an ordered list of test data structures, which + are just dicts with some keys. For example, a test with no + user-specified metadata looks like this: + +.. code-block:: text + + [{'expected': 'pass', + 'path': '/home/mozilla/mozmill/src/manifestparser/manifestparser/tests/testToolbar/testBackForwardButtons.js', + 'relpath': 'testToolbar/testBackForwardButtons.js', + 'name': 'testBackForwardButtons.js', + 'here': '/home/mozilla/mozmill/src/manifestparser/manifestparser/tests', + 'manifest': '/home/mozilla/mozmill/src/manifestparser/manifestparser/tests/manifest.ini',}] + +The keys displayed here (path, relpath, name, here, and manifest) are +reserved keys for manifestparser and any consuming APIs. You can add +additional key, value metadata to each test. + +Why have test manifests? +```````````````````````` + +It is desirable to have a unified format for test manifests for testing +[mozilla-central](http://hg.mozilla.org/mozilla-central), etc. + +* It is desirable to be able to selectively enable or disable tests based on platform or other conditions. This should be easy to do. Currently, since many of the harnesses just crawl directories, there is no effective way of disabling a test except for removal from mozilla-central +* It is desriable to do this in a universal way so that enabling and disabling tests as well as other tasks are easily accessible to a wider audience than just those intimately familiar with the specific test framework. +* It is desirable to have other metadata on top of the test. For instance, let's say a test is marked as skipped. It would be nice to give the reason why. + + +Most Mozilla test harnesses work by crawling a directory structure. +While this is straight-forward, manifests offer several practical +advantages: + +* ability to turn a test off easily: if a test is broken on m-c + currently, the only way to turn it off, generally speaking, is just + removing the test. Often this is undesirable, as if the test should + be dismissed because other people want to land and it can't be + investigated in real time (is it a failure? is the test bad? is no + one around that knows the test?), then backing out a test is at best + problematic. With a manifest, a test may be disabled without + removing it from the tree and a bug filed with the appropriate + reason: + +.. code-block:: text + + [test_broken.js] + disabled = https://bugzilla.mozilla.org/show_bug.cgi?id=123456 + +* ability to run different (subsets of) tests on different + platforms. Traditionally, we've done a bit of magic or had the test + know what platform it would or would not run on. With manifests, you + can mark what platforms a test will or will not run on and change + these without changing the test. + +.. code-block:: text + + [test_works_on_windows_only.js] + skip-if = os != 'win' + +* ability to markup tests with metadata. We have a large, complicated, + and always changing infrastructure. key, value metadata may be used + as an annotation to a test and appropriately curated and mined. For + instance, we could mark certain tests as randomorange with a bug + number, if it were desirable. + +* ability to have sane and well-defined test-runs. You can keep + different manifests for different test runs and ``[include:]`` + (sub)manifests as appropriate to your needs. + +Manifest Format +```````` + +Manifests are .ini file with the section names denoting the path +relative to the manifest: + +.. code-block:: text + + [foo.js] + [bar.js] + [fleem.js] + +The sections are read in order. In addition, tests may include +arbitrary key, value metadata to be used by the harness. You may also +have a `[DEFAULT]` section that will give key, value pairs that will +be inherited by each test unless overridden: + +.. code-block:: text + + [DEFAULT] + type = restart + + [lilies.js] + color = white + + [daffodils.js] + color = yellow + type = other + # override type from DEFAULT + + [roses.js] + color = red + +You can also include other manifests: + +.. code-block:: text + + [include:subdir/anothermanifest.ini] + +And reference parent manifests to inherit keys and values from the DEFAULT +section, without adding possible included tests. + +.. code-block:: text + + [parent:../manifest.ini] + +Manifests are included relative to the directory of the manifest with +the `[include:]` directive unless they are absolute paths. + +By default you can use both '#' and ';' as comment characters. Comments +must start on a new line, inline comments are not supported. + +.. code-block:: text + + [roses.js] + # a valid comment + ; another valid comment + color = red # not a valid comment + +In the example above, the 'color' property will have the value 'red # +not a valid comment'. + +Special variable server-root +```````````````````````````` +There is a special variable called `server-root` used for paths on the system. +This variable is deemed a path and will be expanded into its absolute form. + +Because of the inheritant nature of the key/value pairs, if one requires a +system path, it must be absolute for it to be of any use in any included file. + +.. code-block:: text + + [DEFAULTS] + server-root = ../data + + [test1.js] + server-root = test1/data + +Manifest Conditional Expressions +```````````````````````````````` +The conditional expressions used in manifests are parsed using the *ExpressionParser* class. + +.. autoclass:: manifestparser.ExpressionParser + +Consumers of this module are expected to pass in a value dictionary +for evaluating conditional expressions. A common pattern is to pass +the dictionary from the :mod:`mozinfo` module. + +Data +```` + +Manifest Destiny gives tests as a list of dictionaries (in python +terms). + +* path: full path to the test +* relpath: relative path starting from the root directory. The root directory + is typically the location of the root manifest, or the source + repository. It can be specified at runtime by passing in `rootdir` + to `TestManifest`. Defaults to the directory containing the test's + ancestor manifest. +* name: file name of the test +* here: the parent directory of the manifest +* manifest: the path to the manifest containing the test + +This data corresponds to a one-line manifest: + +.. code-block:: text + + [testToolbar/testBackForwardButtons.js] + +If additional key, values were specified, they would be in this dict +as well. + +Outside of the reserved keys, the remaining key, values +are up to convention to use. There is a (currently very minimal) +generic integration layer in manifestparser for use of all harnesses, +`manifestparser.TestManifest`. +For instance, if the 'disabled' key is present, you can get the set of +tests without disabled (various other queries are doable as well). + +Since the system is convention-based, the harnesses may do whatever +they want with the data. They may ignore it completely, they may use +the provided integration layer, or they may provide their own +integration layer. This should allow whatever sort of logic is +desired. For instance, if in yourtestharness you wanted to run only on +mondays for a certain class of tests: + +.. code-block:: text + + tests = [] + for test in manifests.tests: + if 'runOnDay' in test: + if calendar.day_name[calendar.weekday(*datetime.datetime.now().timetuple()[:3])].lower() == test['runOnDay'].lower(): + tests.append(test) + else: + tests.append(test) + +To recap: +* the manifests allow you to specify test data +* the parser gives you this data +* you can use it however you want or process it further as you need + +Tests are denoted by sections in an .ini file (see +http://hg.mozilla.org/automation/manifestparser/file/tip/manifestparser/tests/mozmill-example.ini). + +Additional manifest files may be included with an `[include:]` directive: + +.. code-block:: text + + [include:path-to-additional-file.manifest] + +The path to included files is relative to the current manifest. + +The `[DEFAULT]` section contains variables that all tests inherit from. + +Included files will inherit the top-level variables but may override +in their own `[DEFAULT]` section. + +manifestparser Architecture +```````````````````````````` + +There is a two- or three-layered approach to the manifestparser +architecture, depending on your needs: + +1. ManifestParser: this is a generic parser for .ini manifests that +facilitates the `[include:]` logic and the inheritence of +metadata. Despite the internal variable being called `self.tests` +(an oversight), this layer has nothing in particular to do with tests. + +2. TestManifest: this is a harness-agnostic integration layer that is +test-specific. TestManifest faciliates `skip-if` logic. + +3. Optionally, a harness will have an integration layer than inherits +from TestManifest if more harness-specific customization is desired at +the manifest level. + +See the source code at https://github.com/mozilla/mozbase/tree/master/manifestparser +and +https://github.com/mozilla/mozbase/blob/master/manifestparser/manifestparser.py +in particular. + +Filtering Manifests +``````````````````` + +After creating a `TestManifest` object, all manifest files are read and a list +of test objects can be accessed via `TestManifest.tests`. However this list contains +all test objects, whether they should be run or not. Normally they need to be +filtered down only to the set of tests that should be run by the test harness. + +To do this, a test harness can call `TestManifest.active_tests`: + +.. code-block:: python + + tests = manifest.active_tests(exists=True, disabled=True, **tags) + +By default, `active_tests` runs the filters found in +:attr:`~.DEFAULT_FILTERS`. It also accepts two convenience arguments: + +1. `exists`: if True (default), filter out tests that do not exist on the local file system. +2. `disabled`: if True (default), do not filter out tests containing the 'disabled' key + (which can be set by `skip-if` manually). + +This works for simple cases, but there are other built-in filters, or even custom filters +that can be applied to the `TestManifest`. To do so, add the filter to `TestManifest.filters`: + +.. code-block:: python + + from manifestparser.filters import subsuite + import mozinfo + + filters = [subsuite('devtools')] + tests = manifest.active_tests(filters=filters, **mozinfo.info) + +.. automodule:: manifestparser.filters + :members: + :exclude-members: filterlist,InstanceFilter,DEFAULT_FILTERS + +.. autodata:: manifestparser.filters.DEFAULT_FILTERS + :annotation: + +For example, suppose we want to introduce a new key called `timeout-if` that adds a +'timeout' property to a test if a certain condition is True. The syntax in the manifest +files will look like this: + +.. code-block:: text + + [test_foo.py] + timeout-if = 300, os == 'win' + +The value is , where condition is the same format as the one in +`skip-if`. In the above case, if os == 'win', a timeout of 300 seconds will be +applied. Otherwise, no timeout will be applied. All we need to do is define the filter +and add it: + +.. code-block:: python + + from manifestparser.expression import parse + import mozinfo + + def timeout_if(tests, values): + for test in tests: + if 'timeout-if' in test: + timeout, condition = test['timeout-if'].split(',', 1) + if parse(condition, **values): + test['timeout'] = timeout + yield test + + tests = manifest.active_tests(filters=[timeout_if], **mozinfo.info) + +Creating Manifests +`````````````````` + +manifestparser comes with a console script, `manifestparser create`, that +may be used to create a seed manifest structure from a directory of +files. Run `manifestparser help create` for usage information. + +Copying Manifests +````````````````` + +To copy tests and manifests from a source: + +.. code-block:: text + + manifestparser [options] copy from_manifest to_directory -tag1 -tag2 `key1=value1 key2=value2 ... + +Updating Tests +`````````````` + +To update the tests associated with with a manifest from a source +directory: + +.. code-block:: text + + manifestparser [options] update manifest from_directory -tag1 -tag2 `key1=value1 `key2=value2 ... + +Usage example +````````````` + +Here is an example of how to create manifests for a directory tree and +update the tests listed in the manifests from an external source. + +Creating Manifests +`````````````````` + +Let's say you want to make a series of manifests for a given directory structure containing `.js` test files: + +.. code-block:: text + + testing/mozmill/tests/firefox/ + testing/mozmill/tests/firefox/testAwesomeBar/ + testing/mozmill/tests/firefox/testPreferences/ + testing/mozmill/tests/firefox/testPrivateBrowsing/ + testing/mozmill/tests/firefox/testSessionStore/ + testing/mozmill/tests/firefox/testTechnicalTools/ + testing/mozmill/tests/firefox/testToolbar/ + testing/mozmill/tests/firefox/restartTests + +You can use `manifestparser create` to do this: + +.. code-block:: text + + $ manifestparser help create + Usage: manifestparser.py [options] create directory <...> + + create a manifest from a list of directories + + Options: + -p PATTERN, `pattern=PATTERN + glob pattern for files + -i IGNORE, `ignore=IGNORE + directories to ignore + -w IN_PLACE, --in-place=IN_PLACE + Write .ini files in place; filename to write to + +We only want `.js` files and we want to skip the `restartTests` directory. +We also want to write a manifest per directory, so I use the `--in-place` +option to write the manifests: + +.. code-block:: text + + manifestparser create . -i restartTests -p '*.js' -w manifest.ini + +This creates a manifest.ini per directory that we care about with the JS test files: + +.. code-block:: text + + testing/mozmill/tests/firefox/manifest.ini + testing/mozmill/tests/firefox/testAwesomeBar/manifest.ini + testing/mozmill/tests/firefox/testPreferences/manifest.ini + testing/mozmill/tests/firefox/testPrivateBrowsing/manifest.ini + testing/mozmill/tests/firefox/testSessionStore/manifest.ini + testing/mozmill/tests/firefox/testTechnicalTools/manifest.ini + testing/mozmill/tests/firefox/testToolbar/manifest.ini + +The top-level `manifest.ini` merely has `[include:]` references to the sub manifests: + +.. code-block:: text + + [include:testAwesomeBar/manifest.ini] + [include:testPreferences/manifest.ini] + [include:testPrivateBrowsing/manifest.ini] + [include:testSessionStore/manifest.ini] + [include:testTechnicalTools/manifest.ini] + [include:testToolbar/manifest.ini] + +Each sub-level manifest contains the (`.js`) test files relative to it. + +Updating the tests from manifests +````````````````````````````````` + +You may need to update tests as given in manifests from a different source directory. +`manifestparser update` was made for just this purpose: + +.. code-block:: text + + Usage: manifestparser [options] update manifest directory -tag1 -tag2 `key1=value1 --key2=value2 ... + + update the tests as listed in a manifest from a directory + +To update from a directory of tests in `~/mozmill/src/mozmill-tests/firefox/` run: + +.. code-block:: text + + manifestparser update manifest.ini ~/mozmill/src/mozmill-tests/firefox/ + +Tests +````` + +manifestparser includes a suite of tests: + +https://github.com/mozilla/mozbase/tree/master/manifestparsery/tests + +`test_manifest.txt` is a doctest that may be helpful in figuring out +how to use the API. Tests are run via `python test.py`. + +Bugs +```` + +Please file any bugs or feature requests at + +https://bugzilla.mozilla.org/enter_bug.cgi?product=Testing&component=ManifestParser + +Or contact jhammel @mozilla.org or in #ateam on irc.mozilla.org + +CLI +``` + +Run `manifestparser help` for usage information. + +To create a manifest from a set of directories: + +.. code-block:: text + + manifestparser [options] create directory <...> [create-options] + +To output a manifest of tests: + +.. code-block:: text + + manifestparser [options] write manifest <...> -tag1 -tag2 --key1=value1 --key2=value2 ... + +To copy tests and manifests from a source: + +.. code-block:: text + + manifestparser [options] copy from_manifest to_manifest -tag1 -tag2 `key1=value1 key2=value2 ... + +To update the tests associated with with a manifest from a source +directory: + +.. code-block:: text + + manifestparser [options] update manifest from_directory -tag1 -tag2 --key1=value1 --key2=value2 ... + +Design Considerations +````````````````````` + +Contrary to some opinion, manifestparser.py and the associated .ini +format were not magically plucked from the sky but were descended upon +through several design considerations. + +* test manifests should be ordered. While python 2.6 and greater has + a ConfigParser that can use an ordered dictionary, it is a + requirement that we support python 2.4 for the build + testing + environment. To that end, a `read_ini` function was implemented + in manifestparser.py that should be the equivalent of the .ini + dialect used by ConfigParser. + +* the manifest format should be easily human readable/writable. While + there was initially some thought of using JSON, there was pushback + that JSON was not easily editable. An ideal manifest format would + degenerate to a line-separated list of files. While .ini format + requires an additional `[]` per line, and while there have been + complaints about this, hopefully this is good enough. + +* python does not have an in-built YAML parser. Since it was + undesirable for manifestparser.py to have any dependencies, YAML was + dismissed as a format. + +* we could have used a proprietary format but decided against it. + Everyone knows .ini and there are good tools to deal with it. + However, since read_ini is the only function that transforms a + manifest to a list of key, value pairs, while the implications for + changing the format impacts downstream code, doing so should be + programmatically simple. + +* there should be a single file that may easily be + transported. Traditionally, test harnesses have lived in + mozilla-central. This is less true these days and it is increasingly + likely that more tests will not live in mozilla-central going + forward. So `manifestparser.py` should be highly consumable. To + this end, it is a single file, as appropriate to mozilla-central, + which is also a working python package deployed to PyPI for easy + installation. + +Historical Reference +```````````````````` + +Date-ordered list of links about how manifests came to be where they are today:: + +* https://wiki.mozilla.org/Auto-tools/Projects/UniversalManifest +* http://alice.nodelman.net/blog/post/2010/05/ +* http://alice.nodelman.net/blog/post/universal-manifest-for-unit-tests-a-proposal/ +* https://elvis314.wordpress.com/2010/07/05/improving-personal-hygiene-by-adjusting-mochitests/ +* https://elvis314.wordpress.com/2010/07/27/types-of-data-we-care-about-in-a-manifest/ +* https://bugzilla.mozilla.org/show_bug.cgi?id=585106 +* http://elvis314.wordpress.com/2011/05/20/converting-xpcshell-from-listing-directories-to-a-manifest/ +* https://bugzilla.mozilla.org/show_bug.cgi?id=616999 +* https://developer.mozilla.org/en/Writing_xpcshell-based_unit_tests#Adding_your_tests_to_the_xpcshell_manifest diff --git a/testing/mozbase/docs/mozcrash.rst b/testing/mozbase/docs/mozcrash.rst new file mode 100644 index 000000000..750c46dd8 --- /dev/null +++ b/testing/mozbase/docs/mozcrash.rst @@ -0,0 +1,8 @@ +:mod:`mozcrash` --- Print stack traces from minidumps left behind by crashed processes +====================================================================================== + +Gets stack traces out of processes that have crashed and left behind +a minidump file using the Google Breakpad library. + +.. automodule:: mozcrash + :members: check_for_crashes diff --git a/testing/mozbase/docs/mozdebug.rst b/testing/mozbase/docs/mozdebug.rst new file mode 100644 index 000000000..6a4be63f4 --- /dev/null +++ b/testing/mozbase/docs/mozdebug.rst @@ -0,0 +1,5 @@ +:mod:`mozdebug` --- Configure and launch compatible debuggers. +====================================================================================== + +.. automodule:: mozdebug + :members: get_debugger_info, get_default_debugger_name, DebuggerSearch diff --git a/testing/mozbase/docs/mozdevice.rst b/testing/mozbase/docs/mozdevice.rst new file mode 100644 index 000000000..5e18b56d2 --- /dev/null +++ b/testing/mozbase/docs/mozdevice.rst @@ -0,0 +1,254 @@ +:mod:`mozdevice` --- Interact with remote devices +================================================= + +Mozdevice provides several interfaces to interact with a remote device +such as an Android- or FirefoxOS-based phone. It allows you to push +files to these types of devices, launch processes, and more. There are +currently two available interfaces: + +* :ref:`DeviceManager`: Works either via ADB or a custom TCP protocol + (the latter requires an agent application running on the device). +* :ref:`ADB`: Uses the Android Debugger Protocol explicitly + +In general, new code should use the ADB abstraction where possible as +it is simpler and more reliable. + +.. automodule:: mozdevice + +.. _DeviceManager: + +DeviceManager interface +----------------------- +.. autoclass:: DeviceManager + +Here's an example script which lists the files in '/mnt/sdcard' and sees if a +process called 'org.mozilla.fennec' is running. In this example, we're +instantiating the DeviceManagerADB implementation, but we could just +as easily have used DeviceManagerSUT (assuming the device had an agent +running speaking the SUT protocol). + +:: + + import mozdevice + + dm = mozdevice.DeviceManagerADB() + print dm.listFiles("/mnt/sdcard") + if dm.processExist("org.mozilla.fennec"): + print "Fennec is running" + +Informational methods +````````````````````` +.. automethod:: DeviceManager.getInfo(self, directive=None) +.. automethod:: DeviceManager.getCurrentTime(self) +.. automethod:: DeviceManager.getIP +.. automethod:: DeviceManager.saveScreenshot +.. automethod:: DeviceManager.recordLogcat +.. automethod:: DeviceManager.getLogcat + +File management methods +``````````````````````` +.. autoattribute:: DeviceManager.deviceRoot +.. automethod:: DeviceManager.getDeviceRoot(self) +.. automethod:: DeviceManager.pushFile(self, localFilename, remoteFilename, retryLimit=1) +.. automethod:: DeviceManager.pushDir(self, localDirname, remoteDirname, retryLimit=1) +.. automethod:: DeviceManager.pullFile(self, remoteFilename) +.. automethod:: DeviceManager.getFile(self, remoteFilename, localFilename) +.. automethod:: DeviceManager.getDirectory(self, remoteDirname, localDirname, checkDir=True) +.. automethod:: DeviceManager.validateFile(self, remoteFilename, localFilename) +.. automethod:: DeviceManager.mkDir(self, remoteDirname) +.. automethod:: DeviceManager.mkDirs(self, filename) +.. automethod:: DeviceManager.dirExists(self, dirpath) +.. automethod:: DeviceManager.fileExists(self, filepath) +.. automethod:: DeviceManager.listFiles(self, rootdir) +.. automethod:: DeviceManager.removeFile(self, filename) +.. automethod:: DeviceManager.removeDir(self, remoteDirname) +.. automethod:: DeviceManager.chmodDir(self, remoteDirname, mask="777") +.. automethod:: DeviceManager.getTempDir(self) + +Process management methods +`````````````````````````` +.. automethod:: DeviceManager.shell(self, cmd, outputfile, env=None, cwd=None, timeout=None, root=False) +.. automethod:: DeviceManager.shellCheckOutput(self, cmd, env=None, cwd=None, timeout=None, root=False) +.. automethod:: DeviceManager.getProcessList(self) +.. automethod:: DeviceManager.processExist(self, processName) +.. automethod:: DeviceManager.killProcess(self, processName) + +System control methods +`````````````````````` +.. automethod:: DeviceManager.reboot(self, ipAddr=None, port=30000) + +Application management methods +`````````````````````````````` +.. automethod:: DeviceManager.uninstallAppAndReboot(self, appName, installPath=None) +.. automethod:: DeviceManager.installApp(self, appBundlePath, destPath=None) +.. automethod:: DeviceManager.uninstallApp(self, appName, installPath=None) +.. automethod:: DeviceManager.updateApp(self, appBundlePath, processName=None, destPath=None, ipAddr=None, port=30000) + +DeviceManagerADB implementation +``````````````````````````````` + +.. autoclass:: mozdevice.DeviceManagerADB + +DeviceManagerADB has several methods that are not present in all +DeviceManager implementations. Please do not use them in code that +is meant to be interoperable. + +.. automethod:: DeviceManagerADB.forward +.. automethod:: DeviceManagerADB.remount +.. automethod:: DeviceManagerADB.devices + +DeviceManagerSUT implementation +``````````````````````````````` + +.. autoclass:: mozdevice.DeviceManagerSUT + +DeviceManagerSUT has several methods that are only used in specific +tests and are not present in all DeviceManager implementations. Please +do not use them in code that is meant to be interoperable. + +.. automethod:: DeviceManagerSUT.unpackFile +.. automethod:: DeviceManagerSUT.adjustResolution + +Android extensions +`````````````````` + +For Android, we provide two variants of the `DeviceManager` interface +with extensions useful for that platform. These classes are called +DroidADB and DroidSUT. They inherit all methods from DeviceManagerADB +and DeviceManagerSUT. Here is the interface for DroidADB: + +.. automethod:: mozdevice.DroidADB.launchApplication +.. automethod:: mozdevice.DroidADB.launchFennec +.. automethod:: mozdevice.DroidADB.getInstalledApps +.. automethod:: mozdevice.DroidADB.getAppRoot + +These methods are also found in the DroidSUT class. + +.. _ADB: + +ADB Interface +------------- + +The following classes provide a basic interface to interact with the +Android Debug Tool (adb) and Android-based devices. It is intended to +provide the basis for a replacement for DeviceManager and +DeviceManagerADB. + +ADBCommand +`````````` + +.. autoclass:: mozdevice.ADBCommand + +.. automethod:: ADBCommand.command(self, cmds, timeout=None) +.. automethod:: ADBCommand.command_output(self, cmds, timeout=None) + +ADBHost +``````` +.. autoclass:: mozdevice.ADBHost + +.. automethod:: ADBHost.command(self, cmds, timeout=None) +.. automethod:: ADBHost.command_output(self, cmds, timeout=None) +.. automethod:: ADBHost.start_server(self, timeout=None) +.. automethod:: ADBHost.kill_server(self, timeout=None) +.. automethod:: ADBHost.devices(self, timeout=None) + +ADBDevice +````````` +.. autoclass:: mozdevice.ADBDevice + +Host Command methods +++++++++++++++++++++ +.. automethod:: ADBDevice.command(self, cmds, timeout=None) +.. automethod:: ADBDevice.command_output(self, cmds, timeout=None) + +Device Shell methods +++++++++++++++++++++ +.. automethod:: ADBDevice.shell(self, cmd, env=None, cwd=None, timeout=None, root=False) +.. automethod:: ADBDevice.shell_bool(self, cmd, env=None, cwd=None, timeout=None, root=False) +.. automethod:: ADBDevice.shell_output(self, cmd, env=None, cwd=None, timeout=None, root=False) + +Informational methods ++++++++++++++++++++++ +.. automethod:: ADBDevice.clear_logcat +.. automethod:: ADBDevice.get_battery_percentage +.. automethod:: ADBDevice.get_info +.. automethod:: ADBDevice.get_logcat +.. automethod:: ADBDevice.get_prop +.. automethod:: ADBDevice.get_state + +System control methods +++++++++++++++++++++++ +.. automethod:: ADBDevice.is_device_ready +.. automethod:: ADBDevice.reboot + +File management methods ++++++++++++++++++++++++ +.. automethod:: ADBDevice.chmod +.. automethod:: ADBDevice.cp +.. automethod:: ADBDevice.exists +.. automethod:: ADBDevice.is_dir +.. automethod:: ADBDevice.is_file +.. automethod:: ADBDevice.list_files +.. automethod:: ADBDevice.mkdir +.. automethod:: ADBDevice.mv +.. automethod:: ADBDevice.push +.. automethod:: ADBDevice.rm +.. automethod:: ADBDevice.rmdir +.. autoattribute:: ADBDevice.test_root + +Process management methods +++++++++++++++++++++++++++ +.. automethod:: ADBDevice.get_process_list +.. automethod:: ADBDevice.kill +.. automethod:: ADBDevice.pkill +.. automethod:: ADBDevice.process_exist + +ADBAndroid +`````````` +.. autoclass:: ADBAndroid + +Informational methods ++++++++++++++++++++++ +.. automethod:: ADBAndroid.get_battery_percentage + +System control methods +++++++++++++++++++++++ +.. automethod:: ADBAndroid.is_device_ready +.. automethod:: ADBAndroid.power_on + +Application management methods +++++++++++++++++++++++++++++++ +.. automethod:: ADBAndroid.install_app +.. automethod:: ADBAndroid.is_app_installed +.. automethod:: ADBAndroid.launch_application +.. automethod:: ADBAndroid.launch_fennec +.. automethod:: ADBAndroid.stop_application +.. automethod:: ADBAndroid.uninstall_app +.. automethod:: ADBAndroid.update_app + +ADBB2G +`````` +.. autoclass:: ADBB2G + +Informational methods ++++++++++++++++++++++ +.. automethod:: ADBB2G.get_battery_percentage +.. automethod:: ADBB2G.get_info +.. automethod:: ADBB2G.get_memory_total + +ADBProcess +`````````` +.. autoclass:: mozdevice.ADBProcess + +ADBError +```````` +.. autoexception:: mozdevice.ADBError + +ADBRootError +```````````` +.. autoexception:: mozdevice.ADBRootError + +ADBTimeoutError +``````````````` +.. autoexception:: mozdevice.ADBTimeoutError + diff --git a/testing/mozbase/docs/mozfile.rst b/testing/mozbase/docs/mozfile.rst new file mode 100644 index 000000000..52e1cd4c9 --- /dev/null +++ b/testing/mozbase/docs/mozfile.rst @@ -0,0 +1,10 @@ +:mod:`mozfile` --- File utilities for use in Mozilla testing +============================================================ + +mozfile is a convenience library for taking care of some common file-related +tasks in automated testing, such as extracting files or recursively removing +directories. + +.. automodule:: mozfile + :members: extract, extract_tarball, extract_zip, move, remove + diff --git a/testing/mozbase/docs/mozhttpd.rst b/testing/mozbase/docs/mozhttpd.rst new file mode 100644 index 000000000..f6ceddb37 --- /dev/null +++ b/testing/mozbase/docs/mozhttpd.rst @@ -0,0 +1,22 @@ + +Serving up content to be consumed by the browser +================================================ + + +.. warning:: The mozhttpd module is considered obsolete. For new code, + please use wptserve_ which can do everything mozhttpd does + and more. + +.. _wptserve: https://pypi.python.org/pypi/wptserve + +:mod:`mozhttpd` --- Simple webserver +------------------------------------ + +.. automodule:: mozhttpd + :members: + +Interface +````````` + +.. autoclass:: MozHttpd + :members: diff --git a/testing/mozbase/docs/mozinfo.rst b/testing/mozbase/docs/mozinfo.rst new file mode 100644 index 000000000..2f9eb5f7e --- /dev/null +++ b/testing/mozbase/docs/mozinfo.rst @@ -0,0 +1,71 @@ +:mod:`mozinfo` --- Get system information +========================================= + +Throughout `mozmill `_ +and other Mozilla python code, checking the underlying +platform is done in many different ways. The various checks needed +lead to a lot of copy+pasting, leaving the reader to wonder....is this +specific check necessary for (e.g.) an operating system? Because +information is not consolidated, checks are not done consistently, nor +is it defined what we are checking for. + +`mozinfo `_ +proposes to solve this problem. mozinfo is a bridge interface, +making the underlying (complex) plethora of OS and architecture +combinations conform to a subset of values of relevance to +Mozilla software. The current implementation exposes relevant keys and +values such as: ``os``, ``version``, ``bits``, and ``processor``. Additionally, the +service pack in use is available on the windows platform. + + +API Usage +--------- + +mozinfo is a python package. Downloading the software and running +``python setup.py develop`` will allow you to do ``import mozinfo`` +from python. +`mozinfo.py `_ +is the only file contained is this package, +so if you need a single-file solution, you can just download or call +this file through the web. + +The top level attributes (``os``, ``version``, ``bits``, ``processor``) are +available as module globals:: + + if mozinfo.os == 'win': ... + +In addition, mozinfo exports a dictionary, ``mozinfo.info``, that +contain these values. mozinfo also exports: + +- ``choices``: a dictionary of possible values for os, bits, and + processor +- ``main``: the console_script entry point for mozinfo +- ``unknown``: a singleton denoting a value that cannot be determined + +``unknown`` has the string representation ``"UNKNOWN"``. +``unknown`` will evaluate as ``False`` in python:: + + if not mozinfo.os: ... # unknown! + + +Command Line Usage +------------------ + +mozinfo comes with a command line program, ``mozinfo`` which may be used to +diagnose one's current system. + +Example output:: + + os: linux + version: Ubuntu 10.10 + bits: 32 + processor: x86 + +Three of these fields, os, bits, and processor, have a finite set of +choices. You may display the value of these choices using +``mozinfo --os``, ``mozinfo --bits``, and ``mozinfo --processor``. +``mozinfo --help`` documents command-line usage. + + +.. automodule:: mozinfo + :members: diff --git a/testing/mozbase/docs/mozinstall.rst b/testing/mozbase/docs/mozinstall.rst new file mode 100644 index 000000000..7db40d73d --- /dev/null +++ b/testing/mozbase/docs/mozinstall.rst @@ -0,0 +1,29 @@ +:mod:`mozinstall` --- Install and uninstall Gecko-based applications +==================================================================== + +mozinstall is a small python module with several convenience methods +useful for installing and uninstalling a gecko-based application +(e.g. Firefox) on the desktop. + +Simple example +-------------- + +:: + + import mozinstall + import tempfile + + tempdir = tempfile.mkdtemp() + firefox_dmg = 'firefox-38.0a1.en-US.mac.dmg' + install_folder = mozinstall.install(src=firefox_dmg, dest=tempdir) + binary = mozinstall.get_binary(install_folder, 'Firefox') + # from here you can execute the binary directly + # ... + mozinstall.uninstall(install_folder) + +API Documentation +----------------- + +.. automodule:: mozinstall + :members: is_installer, install, get_binary, uninstall, + InstallError, InvalidBinary, InvalidSource diff --git a/testing/mozbase/docs/mozlog.rst b/testing/mozbase/docs/mozlog.rst new file mode 100644 index 000000000..db26c78b0 --- /dev/null +++ b/testing/mozbase/docs/mozlog.rst @@ -0,0 +1,486 @@ +:mod:`mozlog` --- Structured logging for test output +=============================================================== + +:py:mod:`mozlog` is a library designed for logging the +execution and results of test harnesses. The internal data model is a +stream of JSON-compatible objects, with one object per log entry. The +default output format is line-based, with one JSON object serialized +per line. + +:py:mod:`mozlog` is *not* based on the stdlib logging +module, although it shares several concepts with it. + +One notable difference between this module and the standard logging +module is the way that loggers are created. The structured logging +module does not require that loggers with a specific name are +singleton objects accessed through a factory function. Instead the +``StructuredLogger`` constructor may be used directly. However all +loggers with the same name share the same internal state (the "Borg" +pattern). In particular the list of handler functions is the same for +all loggers with the same name. + +Typically, you would only instantiate one logger object per +program. Two convenience methods are provided to set and get the +default logger in the program. + +Logging is threadsafe, with access to handlers protected by a +``threading.Lock``. However it is `not` process-safe. This means that +applications using multiple processes, e.g. via the +``multiprocessing`` module, should arrange for all logging to happen in +a single process. + +Data Format +----------- + +Structured loggers produce messages in a simple format designed to be +compatible with the JSON data model. Each message is a single object, +with the type of message indicated by the ``action`` key. It is +intended that the set of ``action`` values be closed; where there are +use cases for additional values they should be integrated into this +module rather than extended in an ad-hoc way. The set of keys present +on on all messages is: + +``action`` + The type of the message (string). + +``time`` + The timestamp of the message in ms since the epoch (int). + +``thread`` + The name of the thread emitting the message (string). + +``pid`` + The pid of the process creating the message (int). + +``source`` + Name of the logger creating the message (string). + +For each ``action`` there are is a further set of specific fields +describing the details of the event that caused the message to be +emitted: + +``suite_start`` + Emitted when the testsuite starts running. + + ``tests`` + A list of test ids. Test ids can either be strings or lists of + strings (an example of the latter is reftests where the id has the + form [test_url, ref_type, ref_url]) and are assumed to be unique + within a given testsuite. In cases where the test list is not + known upfront an empty list may be passed (list). + + ``run_info`` + An optional dictionary describing the properties of the + build and test environment. This contains the information provided + by :doc:`mozinfo `, plus a boolean ``debug`` field indicating + whether the build under test is a debug build. + +``suite_end`` + Emitted when the testsuite is finished and no more results will be produced. + +``test_start`` + Emitted when a test is being started. + + ``test`` + A unique id for the test (string or list of strings). + + ``path`` + Optional path to the test relative to some base (typically the root of the + source tree). Mainly used when ``test`` id is not a path (string). + +``test_status`` + Emitted for a test which has subtests to record the result of a + single subtest. + + ``test`` + The same unique id for the test as in the ``test_start`` message. + + ``subtest`` + Name of the subtest (string). + + ``status`` + Result of the test (string enum; ``PASS``, ``FAIL``, ``TIMEOUT``, + ``NOTRUN``) + + ``expected`` + Expected result of the test. Omitted if the expected result is the + same as the actual result (string enum, same as ``status``). + +``test_end`` + Emitted to give the result of a test with no subtests, or the status + of the overall file when there are subtests. + + ``test`` + The same unique id for the test as in the ``test_start`` message. + + ``status`` + Either result of the test (if there are no subtests) in which case + (string enum ``PASS``, ``FAIL``, ``TIMEOUT``, ``CRASH``, + ``ASSERT``, ``SKIP``) or the status of the overall file where + there are subtests (string enum ``OK``, ``ERROR``, ``TIMEOUT``, + ``CRASH``, ``ASSERT``, ``SKIP``). + + ``expected`` + The expected status, or omitted if the expected status matches the + actual status (string enum, same as ``status``). + +``process_output`` + Output from a managed subprocess. + + ``process`` + pid of the subprocess. + + ``command`` + Command used to launch the subprocess. + + ``data`` + Data output by the subprocess. + +``log`` + General human-readable logging message, used to debug the harnesses + themselves rather than to provide input to other tools. + + ``level`` + Level of the log message (string enum ``CRITICAL``, ``ERROR``, + ``WARNING``, ``INFO``, ``DEBUG``). + + ``message`` + Text of the log message. + +Testsuite Protocol +------------------ + +When used for testsuites, the following structured logging messages must be emitted: + + * One ``suite_start`` message before any ``test_*`` messages + + * One ``test_start`` message per test that is run + + * One ``test_status`` message per subtest that is run. This might be + zero if the test type doesn't have the notion of subtests. + + * One ``test_end`` message per test that is run, after the + ``test_start`` and any ``test_status`` messages for that same test. + + * One ``suite_end`` message after all ``test_*`` messages have been + emitted. + +The above mandatory events may be interspersed with ``process_output`` +and ``log`` events, as required. + +Subtests +~~~~~~~~ + +The purpose of subtests is to deal with situations where a single test +produces more than one result, and the exact details of the number of +results is not known ahead of time. For example consider a test +harness that loads JavaScript-based tests in a browser. Each url +loaded would be a single test, with corresponding ``test_start`` and +``test_end`` messages. If there can be more than one JS-defined test +on a page, however, it it useful to track the results of those tests +seperately. Therefore each of those tests is a subtest, and one +``test_status`` message must be generated for each subtest result. + +Subtests must have a name that is unique within their parent test. + +Whether or not a test has subtests changes the meaning of the +``status`` property on the test itself. When the test does not have +any subtests, this property is the actual test result such as ``PASS`` +or ``FAIL`` . When a test does have subtests, the test itself does not +have a result as-such; it isn't meaningful to describe it as having a +``PASS`` result, especially if the subtests did not all pass. Instead +this property is used to hold information about whether the test ran +without error. If no errors were detected the test must be given the +status ``OK``. Otherwise the test may get the status ``ERROR`` (for +e.g. uncaught JS exceptions), ``TIMEOUT`` (if no results were reported +in the allowed time) or ``CRASH`` (if the test caused the process +under test to crash). + +StructuredLogger Objects +------------------------ + +.. automodule:: mozlog.structuredlog + :members: set_default_logger, get_default_logger + +.. autoclass:: StructuredLogger + :members: add_handler, remove_handler, handlers, suite_start, + suite_end, test_start, test_status, test_end, + process_output, critical, error, warning, info, debug + +.. autoclass:: StructuredLogFileLike + :members: + +ProxyLogger Objects +------------------- + +Since :func:`mozlog.structuredlog.get_default_logger` return None when +the default logger is not initialized, it is not possible to directly +use it at the module level. + +With ProxyLogger, it is possible to write the following code: :: + + from mozlog import get_proxy_logger + + LOG = get_proxy_logger('component_name') + + + def my_function(): + LOG.info('logging with a module level object') + + +.. note:: + + mozlog still needs to be initialized before the first call occurs + to a ProxyLogger instance, for example with + :func:`mozlog.commandline.setup_logging`. + +.. automodule:: mozlog.proxy + :members: get_proxy_logger, ProxyLogger + +Handlers +-------- + +A handler is a callable that is called for each log message produced +and is responsible for handling the processing of that +message. The typical example of this is a ``StreamHandler`` which takes +a log message, invokes a formatter which converts the log to a string, +and writes it to a file. + +.. automodule:: mozlog.handlers + +.. autoclass:: BaseHandler + :members: + +.. autoclass:: StreamHandler + :members: + +.. autoclass:: LogLevelFilter + :members: + +.. autoclass:: BufferHandler + :members: + +Formatters +---------- + +Formatters are callables that take a log message, and return either a +string representation of that message, or ``None`` if that message +should not appear in the output. This allows formatters to both +exclude certain items and create internal buffers of the output so +that, for example, a single string might be returned for a +``test_end`` message indicating the overall result of the test, +including data provided in the ``test_status`` messages. + +Formatter modules are written so that they can take raw input on stdin +and write formatted output on stdout. This allows the formatters to be +invoked as part of a command line for post-processing raw log files. + +.. automodule:: mozlog.formatters.base + +.. autoclass:: BaseFormatter + :members: + +.. automodule:: mozlog.formatters.unittest + +.. autoclass:: UnittestFormatter + :members: + +.. automodule:: mozlog.formatters.xunit + +.. autoclass:: XUnitFormatter + :members: + +.. automodule:: mozlog.formatters.html + +.. autoclass:: HTMLFormatter + :members: + +.. automodule:: mozlog.formatters.machformatter + +.. autoclass:: MachFormatter + :members: + +.. automodule:: mozlog.formatters.tbplformatter + +.. autoclass:: TbplFormatter + :members: + +Processing Log Files +-------------------- + +The ``mozlog.reader`` module provides utilities for working +with structured log files. + +.. automodule:: mozlog.reader + :members: + +Integration with argparse +------------------------- + +The `mozlog.commandline` module provides integration with the `argparse` +module to provide uniform logging-related command line arguments to programs +using `mozlog`. Each known formatter gets a command line argument of the form +``--log-{name}``, which takes the name of a file to log to with that format, +or ``-`` to indicate stdout. + +.. automodule:: mozlog.commandline + :members: + +Simple Examples +--------------- + +Log to stdout:: + + from mozlog import structuredlog + from mozlog import handlers, formatters + logger = structuredlog.StructuredLogger("my-test-suite") + logger.add_handler(handlers.StreamHandler(sys.stdout, + formatters.JSONFormatter())) + logger.suite_start(["test-id-1"]) + logger.test_start("test-id-1") + logger.info("This is a message with action='LOG' and level='INFO'") + logger.test_status("test-id-1", "subtest-1", "PASS") + logger.test_end("test-id-1", "OK") + logger.suite_end() + + +Populate an ``argparse.ArgumentParser`` with logging options, and +create a logger based on the value of those options, defaulting to +JSON output on stdout if nothing else is supplied:: + + import argparse + from mozlog import commandline + + parser = argparse.ArgumentParser() + # Here one would populate the parser with other options + commandline.add_logging_group(parser) + + args = parser.parse_args() + logger = commandline.setup_logging("testsuite-name", args, {"raw": sys.stdout}) + +Count the number of tests that timed out in a testsuite:: + + from mozlog import reader + + count = 0 + + def handle_test_end(data): + global count + if data["status"] == "TIMEOUT": + count += 1 + + reader.each_log(reader.read("my_test_run.log"), + {"test_end": handle_test_end}) + + print count + +More Complete Example +--------------------- + +This example shows a complete toy testharness set up to used +structured logging. It is avaliable as `structured_example.py <_static/structured_example.py>`_: + +.. literalinclude:: _static/structured_example.py + +Each global function with a name starting +``test_`` represents a test. A passing test returns without +throwing. A failing test throws a :py:class:`TestAssertion` exception +via the :py:func:`assert_equals` function. Throwing anything else is +considered an error in the test. There is also a :py:func:`expected` +decorator that is used to annotate tests that are expected to do +something other than pass. + +The main entry point to the test runner is via that :py:func:`main` +function. This is responsible for parsing command line +arguments, and initiating the test run. Although the test harness +itself does not provide any command line arguments, the +:py:class:`ArgumentParser` object is populated by +:py:meth:`commandline.add_logging_group`, which provides a generic +set of structured logging arguments appropriate to all tools producing +structured logging. + +The values of these command line arguments are used to create a +:py:class:`mozlog.StructuredLogger` object populated with the +specified handlers and formatters in +:py:func:`commandline.setup_logging`. The third argument to this +function is the default arguments to use. In this case the default +is to output raw (i.e. JSON-formatted) logs to stdout. + +The main test harness is provided by the :py:class:`TestRunner` +class. This class is responsible for scheduling all the tests and +logging all the results. It is passed the :py:obj:`logger` object +created from the command line arguments. The :py:meth:`run` method +starts the test run. Before the run is started it logs a +``suite_start`` message containing the id of each test that will run, +and after the testrun is done it logs a ``suite_end`` message. + +Individual tests are run in the :py:meth:`run_test` method. For each +test this logs a ``test_start`` message. It then runs the test and +logs a ``test_end`` message containing the test name, status, expected +status, and any informational message about the reason for the +result. In this test harness there are no subtests, so the +``test_end`` message has the status of the test and there are no +``test_status`` messages. + +Example Output +~~~~~~~~~~~~~~ + +When run without providing any command line options, the raw +structured log messages are sent to stdout:: + + $ python structured_example.py + + {"source": "structured-example", "tests": ["test_that_has_an_error", "test_that_fails", "test_expected_fail", "test_that_passes"], "thread": "MainThread", "time": 1401446682787, "action": "suite_start", "pid": 18456} + {"source": "structured-example", "thread": "MainThread", "time": 1401446682787, "action": "log", "message": "Running tests", "level": "INFO", "pid": 18456} + {"source": "structured-example", "test": "test_that_has_an_error", "thread": "MainThread", "time": 1401446682787, "action": "test_start", "pid": 18456} + {"status": "ERROR", "thread": "MainThread", "pid": 18456, "source": "structured-example", "test": "test_that_has_an_error", "time": 1401446682788, "action": "test_end", "message": "Traceback (most recent call last):\n File \"structured_example.py\", line 61, in run_test\n func()\n File \"structured_example.py\", line 31, in test_that_has_an_error\n assert_equals(2, 1 + \"1\")\nTypeError: unsupported operand type(s) for +: 'int' and 'str'\n", "expected": "PASS"} + {"source": "structured-example", "test": "test_that_fails", "thread": "MainThread", "time": 1401446682788, "action": "test_start", "pid": 18456} + {"status": "FAIL", "thread": "MainThread", "pid": 18456, "source": "structured-example", "test": "test_that_fails", "time": 1401446682788, "action": "test_end", "message": "1 not equal to 2", "expected": "PASS"} + {"source": "structured-example", "test": "test_expected_fail", "thread": "MainThread", "time": 1401446682788, "action": "test_start", "pid": 18456} + {"status": "FAIL", "thread": "MainThread", "pid": 18456, "source": "structured-example", "test": "test_expected_fail", "time": 1401446682788, "action": "test_end", "message": "4 not equal to 5"} + {"source": "structured-example", "test": "test_that_passes", "thread": "MainThread", "time": 1401446682788, "action": "test_start", "pid": 18456} + {"status": "PASS", "source": "structured-example", "test": "test_that_passes", "thread": "MainThread", "time": 1401446682789, "action": "test_end", "pid": 18456} + {"action": "suite_end", "source": "structured-example", "pid": 18456, "thread": "MainThread", "time": 1401446682789} + +The structured logging module provides a number of command line +options:: + + $ python structured_example.py --help + + usage: structured_example.py [-h] [--log-unittest LOG_UNITTEST] + [--log-raw LOG_RAW] [--log-html LOG_HTML] + [--log-xunit LOG_XUNIT] + [--log-mach LOG_MACH] + + optional arguments: + -h, --help show this help message and exit + + Output Logging: + Options for logging output. Each option represents a possible logging + format and takes a filename to write that format to, or '-' to write to + stdout. + + --log-unittest LOG_UNITTEST + Unittest style output + --log-raw LOG_RAW Raw structured log messages + --log-html LOG_HTML HTML report + --log-xunit LOG_XUNIT + xUnit compatible XML + --log-mach LOG_MACH Human-readable output + +In order to get human-readable output on stdout and the structured log +data to go to the file ``structured.log``, we would run:: + + $ python structured_example.py --log-mach=- --log-raw=structured.log + + 0:00.00 SUITE_START: MainThread 4 + 0:01.00 LOG: MainThread INFO Running tests + 0:01.00 TEST_START: MainThread test_that_has_an_error + 0:01.00 TEST_END: MainThread Harness status ERROR, expected PASS. Subtests passed 0/0. Unexpected 1 + 0:01.00 TEST_START: MainThread test_that_fails + 0:01.00 TEST_END: MainThread Harness status FAIL, expected PASS. Subtests passed 0/0. Unexpected 1 + 0:01.00 TEST_START: MainThread test_expected_fail + 0:02.00 TEST_END: MainThread Harness status FAIL. Subtests passed 0/0. Unexpected 0 + 0:02.00 TEST_START: MainThread test_that_passes + 0:02.00 TEST_END: MainThread Harness status PASS. Subtests passed 0/0. Unexpected 0 + 0:02.00 SUITE_END: MainThread diff --git a/testing/mozbase/docs/moznetwork.rst b/testing/mozbase/docs/moznetwork.rst new file mode 100644 index 000000000..d6ed54b85 --- /dev/null +++ b/testing/mozbase/docs/moznetwork.rst @@ -0,0 +1,9 @@ +:mod:`moznetwork` --- Get network information +============================================= + +.. automodule:: moznetwork + + .. automethod:: moznetwork.get_ip + + .. autoclass:: moznetwork.NetworkError + diff --git a/testing/mozbase/docs/mozprocess.rst b/testing/mozbase/docs/mozprocess.rst new file mode 100644 index 000000000..5cd23ad0d --- /dev/null +++ b/testing/mozbase/docs/mozprocess.rst @@ -0,0 +1,324 @@ +:mod:`mozprocess` --- Launch and manage processes +================================================= + +Mozprocess is a process-handling module that provides some additional +features beyond those available with python's subprocess: + +* better handling of child processes, especially on Windows +* the ability to timeout the process after some absolute period, or some + period without any data written to stdout/stderr +* the ability to specify output handlers that will be called + for each line of output produced by the process +* the ability to specify handlers that will be called on process timeout + and normal process termination + +Running a process +----------------- + +mozprocess consists of two classes: ProcessHandler inherits from ProcessHandlerMixin. + +Let's see how to run a process. +First, the class should be instanciated with at least one argument which is a command (or a list formed by the command followed by its arguments). +Then the process can be launched using the *run()* method. +Finally the *wait()* method will wait until end of execution. + +.. code-block:: python + + from mozprocess import processhandler + + # under Windows replace by command = ['dir', '/a'] + command = ['ls', '-l'] + p = processhandler.ProcessHandler(command) + print("execute command: %s" % p.commandline) + p.run() + p.wait() + +Note that using *ProcessHandler* instead of *ProcessHandlerMixin* will print the output of executed command. The attribute *commandline* provides the launched command. + +Collecting process output +------------------------- + +Let's now consider a basic shell script that will print numbers from 1 to 5 waiting 1 second between each. +This script will be used as a command to launch in further examples. + +**proc_sleep_echo.sh**: + +.. code-block:: sh + + #!/bin/sh + + for i in 1 2 3 4 5 + do + echo $i + sleep 1 + done + +If you are running under Windows, you won't be able to use the previous script (unless using Cygwin). +So you'll use the following script: + +**proc_sleep_echo.bat**: + +.. code-block:: bat + + @echo off + FOR %%A IN (1 2 3 4 5) DO ( + ECHO %%A + REM if you have TIMEOUT then use it instead of PING + REM TIMEOUT /T 1 /NOBREAK + PING -n 2 127.0.0.1 > NUL + ) + +Mozprocess allows the specification of custom output handlers to gather process output while running. +ProcessHandler will by default write all outputs on stdout. You can also provide (to ProcessHandler or ProcessHandlerMixin) a function or a list of functions that will be used as callbacks on each output line generated by the process. + +In the following example the command's output will be stored in a file *output.log* and printed in stdout: + +.. code-block:: python + + import sys + from mozprocess import processhandler + + fd = open('output.log', 'w') + + def tostdout(line): + sys.stdout.write("<%s>\n" % line) + + def tofile(line): + fd.write("<%s>\n" % line) + + # under Windows you'll replace by 'proc_sleep_echo.bat' + command = './proc_sleep_echo.sh' + outputs = [tostdout, tofile] + + p = processhandler.ProcessHandlerMixin(command, processOutputLine=outputs) + p.run() + p.wait() + + fd.close() + +The process output can be saved (*obj = ProcessHandler(..., storeOutput=True)*) so as it is possible to request it (*obj.output*) at any time. Note that the default value for *stroreOutput* is *True*, so it is not necessary to provide it in the parameters. + +.. code-block:: python + + import time + import sys + from mozprocess import processhandler + + command = './proc_sleep_echo.sh' # Windows: 'proc_sleep_echo.bat' + + p = processhandler.ProcessHandler(command, storeOutput=True) + p.run() + for i in xrange(10): + print(p.output) + time.sleep(0.5) + p.wait() + +In previous example, you will see the *p.output* list growing. + +Execution +--------- + +Status +`````` + +It is possible to query the status of the process via *poll()* that will return None if the process is still running, 0 if it ended without failures and a negative value if it was killed by a signal (Unix-only). + +.. code-block:: python + + import time + import signal + from mozprocess import processhandler + + command = './proc_sleep_echo.sh' + p = processhandler.ProcessHandler(command) + p.run() + time.sleep(2) + print("poll status: %s" % p.poll()) + time.sleep(1) + p.kill(signal.SIGKILL) + print("poll status: %s" % p.poll()) + +Timeout +``````` + +A timeout can be provided to the *run()* method. If the process last more than timeout seconds, it will be stopped. + +After execution, the property *timedOut* will be set to True if a timeout was reached. + +It is also possible to provide functions (*obj = ProcessHandler[Mixin](..., onTimeout=functions)*) that will be called if the timeout was reached. + +.. code-block:: python + + from mozprocess import processhandler + + def ontimeout(): + print("REACHED TIMEOUT") + + command = './proc_sleep_echo.sh' # Windows: 'proc_sleep_echo.bat' + functions = [ontimeout] + p = processhandler.ProcessHandler(command, onTimeout=functions) + p.run(timeout=2) + p.wait() + print("timedOut = %s" % p.timedOut) + +By default the process will be killed on timeout but it is possible to prevent this by setting *kill_on_timeout* to *False*. + +.. code-block:: python + + p = processhandler.ProcessHandler(command, onTimeout=functions, kill_on_timeout=False) + p.run(timeout=2) + p.wait() + print("timedOut = %s" % p.timedOut) + +In this case, no output will be available after the timeout, but the process will still be running. + +Waiting +``````` + +It is possible to wait until the process exits as already seen with the method *wait()*, or until the end of a timeout if given. Note that in last case the process is still alive after the timeout. + +.. code-block:: python + + command = './proc_sleep_echo.sh' # Windows: 'proc_sleep_echo.bat' + p = processhandler.ProcessHandler(command) + p.run() + p.wait(timeout=2) + print("timedOut = %s" % p.timedOut) + p.wait() + +Killing +``````` + +You can request to kill the process with the method *kill*. f the parameter "ignore_children" is set to False when the process handler class is initialized, all the process's children will be killed as well. + +Except on Windows, you can specify the signal with which to kill method the process (e.g.: *kill(signal.SIGKILL)*). + +.. code-block:: python + + import time + from mozprocess import processhandler + + command = './proc_sleep_echo.sh' # Windows: 'proc_sleep_echo.bat' + p = processhandler.ProcessHandler(command) + p.run() + time.sleep(2) + p.kill() + +End of execution +```````````````` + +You can provide a function or a list of functions to call at the end of the process using the initilization parameter *onFinish*. + +.. code-block:: python + + from mozprocess import processhandler + + def finish(): + print("Finished!!") + + command = './proc_sleep_echo.sh' # Windows: 'proc_sleep_echo.bat' + + p = processhandler.ProcessHandler(command, onFinish=finish) + p.run() + p.wait() + +Child management +---------------- + +Consider the following scripts: + +**proc_child.sh**: + +.. code-block:: sh + + #!/bin/sh + for i in a b c d e + do + echo $i + sleep 1 + done + +**proc_parent.sh**: + +.. code-block:: sh + + #!/bin/sh + ./proc_child.sh + for i in 1 2 3 4 5 + do + echo $i + sleep 1 + done + +For windows users consider: + +**proc_child.bat**: + +.. code-block:: bat + + @echo off + FOR %%A IN (a b c d e) DO ( + ECHO %%A + REM TIMEOUT /T 1 /NOBREAK + PING -n 2 127.0.0.1 > NUL + ) + +**proc_parent.bat**: + +.. code-block:: bat + + @echo off + call proc_child.bat + FOR %%A IN (1 2 3 4 5) DO ( + ECHO %%A + REM TIMEOUT /T 1 /NOBREAK + PING -n 2 127.0.0.1 > NUL + ) + +For processes that launch other processes, mozprocess allows you to get child running status, wait for child termination, and kill children. + +Ignoring children +````````````````` + +By default the *ignore_children* option is False. In that case, killing the main process will kill all its children at the same time. + +.. code-block:: python + + import time + from mozprocess import processhandler + + def finish(): + print("Finished") + + command = './proc_parent.sh' + p = processhandler.ProcessHandler(command, ignore_children=False, onFinish=finish) + p.run() + time.sleep(2) + print("kill") + p.kill() + +If *ignore_children* is set to *True*, killing will apply only to the main process that will wait children end of execution before stoping (join). + +.. code-block:: python + + import time + from mozprocess import processhandler + + def finish(): + print("Finished") + + command = './proc_parent.sh' + p = processhandler.ProcessHandler(command, ignore_children=True, onFinish=finish) + p.run() + time.sleep(2) + print("kill") + p.kill() + +API Documentation +----------------- + +.. module:: mozprocess +.. autoclass:: ProcessHandlerMixin + :members: __init__, timedOut, commandline, run, kill, processOutputLine, onTimeout, onFinish, wait +.. autoclass:: ProcessHandler + :members: diff --git a/testing/mozbase/docs/mozprofile.rst b/testing/mozbase/docs/mozprofile.rst new file mode 100644 index 000000000..85428e835 --- /dev/null +++ b/testing/mozbase/docs/mozprofile.rst @@ -0,0 +1,99 @@ +:mod:`mozprofile` --- Create and modify Mozilla application profiles +==================================================================== + +Mozprofile_ is a python tool for creating and managing profiles for Mozilla's +applications (Firefox, Thunderbird, etc.). In addition to creating profiles, +mozprofile can install addons_ and set preferences_ Mozprofile can be utilized +from the command line or as an API. + +The preferred way of setting up profile data (addons, permissions, preferences +etc) is by passing them to the profile_ constructor. + +Addons +------ + +.. automodule:: mozprofile.addons + :members: + +Addons may be installed individually or from a manifest. + +Example:: + + from mozprofile import FirefoxProfile + + # create new profile to pass to mozmill/mozrunner + profile = FirefoxProfile(addons=["adblock.xpi"]) + +Command Line Interface +---------------------- + +.. automodule:: mozprofile.cli + :members: + +The profile to be operated on may be specified with the ``--profile`` +switch. If a profile is not specified, one will be created in a +temporary directory which will be echoed to the terminal:: + + (mozmill)> mozprofile + /tmp/tmp4q1iEU.mozrunner + (mozmill)> ls /tmp/tmp4q1iEU.mozrunner + user.js + +To run mozprofile from the command line enter: +``mozprofile --help`` for a list of options. + +Permissions +----------- + +.. automodule:: mozprofile.permissions + :members: + +You can set permissions by creating a ``ServerLocations`` object that you pass +to the ``Profile`` constructor. Hosts can be added to it with +``add_host(host, port)``. ``port`` can be 0. + +Preferences +----------- + +.. automodule:: mozprofile.prefs + :members: + +Preferences can be set in several ways: + +- using the API: You can make a dictionary with the preferences and pass it to + the ``Profile`` constructor. You can also add more preferences with the + ``Profile.set_preferences`` method. +- using a JSON blob file: ``mozprofile --preferences myprefs.json`` +- using a ``.ini`` file: ``mozprofile --preferences myprefs.ini`` +- via the command line: ``mozprofile --pref key:value --pref key:value [...]`` + +When setting preferences from an ``.ini`` file or the ``--pref`` switch, +the value will be interpolated as an integer or a boolean +(``true``/``false``) if possible. + +Profile +-------------------- + +.. automodule:: mozprofile.profile + :members: + +Resources +----------- +Other Mozilla programs offer additional and overlapping functionality +for profiles. There is also substantive documentation on profiles and +their management. + +- ProfileManager_: XULRunner application for managing profiles. Has a GUI and CLI. +- python-profilemanager_: python CLI interface similar to ProfileManager +- profile documentation_ + + +.. _Mozprofile: https://github.com/mozilla/mozbase/tree/master/mozprofile +.. _addons: https://developer.mozilla.org/en/addons +.. _preferences: https://developer.mozilla.org/En/A_Brief_Guide_to_Mozilla_Preferences +.. _mozprofile.profile: https://github.com/mozilla/mozbase/tree/master/mozprofile/mozprofile/profile.py +.. _AddonManager: https://github.com/mozilla/mozbase/tree/master/mozprofile/mozprofile/addons.py +.. _here: https://github.com/mozilla/mozbase/blob/master/mozprofile/mozprofile/permissions.py +.. _ProfileManager: https://developer.mozilla.org/en/Profile_Manager +.. _python-profilemanager: http://k0s.org/mozilla/hg/profilemanager/ +.. _documentation: http://support.mozilla.com/en-US/kb/Profiles diff --git a/testing/mozbase/docs/mozrunner.rst b/testing/mozbase/docs/mozrunner.rst new file mode 100644 index 000000000..766b02cc4 --- /dev/null +++ b/testing/mozbase/docs/mozrunner.rst @@ -0,0 +1,177 @@ +:mod:`mozrunner` --- Manage remote and local gecko processes +============================================================ + +Mozrunner provides an API to manage a gecko-based application with an +arbitrary configuration profile. It currently supports local desktop +binaries such as Firefox and Thunderbird, as well as Firefox OS on +mobile devices and emulators. + + +Basic usage +----------- + +The simplest way to use mozrunner, is to instantiate a runner, start it +and then wait for it to finish: + +.. code-block:: python + + from mozrunner import FirefoxRunner + binary = 'path/to/firefox/binary' + runner = FirefoxRunner(binary=binary) + runner.start() + runner.wait() + +This automatically creates and uses a default mozprofile object. If you +wish to use a specialized or pre-existing profile, you can create a +:doc:`mozprofile ` object and pass it in: + +.. code-block:: python + + from mozprofile import FirefoxProfile + from mozrunner import FirefoxRunner + import os + + binary = 'path/to/firefox/binary' + profile_path = 'path/to/profile' + if os.path.exists(profile_path): + profile = FirefoxProfile.clone(path_from=profile_path) + else: + profile = FirefoxProfile(profile=profile_path) + runner = FirefoxRunner(binary=binary, profile=profile) + runner.start() + runner.wait() + + +Handling output +--------------- + +By default, mozrunner dumps the output of the gecko process to standard output. +It is possible to add arbitrary output handlers by passing them in via the +`process_args` argument. Be careful, passing in a handler overrides the default +behaviour. So if you want to use a handler in addition to dumping to stdout, you +need to specify that explicitly. For example: + +.. code-block:: python + + from mozrunner import FirefoxRunner + + def handle_output_line(line): + do_something(line) + + binary = 'path/to/firefox/binary' + process_args = { 'stream': sys.stdout, + 'processOutputLine': [handle_output_line] } + runner = FirefoxRunner(binary=binary, process_args=process_args) + +Mozrunner uses :doc:`mozprocess ` to manage the underlying gecko +process and handle output. See the :doc:`mozprocess documentation ` +for all available arguments accepted by `process_args`. + + +Handling timeouts +----------------- + +Sometimes gecko can hang, or maybe it is just taking too long. To handle this case you +may want to set a timeout. Mozrunner has two kinds of timeouts, the +traditional `timeout`, and the `outputTimeout`. These get passed into the +`runner.start()` method. Setting `timeout` will cause gecko to be killed after +the specified number of seconds, no matter what. Setting `outputTimeout` will cause +gecko to be killed after the specified number of seconds with no output. In both +cases the process handler's `onTimeout` callbacks will be triggered. + +.. code-block:: python + + from mozrunner import FirefoxRunner + + def on_timeout(): + print('timed out after 10 seconds with no output!') + + binary = 'path/to/firefox/binary' + process_args = { 'onTimeout': on_timeout } + runner = FirefoxRunner(binary=binary, process_args=process_args) + runner.start(outputTimeout=10) + runner.wait() + +The `runner.wait()` method also accepts a timeout argument. But unlike the arguments +to `runner.start()`, this one simply returns from the wait call and does not kill the +gecko process. + +.. code-block:: python + + runner.start(timeout=100) + + waiting = 0 + while runner.wait(timeout=1) is None: + waiting += 1 + print("Been waiting for %d seconds so far.." % waiting) + assert waiting <= 100 + + +Using a device runner +--------------------- + +The previous examples used a GeckoRuntimeRunner. If you want to control a +gecko process on a remote device, you need to use a DeviceRunner. The api is +nearly identical except you don't pass in a binary, instead you create a device +object. For example, for B2G (Firefox OS) emulators you might do: + +.. code-block:: python + + from mozrunner import B2GEmulatorRunner + + b2g_home = 'path/to/B2G' + runner = B2GEmulatorRunner(arch='arm', b2g_home=b2g_home) + runner.start() + runner.wait() + +Device runners have a `device` object. Remember that the gecko process runs on +the device. In the case of the emulator, it is possible to start the +device independently of the gecko process. + +.. code-block:: python + + runner.device.start() # launches the emulator (which also launches gecko) + runner.start() # stops the gecko process, installs the profile, restarts the gecko process + + +Runner API Documentation +------------------------ + +Application Runners +~~~~~~~~~~~~~~~~~~~ +.. automodule:: mozrunner.runners + :members: + +BaseRunner +~~~~~~~~~~ +.. autoclass:: mozrunner.base.BaseRunner + :members: + +GeckoRuntimeRunner +~~~~~~~~~~~~~~~~~~ +.. autoclass:: mozrunner.base.GeckoRuntimeRunner + :show-inheritance: + :members: + +DeviceRunner +~~~~~~~~~~~~ +.. autoclass:: mozrunner.base.DeviceRunner + :show-inheritance: + :members: + +Device API Documentation +------------------------ + +Generally using the device classes directly shouldn't be required, but in some +cases it may be desirable. + +Device +~~~~~~ +.. autoclass:: mozrunner.devices.Device + :members: + +Emulator +~~~~~~~~ +.. autoclass:: mozrunner.devices.Emulator + :show-inheritance: + :members: diff --git a/testing/mozbase/docs/mozversion.rst b/testing/mozbase/docs/mozversion.rst new file mode 100644 index 000000000..21b028d17 --- /dev/null +++ b/testing/mozbase/docs/mozversion.rst @@ -0,0 +1,112 @@ +:mod:`mozversion` --- Get application information +================================================= + +`mozversion `_ +provides version information such as the application name and the changesets +that it has been built from. This is commonly used in reporting or for +conditional logic based on the application under test. + +Note that mozversion can report the version of remote devices (e.g. Firefox OS) +but it requires the :mod:`mozdevice` dependency in that case. You can require it +along with mozversion by using the extra *device* dependency: + +.. code-block:: bash + + pip install mozversion[device] + + +API Usage +--------- + +.. automodule:: mozversion + :members: get_version + +Examples +```````` + +Firefox:: + + import mozversion + + version = mozversion.get_version(binary='/path/to/firefox-bin') + for (key, value) in sorted(version.items()): + if value: + print '%s: %s' % (key, value) + +Firefox for Android:: + + version = mozversion.get_version(binary='path/to/firefox.apk') + print version['application_changeset'] # gets hg revision of build + +FirefoxOS:: + + version = mozversion.get_version(sources='path/to/sources.xml', dm_type='adb') + print version['gaia_changeset'] # gets gaia git revision + +Command Line Usage +------------------ + +mozversion comes with a command line program, ``mozversion`` which may be used to +get version information from an application. + +Usage:: + + mozversion [options] + +Options +``````` + +---binary +''''''''' + +This is the path to the target application binary or .apk. If this is omitted +then the current directory is checked for the existance of an +application.ini file. If not found, then it is assumed the target +application is a remote Firefox OS instance. + + +---sources +'''''''''' + +The path to the sources.xml that accompanies the target application (Firefox OS +only). If this is omitted then the current directory is checked for the +existance of a sources.xml file. + +Examples +```````` + +Firefox:: + + $ mozversion --binary=/path/to/firefox-bin + application_buildid: 20131205075310 + application_changeset: 39faf812aaec + application_name: Firefox + application_repository: http://hg.mozilla.org/releases/mozilla-release + application_version: 26.0 + platform_buildid: 20131205075310 + platform_changeset: 39faf812aaec + platform_repository: http://hg.mozilla.org/releases/mozilla-release + +Firefox for Android:: + + $ mozversion --binary=/path/to/firefox.apk + +Firefox OS:: + + $ mozversion --sources=/path/to/sources.xml + application_buildid: 20140106040201 + application_changeset: 14ac61461f2a + application_name: B2G + application_repository: http://hg.mozilla.org/mozilla-central + application_version: 29.0a1 + build_changeset: 59605a7c026ff06cc1613af3938579b1dddc6cfe + device_firmware_date: 1380051975 + device_firmware_version_incremental: 139 + device_firmware_version_release: 4.0.4 + device_id: msm7627a + gaia_changeset: 9a222ac02db176e47299bb37112ae40aeadbeca7 + gaia_date: 1389005812 + gecko_changeset: 3a2d8af198510726b063a217438fcf2591f4dfcf + platform_buildid: 20140106040201 + platform_changeset: 14ac61461f2a + platform_repository: http://hg.mozilla.org/mozilla-central diff --git a/testing/mozbase/docs/requirements.txt b/testing/mozbase/docs/requirements.txt new file mode 100644 index 000000000..53dd4ca67 --- /dev/null +++ b/testing/mozbase/docs/requirements.txt @@ -0,0 +1 @@ +marionette_client diff --git a/testing/mozbase/docs/setuprunning.rst b/testing/mozbase/docs/setuprunning.rst new file mode 100644 index 000000000..c1e781bc0 --- /dev/null +++ b/testing/mozbase/docs/setuprunning.rst @@ -0,0 +1,18 @@ +Set up and running +------------------ + +Activities under this domain include installing the software, creating +a profile (a set of configuration settings), running a program in a +controlled environment such that it can be shut down safely, and +correctly handling the case where the system crashes. + +.. toctree:: + :maxdepth: 2 + + mozfile + mozinstall + mozprofile + mozprocess + mozrunner + mozcrash + mozdebug diff --git a/testing/mozbase/manifestparser/manifestparser/__init__.py b/testing/mozbase/manifestparser/manifestparser/__init__.py new file mode 100644 index 000000000..43c58ae79 --- /dev/null +++ b/testing/mozbase/manifestparser/manifestparser/__init__.py @@ -0,0 +1,8 @@ +# flake8: noqa +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this file, +# You can obtain one at http://mozilla.org/MPL/2.0/. + +from .manifestparser import * +from .expression import * +from .ini import * diff --git a/testing/mozbase/manifestparser/manifestparser/cli.py b/testing/mozbase/manifestparser/manifestparser/cli.py new file mode 100644 index 000000000..482575d29 --- /dev/null +++ b/testing/mozbase/manifestparser/manifestparser/cli.py @@ -0,0 +1,246 @@ +#!/usr/bin/env python +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this file, +# You can obtain one at http://mozilla.org/MPL/2.0/. + +""" +Mozilla universal manifest parser +""" + +from optparse import OptionParser +import os +import sys + +from .manifestparser import ( + convert, + ManifestParser, +) + + +class ParserError(Exception): + """error for exceptions while parsing the command line""" + + +def parse_args(_args): + """ + parse and return: + --keys=value (or --key value) + -tags + args + """ + + # return values + _dict = {} + tags = [] + args = [] + + # parse the arguments + key = None + for arg in _args: + if arg.startswith('---'): + raise ParserError("arguments should start with '-' or '--' only") + elif arg.startswith('--'): + if key: + raise ParserError("Key %s still open" % key) + key = arg[2:] + if '=' in key: + key, value = key.split('=', 1) + _dict[key] = value + key = None + continue + elif arg.startswith('-'): + if key: + raise ParserError("Key %s still open" % key) + tags.append(arg[1:]) + continue + else: + if key: + _dict[key] = arg + continue + args.append(arg) + + # return values + return (_dict, tags, args) + + +class CLICommand(object): + usage = '%prog [options] command' + + def __init__(self, parser): + self._parser = parser # master parser + + def parser(self): + return OptionParser(usage=self.usage, description=self.__doc__, + add_help_option=False) + + +class Copy(CLICommand): + usage = '%prog [options] copy manifest directory -tag1 -tag2 --key1=value1 --key2=value2 ...' + + def __call__(self, options, args): + # parse the arguments + try: + kwargs, tags, args = parse_args(args) + except ParserError, e: + self._parser.error(e.message) + + # make sure we have some manifests, otherwise it will + # be quite boring + if not len(args) == 2: + HelpCLI(self._parser)(options, ['copy']) + return + + # read the manifests + # TODO: should probably ensure these exist here + manifests = ManifestParser() + manifests.read(args[0]) + + # print the resultant query + manifests.copy(args[1], None, *tags, **kwargs) + + +class CreateCLI(CLICommand): + """ + create a manifest from a list of directories + """ + usage = '%prog [options] create directory <...>' + + def parser(self): + parser = CLICommand.parser(self) + parser.add_option('-p', '--pattern', dest='pattern', + help="glob pattern for files") + parser.add_option('-i', '--ignore', dest='ignore', + default=[], action='append', + help='directories to ignore') + parser.add_option('-w', '--in-place', dest='in_place', + help='Write .ini files in place; filename to write to') + return parser + + def __call__(self, _options, args): + parser = self.parser() + options, args = parser.parse_args(args) + + # need some directories + if not len(args): + parser.print_usage() + return + + # add the directories to the manifest + for arg in args: + assert os.path.exists(arg) + assert os.path.isdir(arg) + manifest = convert(args, pattern=options.pattern, ignore=options.ignore, + write=options.in_place) + if manifest: + print manifest + + +class WriteCLI(CLICommand): + """ + write a manifest based on a query + """ + usage = '%prog [options] write manifest -tag1 -tag2 --key1=value1 --key2=value2 ...' + + def __call__(self, options, args): + + # parse the arguments + try: + kwargs, tags, args = parse_args(args) + except ParserError, e: + self._parser.error(e.message) + + # make sure we have some manifests, otherwise it will + # be quite boring + if not args: + HelpCLI(self._parser)(options, ['write']) + return + + # read the manifests + # TODO: should probably ensure these exist here + manifests = ManifestParser() + manifests.read(*args) + + # print the resultant query + manifests.write(global_tags=tags, global_kwargs=kwargs) + + +class HelpCLI(CLICommand): + """ + get help on a command + """ + usage = '%prog [options] help [command]' + + def __call__(self, options, args): + if len(args) == 1 and args[0] in commands: + commands[args[0]](self._parser).parser().print_help() + else: + self._parser.print_help() + print '\nCommands:' + for command in sorted(commands): + print ' %s : %s' % (command, commands[command].__doc__.strip()) + + +class UpdateCLI(CLICommand): + """ + update the tests as listed in a manifest from a directory + """ + usage = '%prog [options] update manifest directory -tag1 -tag2 --key1=value1 --key2=value2 ...' + + def __call__(self, options, args): + # parse the arguments + try: + kwargs, tags, args = parse_args(args) + except ParserError, e: + self._parser.error(e.message) + + # make sure we have some manifests, otherwise it will + # be quite boring + if not len(args) == 2: + HelpCLI(self._parser)(options, ['update']) + return + + # read the manifests + # TODO: should probably ensure these exist here + manifests = ManifestParser() + manifests.read(args[0]) + + # print the resultant query + manifests.update(args[1], None, *tags, **kwargs) + + +# command -> class mapping +commands = {'create': CreateCLI, + 'help': HelpCLI, + 'update': UpdateCLI, + 'write': WriteCLI} + + +def main(args=sys.argv[1:]): + """console_script entry point""" + + # set up an option parser + usage = '%prog [options] [command] ...' + description = "%s. Use `help` to display commands" % __doc__.strip() + parser = OptionParser(usage=usage, description=description) + parser.add_option('-s', '--strict', dest='strict', + action='store_true', default=False, + help='adhere strictly to errors') + parser.disable_interspersed_args() + + options, args = parser.parse_args(args) + + if not args: + HelpCLI(parser)(options, args) + parser.exit() + + # get the command + command = args[0] + if command not in commands: + parser.error("Command must be one of %s (you gave '%s')" % + (', '.join(sorted(commands.keys())), command)) + + handler = commands[command](parser) + handler(options, args[1:]) + +if __name__ == '__main__': + main() diff --git a/testing/mozbase/manifestparser/manifestparser/expression.py b/testing/mozbase/manifestparser/manifestparser/expression.py new file mode 100644 index 000000000..6b705ead9 --- /dev/null +++ b/testing/mozbase/manifestparser/manifestparser/expression.py @@ -0,0 +1,324 @@ +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this file, +# You can obtain one at http://mozilla.org/MPL/2.0/. + +import re +import sys +import traceback + +__all__ = ['parse', 'ParseError', 'ExpressionParser'] + +# expr.py +# from: +# http://k0s.org/mozilla/hg/expressionparser +# http://hg.mozilla.org/users/tmielczarek_mozilla.com/expressionparser + +# Implements a top-down parser/evaluator for simple boolean expressions. +# ideas taken from http://effbot.org/zone/simple-top-down-parsing.htm +# +# Rough grammar: +# expr := literal +# | '(' expr ')' +# | expr '&&' expr +# | expr '||' expr +# | expr '==' expr +# | expr '!=' expr +# | expr '<' expr +# | expr '>' expr +# | expr '<=' expr +# | expr '>=' expr +# literal := BOOL +# | INT +# | STRING +# | IDENT +# BOOL := true|false +# INT := [0-9]+ +# STRING := "[^"]*" +# IDENT := [A-Za-z_]\w* + +# Identifiers take their values from a mapping dictionary passed as the second +# argument. + +# Glossary (see above URL for details): +# - nud: null denotation +# - led: left detonation +# - lbp: left binding power +# - rbp: right binding power + + +class ident_token(object): + + def __init__(self, scanner, value): + self.value = value + + def nud(self, parser): + # identifiers take their value from the value mappings passed + # to the parser + return parser.value(self.value) + + +class literal_token(object): + + def __init__(self, scanner, value): + self.value = value + + def nud(self, parser): + return self.value + + +class eq_op_token(object): + "==" + + def led(self, parser, left): + return left == parser.expression(self.lbp) + + +class neq_op_token(object): + "!=" + + def led(self, parser, left): + return left != parser.expression(self.lbp) + + +class lt_op_token(object): + "<" + + def led(self, parser, left): + return left < parser.expression(self.lbp) + + +class gt_op_token(object): + ">" + + def led(self, parser, left): + return left > parser.expression(self.lbp) + + +class le_op_token(object): + "<=" + + def led(self, parser, left): + return left <= parser.expression(self.lbp) + + +class ge_op_token(object): + ">=" + + def led(self, parser, left): + return left >= parser.expression(self.lbp) + + +class not_op_token(object): + "!" + + def nud(self, parser): + return not parser.expression(100) + + +class and_op_token(object): + "&&" + + def led(self, parser, left): + right = parser.expression(self.lbp) + return left and right + + +class or_op_token(object): + "||" + + def led(self, parser, left): + right = parser.expression(self.lbp) + return left or right + + +class lparen_token(object): + "(" + + def nud(self, parser): + expr = parser.expression() + parser.advance(rparen_token) + return expr + + +class rparen_token(object): + ")" + + +class end_token(object): + """always ends parsing""" + +# derived literal tokens + + +class bool_token(literal_token): + + def __init__(self, scanner, value): + value = {'true': True, 'false': False}[value] + literal_token.__init__(self, scanner, value) + + +class int_token(literal_token): + + def __init__(self, scanner, value): + literal_token.__init__(self, scanner, int(value)) + + +class string_token(literal_token): + + def __init__(self, scanner, value): + literal_token.__init__(self, scanner, value[1:-1]) + +precedence = [(end_token, rparen_token), + (or_op_token,), + (and_op_token,), + (lt_op_token, gt_op_token, le_op_token, ge_op_token, + eq_op_token, neq_op_token), + (lparen_token,), + ] +for index, rank in enumerate(precedence): + for token in rank: + token.lbp = index # lbp = lowest left binding power + + +class ParseError(Exception): + """error parsing conditional expression""" + + +class ExpressionParser(object): + """ + A parser for a simple expression language. + + The expression language can be described as follows:: + + EXPRESSION ::= LITERAL | '(' EXPRESSION ')' | '!' EXPRESSION | EXPRESSION OP EXPRESSION + OP ::= '==' | '!=' | '<' | '>' | '<=' | '>=' | '&&' | '||' + LITERAL ::= BOOL | INT | IDENT | STRING + BOOL ::= 'true' | 'false' + INT ::= [0-9]+ + IDENT ::= [a-zA-Z_]\w* + STRING ::= '"' [^\"] '"' | ''' [^\'] ''' + + At its core, expressions consist of booleans, integers, identifiers and. + strings. Booleans are one of *true* or *false*. Integers are a series + of digits. Identifiers are a series of English letters and underscores. + Strings are a pair of matching quote characters (single or double) with + zero or more characters inside. + + Expressions can be combined with operators: the equals (==) and not + equals (!=) operators compare two expressions and produce a boolean. The + and (&&) and or (||) operators take two expressions and produce the logical + AND or OR value of them, respectively. An expression can also be prefixed + with the not (!) operator, which produces its logical negation. + + Finally, any expression may be contained within parentheses for grouping. + + Identifiers take their values from the mapping provided. + """ + + scanner = None + + def __init__(self, text, valuemapping, strict=False): + """ + Initialize the parser + :param text: The expression to parse as a string. + :param valuemapping: A dict mapping identifier names to values. + :param strict: If true, referencing an identifier that was not + provided in :valuemapping: will raise an error. + """ + self.text = text + self.valuemapping = valuemapping + self.strict = strict + + def _tokenize(self): + """ + Lex the input text into tokens and yield them in sequence. + """ + if not ExpressionParser.scanner: + ExpressionParser.scanner = re.Scanner([ + # Note: keep these in sync with the class docstring above. + (r"true|false", bool_token), + (r"[a-zA-Z_]\w*", ident_token), + (r"[0-9]+", int_token), + (r'("[^"]*")|(\'[^\']*\')', string_token), + (r"==", eq_op_token()), + (r"!=", neq_op_token()), + (r"<=", le_op_token()), + (r">=", ge_op_token()), + (r"<", lt_op_token()), + (r">", gt_op_token()), + (r"\|\|", or_op_token()), + (r"!", not_op_token()), + (r"&&", and_op_token()), + (r"\(", lparen_token()), + (r"\)", rparen_token()), + (r"\s+", None), # skip whitespace + ]) + tokens, remainder = ExpressionParser.scanner.scan(self.text) + for t in tokens: + yield t + yield end_token() + + def value(self, ident): + """ + Look up the value of |ident| in the value mapping passed in the + constructor. + """ + if self.strict: + return self.valuemapping[ident] + else: + return self.valuemapping.get(ident, None) + + def advance(self, expected): + """ + Assert that the next token is an instance of |expected|, and advance + to the next token. + """ + if not isinstance(self.token, expected): + raise Exception("Unexpected token!") + self.token = self.iter.next() + + def expression(self, rbp=0): + """ + Parse and return the value of an expression until a token with + right binding power greater than rbp is encountered. + """ + t = self.token + self.token = self.iter.next() + left = t.nud(self) + while rbp < self.token.lbp: + t = self.token + self.token = self.iter.next() + left = t.led(self, left) + return left + + def parse(self): + """ + Parse and return the value of the expression in the text + passed to the constructor. Raises a ParseError if the expression + could not be parsed. + """ + try: + self.iter = self._tokenize() + self.token = self.iter.next() + return self.expression() + except: + extype, ex, tb = sys.exc_info() + formatted = ''.join(traceback.format_exception_only(extype, ex)) + raise ParseError("could not parse: " + "%s\nexception: %svariables: %s" % (self.text, + formatted, + self.valuemapping)), None, tb + + __call__ = parse + + +def parse(text, **values): + """ + Parse and evaluate a boolean expression. + :param text: The expression to parse, as a string. + :param values: A dict containing a name to value mapping for identifiers + referenced in *text*. + :rtype: the final value of the expression. + :raises: :py:exc::ParseError: will be raised if parsing fails. + """ + return ExpressionParser(text, values).parse() diff --git a/testing/mozbase/manifestparser/manifestparser/filters.py b/testing/mozbase/manifestparser/manifestparser/filters.py new file mode 100644 index 000000000..e832c0da6 --- /dev/null +++ b/testing/mozbase/manifestparser/manifestparser/filters.py @@ -0,0 +1,421 @@ +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this file, +# You can obtain one at http://mozilla.org/MPL/2.0/. + +""" +A filter is a callable that accepts an iterable of test objects and a +dictionary of values, and returns a new iterable of test objects. It is +possible to define custom filters if the built-in ones are not enough. +""" + +from collections import defaultdict, MutableSequence +import itertools +import os + +from .expression import ( + parse, + ParseError, +) + + +# built-in filters + +def skip_if(tests, values): + """ + Sets disabled on all tests containing the `skip-if` tag and whose condition + is True. This filter is added by default. + """ + tag = 'skip-if' + for test in tests: + if tag in test and parse(test[tag], **values): + test.setdefault('disabled', '{}: {}'.format(tag, test[tag])) + yield test + + +def run_if(tests, values): + """ + Sets disabled on all tests containing the `run-if` tag and whose condition + is False. This filter is added by default. + """ + tag = 'run-if' + for test in tests: + if tag in test and not parse(test[tag], **values): + test.setdefault('disabled', '{}: {}'.format(tag, test[tag])) + yield test + + +def fail_if(tests, values): + """ + Sets expected to 'fail' on all tests containing the `fail-if` tag and whose + condition is True. This filter is added by default. + """ + tag = 'fail-if' + for test in tests: + if tag in test and parse(test[tag], **values): + test['expected'] = 'fail' + yield test + + +def enabled(tests, values): + """ + Removes all tests containing the `disabled` key. This filter can be + added by passing `disabled=False` into `active_tests`. + """ + for test in tests: + if 'disabled' not in test: + yield test + + +def exists(tests, values): + """ + Removes all tests that do not exist on the file system. This filter is + added by default, but can be removed by passing `exists=False` into + `active_tests`. + """ + for test in tests: + if os.path.exists(test['path']): + yield test + + +# built-in instance filters + +class InstanceFilter(object): + """ + Generally only one instance of a class filter should be applied at a time. + Two instances of `InstanceFilter` are considered equal if they have the + same class name. This ensures only a single instance is ever added to + `filterlist`. This class also formats filters' __str__ method for easier + debugging. + """ + unique = True + + def __init__(self, *args, **kwargs): + self.fmt_args = ', '.join(itertools.chain( + [str(a) for a in args], + ['{}={}'.format(k, v) for k, v in kwargs.iteritems()])) + + def __eq__(self, other): + if self.unique: + return self.__class__ == other.__class__ + return self.__hash__() == other.__hash__() + + def __str__(self): + return "{}({})".format(self.__class__.__name__, self.fmt_args) + + +class subsuite(InstanceFilter): + """ + If `name` is None, removes all tests that have a `subsuite` key. + Otherwise removes all tests that do not have a subsuite matching `name`. + + It is possible to specify conditional subsuite keys using: + subsuite = foo,condition + + where 'foo' is the subsuite name, and 'condition' is the same type of + condition used for skip-if. If the condition doesn't evaluate to true, + the subsuite designation will be removed from the test. + + :param name: The name of the subsuite to run (default None) + """ + + def __init__(self, name=None): + InstanceFilter.__init__(self, name=name) + self.name = name + + def __call__(self, tests, values): + # Look for conditional subsuites, and replace them with the subsuite + # itself (if the condition is true), or nothing. + for test in tests: + subsuite = test.get('subsuite', '') + if ',' in subsuite: + try: + subsuite, cond = subsuite.split(',') + except ValueError: + raise ParseError("subsuite condition can't contain commas") + matched = parse(cond, **values) + if matched: + test['subsuite'] = subsuite + else: + test['subsuite'] = '' + + # Filter on current subsuite + if self.name is None: + if not test.get('subsuite'): + yield test + else: + if test.get('subsuite', '') == self.name: + yield test + + +class chunk_by_slice(InstanceFilter): + """ + Basic chunking algorithm that splits tests evenly across total chunks. + + :param this_chunk: the current chunk, 1 <= this_chunk <= total_chunks + :param total_chunks: the total number of chunks + :param disabled: Whether to include disabled tests in the chunking + algorithm. If False, each chunk contains an equal number + of non-disabled tests. If True, each chunk contains an + equal number of tests (default False) + """ + + def __init__(self, this_chunk, total_chunks, disabled=False): + assert 1 <= this_chunk <= total_chunks + InstanceFilter.__init__(self, this_chunk, total_chunks, + disabled=disabled) + self.this_chunk = this_chunk + self.total_chunks = total_chunks + self.disabled = disabled + + def __call__(self, tests, values): + tests = list(tests) + if self.disabled: + chunk_tests = tests[:] + else: + chunk_tests = [t for t in tests if 'disabled' not in t] + + tests_per_chunk = float(len(chunk_tests)) / self.total_chunks + start = int(round((self.this_chunk - 1) * tests_per_chunk)) + end = int(round(self.this_chunk * tests_per_chunk)) + + if not self.disabled: + # map start and end back onto original list of tests. Disabled + # tests will still be included in the returned list, but each + # chunk will contain an equal number of enabled tests. + if self.this_chunk == 1: + start = 0 + elif start < len(chunk_tests): + start = tests.index(chunk_tests[start]) + + if self.this_chunk == self.total_chunks: + end = len(tests) + elif end < len(chunk_tests): + end = tests.index(chunk_tests[end]) + return (t for t in tests[start:end]) + + +class chunk_by_dir(InstanceFilter): + """ + Basic chunking algorithm that splits directories of tests evenly at a + given depth. + + For example, a depth of 2 means all test directories two path nodes away + from the base are gathered, then split evenly across the total number of + chunks. The number of tests in each of the directories is not taken into + account (so chunks will not contain an even number of tests). All test + paths must be relative to the same root (typically the root of the source + repository). + + :param this_chunk: the current chunk, 1 <= this_chunk <= total_chunks + :param total_chunks: the total number of chunks + :param depth: the minimum depth of a subdirectory before it will be + considered unique + """ + + def __init__(self, this_chunk, total_chunks, depth): + InstanceFilter.__init__(self, this_chunk, total_chunks, depth) + self.this_chunk = this_chunk + self.total_chunks = total_chunks + self.depth = depth + + def __call__(self, tests, values): + tests_by_dir = defaultdict(list) + ordered_dirs = [] + for test in tests: + path = test['relpath'] + + if path.startswith(os.sep): + path = path[1:] + + dirs = path.split(os.sep) + dirs = dirs[:min(self.depth, len(dirs) - 1)] + path = os.sep.join(dirs) + + # don't count directories that only have disabled tests in them, + # but still yield disabled tests that are alongside enabled tests + if path not in ordered_dirs and 'disabled' not in test: + ordered_dirs.append(path) + tests_by_dir[path].append(test) + + tests_per_chunk = float(len(ordered_dirs)) / self.total_chunks + start = int(round((self.this_chunk - 1) * tests_per_chunk)) + end = int(round(self.this_chunk * tests_per_chunk)) + + for i in range(start, end): + for test in tests_by_dir.pop(ordered_dirs[i]): + yield test + + # find directories that only contain disabled tests. They still need to + # be yielded for reporting purposes. Put them all in chunk 1 for + # simplicity. + if self.this_chunk == 1: + disabled_dirs = [v for k, v in tests_by_dir.iteritems() + if k not in ordered_dirs] + for disabled_test in itertools.chain(*disabled_dirs): + yield disabled_test + + +class chunk_by_runtime(InstanceFilter): + """ + Chunking algorithm that attempts to group tests into chunks based on their + average runtimes. It keeps manifests of tests together and pairs slow + running manifests with fast ones. + + :param this_chunk: the current chunk, 1 <= this_chunk <= total_chunks + :param total_chunks: the total number of chunks + :param runtimes: dictionary of test runtime data, of the form + {: } + :param default_runtime: value in seconds to assign tests that don't exist + in the runtimes file + """ + + def __init__(self, this_chunk, total_chunks, runtimes, default_runtime=0): + InstanceFilter.__init__(self, this_chunk, total_chunks, runtimes, + default_runtime=default_runtime) + self.this_chunk = this_chunk + self.total_chunks = total_chunks + + # defaultdict(lambda:) assigns all non-existent keys the value of + # . This means all tests we encounter that don't exist in the + # runtimes file will be assigned `default_runtime`. + self.runtimes = defaultdict(lambda: default_runtime) + self.runtimes.update(runtimes) + + def __call__(self, tests, values): + tests = list(tests) + manifests = set(t['manifest'] for t in tests) + + def total_runtime(tests): + return sum(self.runtimes[t['relpath']] for t in tests + if 'disabled' not in t) + + tests_by_manifest = [] + for manifest in manifests: + mtests = [t for t in tests if t['manifest'] == manifest] + tests_by_manifest.append((total_runtime(mtests), mtests)) + tests_by_manifest.sort(reverse=True) + + tests_by_chunk = [[0, []] for i in range(self.total_chunks)] + for runtime, batch in tests_by_manifest: + # sort first by runtime, then by number of tests in case of a tie. + # This guarantees the chunk with the fastest runtime will always + # get the next batch of tests. + tests_by_chunk.sort(key=lambda x: (x[0], len(x[1]))) + tests_by_chunk[0][0] += runtime + tests_by_chunk[0][1].extend(batch) + + return (t for t in tests_by_chunk[self.this_chunk - 1][1]) + + +class tags(InstanceFilter): + """ + Removes tests that don't contain any of the given tags. This overrides + InstanceFilter's __eq__ method, so multiple instances can be added. + Multiple tag filters is equivalent to joining tags with the AND operator. + + To define a tag in a manifest, add a `tags` attribute to a test or DEFAULT + section. Tests can have multiple tags, in which case they should be + whitespace delimited. For example: + + [test_foobar.html] + tags = foo bar + + :param tags: A tag or list of tags to filter tests on + """ + unique = False + + def __init__(self, tags): + InstanceFilter.__init__(self, tags) + if isinstance(tags, basestring): + tags = [tags] + self.tags = tags + + def __call__(self, tests, values): + for test in tests: + if 'tags' not in test: + continue + + test_tags = [t.strip() for t in test['tags'].split()] + if any(t in self.tags for t in test_tags): + yield test + + +class pathprefix(InstanceFilter): + """ + Removes tests that don't start with any of the given test paths. + + :param paths: A list of test paths to filter on + """ + + def __init__(self, paths): + InstanceFilter.__init__(self, paths) + if isinstance(paths, basestring): + paths = [paths] + self.paths = paths + + def __call__(self, tests, values): + for test in tests: + for tp in self.paths: + tp = os.path.normpath(tp) + + path = test['relpath'] + if os.path.isabs(tp): + path = test['path'] + + if not os.path.normpath(path).startswith(tp): + continue + + # any test path that points to a single file will be run no + # matter what, even if it's disabled + if 'disabled' in test and os.path.normpath(test['relpath']) == tp: + del test['disabled'] + yield test + break + + +# filter container + +DEFAULT_FILTERS = ( + skip_if, + run_if, + fail_if, +) +""" +By default :func:`~.active_tests` will run the :func:`~.skip_if`, +:func:`~.run_if` and :func:`~.fail_if` filters. +""" + + +class filterlist(MutableSequence): + """ + A MutableSequence that raises TypeError when adding a non-callable and + ValueError if the item is already added. + """ + + def __init__(self, items=None): + self.items = [] + if items: + self.items = list(items) + + def _validate(self, item): + if not callable(item): + raise TypeError("Filters must be callable!") + if item in self: + raise ValueError("Filter {} is already applied!".format(item)) + + def __getitem__(self, key): + return self.items[key] + + def __setitem__(self, key, value): + self._validate(value) + self.items[key] = value + + def __delitem__(self, key): + del self.items[key] + + def __len__(self): + return len(self.items) + + def insert(self, index, value): + self._validate(value) + self.items.insert(index, value) diff --git a/testing/mozbase/manifestparser/manifestparser/ini.py b/testing/mozbase/manifestparser/manifestparser/ini.py new file mode 100644 index 000000000..5117dd1ae --- /dev/null +++ b/testing/mozbase/manifestparser/manifestparser/ini.py @@ -0,0 +1,142 @@ +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this file, +# You can obtain one at http://mozilla.org/MPL/2.0/. + +import os + +__all__ = ['read_ini', 'combine_fields'] + + +def read_ini(fp, variables=None, default='DEFAULT', defaults_only=False, + comments=';#', separators=('=', ':'), strict=True, + handle_defaults=True): + """ + read an .ini file and return a list of [(section, values)] + - fp : file pointer or path to read + - variables : default set of variables + - default : name of the section for the default section + - defaults_only : if True, return the default section only + - comments : characters that if they start a line denote a comment + - separators : strings that denote key, value separation in order + - strict : whether to be strict about parsing + - handle_defaults : whether to incorporate defaults into each section + """ + + # variables + variables = variables or {} + sections = [] + key = value = None + section_names = set() + if isinstance(fp, basestring): + fp = file(fp) + + # read the lines + for (linenum, line) in enumerate(fp.read().splitlines(), start=1): + + stripped = line.strip() + + # ignore blank lines + if not stripped: + # reset key and value to avoid continuation lines + key = value = None + continue + + # ignore comment lines + if stripped[0] in comments: + continue + + # check for a new section + if len(stripped) > 2 and stripped[0] == '[' and stripped[-1] == ']': + section = stripped[1:-1].strip() + key = value = None + + # deal with DEFAULT section + if section.lower() == default.lower(): + if strict: + assert default not in section_names + section_names.add(default) + current_section = variables + continue + + if strict: + # make sure this section doesn't already exist + assert section not in section_names, "Section '%s' already found in '%s'" % ( + section, section_names) + + section_names.add(section) + current_section = {} + sections.append((section, current_section)) + continue + + # if there aren't any sections yet, something bad happen + if not section_names: + raise Exception('No sections found') + + # (key, value) pair + for separator in separators: + if separator in stripped: + key, value = stripped.split(separator, 1) + key = key.strip() + value = value.strip() + + if strict: + # make sure this key isn't already in the section or empty + assert key + if current_section is not variables: + assert key not in current_section + + current_section[key] = value + break + else: + # continuation line ? + if line[0].isspace() and key: + value = '%s%s%s' % (value, os.linesep, stripped) + current_section[key] = value + else: + # something bad happened! + if hasattr(fp, 'name'): + filename = fp.name + else: + filename = 'unknown' + raise Exception("Error parsing manifest file '%s', line %s" % + (filename, linenum)) + + # server-root is a special os path declared relative to the manifest file. + # inheritance demands we expand it as absolute + if 'server-root' in variables: + root = os.path.join(os.path.dirname(fp.name), + variables['server-root']) + variables['server-root'] = os.path.abspath(root) + + # return the default section only if requested + if defaults_only: + return [(default, variables)] + + global_vars = variables if handle_defaults else {} + sections = [(i, combine_fields(global_vars, j)) for i, j in sections] + return sections + + +def combine_fields(global_vars, local_vars): + """ + Combine the given manifest entries according to the semantics of specific fields. + This is used to combine manifest level defaults with a per-test definition. + """ + if not global_vars: + return local_vars + if not local_vars: + return global_vars + field_patterns = { + 'skip-if': '(%s) || (%s)', + 'support-files': '%s %s', + } + final_mapping = global_vars.copy() + for field_name, value in local_vars.items(): + if field_name not in field_patterns or field_name not in global_vars: + final_mapping[field_name] = value + continue + global_value = global_vars[field_name] + pattern = field_patterns[field_name] + final_mapping[field_name] = pattern % ( + global_value.split('#')[0], value.split('#')[0]) + return final_mapping diff --git a/testing/mozbase/manifestparser/manifestparser/manifestparser.py b/testing/mozbase/manifestparser/manifestparser/manifestparser.py new file mode 100644 index 000000000..23f14d3f8 --- /dev/null +++ b/testing/mozbase/manifestparser/manifestparser/manifestparser.py @@ -0,0 +1,804 @@ +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this file, +# You can obtain one at http://mozilla.org/MPL/2.0/. + +from StringIO import StringIO +import json +import fnmatch +import os +import shutil +import sys +import types + +from .ini import read_ini +from .filters import ( + DEFAULT_FILTERS, + enabled, + exists as _exists, + filterlist, +) + +__all__ = ['ManifestParser', 'TestManifest', 'convert'] + +relpath = os.path.relpath +string = (basestring,) + + +# path normalization + +def normalize_path(path): + """normalize a relative path""" + if sys.platform.startswith('win'): + return path.replace('/', os.path.sep) + return path + + +def denormalize_path(path): + """denormalize a relative path""" + if sys.platform.startswith('win'): + return path.replace(os.path.sep, '/') + return path + + +# objects for parsing manifests + +class ManifestParser(object): + """read .ini manifests""" + + def __init__(self, manifests=(), defaults=None, strict=True, rootdir=None, + finder=None, handle_defaults=True): + """Creates a ManifestParser from the given manifest files. + + :param manifests: An iterable of file paths or file objects corresponding + to manifests. If a file path refers to a manifest file that + does not exist, an IOError is raised. + :param defaults: Variables to pre-define in the environment for evaluating + expressions in manifests. + :param strict: If False, the provided manifests may contain references to + listed (test) files that do not exist without raising an + IOError during reading, and certain errors in manifests + are not considered fatal. Those errors include duplicate + section names, redefining variables, and defining empty + variables. + :param rootdir: The directory used as the basis for conversion to and from + relative paths during manifest reading. + :param finder: If provided, this finder object will be used for filesystem + interactions. Finder objects are part of the mozpack package, + documented at + http://gecko.readthedocs.org/en/latest/python/mozpack.html#module-mozpack.files + :param handle_defaults: If not set, do not propagate manifest defaults to individual + test objects. Callers are expected to manage per-manifest + defaults themselves via the manifest_defaults member + variable in this case. + """ + self._defaults = defaults or {} + self._ancestor_defaults = {} + self.tests = [] + self.manifest_defaults = {} + self.strict = strict + self.rootdir = rootdir + self.relativeRoot = None + self.finder = finder + self._handle_defaults = handle_defaults + if manifests: + self.read(*manifests) + + def path_exists(self, path): + if self.finder: + return self.finder.get(path) is not None + return os.path.exists(path) + + # methods for reading manifests + + def _read(self, root, filename, defaults, defaults_only=False, parentmanifest=None): + """ + Internal recursive method for reading and parsing manifests. + Stores all found tests in self.tests + :param root: The base path + :param filename: File object or string path for the base manifest file + :param defaults: Options that apply to all items + :param defaults_only: If True will only gather options, not include + tests. Used for upstream parent includes + (default False) + :param parentmanifest: Filename of the parent manifest (default None) + """ + def read_file(type): + include_file = section.split(type, 1)[-1] + include_file = normalize_path(include_file) + if not os.path.isabs(include_file): + include_file = os.path.join(here, include_file) + if not self.path_exists(include_file): + message = "Included file '%s' does not exist" % include_file + if self.strict: + raise IOError(message) + else: + sys.stderr.write("%s\n" % message) + return + return include_file + + # get directory of this file if not file-like object + if isinstance(filename, string): + # If we're using mercurial as our filesystem via a finder + # during manifest reading, the getcwd() calls that happen + # with abspath calls will not be meaningful, so absolute + # paths are required. + if self.finder: + assert os.path.isabs(filename) + filename = os.path.abspath(filename) + if self.finder: + fp = self.finder.get(filename) + else: + fp = open(filename) + here = os.path.dirname(filename) + else: + fp = filename + filename = here = None + defaults['here'] = here + + # Rootdir is needed for relative path calculation. Precompute it for + # the microoptimization used below. + if self.rootdir is None: + rootdir = "" + else: + assert os.path.isabs(self.rootdir) + rootdir = self.rootdir + os.path.sep + + # read the configuration + sections = read_ini(fp=fp, variables=defaults, strict=self.strict, + handle_defaults=self._handle_defaults) + self.manifest_defaults[filename] = defaults + + parent_section_found = False + + # get the tests + for section, data in sections: + # In case of defaults only, no other section than parent: has to + # be processed. + if defaults_only and not section.startswith('parent:'): + continue + + # read the parent manifest if specified + if section.startswith('parent:'): + parent_section_found = True + + include_file = read_file('parent:') + if include_file: + self._read(root, include_file, {}, True) + continue + + # a file to include + # TODO: keep track of included file structure: + # self.manifests = {'manifest.ini': 'relative/path.ini'} + if section.startswith('include:'): + include_file = read_file('include:') + if include_file: + include_defaults = data.copy() + self._read(root, include_file, include_defaults, parentmanifest=filename) + continue + + # otherwise an item + # apply ancestor defaults, while maintaining current file priority + data = dict(self._ancestor_defaults.items() + data.items()) + + test = data + test['name'] = section + + # Will be None if the manifest being read is a file-like object. + test['manifest'] = filename + + # determine the path + path = test.get('path', section) + _relpath = path + if '://' not in path: # don't futz with URLs + path = normalize_path(path) + if here and not os.path.isabs(path): + # Profiling indicates 25% of manifest parsing is spent + # in this call to normpath, but almost all calls return + # their argument unmodified, so we avoid the call if + # '..' if not present in the path. + path = os.path.join(here, path) + if '..' in path: + path = os.path.normpath(path) + + # Microoptimization, because relpath is quite expensive. + # We know that rootdir is an absolute path or empty. If path + # starts with rootdir, then path is also absolute and the tail + # of the path is the relative path (possibly non-normalized, + # when here is unknown). + # For this to work rootdir needs to be terminated with a path + # separator, so that references to sibling directories with + # a common prefix don't get misscomputed (e.g. /root and + # /rootbeer/file). + # When the rootdir is unknown, the relpath needs to be left + # unchanged. We use an empty string as rootdir in that case, + # which leaves relpath unchanged after slicing. + if path.startswith(rootdir): + _relpath = path[len(rootdir):] + else: + _relpath = relpath(path, rootdir) + + test['path'] = path + test['relpath'] = _relpath + + if parentmanifest is not None: + # If a test was included by a parent manifest we may need to + # indicate that in the test object for the sake of identifying + # a test, particularly in the case a test file is included by + # multiple manifests. + test['ancestor-manifest'] = parentmanifest + + # append the item + self.tests.append(test) + + # if no parent: section was found for defaults-only, only read the + # defaults section of the manifest without interpreting variables + if defaults_only and not parent_section_found: + sections = read_ini(fp=fp, variables=defaults, defaults_only=True, + strict=self.strict) + (section, self._ancestor_defaults) = sections[0] + + def read(self, *filenames, **defaults): + """ + read and add manifests from file paths or file-like objects + + filenames -- file paths or file-like objects to read as manifests + defaults -- default variables + """ + + # ensure all files exist + missing = [filename for filename in filenames + if isinstance(filename, string) and not self.path_exists(filename)] + if missing: + raise IOError('Missing files: %s' % ', '.join(missing)) + + # default variables + _defaults = defaults.copy() or self._defaults.copy() + _defaults.setdefault('here', None) + + # process each file + for filename in filenames: + # set the per file defaults + defaults = _defaults.copy() + here = None + if isinstance(filename, string): + here = os.path.dirname(os.path.abspath(filename)) + defaults['here'] = here # directory of master .ini file + + if self.rootdir is None: + # set the root directory + # == the directory of the first manifest given + self.rootdir = here + + self._read(here, filename, defaults) + + # methods for querying manifests + + def query(self, *checks, **kw): + """ + general query function for tests + - checks : callable conditions to test if the test fulfills the query + """ + tests = kw.get('tests', None) + if tests is None: + tests = self.tests + retval = [] + for test in tests: + for check in checks: + if not check(test): + break + else: + retval.append(test) + return retval + + def get(self, _key=None, inverse=False, tags=None, tests=None, **kwargs): + # TODO: pass a dict instead of kwargs since you might hav + # e.g. 'inverse' as a key in the dict + + # TODO: tags should just be part of kwargs with None values + # (None == any is kinda weird, but probably still better) + + # fix up tags + if tags: + tags = set(tags) + else: + tags = set() + + # make some check functions + if inverse: + def has_tags(test): + return not tags.intersection(test.keys()) + + def dict_query(test): + for key, value in kwargs.items(): + if test.get(key) == value: + return False + return True + else: + def has_tags(test): + return tags.issubset(test.keys()) + + def dict_query(test): + for key, value in kwargs.items(): + if test.get(key) != value: + return False + return True + + # query the tests + tests = self.query(has_tags, dict_query, tests=tests) + + # if a key is given, return only a list of that key + # useful for keys like 'name' or 'path' + if _key: + return [test[_key] for test in tests] + + # return the tests + return tests + + def manifests(self, tests=None): + """ + return manifests in order in which they appear in the tests + """ + if tests is None: + # Make sure to return all the manifests, even ones without tests. + return self.manifest_defaults.keys() + + manifests = [] + for test in tests: + manifest = test.get('manifest') + if not manifest: + continue + if manifest not in manifests: + manifests.append(manifest) + return manifests + + def paths(self): + return [i['path'] for i in self.tests] + + # methods for auditing + + def missing(self, tests=None): + """ + return list of tests that do not exist on the filesystem + """ + if tests is None: + tests = self.tests + existing = list(_exists(tests, {})) + return [t for t in tests if t not in existing] + + def check_missing(self, tests=None): + missing = self.missing(tests=tests) + if missing: + missing_paths = [test['path'] for test in missing] + if self.strict: + raise IOError("Strict mode enabled, test paths must exist. " + "The following test(s) are missing: %s" % + json.dumps(missing_paths, indent=2)) + print >> sys.stderr, "Warning: The following test(s) are missing: %s" % \ + json.dumps(missing_paths, indent=2) + return missing + + def verifyDirectory(self, directories, pattern=None, extensions=None): + """ + checks what is on the filesystem vs what is in a manifest + returns a 2-tuple of sets: + (missing_from_filesystem, missing_from_manifest) + """ + + files = set([]) + if isinstance(directories, basestring): + directories = [directories] + + # get files in directories + for directory in directories: + for dirpath, dirnames, filenames in os.walk(directory, topdown=True): + + # only add files that match a pattern + if pattern: + filenames = fnmatch.filter(filenames, pattern) + + # only add files that have one of the extensions + if extensions: + filenames = [filename for filename in filenames + if os.path.splitext(filename)[-1] in extensions] + + files.update([os.path.join(dirpath, filename) for filename in filenames]) + + paths = set(self.paths()) + missing_from_filesystem = paths.difference(files) + missing_from_manifest = files.difference(paths) + return (missing_from_filesystem, missing_from_manifest) + + # methods for output + + def write(self, fp=sys.stdout, rootdir=None, + global_tags=None, global_kwargs=None, + local_tags=None, local_kwargs=None): + """ + write a manifest given a query + global and local options will be munged to do the query + globals will be written to the top of the file + locals (if given) will be written per test + """ + + # open file if `fp` given as string + close = False + if isinstance(fp, string): + fp = file(fp, 'w') + close = True + + # root directory + if rootdir is None: + rootdir = self.rootdir + + # sanitize input + global_tags = global_tags or set() + local_tags = local_tags or set() + global_kwargs = global_kwargs or {} + local_kwargs = local_kwargs or {} + + # create the query + tags = set([]) + tags.update(global_tags) + tags.update(local_tags) + kwargs = {} + kwargs.update(global_kwargs) + kwargs.update(local_kwargs) + + # get matching tests + tests = self.get(tags=tags, **kwargs) + + # print the .ini manifest + if global_tags or global_kwargs: + print >> fp, '[DEFAULT]' + for tag in global_tags: + print >> fp, '%s =' % tag + for key, value in global_kwargs.items(): + print >> fp, '%s = %s' % (key, value) + print >> fp + + for test in tests: + test = test.copy() # don't overwrite + + path = test['name'] + if not os.path.isabs(path): + path = test['path'] + if self.rootdir: + path = relpath(test['path'], self.rootdir) + path = denormalize_path(path) + print >> fp, '[%s]' % path + + # reserved keywords: + reserved = ['path', 'name', 'here', 'manifest', 'relpath', 'ancestor-manifest'] + for key in sorted(test.keys()): + if key in reserved: + continue + if key in global_kwargs: + continue + if key in global_tags and not test[key]: + continue + print >> fp, '%s = %s' % (key, test[key]) + print >> fp + + if close: + # close the created file + fp.close() + + def __str__(self): + fp = StringIO() + self.write(fp=fp) + value = fp.getvalue() + return value + + def copy(self, directory, rootdir=None, *tags, **kwargs): + """ + copy the manifests and associated tests + - directory : directory to copy to + - rootdir : root directory to copy to (if not given from manifests) + - tags : keywords the tests must have + - kwargs : key, values the tests must match + """ + # XXX note that copy does *not* filter the tests out of the + # resulting manifest; it just stupidly copies them over. + # ideally, it would reread the manifests and filter out the + # tests that don't match *tags and **kwargs + + # destination + if not os.path.exists(directory): + os.path.makedirs(directory) + else: + # sanity check + assert os.path.isdir(directory) + + # tests to copy + tests = self.get(tags=tags, **kwargs) + if not tests: + return # nothing to do! + + # root directory + if rootdir is None: + rootdir = self.rootdir + + # copy the manifests + tests + manifests = [relpath(manifest, rootdir) for manifest in self.manifests()] + for manifest in manifests: + destination = os.path.join(directory, manifest) + dirname = os.path.dirname(destination) + if not os.path.exists(dirname): + os.makedirs(dirname) + else: + # sanity check + assert os.path.isdir(dirname) + shutil.copy(os.path.join(rootdir, manifest), destination) + + missing = self.check_missing(tests) + tests = [test for test in tests if test not in missing] + for test in tests: + if os.path.isabs(test['name']): + continue + source = test['path'] + destination = os.path.join(directory, relpath(test['path'], rootdir)) + shutil.copy(source, destination) + # TODO: ensure that all of the tests are below the from_dir + + def update(self, from_dir, rootdir=None, *tags, **kwargs): + """ + update the tests as listed in a manifest from a directory + - from_dir : directory where the tests live + - rootdir : root directory to copy to (if not given from manifests) + - tags : keys the tests must have + - kwargs : key, values the tests must match + """ + + # get the tests + tests = self.get(tags=tags, **kwargs) + + # get the root directory + if not rootdir: + rootdir = self.rootdir + + # copy them! + for test in tests: + if not os.path.isabs(test['name']): + _relpath = relpath(test['path'], rootdir) + source = os.path.join(from_dir, _relpath) + if not os.path.exists(source): + message = "Missing test: '%s' does not exist!" + if self.strict: + raise IOError(message) + print >> sys.stderr, message + " Skipping." + continue + destination = os.path.join(rootdir, _relpath) + shutil.copy(source, destination) + + # directory importers + + @classmethod + def _walk_directories(cls, directories, callback, pattern=None, ignore=()): + """ + internal function to import directories + """ + + if isinstance(pattern, basestring): + patterns = [pattern] + else: + patterns = pattern + ignore = set(ignore) + + if not patterns: + def accept_filename(filename): + return True + else: + def accept_filename(filename): + for pattern in patterns: + if fnmatch.fnmatch(filename, pattern): + return True + + if not ignore: + def accept_dirname(dirname): + return True + else: + def accept_dirname(dirname): + return dirname not in ignore + + rootdirectories = directories[:] + seen_directories = set() + for rootdirectory in rootdirectories: + # let's recurse directories using list + directories = [os.path.realpath(rootdirectory)] + while directories: + directory = directories.pop(0) + if directory in seen_directories: + # eliminate possible infinite recursion due to + # symbolic links + continue + seen_directories.add(directory) + + files = [] + subdirs = [] + for name in sorted(os.listdir(directory)): + path = os.path.join(directory, name) + if os.path.isfile(path): + # os.path.isfile follow symbolic links, we don't + # need to handle them here. + if accept_filename(name): + files.append(name) + continue + elif os.path.islink(path): + # eliminate symbolic links + path = os.path.realpath(path) + + # we must have a directory here + if accept_dirname(name): + subdirs.append(name) + # this subdir is added for recursion + directories.insert(0, path) + + # here we got all subdirs and files filtered, we can + # call the callback function if directory is not empty + if subdirs or files: + callback(rootdirectory, directory, subdirs, files) + + @classmethod + def populate_directory_manifests(cls, directories, filename, pattern=None, ignore=(), + overwrite=False): + """ + walks directories and writes manifests of name `filename` in-place; + returns `cls` instance populated with the given manifests + + filename -- filename of manifests to write + pattern -- shell pattern (glob) or patterns of filenames to match + ignore -- directory names to ignore + overwrite -- whether to overwrite existing files of given name + """ + + manifest_dict = {} + + if os.path.basename(filename) != filename: + raise IOError("filename should not include directory name") + + # no need to hit directories more than once + _directories = directories + directories = [] + for directory in _directories: + if directory not in directories: + directories.append(directory) + + def callback(directory, dirpath, dirnames, filenames): + """write a manifest for each directory""" + + manifest_path = os.path.join(dirpath, filename) + if (dirnames or filenames) and not (os.path.exists(manifest_path) and overwrite): + with file(manifest_path, 'w') as manifest: + for dirname in dirnames: + print >> manifest, '[include:%s]' % os.path.join(dirname, filename) + for _filename in filenames: + print >> manifest, '[%s]' % _filename + + # add to list of manifests + manifest_dict.setdefault(directory, manifest_path) + + # walk the directories to gather files + cls._walk_directories(directories, callback, pattern=pattern, ignore=ignore) + # get manifests + manifests = [manifest_dict[directory] for directory in _directories] + + # create a `cls` instance with the manifests + return cls(manifests=manifests) + + @classmethod + def from_directories(cls, directories, pattern=None, ignore=(), write=None, relative_to=None): + """ + convert directories to a simple manifest; returns ManifestParser instance + + pattern -- shell pattern (glob) or patterns of filenames to match + ignore -- directory names to ignore + write -- filename or file-like object of manifests to write; + if `None` then a StringIO instance will be created + relative_to -- write paths relative to this path; + if false then the paths are absolute + """ + + # determine output + opened_manifest_file = None # name of opened manifest file + absolute = not relative_to # whether to output absolute path names as names + if isinstance(write, string): + opened_manifest_file = write + write = file(write, 'w') + if write is None: + write = StringIO() + + # walk the directories, generating manifests + def callback(directory, dirpath, dirnames, filenames): + + # absolute paths + filenames = [os.path.join(dirpath, filename) + for filename in filenames] + # ensure new manifest isn't added + filenames = [filename for filename in filenames + if filename != opened_manifest_file] + # normalize paths + if not absolute and relative_to: + filenames = [relpath(filename, relative_to) + for filename in filenames] + + # write to manifest + print >> write, '\n'.join(['[%s]' % denormalize_path(filename) + for filename in filenames]) + + cls._walk_directories(directories, callback, pattern=pattern, ignore=ignore) + + if opened_manifest_file: + # close file + write.close() + manifests = [opened_manifest_file] + else: + # manifests/write is a file-like object; + # rewind buffer + write.flush() + write.seek(0) + manifests = [write] + + # make a ManifestParser instance + return cls(manifests=manifests) + +convert = ManifestParser.from_directories + + +class TestManifest(ManifestParser): + """ + apply logic to manifests; this is your integration layer :) + specific harnesses may subclass from this if they need more logic + """ + + def __init__(self, *args, **kwargs): + ManifestParser.__init__(self, *args, **kwargs) + self.filters = filterlist(DEFAULT_FILTERS) + self.last_used_filters = [] + + def active_tests(self, exists=True, disabled=True, filters=None, **values): + """ + Run all applied filters on the set of tests. + + :param exists: filter out non-existing tests (default True) + :param disabled: whether to return disabled tests (default True) + :param values: keys and values to filter on (e.g. `os = linux mac`) + :param filters: list of filters to apply to the tests + :returns: list of test objects that were not filtered out + """ + tests = [i.copy() for i in self.tests] # shallow copy + + # mark all tests as passing + for test in tests: + test['expected'] = test.get('expected', 'pass') + + # make a copy so original doesn't get modified + fltrs = self.filters[:] + if exists: + if self.strict: + self.check_missing(tests) + else: + fltrs.append(_exists) + + if not disabled: + fltrs.append(enabled) + + if filters: + fltrs += filters + + self.last_used_filters = fltrs[:] + for fn in fltrs: + tests = fn(tests, values) + return list(tests) + + def test_paths(self): + return [test['path'] for test in self.active_tests()] + + def fmt_filters(self, filters=None): + filters = filters or self.last_used_filters + names = [] + for f in filters: + if isinstance(f, types.FunctionType): + names.append(f.__name__) + else: + names.append(str(f)) + return ', '.join(names) diff --git a/testing/mozbase/manifestparser/setup.py b/testing/mozbase/manifestparser/setup.py new file mode 100644 index 000000000..b34f9cea7 --- /dev/null +++ b/testing/mozbase/manifestparser/setup.py @@ -0,0 +1,27 @@ +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this file, +# You can obtain one at http://mozilla.org/MPL/2.0/. + +from setuptools import setup + +PACKAGE_NAME = "manifestparser" +PACKAGE_VERSION = '1.1' + +setup(name=PACKAGE_NAME, + version=PACKAGE_VERSION, + description="Library to create and manage test manifests", + long_description="see http://mozbase.readthedocs.org/", + classifiers=[], # Get strings from http://pypi.python.org/pypi?%3Aaction=list_classifiers + keywords='mozilla manifests', + author='Mozilla Automation and Testing Team', + author_email='tools@lists.mozilla.org', + url='https://wiki.mozilla.org/Auto-tools/Projects/Mozbase', + license='MPL', + zip_safe=False, + packages=['manifestparser'], + install_requires=[], + entry_points=""" + [console_scripts] + manifestparser = manifestparser.cli:main + """, + ) diff --git a/testing/mozbase/manifestparser/tests/comment-example.ini b/testing/mozbase/manifestparser/tests/comment-example.ini new file mode 100644 index 000000000..030ceffdb --- /dev/null +++ b/testing/mozbase/manifestparser/tests/comment-example.ini @@ -0,0 +1,11 @@ +; See https://bugzilla.mozilla.org/show_bug.cgi?id=813674 + +[test_0180_fileInUse_xp_win_complete.js] +[test_0181_fileInUse_xp_win_partial.js] +[test_0182_rmrfdirFileInUse_xp_win_complete.js] +[test_0183_rmrfdirFileInUse_xp_win_partial.js] +[test_0184_fileInUse_xp_win_complete.js] +[test_0185_fileInUse_xp_win_partial.js] +[test_0186_rmrfdirFileInUse_xp_win_complete.js] +[test_0187_rmrfdirFileInUse_xp_win_partial.js] +; [test_0202_app_launch_apply_update_dirlocked.js] # Test disabled, bug 757632 \ No newline at end of file diff --git a/testing/mozbase/manifestparser/tests/default-skipif.ini b/testing/mozbase/manifestparser/tests/default-skipif.ini new file mode 100644 index 000000000..d3c268733 --- /dev/null +++ b/testing/mozbase/manifestparser/tests/default-skipif.ini @@ -0,0 +1,22 @@ +[DEFAULT] +skip-if = os == 'win' && debug # a pesky comment + + +[test1] +skip-if = debug + +[test2] +skip-if = os == 'linux' + +[test3] +skip-if = os == 'win' + +[test4] +skip-if = os == 'win' && debug + +[test5] +foo = bar + +[test6] +skip-if = debug # a second pesky comment + diff --git a/testing/mozbase/manifestparser/tests/default-suppfiles.ini b/testing/mozbase/manifestparser/tests/default-suppfiles.ini new file mode 100644 index 000000000..12af247b8 --- /dev/null +++ b/testing/mozbase/manifestparser/tests/default-suppfiles.ini @@ -0,0 +1,9 @@ +[DEFAULT] +support-files = foo.js # a comment + +[test7] +[test8] +support-files = bar.js # another comment +[test9] +foo = bar + diff --git a/testing/mozbase/manifestparser/tests/filter-example.ini b/testing/mozbase/manifestparser/tests/filter-example.ini new file mode 100644 index 000000000..13a8734c3 --- /dev/null +++ b/testing/mozbase/manifestparser/tests/filter-example.ini @@ -0,0 +1,11 @@ +# illustrate test filters based on various categories + +[windowstest] +skip-if = os != 'win' + +[fleem] +skip-if = os == 'mac' + +[linuxtest] +skip-if = (os == 'mac') || (os == 'win') +fail-if = toolkit == 'cocoa' diff --git a/testing/mozbase/manifestparser/tests/fleem b/testing/mozbase/manifestparser/tests/fleem new file mode 100644 index 000000000..744817b82 --- /dev/null +++ b/testing/mozbase/manifestparser/tests/fleem @@ -0,0 +1 @@ +# dummy spot for "fleem" test diff --git a/testing/mozbase/manifestparser/tests/include-example.ini b/testing/mozbase/manifestparser/tests/include-example.ini new file mode 100644 index 000000000..69e728c3b --- /dev/null +++ b/testing/mozbase/manifestparser/tests/include-example.ini @@ -0,0 +1,11 @@ +[DEFAULT] +foo = bar + +[include:include/bar.ini] + +[fleem] + +[include:include/foo.ini] +red = roses +blue = violets +yellow = daffodils \ No newline at end of file diff --git a/testing/mozbase/manifestparser/tests/include-invalid.ini b/testing/mozbase/manifestparser/tests/include-invalid.ini new file mode 100644 index 000000000..e3ed0dd6b --- /dev/null +++ b/testing/mozbase/manifestparser/tests/include-invalid.ini @@ -0,0 +1 @@ +[include:invalid.ini] diff --git a/testing/mozbase/manifestparser/tests/include/bar.ini b/testing/mozbase/manifestparser/tests/include/bar.ini new file mode 100644 index 000000000..bcb312d1d --- /dev/null +++ b/testing/mozbase/manifestparser/tests/include/bar.ini @@ -0,0 +1,4 @@ +[DEFAULT] +foo = fleem + +[crash-handling] \ No newline at end of file diff --git a/testing/mozbase/manifestparser/tests/include/crash-handling b/testing/mozbase/manifestparser/tests/include/crash-handling new file mode 100644 index 000000000..8e19a6375 --- /dev/null +++ b/testing/mozbase/manifestparser/tests/include/crash-handling @@ -0,0 +1 @@ +# dummy spot for "crash-handling" test diff --git a/testing/mozbase/manifestparser/tests/include/flowers b/testing/mozbase/manifestparser/tests/include/flowers new file mode 100644 index 000000000..a25acfbe2 --- /dev/null +++ b/testing/mozbase/manifestparser/tests/include/flowers @@ -0,0 +1 @@ +# dummy spot for "flowers" test diff --git a/testing/mozbase/manifestparser/tests/include/foo.ini b/testing/mozbase/manifestparser/tests/include/foo.ini new file mode 100644 index 000000000..cfc90ace8 --- /dev/null +++ b/testing/mozbase/manifestparser/tests/include/foo.ini @@ -0,0 +1,5 @@ +[DEFAULT] +blue = ocean + +[flowers] +yellow = submarine \ No newline at end of file diff --git a/testing/mozbase/manifestparser/tests/just-defaults.ini b/testing/mozbase/manifestparser/tests/just-defaults.ini new file mode 100644 index 000000000..83a0cec0c --- /dev/null +++ b/testing/mozbase/manifestparser/tests/just-defaults.ini @@ -0,0 +1,2 @@ +[DEFAULT] +foo = bar diff --git a/testing/mozbase/manifestparser/tests/manifest.ini b/testing/mozbase/manifestparser/tests/manifest.ini new file mode 100644 index 000000000..dfa185649 --- /dev/null +++ b/testing/mozbase/manifestparser/tests/manifest.ini @@ -0,0 +1,11 @@ +# test manifest for manifestparser +[test_expressionparser.py] +[test_manifestparser.py] +[test_testmanifest.py] +[test_read_ini.py] +[test_convert_directory.py] +[test_filters.py] +[test_chunking.py] + +[test_convert_symlinks.py] +disabled = https://bugzilla.mozilla.org/show_bug.cgi?id=920938 diff --git a/testing/mozbase/manifestparser/tests/missing-path.ini b/testing/mozbase/manifestparser/tests/missing-path.ini new file mode 100644 index 000000000..919d8e04d --- /dev/null +++ b/testing/mozbase/manifestparser/tests/missing-path.ini @@ -0,0 +1,2 @@ +[foo] +[bar] diff --git a/testing/mozbase/manifestparser/tests/mozmill-example.ini b/testing/mozbase/manifestparser/tests/mozmill-example.ini new file mode 100644 index 000000000..114cf48c4 --- /dev/null +++ b/testing/mozbase/manifestparser/tests/mozmill-example.ini @@ -0,0 +1,80 @@ +[testAddons/testDisableEnablePlugin.js] +[testAddons/testGetAddons.js] +[testAddons/testSearchAddons.js] +[testAwesomeBar/testAccessLocationBar.js] +[testAwesomeBar/testCheckItemHighlight.js] +[testAwesomeBar/testEscapeAutocomplete.js] +[testAwesomeBar/testFaviconInAutocomplete.js] +[testAwesomeBar/testGoButton.js] +[testAwesomeBar/testLocationBarSearches.js] +[testAwesomeBar/testPasteLocationBar.js] +[testAwesomeBar/testSuggestHistoryBookmarks.js] +[testAwesomeBar/testVisibleItemsMax.js] +[testBookmarks/testAddBookmarkToMenu.js] +[testCookies/testDisableCookies.js] +[testCookies/testEnableCookies.js] +[testCookies/testRemoveAllCookies.js] +[testCookies/testRemoveCookie.js] +[testDownloading/testCloseDownloadManager.js] +[testDownloading/testDownloadStates.js] +[testDownloading/testOpenDownloadManager.js] +[testFindInPage/testFindInPage.js] +[testFormManager/testAutoCompleteOff.js] +[testFormManager/testBasicFormCompletion.js] +[testFormManager/testClearFormHistory.js] +[testFormManager/testDisableFormManager.js] +[testGeneral/testGoogleSuggestions.js] +[testGeneral/testStopReloadButtons.js] +[testInstallation/testBreakpadInstalled.js] +[testLayout/testNavigateFTP.js] +[testPasswordManager/testPasswordNotSaved.js] +[testPasswordManager/testPasswordSavedAndDeleted.js] +[testPopups/testPopupsAllowed.js] +[testPopups/testPopupsBlocked.js] +[testPreferences/testPaneRetention.js] +[testPreferences/testPreferredLanguage.js] +[testPreferences/testRestoreHomepageToDefault.js] +[testPreferences/testSetToCurrentPage.js] +[testPreferences/testSwitchPanes.js] +[testPrivateBrowsing/testAboutPrivateBrowsing.js] +[testPrivateBrowsing/testCloseWindow.js] +[testPrivateBrowsing/testDisabledElements.js] +[testPrivateBrowsing/testDisabledPermissions.js] +[testPrivateBrowsing/testDownloadManagerClosed.js] +[testPrivateBrowsing/testGeolocation.js] +[testPrivateBrowsing/testStartStopPBMode.js] +[testPrivateBrowsing/testTabRestoration.js] +[testPrivateBrowsing/testTabsDismissedOnStop.js] +[testSearch/testAddMozSearchProvider.js] +[testSearch/testFocusAndSearch.js] +[testSearch/testGetMoreSearchEngines.js] +[testSearch/testOpenSearchAutodiscovery.js] +[testSearch/testRemoveSearchEngine.js] +[testSearch/testReorderSearchEngines.js] +[testSearch/testRestoreDefaults.js] +[testSearch/testSearchSelection.js] +[testSearch/testSearchSuggestions.js] +[testSecurity/testBlueLarry.js] +[testSecurity/testDefaultPhishingEnabled.js] +[testSecurity/testDefaultSecurityPrefs.js] +[testSecurity/testEncryptedPageWarning.js] +[testSecurity/testGreenLarry.js] +[testSecurity/testGreyLarry.js] +[testSecurity/testIdentityPopupOpenClose.js] +[testSecurity/testSSLDisabledErrorPage.js] +[testSecurity/testSafeBrowsingNotificationBar.js] +[testSecurity/testSafeBrowsingWarningPages.js] +[testSecurity/testSecurityInfoViaMoreInformation.js] +[testSecurity/testSecurityNotification.js] +[testSecurity/testSubmitUnencryptedInfoWarning.js] +[testSecurity/testUnknownIssuer.js] +[testSecurity/testUntrustedConnectionErrorPage.js] +[testSessionStore/testUndoTabFromContextMenu.js] +[testTabbedBrowsing/testBackgroundTabScrolling.js] +[testTabbedBrowsing/testCloseTab.js] +[testTabbedBrowsing/testNewTab.js] +[testTabbedBrowsing/testNewWindow.js] +[testTabbedBrowsing/testOpenInBackground.js] +[testTabbedBrowsing/testOpenInForeground.js] +[testTechnicalTools/testAccessPageInfoDialog.js] +[testToolbar/testBackForwardButtons.js] diff --git a/testing/mozbase/manifestparser/tests/mozmill-restart-example.ini b/testing/mozbase/manifestparser/tests/mozmill-restart-example.ini new file mode 100644 index 000000000..e27ae9b93 --- /dev/null +++ b/testing/mozbase/manifestparser/tests/mozmill-restart-example.ini @@ -0,0 +1,26 @@ +[DEFAULT] +type = restart + +[restartTests/testExtensionInstallUninstall/test2.js] +foo = bar + +[restartTests/testExtensionInstallUninstall/test1.js] +foo = baz + +[restartTests/testExtensionInstallUninstall/test3.js] +[restartTests/testSoftwareUpdateAutoProxy/test2.js] +[restartTests/testSoftwareUpdateAutoProxy/test1.js] +[restartTests/testMasterPassword/test1.js] +[restartTests/testExtensionInstallGetAddons/test2.js] +[restartTests/testExtensionInstallGetAddons/test1.js] +[restartTests/testMultipleExtensionInstallation/test2.js] +[restartTests/testMultipleExtensionInstallation/test1.js] +[restartTests/testThemeInstallUninstall/test2.js] +[restartTests/testThemeInstallUninstall/test1.js] +[restartTests/testThemeInstallUninstall/test3.js] +[restartTests/testDefaultBookmarks/test1.js] +[softwareUpdate/testFallbackUpdate/test2.js] +[softwareUpdate/testFallbackUpdate/test1.js] +[softwareUpdate/testFallbackUpdate/test3.js] +[softwareUpdate/testDirectUpdate/test2.js] +[softwareUpdate/testDirectUpdate/test1.js] diff --git a/testing/mozbase/manifestparser/tests/no-tests.ini b/testing/mozbase/manifestparser/tests/no-tests.ini new file mode 100644 index 000000000..83a0cec0c --- /dev/null +++ b/testing/mozbase/manifestparser/tests/no-tests.ini @@ -0,0 +1,2 @@ +[DEFAULT] +foo = bar diff --git a/testing/mozbase/manifestparser/tests/parent/include/first/manifest.ini b/testing/mozbase/manifestparser/tests/parent/include/first/manifest.ini new file mode 100644 index 000000000..828525c18 --- /dev/null +++ b/testing/mozbase/manifestparser/tests/parent/include/first/manifest.ini @@ -0,0 +1,3 @@ +[parent:../manifest.ini] + +[testFirst.js] diff --git a/testing/mozbase/manifestparser/tests/parent/include/manifest.ini b/testing/mozbase/manifestparser/tests/parent/include/manifest.ini new file mode 100644 index 000000000..fb9756d6a --- /dev/null +++ b/testing/mozbase/manifestparser/tests/parent/include/manifest.ini @@ -0,0 +1,8 @@ +[DEFAULT] +top = data + +[include:first/manifest.ini] +disabled = YES + +[include:second/manifest.ini] +disabled = NO diff --git a/testing/mozbase/manifestparser/tests/parent/include/second/manifest.ini b/testing/mozbase/manifestparser/tests/parent/include/second/manifest.ini new file mode 100644 index 000000000..31f053756 --- /dev/null +++ b/testing/mozbase/manifestparser/tests/parent/include/second/manifest.ini @@ -0,0 +1,3 @@ +[parent:../manifest.ini] + +[testSecond.js] diff --git a/testing/mozbase/manifestparser/tests/parent/level_1/level_1.ini b/testing/mozbase/manifestparser/tests/parent/level_1/level_1.ini new file mode 100644 index 000000000..ac7c370c3 --- /dev/null +++ b/testing/mozbase/manifestparser/tests/parent/level_1/level_1.ini @@ -0,0 +1,5 @@ +[DEFAULT] +x = level_1 + +[test_1] +[test_2] diff --git a/testing/mozbase/manifestparser/tests/parent/level_1/level_1_server-root.ini b/testing/mozbase/manifestparser/tests/parent/level_1/level_1_server-root.ini new file mode 100644 index 000000000..486a9596e --- /dev/null +++ b/testing/mozbase/manifestparser/tests/parent/level_1/level_1_server-root.ini @@ -0,0 +1,5 @@ +[DEFAULT] +server-root = ../root +other-root = ../root + +[test_1] diff --git a/testing/mozbase/manifestparser/tests/parent/level_1/level_2/level_2.ini b/testing/mozbase/manifestparser/tests/parent/level_1/level_2/level_2.ini new file mode 100644 index 000000000..ada6a510d --- /dev/null +++ b/testing/mozbase/manifestparser/tests/parent/level_1/level_2/level_2.ini @@ -0,0 +1,3 @@ +[parent:../level_1.ini] + +[test_2] diff --git a/testing/mozbase/manifestparser/tests/parent/level_1/level_2/level_2_server-root.ini b/testing/mozbase/manifestparser/tests/parent/level_1/level_2/level_2_server-root.ini new file mode 100644 index 000000000..218789784 --- /dev/null +++ b/testing/mozbase/manifestparser/tests/parent/level_1/level_2/level_2_server-root.ini @@ -0,0 +1,3 @@ +[parent:../level_1_server-root.ini] + +[test_2] diff --git a/testing/mozbase/manifestparser/tests/parent/level_1/level_2/level_3/level_3.ini b/testing/mozbase/manifestparser/tests/parent/level_1/level_2/level_3/level_3.ini new file mode 100644 index 000000000..2edd647fc --- /dev/null +++ b/testing/mozbase/manifestparser/tests/parent/level_1/level_2/level_3/level_3.ini @@ -0,0 +1,3 @@ +[parent:../level_2.ini] + +[test_3] diff --git a/testing/mozbase/manifestparser/tests/parent/level_1/level_2/level_3/level_3_default.ini b/testing/mozbase/manifestparser/tests/parent/level_1/level_2/level_3/level_3_default.ini new file mode 100644 index 000000000..d6aae60ae --- /dev/null +++ b/testing/mozbase/manifestparser/tests/parent/level_1/level_2/level_3/level_3_default.ini @@ -0,0 +1,6 @@ +[parent:../level_2.ini] + +[DEFAULT] +x = level_3 + +[test_3] diff --git a/testing/mozbase/manifestparser/tests/parent/level_1/level_2/level_3/level_3_server-root.ini b/testing/mozbase/manifestparser/tests/parent/level_1/level_2/level_3/level_3_server-root.ini new file mode 100644 index 000000000..0427087b4 --- /dev/null +++ b/testing/mozbase/manifestparser/tests/parent/level_1/level_2/level_3/level_3_server-root.ini @@ -0,0 +1,3 @@ +[parent:../level_2_server-root.ini] + +[test_3] diff --git a/testing/mozbase/manifestparser/tests/parent/level_1/level_2/level_3/test_3 b/testing/mozbase/manifestparser/tests/parent/level_1/level_2/level_3/test_3 new file mode 100644 index 000000000..f5de58752 --- /dev/null +++ b/testing/mozbase/manifestparser/tests/parent/level_1/level_2/level_3/test_3 @@ -0,0 +1 @@ +# dummy spot for "test_3" test diff --git a/testing/mozbase/manifestparser/tests/parent/level_1/level_2/test_2 b/testing/mozbase/manifestparser/tests/parent/level_1/level_2/test_2 new file mode 100644 index 000000000..5b77e04f3 --- /dev/null +++ b/testing/mozbase/manifestparser/tests/parent/level_1/level_2/test_2 @@ -0,0 +1 @@ +# dummy spot for "test_2" test diff --git a/testing/mozbase/manifestparser/tests/parent/level_1/test_1 b/testing/mozbase/manifestparser/tests/parent/level_1/test_1 new file mode 100644 index 000000000..dccbf04e4 --- /dev/null +++ b/testing/mozbase/manifestparser/tests/parent/level_1/test_1 @@ -0,0 +1 @@ +# dummy spot for "test_1" test diff --git a/testing/mozbase/manifestparser/tests/parent/root/dummy b/testing/mozbase/manifestparser/tests/parent/root/dummy new file mode 100644 index 000000000..e69de29bb diff --git a/testing/mozbase/manifestparser/tests/path-example.ini b/testing/mozbase/manifestparser/tests/path-example.ini new file mode 100644 index 000000000..366782d95 --- /dev/null +++ b/testing/mozbase/manifestparser/tests/path-example.ini @@ -0,0 +1,2 @@ +[foo] +path = fleem \ No newline at end of file diff --git a/testing/mozbase/manifestparser/tests/relative-path.ini b/testing/mozbase/manifestparser/tests/relative-path.ini new file mode 100644 index 000000000..57105489b --- /dev/null +++ b/testing/mozbase/manifestparser/tests/relative-path.ini @@ -0,0 +1,5 @@ +[foo] +path = ../fleem + +[bar] +path = ../testsSIBLING/example diff --git a/testing/mozbase/manifestparser/tests/subsuite.ini b/testing/mozbase/manifestparser/tests/subsuite.ini new file mode 100644 index 000000000..c1a70bd44 --- /dev/null +++ b/testing/mozbase/manifestparser/tests/subsuite.ini @@ -0,0 +1,13 @@ +[test1] +subsuite=bar,foo=="bar" # this has a comment + +[test2] +subsuite=bar,foo=="bar" + +[test3] +subsuite=baz + +[test4] +[test5] +[test6] +subsuite=bar,foo=="szy" || foo=="bar" \ No newline at end of file diff --git a/testing/mozbase/manifestparser/tests/test_chunking.py b/testing/mozbase/manifestparser/tests/test_chunking.py new file mode 100644 index 000000000..719bbca80 --- /dev/null +++ b/testing/mozbase/manifestparser/tests/test_chunking.py @@ -0,0 +1,302 @@ +#!/usr/bin/env python + +from itertools import chain +from unittest import TestCase +import os +import random + +from manifestparser.filters import ( + chunk_by_dir, + chunk_by_runtime, + chunk_by_slice, +) + +here = os.path.dirname(os.path.abspath(__file__)) + + +class ChunkBySlice(TestCase): + """Test chunking related filters""" + + def generate_tests(self, num, disabled=None): + disabled = disabled or [] + tests = [] + for i in range(num): + test = {'name': 'test%i' % i} + if i in disabled: + test['disabled'] = '' + tests.append(test) + return tests + + def run_all_combos(self, num_tests, disabled=None): + tests = self.generate_tests(num_tests, disabled=disabled) + + for total in range(1, num_tests + 1): + res = [] + res_disabled = [] + for chunk in range(1, total + 1): + f = chunk_by_slice(chunk, total) + res.append(list(f(tests, {}))) + if disabled: + f.disabled = True + res_disabled.append(list(f(tests, {}))) + + lengths = [len([t for t in c if 'disabled' not in t]) for c in res] + # the chunk with the most tests should have at most one more test + # than the chunk with the least tests + self.assertLessEqual(max(lengths) - min(lengths), 1) + + # chaining all chunks back together should equal the original list + # of tests + self.assertEqual(list(chain.from_iterable(res)), list(tests)) + + if disabled: + lengths = [len(c) for c in res_disabled] + self.assertLessEqual(max(lengths) - min(lengths), 1) + self.assertEqual(list(chain.from_iterable(res_disabled)), + list(tests)) + + def test_chunk_by_slice(self): + chunk = chunk_by_slice(1, 1) + self.assertEqual(list(chunk([], {})), []) + + self.run_all_combos(num_tests=1) + self.run_all_combos(num_tests=10, disabled=[1, 2]) + + num_tests = 67 + disabled = list(i for i in xrange(num_tests) if i % 4 == 0) + self.run_all_combos(num_tests=num_tests, disabled=disabled) + + def test_two_times_more_chunks_than_tests(self): + # test case for bug 1182817 + tests = self.generate_tests(5) + + total_chunks = 10 + for i in range(1, total_chunks + 1): + # ensure IndexError is not raised + chunk_by_slice(i, total_chunks)(tests, {}) + + +class ChunkByDir(TestCase): + """Test chunking related filters""" + + def generate_tests(self, dirs): + """ + :param dirs: dict of the form, + { : } + """ + i = 0 + for d, num in dirs.iteritems(): + for j in range(num): + i += 1 + name = 'test%i' % i + test = {'name': name, + 'relpath': os.path.join(d, name)} + yield test + + def run_all_combos(self, dirs): + tests = list(self.generate_tests(dirs)) + + deepest = max(len(t['relpath'].split(os.sep)) - 1 for t in tests) + for depth in range(1, deepest + 1): + + def num_groups(tests): + unique = set() + for p in [t['relpath'] for t in tests]: + p = p.split(os.sep) + p = p[:min(depth, len(p) - 1)] + unique.add(os.sep.join(p)) + return len(unique) + + for total in range(1, num_groups(tests) + 1): + res = [] + for this in range(1, total + 1): + f = chunk_by_dir(this, total, depth) + res.append(list(f(tests, {}))) + + lengths = map(num_groups, res) + # the chunk with the most dirs should have at most one more + # dir than the chunk with the least dirs + self.assertLessEqual(max(lengths) - min(lengths), 1) + + all_chunks = list(chain.from_iterable(res)) + # chunk_by_dir will mess up order, but chained chunks should + # contain all of the original tests and be the same length + self.assertEqual(len(all_chunks), len(tests)) + for t in tests: + self.assertIn(t, all_chunks) + + def test_chunk_by_dir(self): + chunk = chunk_by_dir(1, 1, 1) + self.assertEqual(list(chunk([], {})), []) + + dirs = { + 'a': 2, + } + self.run_all_combos(dirs) + + dirs = { + '': 1, + 'foo': 1, + 'bar': 0, + '/foobar': 1, + } + self.run_all_combos(dirs) + + dirs = { + 'a': 1, + 'b': 1, + 'a/b': 2, + 'a/c': 1, + } + self.run_all_combos(dirs) + + dirs = { + 'a': 5, + 'a/b': 4, + 'a/b/c': 7, + 'a/b/c/d': 1, + 'a/b/c/e': 3, + 'b/c': 2, + 'b/d': 5, + 'b/d/e': 6, + 'c': 8, + 'c/d/e/f/g/h/i/j/k/l': 5, + 'c/d/e/f/g/i/j/k/l/m/n': 2, + 'c/e': 1, + } + self.run_all_combos(dirs) + + +class ChunkByRuntime(TestCase): + """Test chunking related filters""" + + def generate_tests(self, dirs): + """ + :param dirs: dict of the form, + { : } + """ + i = 0 + for d, num in dirs.iteritems(): + for j in range(num): + i += 1 + name = 'test%i' % i + test = {'name': name, + 'relpath': os.path.join(d, name), + 'manifest': os.path.join(d, 'manifest.ini')} + yield test + + def get_runtimes(self, tests): + runtimes = {} + for test in tests: + runtimes[test['relpath']] = random.randint(0, 100) + return runtimes + + def chunk_by_round_robin(self, tests, runtimes): + manifests = set(t['manifest'] for t in tests) + tests_by_manifest = [] + for manifest in manifests: + mtests = [t for t in tests if t['manifest'] == manifest] + total = sum(runtimes[t['relpath']] for t in mtests + if 'disabled' not in t) + tests_by_manifest.append((total, mtests)) + tests_by_manifest.sort() + + chunks = [[] for i in range(total)] + d = 1 # direction + i = 0 + for runtime, batch in tests_by_manifest: + chunks[i].extend(batch) + + # "draft" style (last pick goes first in the next round) + if (i == 0 and d == -1) or (i == total - 1 and d == 1): + d = -d + else: + i += d + + # make sure this test algorithm is valid + all_chunks = list(chain.from_iterable(chunks)) + self.assertEqual(len(all_chunks), len(tests)) + for t in tests: + self.assertIn(t, all_chunks) + + return chunks + + def run_all_combos(self, dirs): + tests = list(self.generate_tests(dirs)) + runtimes = self.get_runtimes(tests) + + for total in range(1, len(dirs) + 1): + chunks = [] + for this in range(1, total + 1): + f = chunk_by_runtime(this, total, runtimes) + ret = list(f(tests, {})) + chunks.append(ret) + + # chunk_by_runtime will mess up order, but chained chunks should + # contain all of the original tests and be the same length + all_chunks = list(chain.from_iterable(chunks)) + self.assertEqual(len(all_chunks), len(tests)) + for t in tests: + self.assertIn(t, all_chunks) + + # calculate delta between slowest and fastest chunks + def runtime_delta(chunks): + totals = [] + for chunk in chunks: + total = sum(runtimes[t['relpath']] for t in chunk + if 'disabled' not in t) + totals.append(total) + return max(totals) - min(totals) + delta = runtime_delta(chunks) + + # redo the chunking a second time using a round robin style + # algorithm + chunks = self.chunk_by_round_robin(tests, runtimes) + + # since chunks will never have exactly equal runtimes, it's hard + # to tell if they were chunked optimally. Make sure it at least + # beats a naive round robin approach. + self.assertLessEqual(delta, runtime_delta(chunks)) + + def test_chunk_by_runtime(self): + random.seed(42) + + chunk = chunk_by_runtime(1, 1, {}) + self.assertEqual(list(chunk([], {})), []) + + dirs = { + 'a': 2, + } + self.run_all_combos(dirs) + + dirs = { + '': 1, + 'foo': 1, + 'bar': 0, + '/foobar': 1, + } + self.run_all_combos(dirs) + + dirs = { + 'a': 1, + 'b': 1, + 'a/b': 2, + 'a/c': 1, + } + self.run_all_combos(dirs) + + dirs = { + 'a': 5, + 'a/b': 4, + 'a/b/c': 7, + 'a/b/c/d': 1, + 'a/b/c/e': 3, + 'b/c': 2, + 'b/d': 5, + 'b/d/e': 6, + 'c': 8, + 'c/d/e/f/g/h/i/j/k/l': 5, + 'c/d/e/f/g/i/j/k/l/m/n': 2, + 'c/e': 1, + } + self.run_all_combos(dirs) diff --git a/testing/mozbase/manifestparser/tests/test_convert_directory.py b/testing/mozbase/manifestparser/tests/test_convert_directory.py new file mode 100755 index 000000000..12776e4e4 --- /dev/null +++ b/testing/mozbase/manifestparser/tests/test_convert_directory.py @@ -0,0 +1,181 @@ +#!/usr/bin/env python + +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this file, +# You can obtain one at http://mozilla.org/MPL/2.0/. + +import os +import shutil +import tempfile +import unittest + +from manifestparser import convert +from manifestparser import ManifestParser + +here = os.path.dirname(os.path.abspath(__file__)) + +# In some cases tempfile.mkdtemp() may returns a path which contains +# symlinks. Some tests here will then break, as the manifestparser.convert +# function returns paths that does not contains symlinks. +# +# Workaround is to use the following function, if absolute path of temp dir +# must be compared. + + +def create_realpath_tempdir(): + """ + Create a tempdir without symlinks. + """ + return os.path.realpath(tempfile.mkdtemp()) + + +class TestDirectoryConversion(unittest.TestCase): + """test conversion of a directory tree to a manifest structure""" + + def create_stub(self, directory=None): + """stub out a directory with files in it""" + + files = ('foo', 'bar', 'fleem') + if directory is None: + directory = create_realpath_tempdir() + for i in files: + file(os.path.join(directory, i), 'w').write(i) + subdir = os.path.join(directory, 'subdir') + os.mkdir(subdir) + file(os.path.join(subdir, 'subfile'), 'w').write('baz') + return directory + + def test_directory_to_manifest(self): + """ + Test our ability to convert a static directory structure to a + manifest. + """ + + # create a stub directory + stub = self.create_stub() + try: + stub = stub.replace(os.path.sep, "/") + self.assertTrue(os.path.exists(stub) and os.path.isdir(stub)) + + # Make a manifest for it + manifest = convert([stub]) + out_tmpl = """[%(stub)s/bar] + +[%(stub)s/fleem] + +[%(stub)s/foo] + +[%(stub)s/subdir/subfile] + +""" # noqa + self.assertEqual(str(manifest), out_tmpl % dict(stub=stub)) + except: + raise + finally: + shutil.rmtree(stub) # cleanup + + def test_convert_directory_manifests_in_place(self): + """ + keep the manifests in place + """ + + stub = self.create_stub() + try: + ManifestParser.populate_directory_manifests([stub], filename='manifest.ini') + self.assertEqual(sorted(os.listdir(stub)), + ['bar', 'fleem', 'foo', 'manifest.ini', 'subdir']) + parser = ManifestParser() + parser.read(os.path.join(stub, 'manifest.ini')) + self.assertEqual([i['name'] for i in parser.tests], + ['subfile', 'bar', 'fleem', 'foo']) + parser = ManifestParser() + parser.read(os.path.join(stub, 'subdir', 'manifest.ini')) + self.assertEqual(len(parser.tests), 1) + self.assertEqual(parser.tests[0]['name'], 'subfile') + except: + raise + finally: + shutil.rmtree(stub) + + def test_manifest_ignore(self): + """test manifest `ignore` parameter for ignoring directories""" + + stub = self.create_stub() + try: + ManifestParser.populate_directory_manifests( + [stub], filename='manifest.ini', ignore=('subdir',)) + parser = ManifestParser() + parser.read(os.path.join(stub, 'manifest.ini')) + self.assertEqual([i['name'] for i in parser.tests], + ['bar', 'fleem', 'foo']) + self.assertFalse(os.path.exists(os.path.join(stub, 'subdir', 'manifest.ini'))) + except: + raise + finally: + shutil.rmtree(stub) + + def test_pattern(self): + """test directory -> manifest with a file pattern""" + + stub = self.create_stub() + try: + parser = convert([stub], pattern='f*', relative_to=stub) + self.assertEqual([i['name'] for i in parser.tests], + ['fleem', 'foo']) + + # test multiple patterns + parser = convert([stub], pattern=('f*', 's*'), relative_to=stub) + self.assertEqual([i['name'] for i in parser.tests], + ['fleem', 'foo', 'subdir/subfile']) + except: + raise + finally: + shutil.rmtree(stub) + + def test_update(self): + """ + Test our ability to update tests from a manifest and a directory of + files + """ + + # boilerplate + tempdir = create_realpath_tempdir() + for i in range(10): + file(os.path.join(tempdir, str(i)), 'w').write(str(i)) + + # otherwise empty directory with a manifest file + newtempdir = create_realpath_tempdir() + manifest_file = os.path.join(newtempdir, 'manifest.ini') + manifest_contents = str(convert([tempdir], relative_to=tempdir)) + with file(manifest_file, 'w') as f: + f.write(manifest_contents) + + # get the manifest + manifest = ManifestParser(manifests=(manifest_file,)) + + # All of the tests are initially missing: + paths = [str(i) for i in range(10)] + self.assertEqual([i['name'] for i in manifest.missing()], + paths) + + # But then we copy one over: + self.assertEqual(manifest.get('name', name='1'), ['1']) + manifest.update(tempdir, name='1') + self.assertEqual(sorted(os.listdir(newtempdir)), + ['1', 'manifest.ini']) + + # Update that one file and copy all the "tests": + file(os.path.join(tempdir, '1'), 'w').write('secret door') + manifest.update(tempdir) + self.assertEqual(sorted(os.listdir(newtempdir)), + ['0', '1', '2', '3', '4', '5', '6', '7', '8', '9', 'manifest.ini']) + self.assertEqual(file(os.path.join(newtempdir, '1')).read().strip(), + 'secret door') + + # clean up: + shutil.rmtree(tempdir) + shutil.rmtree(newtempdir) + + +if __name__ == '__main__': + unittest.main() diff --git a/testing/mozbase/manifestparser/tests/test_convert_symlinks.py b/testing/mozbase/manifestparser/tests/test_convert_symlinks.py new file mode 100755 index 000000000..9a0640b4b --- /dev/null +++ b/testing/mozbase/manifestparser/tests/test_convert_symlinks.py @@ -0,0 +1,139 @@ +#!/usr/bin/env python + +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this file, +# You can obtain one at http://mozilla.org/MPL/2.0/. + +import os +import shutil +import tempfile +import unittest + +from manifestparser import convert, ManifestParser + + +class TestSymlinkConversion(unittest.TestCase): + """ + test conversion of a directory tree with symlinks to a manifest structure + """ + + def create_stub(self, directory=None): + """stub out a directory with files in it""" + + files = ('foo', 'bar', 'fleem') + if directory is None: + directory = tempfile.mkdtemp() + for i in files: + file(os.path.join(directory, i), 'w').write(i) + subdir = os.path.join(directory, 'subdir') + os.mkdir(subdir) + file(os.path.join(subdir, 'subfile'), 'w').write('baz') + return directory + + def test_relpath(self): + """test convert `relative_to` functionality""" + + oldcwd = os.getcwd() + stub = self.create_stub() + try: + # subdir with in-memory manifest + files = ['../bar', '../fleem', '../foo', 'subfile'] + subdir = os.path.join(stub, 'subdir') + os.chdir(subdir) + parser = convert([stub], relative_to='.') + self.assertEqual([i['name'] for i in parser.tests], + files) + except: + raise + finally: + shutil.rmtree(stub) + os.chdir(oldcwd) + + @unittest.skipIf(not hasattr(os, 'symlink'), + "symlinks unavailable on this platform") + def test_relpath_symlink(self): + """ + Ensure `relative_to` works in a symlink. + Not available on windows. + """ + + oldcwd = os.getcwd() + workspace = tempfile.mkdtemp() + try: + tmpdir = os.path.join(workspace, 'directory') + os.makedirs(tmpdir) + linkdir = os.path.join(workspace, 'link') + os.symlink(tmpdir, linkdir) + self.create_stub(tmpdir) + + # subdir with in-memory manifest + files = ['../bar', '../fleem', '../foo', 'subfile'] + subdir = os.path.join(linkdir, 'subdir') + os.chdir(os.path.realpath(subdir)) + for directory in (tmpdir, linkdir): + parser = convert([directory], relative_to='.') + self.assertEqual([i['name'] for i in parser.tests], + files) + finally: + shutil.rmtree(workspace) + os.chdir(oldcwd) + + # a more complicated example + oldcwd = os.getcwd() + workspace = tempfile.mkdtemp() + try: + tmpdir = os.path.join(workspace, 'directory') + os.makedirs(tmpdir) + linkdir = os.path.join(workspace, 'link') + os.symlink(tmpdir, linkdir) + self.create_stub(tmpdir) + files = ['../bar', '../fleem', '../foo', 'subfile'] + subdir = os.path.join(linkdir, 'subdir') + subsubdir = os.path.join(subdir, 'sub') + os.makedirs(subsubdir) + linksubdir = os.path.join(linkdir, 'linky') + linksubsubdir = os.path.join(subsubdir, 'linky') + os.symlink(subdir, linksubdir) + os.symlink(subdir, linksubsubdir) + for dest in (subdir,): + os.chdir(dest) + for directory in (tmpdir, linkdir): + parser = convert([directory], relative_to='.') + self.assertEqual([i['name'] for i in parser.tests], + files) + finally: + shutil.rmtree(workspace) + os.chdir(oldcwd) + + @unittest.skipIf(not hasattr(os, 'symlink'), + "symlinks unavailable on this platform") + def test_recursion_symlinks(self): + workspace = tempfile.mkdtemp() + self.addCleanup(shutil.rmtree, workspace) + + # create two dirs + os.makedirs(os.path.join(workspace, 'dir1')) + os.makedirs(os.path.join(workspace, 'dir2')) + + # create cyclical symlinks + os.symlink(os.path.join('..', 'dir1'), + os.path.join(workspace, 'dir2', 'ldir1')) + os.symlink(os.path.join('..', 'dir2'), + os.path.join(workspace, 'dir1', 'ldir2')) + + # create one file in each dir + open(os.path.join(workspace, 'dir1', 'f1.txt'), 'a').close() + open(os.path.join(workspace, 'dir1', 'ldir2', 'f2.txt'), 'a').close() + + data = [] + + def callback(rootdirectory, directory, subdirs, files): + for f in files: + data.append(f) + + ManifestParser._walk_directories([workspace], callback) + self.assertEqual(sorted(data), ['f1.txt', 'f2.txt']) + + +if __name__ == '__main__': + unittest.main() diff --git a/testing/mozbase/manifestparser/tests/test_default_overrides.py b/testing/mozbase/manifestparser/tests/test_default_overrides.py new file mode 100755 index 000000000..3341c4bd8 --- /dev/null +++ b/testing/mozbase/manifestparser/tests/test_default_overrides.py @@ -0,0 +1,115 @@ +#!/usr/bin/env python + +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this file, +# You can obtain one at http://mozilla.org/MPL/2.0/. + +import os +import unittest +from manifestparser import ManifestParser +from manifestparser import combine_fields + +here = os.path.dirname(os.path.abspath(__file__)) + + +class TestDefaultSkipif(unittest.TestCase): + """Tests applying a skip-if condition in [DEFAULT] and || with the value for the test""" + + def test_defaults(self): + + default = os.path.join(here, 'default-skipif.ini') + parser = ManifestParser(manifests=(default,)) + for test in parser.tests: + if test['name'] == 'test1': + self.assertEqual(test['skip-if'], "(os == 'win' && debug ) || (debug)") + elif test['name'] == 'test2': + self.assertEqual(test['skip-if'], "(os == 'win' && debug ) || (os == 'linux')") + elif test['name'] == 'test3': + self.assertEqual(test['skip-if'], "(os == 'win' && debug ) || (os == 'win')") + elif test['name'] == 'test4': + self.assertEqual( + test['skip-if'], "(os == 'win' && debug ) || (os == 'win' && debug)") + elif test['name'] == 'test5': + self.assertEqual(test['skip-if'], "os == 'win' && debug # a pesky comment") + elif test['name'] == 'test6': + self.assertEqual(test['skip-if'], "(os == 'win' && debug ) || (debug )") + + +class TestDefaultSupportFiles(unittest.TestCase): + """Tests combining support-files field in [DEFAULT] with the value for a test""" + + def test_defaults(self): + + default = os.path.join(here, 'default-suppfiles.ini') + parser = ManifestParser(manifests=(default,)) + expected_supp_files = { + 'test7': 'foo.js # a comment', + 'test8': 'foo.js bar.js ', + 'test9': 'foo.js # a comment', + } + for test in parser.tests: + expected = expected_supp_files[test['name']] + self.assertEqual(test['support-files'], expected) + + +class TestOmitDefaults(unittest.TestCase): + """Tests passing omit-defaults prevents defaults from propagating to definitions. + """ + + def test_defaults(self): + manifests = (os.path.join(here, 'default-suppfiles.ini'), + os.path.join(here, 'default-skipif.ini')) + parser = ManifestParser(manifests=manifests, handle_defaults=False) + expected_supp_files = { + 'test8': 'bar.js # another comment', + } + expected_skip_ifs = { + 'test1': "debug", + 'test2': "os == 'linux'", + 'test3': "os == 'win'", + 'test4': "os == 'win' && debug", + 'test6': "debug # a second pesky comment", + } + for test in parser.tests: + for field, expectations in (('support-files', expected_supp_files), + ('skip-if', expected_skip_ifs)): + expected = expectations.get(test['name']) + if not expected: + self.assertNotIn(field, test) + else: + self.assertEqual(test[field], expected) + + expected_defaults = { + os.path.join(here, 'default-suppfiles.ini'): { + "support-files": "foo.js # a comment", + }, + os.path.join(here, 'default-skipif.ini'): { + "skip-if": "os == 'win' && debug # a pesky comment", + }, + } + for path, defaults in expected_defaults.items(): + self.assertIn(path, parser.manifest_defaults) + actual_defaults = parser.manifest_defaults[path] + for key, value in defaults.items(): + self.assertIn(key, actual_defaults) + self.assertEqual(value, actual_defaults[key]) + + +class TestSubsuiteDefaults(unittest.TestCase): + """Test that subsuites are handled correctly when managing defaults + outside of the manifest parser.""" + def test_subsuite_defaults(self): + manifest = os.path.join(here, 'default-subsuite.ini') + parser = ManifestParser(manifests=(manifest,), handle_defaults=False) + expected_subsuites = { + 'test1': 'baz', + 'test2': 'foo', + } + defaults = parser.manifest_defaults[manifest] + for test in parser.tests: + value = combine_fields(defaults, test) + self.assertEqual(expected_subsuites[value['name']], + value['subsuite']) + +if __name__ == '__main__': + unittest.main() diff --git a/testing/mozbase/manifestparser/tests/test_expressionparser.py b/testing/mozbase/manifestparser/tests/test_expressionparser.py new file mode 100755 index 000000000..dc3f2fd3d --- /dev/null +++ b/testing/mozbase/manifestparser/tests/test_expressionparser.py @@ -0,0 +1,152 @@ +#!/usr/bin/env python + +import unittest +from manifestparser import parse + + +class ExpressionParserTest(unittest.TestCase): + """Test the conditional expression parser.""" + + def test_basic(self): + + self.assertEqual(parse("1"), 1) + self.assertEqual(parse("100"), 100) + self.assertEqual(parse("true"), True) + self.assertEqual(parse("false"), False) + self.assertEqual('', parse('""')) + self.assertEqual(parse('"foo bar"'), 'foo bar') + self.assertEqual(parse("'foo bar'"), 'foo bar') + self.assertEqual(parse("foo", foo=1), 1) + self.assertEqual(parse("bar", bar=True), True) + self.assertEqual(parse("abc123", abc123="xyz"), 'xyz') + + def test_equality(self): + + self.assertTrue(parse("true == true")) + self.assertTrue(parse("false == false")) + self.assertTrue(parse("1 == 1")) + self.assertTrue(parse("100 == 100")) + self.assertTrue(parse('"some text" == "some text"')) + self.assertTrue(parse("true != false")) + self.assertTrue(parse("1 != 2")) + self.assertTrue(parse('"text" != "other text"')) + self.assertTrue(parse("foo == true", foo=True)) + self.assertTrue(parse("foo == 1", foo=1)) + self.assertTrue(parse('foo == "bar"', foo='bar')) + self.assertTrue(parse("foo == bar", foo=True, bar=True)) + self.assertTrue(parse("true == foo", foo=True)) + self.assertTrue(parse("foo != true", foo=False)) + self.assertTrue(parse("foo != 2", foo=1)) + self.assertTrue(parse('foo != "bar"', foo='abc')) + self.assertTrue(parse("foo != bar", foo=True, bar=False)) + self.assertTrue(parse("true != foo", foo=False)) + self.assertTrue(parse("!false")) + + def test_conjunctures(self): + self.assertTrue(parse("true && true")) + self.assertTrue(parse("true || false")) + self.assertFalse(parse("false || false")) + self.assertFalse(parse("true && false")) + self.assertTrue(parse("true || false && false")) + + def test_parentheses(self): + self.assertTrue(parse("(true)")) + self.assertEqual(parse("(10)"), 10) + self.assertEqual(parse('("foo")'), 'foo') + self.assertEqual(parse("(foo)", foo=1), 1) + self.assertTrue(parse("(true == true)"), True) + self.assertTrue(parse("(true != false)")) + self.assertTrue(parse("(true && true)")) + self.assertTrue(parse("(true || false)")) + self.assertTrue(parse("(true && true || false)")) + self.assertFalse(parse("(true || false) && false")) + self.assertTrue(parse("(true || false) && true")) + self.assertTrue(parse("true && (true || false)")) + self.assertTrue(parse("true && (true || false)")) + self.assertTrue(parse("(true && false) || (true && (true || false))")) + + def test_comments(self): + # comments in expressions work accidentally, via an implementation + # detail - the '#' character doesn't match any of the regular + # expressions we specify as tokens, and thus are ignored. + # However, having explicit tests for them means that should the + # implementation ever change, comments continue to work, even if that + # means a new implementation must handle them explicitly. + self.assertTrue(parse("true == true # it does!")) + self.assertTrue(parse("false == false # it does")) + self.assertTrue(parse("false != true # it doesnt")) + self.assertTrue(parse('"string with #" == "string with #" # really, it does')) + self.assertTrue(parse('"string with #" != "string with # but not the same" # no match!')) + + def test_not(self): + """ + Test the ! operator. + """ + self.assertTrue(parse("!false")) + self.assertTrue(parse("!(false)")) + self.assertFalse(parse("!true")) + self.assertFalse(parse("!(true)")) + self.assertTrue(parse("!true || true)")) + self.assertTrue(parse("true || !true)")) + self.assertFalse(parse("!true && true")) + self.assertFalse(parse("true && !true")) + + def test_lesser_than(self): + """ + Test the < operator. + """ + self.assertTrue(parse("1 < 2")) + self.assertFalse(parse("3 < 2")) + self.assertTrue(parse("false || (1 < 2)")) + self.assertTrue(parse("1 < 2 && true")) + self.assertTrue(parse("true && 1 < 2")) + self.assertTrue(parse("!(5 < 1)")) + self.assertTrue(parse("'abc' < 'def'")) + self.assertFalse(parse("1 < 1")) + self.assertFalse(parse("'abc' < 'abc'")) + + def test_greater_than(self): + """ + Test the > operator. + """ + self.assertTrue(parse("2 > 1")) + self.assertFalse(parse("2 > 3")) + self.assertTrue(parse("false || (2 > 1)")) + self.assertTrue(parse("2 > 1 && true")) + self.assertTrue(parse("true && 2 > 1")) + self.assertTrue(parse("!(1 > 5)")) + self.assertTrue(parse("'def' > 'abc'")) + self.assertFalse(parse("1 > 1")) + self.assertFalse(parse("'abc' > 'abc'")) + + def test_lesser_or_equals_than(self): + """ + Test the <= operator. + """ + self.assertTrue(parse("1 <= 2")) + self.assertFalse(parse("3 <= 2")) + self.assertTrue(parse("false || (1 <= 2)")) + self.assertTrue(parse("1 < 2 && true")) + self.assertTrue(parse("true && 1 <= 2")) + self.assertTrue(parse("!(5 <= 1)")) + self.assertTrue(parse("'abc' <= 'def'")) + self.assertTrue(parse("1 <= 1")) + self.assertTrue(parse("'abc' <= 'abc'")) + + def test_greater_or_equals_than(self): + """ + Test the > operator. + """ + self.assertTrue(parse("2 >= 1")) + self.assertFalse(parse("2 >= 3")) + self.assertTrue(parse("false || (2 >= 1)")) + self.assertTrue(parse("2 >= 1 && true")) + self.assertTrue(parse("true && 2 >= 1")) + self.assertTrue(parse("!(1 >= 5)")) + self.assertTrue(parse("'def' >= 'abc'")) + self.assertTrue(parse("1 >= 1")) + self.assertTrue(parse("'abc' >= 'abc'")) + + +if __name__ == '__main__': + unittest.main() diff --git a/testing/mozbase/manifestparser/tests/test_filters.py b/testing/mozbase/manifestparser/tests/test_filters.py new file mode 100644 index 000000000..5b0772492 --- /dev/null +++ b/testing/mozbase/manifestparser/tests/test_filters.py @@ -0,0 +1,182 @@ +#!/usr/bin/env python +# flake8: noqa + +from copy import deepcopy +import os +import unittest + +from manifestparser.filters import ( + subsuite, + tags, + skip_if, + run_if, + fail_if, + enabled, + filterlist, +) + +here = os.path.dirname(os.path.abspath(__file__)) + + +class FilterList(unittest.TestCase): + """Test filterlist datatype""" + + def test_data_model(self): + foo = lambda x, y: x + bar = lambda x, y: x + baz = lambda x, y: x + fl = filterlist() + + fl.extend([foo, bar]) + self.assertEquals(len(fl), 2) + self.assertTrue(foo in fl) + + fl.append(baz) + self.assertEquals(fl[2], baz) + + fl.remove(baz) + self.assertFalse(baz in fl) + + item = fl.pop() + self.assertEquals(item, bar) + + self.assertEquals(fl.index(foo), 0) + + del fl[0] + self.assertFalse(foo in fl) + with self.assertRaises(IndexError): + fl[0] + + def test_add_non_callable_to_list(self): + fl = filterlist() + with self.assertRaises(TypeError): + fl.append('foo') + + def test_add_duplicates_to_list(self): + foo = lambda x, y: x + bar = lambda x, y: x + sub = subsuite('foo') + fl = filterlist([foo, bar, sub]) + self.assertEquals(len(fl), 3) + self.assertEquals(fl[0], foo) + + with self.assertRaises(ValueError): + fl.append(foo) + + with self.assertRaises(ValueError): + fl.append(subsuite('bar')) + + def test_add_two_tags_filters(self): + tag1 = tags('foo') + tag2 = tags('bar') + fl = filterlist([tag1]) + + with self.assertRaises(ValueError): + fl.append(tag1) + + fl.append(tag2) + self.assertEquals(len(fl), 2) + + def test_filters_run_in_order(self): + a = lambda x, y: x + b = lambda x, y: x + c = lambda x, y: x + d = lambda x, y: x + e = lambda x, y: x + f = lambda x, y: x + + fl = filterlist([a, b]) + fl.append(c) + fl.extend([d, e]) + fl += [f] + self.assertEquals([i for i in fl], [a, b, c, d, e, f]) + + +class BuiltinFilters(unittest.TestCase): + """Test the built-in filters""" + + tests = ( + {"name": "test0"}, + {"name": "test1", "skip-if": "foo == 'bar'"}, + {"name": "test2", "run-if": "foo == 'bar'"}, + {"name": "test3", "fail-if": "foo == 'bar'"}, + {"name": "test4", "disabled": "some reason"}, + {"name": "test5", "subsuite": "baz"}, + {"name": "test6", "subsuite": "baz,foo == 'bar'"}, + {"name": "test7", "tags": "foo bar"}, + ) + + def test_skip_if(self): + tests = deepcopy(self.tests) + tests = list(skip_if(tests, {})) + self.assertEquals(len(tests), len(self.tests)) + + tests = deepcopy(self.tests) + tests = list(skip_if(tests, {'foo': 'bar'})) + self.assertNotIn(self.tests[1], tests) + + def test_run_if(self): + tests = deepcopy(self.tests) + tests = list(run_if(tests, {})) + self.assertNotIn(self.tests[2], tests) + + tests = deepcopy(self.tests) + tests = list(run_if(tests, {'foo': 'bar'})) + self.assertEquals(len(tests), len(self.tests)) + + def test_fail_if(self): + tests = deepcopy(self.tests) + tests = list(fail_if(tests, {})) + self.assertNotIn('expected', tests[3]) + + tests = deepcopy(self.tests) + tests = list(fail_if(tests, {'foo': 'bar'})) + self.assertEquals(tests[3]['expected'], 'fail') + + def test_enabled(self): + tests = deepcopy(self.tests) + tests = list(enabled(tests, {})) + self.assertNotIn(self.tests[4], tests) + + def test_subsuite(self): + sub1 = subsuite() + sub2 = subsuite('baz') + + tests = deepcopy(self.tests) + tests = list(sub1(tests, {})) + self.assertNotIn(self.tests[5], tests) + self.assertEquals(len(tests), len(self.tests) - 1) + + tests = deepcopy(self.tests) + tests = list(sub2(tests, {})) + self.assertEquals(len(tests), 1) + self.assertIn(self.tests[5], tests) + + def test_subsuite_condition(self): + sub1 = subsuite() + sub2 = subsuite('baz') + + tests = deepcopy(self.tests) + + tests = list(sub1(tests, {'foo': 'bar'})) + self.assertNotIn(self.tests[5], tests) + self.assertNotIn(self.tests[6], tests) + + tests = deepcopy(self.tests) + tests = list(sub2(tests, {'foo': 'bar'})) + self.assertEquals(len(tests), 2) + self.assertEquals(tests[0]['name'], 'test5') + self.assertEquals(tests[1]['name'], 'test6') + + def test_tags(self): + ftags1 = tags([]) + ftags2 = tags(['bar', 'baz']) + + tests = deepcopy(self.tests) + tests = list(ftags1(tests, {})) + self.assertEquals(len(tests), 0) + + tests = deepcopy(self.tests) + tests = list(ftags2(tests, {})) + self.assertEquals(len(tests), 1) + self.assertIn(self.tests[7], tests) diff --git a/testing/mozbase/manifestparser/tests/test_manifestparser.py b/testing/mozbase/manifestparser/tests/test_manifestparser.py new file mode 100755 index 000000000..ca80911fb --- /dev/null +++ b/testing/mozbase/manifestparser/tests/test_manifestparser.py @@ -0,0 +1,325 @@ +#!/usr/bin/env python + +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this file, +# You can obtain one at http://mozilla.org/MPL/2.0/. + +import os +import shutil +import tempfile +import unittest +from manifestparser import ManifestParser +from StringIO import StringIO + +here = os.path.dirname(os.path.abspath(__file__)) + + +class TestManifestParser(unittest.TestCase): + """ + Test the manifest parser + + You must have manifestparser installed before running these tests. + Run ``python manifestparser.py setup develop`` with setuptools installed. + """ + + def test_sanity(self): + """Ensure basic parser is sane""" + + parser = ManifestParser() + mozmill_example = os.path.join(here, 'mozmill-example.ini') + parser.read(mozmill_example) + tests = parser.tests + self.assertEqual(len(tests), len(file(mozmill_example).read().strip().splitlines())) + + # Ensure that capitalization and order aren't an issue: + lines = ['[%s]' % test['name'] for test in tests] + self.assertEqual(lines, file(mozmill_example).read().strip().splitlines()) + + # Show how you select subsets of tests: + mozmill_restart_example = os.path.join(here, 'mozmill-restart-example.ini') + parser.read(mozmill_restart_example) + restart_tests = parser.get(type='restart') + self.assertTrue(len(restart_tests) < len(parser.tests)) + self.assertEqual(len(restart_tests), len(parser.get(manifest=mozmill_restart_example))) + self.assertFalse([test for test in restart_tests + if test['manifest'] != os.path.join(here, + 'mozmill-restart-example.ini')]) + self.assertEqual(parser.get('name', tags=['foo']), + ['restartTests/testExtensionInstallUninstall/test2.js', + 'restartTests/testExtensionInstallUninstall/test1.js']) + self.assertEqual(parser.get('name', foo='bar'), + ['restartTests/testExtensionInstallUninstall/test2.js']) + + def test_include(self): + """Illustrate how include works""" + + include_example = os.path.join(here, 'include-example.ini') + parser = ManifestParser(manifests=(include_example,)) + + # All of the tests should be included, in order: + self.assertEqual(parser.get('name'), + ['crash-handling', 'fleem', 'flowers']) + self.assertEqual([(test['name'], os.path.basename(test['manifest'])) + for test in parser.tests], + [('crash-handling', 'bar.ini'), + ('fleem', 'include-example.ini'), + ('flowers', 'foo.ini')]) + + # The including manifest is always reported as a part of the generated test object. + self.assertTrue(all([t['ancestor-manifest'] == include_example + for t in parser.tests if t['name'] != 'fleem'])) + + # The manifests should be there too: + self.assertEqual(len(parser.manifests()), 3) + + # We already have the root directory: + self.assertEqual(here, parser.rootdir) + + # DEFAULT values should persist across includes, unless they're + # overwritten. In this example, include-example.ini sets foo=bar, but + # it's overridden to fleem in bar.ini + self.assertEqual(parser.get('name', foo='bar'), + ['fleem', 'flowers']) + self.assertEqual(parser.get('name', foo='fleem'), + ['crash-handling']) + + # Passing parameters in the include section allows defining variables in + # the submodule scope: + self.assertEqual(parser.get('name', tags=['red']), + ['flowers']) + + # However, this should be overridable from the DEFAULT section in the + # included file and that overridable via the key directly connected to + # the test: + self.assertEqual(parser.get(name='flowers')[0]['blue'], + 'ocean') + self.assertEqual(parser.get(name='flowers')[0]['yellow'], + 'submarine') + + # You can query multiple times if you need to: + flowers = parser.get(foo='bar') + self.assertEqual(len(flowers), 2) + + # Using the inverse flag should invert the set of tests returned: + self.assertEqual(parser.get('name', inverse=True, tags=['red']), + ['crash-handling', 'fleem']) + + # All of the included tests actually exist: + self.assertEqual([i['name'] for i in parser.missing()], []) + + # Write the output to a manifest: + buffer = StringIO() + parser.write(fp=buffer, global_kwargs={'foo': 'bar'}) + expected_output = """[DEFAULT] +foo = bar + +[fleem] + +[include/flowers] +blue = ocean +red = roses +yellow = submarine""" # noqa + + self.assertEqual(buffer.getvalue().strip(), + expected_output) + + def test_invalid_path(self): + """ + Test invalid path should not throw when not strict + """ + manifest = os.path.join(here, 'include-invalid.ini') + ManifestParser(manifests=(manifest,), strict=False) + + def test_parent_inheritance(self): + """ + Test parent manifest variable inheritance + Specifically tests that inherited variables from parent includes + properly propagate downstream + """ + parent_example = os.path.join(here, 'parent', 'level_1', 'level_2', + 'level_3', 'level_3.ini') + parser = ManifestParser(manifests=(parent_example,)) + + # Parent manifest test should not be included + self.assertEqual(parser.get('name'), + ['test_3']) + self.assertEqual([(test['name'], os.path.basename(test['manifest'])) + for test in parser.tests], + [('test_3', 'level_3.ini')]) + + # DEFAULT values should be the ones from level 1 + self.assertEqual(parser.get('name', x='level_1'), + ['test_3']) + + # Write the output to a manifest: + buffer = StringIO() + parser.write(fp=buffer, global_kwargs={'x': 'level_1'}) + self.assertEqual(buffer.getvalue().strip(), + '[DEFAULT]\nx = level_1\n\n[test_3]') + + def test_parent_defaults(self): + """ + Test downstream variables should overwrite upstream variables + """ + parent_example = os.path.join(here, 'parent', 'level_1', 'level_2', + 'level_3', 'level_3_default.ini') + parser = ManifestParser(manifests=(parent_example,)) + + # Parent manifest test should not be included + self.assertEqual(parser.get('name'), + ['test_3']) + self.assertEqual([(test['name'], os.path.basename(test['manifest'])) + for test in parser.tests], + [('test_3', 'level_3_default.ini')]) + + # DEFAULT values should be the ones from level 3 + self.assertEqual(parser.get('name', x='level_3'), + ['test_3']) + + # Write the output to a manifest: + buffer = StringIO() + parser.write(fp=buffer, global_kwargs={'x': 'level_3'}) + self.assertEqual(buffer.getvalue().strip(), + '[DEFAULT]\nx = level_3\n\n[test_3]') + + def test_parent_defaults_include(self): + parent_example = os.path.join(here, 'parent', 'include', 'manifest.ini') + parser = ManifestParser(manifests=(parent_example,)) + + # global defaults should inherit all includes + self.assertEqual(parser.get('name', top='data'), + ['testFirst.js', 'testSecond.js']) + + # include specific defaults should only inherit the actual include + self.assertEqual(parser.get('name', disabled='YES'), + ['testFirst.js']) + self.assertEqual(parser.get('name', disabled='NO'), + ['testSecond.js']) + + def test_server_root(self): + """ + Test server_root properly expands as an absolute path + """ + server_example = os.path.join(here, 'parent', 'level_1', 'level_2', + 'level_3', 'level_3_server-root.ini') + parser = ManifestParser(manifests=(server_example,)) + + # A regular variable will inherit its value directly + self.assertEqual(parser.get('name', **{'other-root': '../root'}), + ['test_3']) + + # server-root will expand its value as an absolute path + # we will not find anything for the original value + self.assertEqual(parser.get('name', **{'server-root': '../root'}), []) + + # check that the path has expanded + self.assertEqual(parser.get('server-root')[0], + os.path.join(here, 'parent', 'root')) + + def test_copy(self): + """Test our ability to copy a set of manifests""" + + tempdir = tempfile.mkdtemp() + include_example = os.path.join(here, 'include-example.ini') + manifest = ManifestParser(manifests=(include_example,)) + manifest.copy(tempdir) + self.assertEqual(sorted(os.listdir(tempdir)), + ['fleem', 'include', 'include-example.ini']) + self.assertEqual(sorted(os.listdir(os.path.join(tempdir, 'include'))), + ['bar.ini', 'crash-handling', 'flowers', 'foo.ini']) + from_manifest = ManifestParser(manifests=(include_example,)) + to_manifest = os.path.join(tempdir, 'include-example.ini') + to_manifest = ManifestParser(manifests=(to_manifest,)) + self.assertEqual(to_manifest.get('name'), from_manifest.get('name')) + shutil.rmtree(tempdir) + + def test_path_override(self): + """You can override the path in the section too. + This shows that you can use a relative path""" + path_example = os.path.join(here, 'path-example.ini') + manifest = ManifestParser(manifests=(path_example,)) + self.assertEqual(manifest.tests[0]['path'], + os.path.join(here, 'fleem')) + + def test_relative_path(self): + """ + Relative test paths are correctly calculated. + """ + relative_path = os.path.join(here, 'relative-path.ini') + manifest = ManifestParser(manifests=(relative_path,)) + self.assertEqual(manifest.tests[0]['path'], + os.path.join(os.path.dirname(here), 'fleem')) + self.assertEqual(manifest.tests[0]['relpath'], + os.path.join('..', 'fleem')) + self.assertEqual(manifest.tests[1]['relpath'], + os.path.join('..', 'testsSIBLING', 'example')) + + def test_path_from_fd(self): + """ + Test paths are left untouched when manifest is a file-like object. + """ + fp = StringIO("[section]\npath=fleem") + manifest = ManifestParser(manifests=(fp,)) + self.assertEqual(manifest.tests[0]['path'], 'fleem') + self.assertEqual(manifest.tests[0]['relpath'], 'fleem') + self.assertEqual(manifest.tests[0]['manifest'], None) + + def test_comments(self): + """ + ensure comments work, see + https://bugzilla.mozilla.org/show_bug.cgi?id=813674 + """ + comment_example = os.path.join(here, 'comment-example.ini') + manifest = ManifestParser(manifests=(comment_example,)) + self.assertEqual(len(manifest.tests), 8) + names = [i['name'] for i in manifest.tests] + self.assertFalse('test_0202_app_launch_apply_update_dirlocked.js' in names) + + def test_verifyDirectory(self): + + directory = os.path.join(here, 'verifyDirectory') + + # correct manifest + manifest_path = os.path.join(directory, 'verifyDirectory.ini') + manifest = ManifestParser(manifests=(manifest_path,)) + missing = manifest.verifyDirectory(directory, extensions=('.js',)) + self.assertEqual(missing, (set(), set())) + + # manifest is missing test_1.js + test_1 = os.path.join(directory, 'test_1.js') + manifest_path = os.path.join(directory, 'verifyDirectory_incomplete.ini') + manifest = ManifestParser(manifests=(manifest_path,)) + missing = manifest.verifyDirectory(directory, extensions=('.js',)) + self.assertEqual(missing, (set(), set([test_1]))) + + # filesystem is missing test_notappearinginthisfilm.js + missing_test = os.path.join(directory, 'test_notappearinginthisfilm.js') + manifest_path = os.path.join(directory, 'verifyDirectory_toocomplete.ini') + manifest = ManifestParser(manifests=(manifest_path,)) + missing = manifest.verifyDirectory(directory, extensions=('.js',)) + self.assertEqual(missing, (set([missing_test]), set())) + + def test_just_defaults(self): + """Ensure a manifest with just a DEFAULT section exposes that data.""" + + parser = ManifestParser() + manifest = os.path.join(here, 'just-defaults.ini') + parser.read(manifest) + self.assertEqual(len(parser.tests), 0) + self.assertTrue(manifest in parser.manifest_defaults) + self.assertEquals(parser.manifest_defaults[manifest]['foo'], 'bar') + + def test_manifest_list(self): + """ + Ensure a manifest with just a DEFAULT section still returns + itself from the manifests() method. + """ + + parser = ManifestParser() + manifest = os.path.join(here, 'no-tests.ini') + parser.read(manifest) + self.assertEqual(len(parser.tests), 0) + self.assertTrue(len(parser.manifests()) == 1) + +if __name__ == '__main__': + unittest.main() diff --git a/testing/mozbase/manifestparser/tests/test_read_ini.py b/testing/mozbase/manifestparser/tests/test_read_ini.py new file mode 100755 index 000000000..df4a8973b --- /dev/null +++ b/testing/mozbase/manifestparser/tests/test_read_ini.py @@ -0,0 +1,70 @@ +#!/usr/bin/env python + +""" +test .ini parsing + +ensure our .ini parser is doing what we want; to be deprecated for +python's standard ConfigParser when 2.7 is reality so OrderedDict +is the default: + +http://docs.python.org/2/library/configparser.html +""" + +import unittest +from manifestparser import read_ini +from ConfigParser import ConfigParser +from StringIO import StringIO + + +class IniParserTest(unittest.TestCase): + + def test_inline_comments(self): + """ + We have no inline comments; so we're testing to ensure we don't: + https://bugzilla.mozilla.org/show_bug.cgi?id=855288 + """ + + # test '#' inline comments (really, the lack thereof) + string = """[test_felinicity.py] +kittens = true # This test requires kittens +""" + buffer = StringIO() + buffer.write(string) + buffer.seek(0) + result = read_ini(buffer)[0][1]['kittens'] + self.assertEqual(result, "true # This test requires kittens") + + # compare this to ConfigParser + # python 2.7 ConfigParser does not support '#' as an + # inline comment delimeter (for "backwards compatability"): + # http://docs.python.org/2/library/configparser.html + buffer.seek(0) + parser = ConfigParser() + parser.readfp(buffer) + control = parser.get('test_felinicity.py', 'kittens') + self.assertEqual(result, control) + + # test ';' inline comments (really, the lack thereof) + string = string.replace('#', ';') + buffer = StringIO() + buffer.write(string) + buffer.seek(0) + result = read_ini(buffer)[0][1]['kittens'] + self.assertEqual(result, "true ; This test requires kittens") + + # compare this to ConfigParser + # python 2.7 ConfigParser *does* support ';' as an + # inline comment delimeter (ibid). + # Python 3.x configparser, OTOH, does not support + # inline-comments by default. It does support their specification, + # though they are weakly discouraged: + # http://docs.python.org/dev/library/configparser.html + buffer.seek(0) + parser = ConfigParser() + parser.readfp(buffer) + control = parser.get('test_felinicity.py', 'kittens') + self.assertNotEqual(result, control) + + +if __name__ == '__main__': + unittest.main() diff --git a/testing/mozbase/manifestparser/tests/test_testmanifest.py b/testing/mozbase/manifestparser/tests/test_testmanifest.py new file mode 100644 index 000000000..5f79dd48a --- /dev/null +++ b/testing/mozbase/manifestparser/tests/test_testmanifest.py @@ -0,0 +1,122 @@ +#!/usr/bin/env python + +import os +import shutil +import tempfile +import unittest + +from manifestparser import TestManifest, ParseError +from manifestparser.filters import subsuite + +here = os.path.dirname(os.path.abspath(__file__)) + + +class TestTestManifest(unittest.TestCase): + """Test the Test Manifest""" + + def test_testmanifest(self): + # Test filtering based on platform: + filter_example = os.path.join(here, 'filter-example.ini') + manifest = TestManifest(manifests=(filter_example,), strict=False) + self.assertEqual([i['name'] for i in manifest.active_tests(os='win', disabled=False, + exists=False)], + ['windowstest', 'fleem']) + self.assertEqual([i['name'] for i in manifest.active_tests(os='linux', disabled=False, + exists=False)], + ['fleem', 'linuxtest']) + + # Look for existing tests. There is only one: + self.assertEqual([i['name'] for i in manifest.active_tests()], + ['fleem']) + + # You should be able to expect failures: + last = manifest.active_tests(exists=False, toolkit='gtk2')[-1] + self.assertEqual(last['name'], 'linuxtest') + self.assertEqual(last['expected'], 'pass') + last = manifest.active_tests(exists=False, toolkit='cocoa')[-1] + self.assertEqual(last['expected'], 'fail') + + def test_missing_paths(self): + """ + Test paths that don't exist raise an exception in strict mode. + """ + tempdir = tempfile.mkdtemp() + + missing_path = os.path.join(here, 'missing-path.ini') + manifest = TestManifest(manifests=(missing_path,), strict=True) + self.assertRaises(IOError, manifest.active_tests) + self.assertRaises(IOError, manifest.copy, tempdir) + self.assertRaises(IOError, manifest.update, tempdir) + + shutil.rmtree(tempdir) + + def test_comments(self): + """ + ensure comments work, see + https://bugzilla.mozilla.org/show_bug.cgi?id=813674 + """ + comment_example = os.path.join(here, 'comment-example.ini') + manifest = TestManifest(manifests=(comment_example,)) + self.assertEqual(len(manifest.tests), 8) + names = [i['name'] for i in manifest.tests] + self.assertFalse('test_0202_app_launch_apply_update_dirlocked.js' in names) + + def test_manifest_subsuites(self): + """ + test subsuites and conditional subsuites + """ + relative_path = os.path.join(here, 'subsuite.ini') + manifest = TestManifest(manifests=(relative_path,)) + info = {'foo': 'bar'} + + # 6 tests total + tests = manifest.active_tests(exists=False, **info) + self.assertEquals(len(tests), 6) + + # only 3 tests for subsuite bar when foo==bar + tests = manifest.active_tests(exists=False, + filters=[subsuite('bar')], + **info) + self.assertEquals(len(tests), 3) + + # only 1 test for subsuite baz, regardless of conditions + other = {'something': 'else'} + tests = manifest.active_tests(exists=False, + filters=[subsuite('baz')], + **info) + self.assertEquals(len(tests), 1) + tests = manifest.active_tests(exists=False, + filters=[subsuite('baz')], + **other) + self.assertEquals(len(tests), 1) + + # 4 tests match when the condition doesn't match (all tests except + # the unconditional subsuite) + info = {'foo': 'blah'} + tests = manifest.active_tests(exists=False, + filters=[subsuite()], + **info) + self.assertEquals(len(tests), 5) + + # test for illegal subsuite value + manifest.tests[0]['subsuite'] = 'subsuite=bar,foo=="bar",type="nothing"' + with self.assertRaises(ParseError): + manifest.active_tests(exists=False, + filters=[subsuite('foo')], + **info) + + def test_none_and_empty_manifest(self): + """ + Test TestManifest for None and empty manifest, see + https://bugzilla.mozilla.org/show_bug.cgi?id=1087682 + """ + none_manifest = TestManifest(manifests=None, strict=False) + self.assertEqual(len(none_manifest.test_paths()), 0) + self.assertEqual(len(none_manifest.active_tests()), 0) + + empty_manifest = TestManifest(manifests=[], strict=False) + self.assertEqual(len(empty_manifest.test_paths()), 0) + self.assertEqual(len(empty_manifest.active_tests()), 0) + +if __name__ == '__main__': + unittest.main() diff --git a/testing/mozbase/manifestparser/tests/verifyDirectory/subdir/manifest.ini b/testing/mozbase/manifestparser/tests/verifyDirectory/subdir/manifest.ini new file mode 100644 index 000000000..509ebd62e --- /dev/null +++ b/testing/mozbase/manifestparser/tests/verifyDirectory/subdir/manifest.ini @@ -0,0 +1 @@ +[test_sub.js] diff --git a/testing/mozbase/manifestparser/tests/verifyDirectory/subdir/test_sub.js b/testing/mozbase/manifestparser/tests/verifyDirectory/subdir/test_sub.js new file mode 100644 index 000000000..df48720d9 --- /dev/null +++ b/testing/mozbase/manifestparser/tests/verifyDirectory/subdir/test_sub.js @@ -0,0 +1 @@ +// test_sub.js diff --git a/testing/mozbase/manifestparser/tests/verifyDirectory/test_1.js b/testing/mozbase/manifestparser/tests/verifyDirectory/test_1.js new file mode 100644 index 000000000..c5a966f46 --- /dev/null +++ b/testing/mozbase/manifestparser/tests/verifyDirectory/test_1.js @@ -0,0 +1 @@ +// test_1.js diff --git a/testing/mozbase/manifestparser/tests/verifyDirectory/test_2.js b/testing/mozbase/manifestparser/tests/verifyDirectory/test_2.js new file mode 100644 index 000000000..d8648599c --- /dev/null +++ b/testing/mozbase/manifestparser/tests/verifyDirectory/test_2.js @@ -0,0 +1 @@ +// test_2.js diff --git a/testing/mozbase/manifestparser/tests/verifyDirectory/test_3.js b/testing/mozbase/manifestparser/tests/verifyDirectory/test_3.js new file mode 100644 index 000000000..794bc2c34 --- /dev/null +++ b/testing/mozbase/manifestparser/tests/verifyDirectory/test_3.js @@ -0,0 +1 @@ +// test_3.js diff --git a/testing/mozbase/manifestparser/tests/verifyDirectory/verifyDirectory.ini b/testing/mozbase/manifestparser/tests/verifyDirectory/verifyDirectory.ini new file mode 100644 index 000000000..10e0c79c8 --- /dev/null +++ b/testing/mozbase/manifestparser/tests/verifyDirectory/verifyDirectory.ini @@ -0,0 +1,4 @@ +[test_1.js] +[test_2.js] +[test_3.js] +[include:subdir/manifest.ini] diff --git a/testing/mozbase/manifestparser/tests/verifyDirectory/verifyDirectory_incomplete.ini b/testing/mozbase/manifestparser/tests/verifyDirectory/verifyDirectory_incomplete.ini new file mode 100644 index 000000000..cde526acf --- /dev/null +++ b/testing/mozbase/manifestparser/tests/verifyDirectory/verifyDirectory_incomplete.ini @@ -0,0 +1,3 @@ +[test_2.js] +[test_3.js] +[include:subdir/manifest.ini] diff --git a/testing/mozbase/manifestparser/tests/verifyDirectory/verifyDirectory_toocomplete.ini b/testing/mozbase/manifestparser/tests/verifyDirectory/verifyDirectory_toocomplete.ini new file mode 100644 index 000000000..88994ae26 --- /dev/null +++ b/testing/mozbase/manifestparser/tests/verifyDirectory/verifyDirectory_toocomplete.ini @@ -0,0 +1,5 @@ +[test_1.js] +[test_2.js] +[test_3.js] +[test_notappearinginthisfilm.js] +[include:subdir/manifest.ini] diff --git a/testing/mozbase/moz.build b/testing/mozbase/moz.build new file mode 100644 index 000000000..172f4e728 --- /dev/null +++ b/testing/mozbase/moz.build @@ -0,0 +1,38 @@ +# -*- Mode: python; indent-tabs-mode: nil; tab-width: 40 -*- +# vim: set filetype=python: +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, You can obtain one at http://mozilla.org/MPL/2.0/. + +PYTHON_UNIT_TESTS += [ + 'test.py', +] + +python_modules = [ + 'manifestparser', + 'mozcrash', + 'mozdebug', + 'mozdevice', + 'mozfile', + 'mozhttpd', + 'mozinfo', + 'mozinstall', + 'mozleak', + 'mozlog', + 'moznetwork', + 'mozprocess', + 'mozprofile', + 'mozrunner', + 'mozscreenshot', + 'mozsystemmonitor', + 'moztest', + 'mozversion', +] + +TEST_HARNESS_FILES.mozbase += [m + '/**' for m in python_modules] + +TEST_HARNESS_FILES.mozbase += [ + 'setup_development.py', + 'test-manifest.ini', + 'test.py', +] diff --git a/testing/mozbase/mozcrash/mozcrash/__init__.py b/testing/mozbase/mozcrash/mozcrash/__init__.py new file mode 100644 index 000000000..ec95442cf --- /dev/null +++ b/testing/mozbase/mozcrash/mozcrash/__init__.py @@ -0,0 +1,10 @@ +# flake8: noqa +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this file, +# You can obtain one at http://mozilla.org/MPL/2.0/. +""" +mozcrash is a library for getting a stack trace out of processes that have crashed +and left behind a minidump file using the Google Breakpad library. +""" + +from mozcrash import * diff --git a/testing/mozbase/mozcrash/mozcrash/mozcrash.py b/testing/mozbase/mozcrash/mozcrash/mozcrash.py new file mode 100644 index 000000000..c39e68f3a --- /dev/null +++ b/testing/mozbase/mozcrash/mozcrash/mozcrash.py @@ -0,0 +1,557 @@ +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this file, +# You can obtain one at http://mozilla.org/MPL/2.0/. + +import glob +import os +import re +import shutil +import signal +import subprocess +import sys +import tempfile +import urllib2 +import zipfile +from collections import namedtuple + +import mozfile +import mozinfo +import mozlog + +__all__ = [ + 'check_for_crashes', + 'check_for_java_exception', + 'kill_and_get_minidump', + 'log_crashes', + 'cleanup_pending_crash_reports', +] + + +StackInfo = namedtuple("StackInfo", + ["minidump_path", + "signature", + "stackwalk_stdout", + "stackwalk_stderr", + "stackwalk_retcode", + "stackwalk_errors", + "extra"]) + + +def get_logger(): + structured_logger = mozlog.get_default_logger("mozcrash") + if structured_logger is None: + return mozlog.unstructured.getLogger('mozcrash') + return structured_logger + + +def check_for_crashes(dump_directory, + symbols_path=None, + stackwalk_binary=None, + dump_save_path=None, + test_name=None, + quiet=False): + """ + Print a stack trace for minidump files left behind by a crashing program. + + `dump_directory` will be searched for minidump files. Any minidump files found will + have `stackwalk_binary` executed on them, with `symbols_path` passed as an extra + argument. + + `stackwalk_binary` should be a path to the minidump_stackwalk binary. + If `stackwalk_binary` is not set, the MINIDUMP_STACKWALK environment variable + will be checked and its value used if it is not empty. + + `symbols_path` should be a path to a directory containing symbols to use for + dump processing. This can either be a path to a directory containing Breakpad-format + symbols, or a URL to a zip file containing a set of symbols. + + If `dump_save_path` is set, it should be a path to a directory in which to copy minidump + files for safekeeping after a stack trace has been printed. If not set, the environment + variable MINIDUMP_SAVE_PATH will be checked and its value used if it is not empty. + + If `test_name` is set it will be used as the test name in log output. If not set the + filename of the calling function will be used. + + If `quiet` is set, no PROCESS-CRASH message will be printed to stdout if a + crash is detected. + + Returns number of minidump files found. + """ + + # try to get the caller's filename if no test name is given + if test_name is None: + try: + test_name = os.path.basename(sys._getframe(1).f_code.co_filename) + except: + test_name = "unknown" + + crash_info = CrashInfo(dump_directory, symbols_path, dump_save_path=dump_save_path, + stackwalk_binary=stackwalk_binary) + + if not crash_info.has_dumps: + return False + + crash_count = 0 + for info in crash_info: + crash_count += 1 + if not quiet: + stackwalk_output = ["Crash dump filename: %s" % info.minidump_path] + if info.stackwalk_stderr: + stackwalk_output.append("stderr from minidump_stackwalk:") + stackwalk_output.append(info.stackwalk_stderr) + elif info.stackwalk_stdout is not None: + stackwalk_output.append(info.stackwalk_stdout) + if info.stackwalk_retcode is not None and info.stackwalk_retcode != 0: + stackwalk_output.append("minidump_stackwalk exited with return code %d" % + info.stackwalk_retcode) + signature = info.signature if info.signature else "unknown top frame" + print "PROCESS-CRASH | %s | application crashed [%s]" % (test_name, + signature) + print '\n'.join(stackwalk_output) + print '\n'.join(info.stackwalk_errors) + + return crash_count + + +def log_crashes(logger, + dump_directory, + symbols_path, + process=None, + test=None, + stackwalk_binary=None, + dump_save_path=None): + """Log crashes using a structured logger""" + crash_count = 0 + for info in CrashInfo(dump_directory, symbols_path, dump_save_path=dump_save_path, + stackwalk_binary=stackwalk_binary): + crash_count += 1 + kwargs = info._asdict() + kwargs.pop("extra") + logger.crash(process=process, test=test, **kwargs) + return crash_count + + +class CrashInfo(object): + """Get information about a crash based on dump files. + + Typical usage is to iterate over the CrashInfo object. This returns StackInfo + objects, one for each crash dump file that is found in the dump_directory. + + :param dump_directory: Path to search for minidump files + :param symbols_path: Path to a path to a directory containing symbols to use for + dump processing. This can either be a path to a directory + containing Breakpad-format symbols, or a URL to a zip file + containing a set of symbols. + :param dump_save_path: Path to which to save the dump files. If this is None, + the MINIDUMP_SAVE_PATH environment variable will be used. + :param stackwalk_binary: Path to the minidump_stackwalk binary. If this is None, + the MINIDUMP_STACKWALK environment variable will be used + as the path to the minidump binary.""" + + def __init__(self, dump_directory, symbols_path, dump_save_path=None, + stackwalk_binary=None): + self.dump_directory = dump_directory + self.symbols_path = symbols_path + self.remove_symbols = False + + if dump_save_path is None: + dump_save_path = os.environ.get('MINIDUMP_SAVE_PATH', None) + self.dump_save_path = dump_save_path + + if stackwalk_binary is None: + stackwalk_binary = os.environ.get('MINIDUMP_STACKWALK', None) + self.stackwalk_binary = stackwalk_binary + + self.logger = get_logger() + self._dump_files = None + + def _get_symbols(self): + # If no symbols path has been set create a temporary folder to let the + # minidump stackwalk download the symbols. + if not self.symbols_path: + self.symbols_path = tempfile.mkdtemp() + self.remove_symbols = True + + # This updates self.symbols_path so we only download once. + if mozfile.is_url(self.symbols_path): + self.remove_symbols = True + self.logger.info("Downloading symbols from: %s" % self.symbols_path) + # Get the symbols and write them to a temporary zipfile + data = urllib2.urlopen(self.symbols_path) + with tempfile.TemporaryFile() as symbols_file: + symbols_file.write(data.read()) + # extract symbols to a temporary directory (which we'll delete after + # processing all crashes) + self.symbols_path = tempfile.mkdtemp() + with zipfile.ZipFile(symbols_file, 'r') as zfile: + mozfile.extract_zip(zfile, self.symbols_path) + + @property + def dump_files(self): + """List of tuple (path_to_dump_file, path_to_extra_file) for each dump + file in self.dump_directory. The extra files may not exist.""" + if self._dump_files is None: + self._dump_files = [(path, os.path.splitext(path)[0] + '.extra') for path in + glob.glob(os.path.join(self.dump_directory, '*.dmp'))] + max_dumps = 10 + if len(self._dump_files) > max_dumps: + self.logger.warning("Found %d dump files -- limited to %d!" % + (len(self._dump_files), max_dumps)) + del self._dump_files[max_dumps:] + + return self._dump_files + + @property + def has_dumps(self): + """Boolean indicating whether any crash dump files were found in the + current directory""" + return len(self.dump_files) > 0 + + def __iter__(self): + for path, extra in self.dump_files: + rv = self._process_dump_file(path, extra) + yield rv + + if self.remove_symbols: + mozfile.remove(self.symbols_path) + + def _process_dump_file(self, path, extra): + """Process a single dump file using self.stackwalk_binary, and return a + tuple containing properties of the crash dump. + + :param path: Path to the minidump file to analyse + :return: A StackInfo tuple with the fields:: + minidump_path: Path of the dump file + signature: The top frame of the stack trace, or None if it + could not be determined. + stackwalk_stdout: String of stdout data from stackwalk + stackwalk_stderr: String of stderr data from stackwalk or + None if it succeeded + stackwalk_retcode: Return code from stackwalk + stackwalk_errors: List of errors in human-readable form that prevented + stackwalk being launched. + """ + self._get_symbols() + + errors = [] + signature = None + include_stderr = False + out = None + err = None + retcode = None + if (self.symbols_path and self.stackwalk_binary and + os.path.exists(self.stackwalk_binary) and + os.access(self.stackwalk_binary, os.X_OK)): + + command = [ + self.stackwalk_binary, + path, + self.symbols_path + ] + self.logger.info('Copy/paste: ' + ' '.join(command)) + # run minidump_stackwalk + p = subprocess.Popen( + command, + stdout=subprocess.PIPE, + stderr=subprocess.PIPE + ) + (out, err) = p.communicate() + retcode = p.returncode + + if len(out) > 3: + # minidump_stackwalk is chatty, + # so ignore stderr when it succeeds. + # The top frame of the crash is always the line after "Thread N (crashed)" + # Examples: + # 0 libc.so + 0xa888 + # 0 libnss3.so!nssCertificate_Destroy [certificate.c : 102 + 0x0] + # 0 mozjs.dll!js::GlobalObject::getDebuggers() [GlobalObject.cpp:89df18f9b6da : 580 + 0x0] # noqa + # 0 libxul.so!void js::gc::MarkInternal(JSTracer*, JSObject**) + # [Marking.cpp : 92 + 0x28] + lines = out.splitlines() + for i, line in enumerate(lines): + if "(crashed)" in line: + match = re.search(r"^ 0 (?:.*!)?(?:void )?([^\[]+)", lines[i + 1]) + if match: + signature = "@ %s" % match.group(1).strip() + break + else: + include_stderr = True + + else: + if not self.symbols_path: + errors.append("No symbols path given, can't process dump.") + if not self.stackwalk_binary: + errors.append("MINIDUMP_STACKWALK not set, can't process dump.") + elif self.stackwalk_binary and not os.path.exists(self.stackwalk_binary): + errors.append("MINIDUMP_STACKWALK binary not found: %s" % self.stackwalk_binary) + elif not os.access(self.stackwalk_binary, os.X_OK): + errors.append('This user cannot execute the MINIDUMP_STACKWALK binary.') + + if self.dump_save_path: + self._save_dump_file(path, extra) + + if os.path.exists(path): + mozfile.remove(path) + if os.path.exists(extra): + mozfile.remove(extra) + + return StackInfo(path, + signature, + out, + err if include_stderr else None, + retcode, + errors, + extra) + + def _save_dump_file(self, path, extra): + if os.path.isfile(self.dump_save_path): + os.unlink(self.dump_save_path) + if not os.path.isdir(self.dump_save_path): + try: + os.makedirs(self.dump_save_path) + except OSError: + pass + + shutil.move(path, self.dump_save_path) + self.logger.info("Saved minidump as %s" % + os.path.join(self.dump_save_path, os.path.basename(path))) + + if os.path.isfile(extra): + shutil.move(extra, self.dump_save_path) + self.logger.info("Saved app info as %s" % + os.path.join(self.dump_save_path, os.path.basename(extra))) + + +def check_for_java_exception(logcat, test_name=None, quiet=False): + """ + Print a summary of a fatal Java exception, if present in the provided + logcat output. + + Example: + PROCESS-CRASH | | java-exception java.lang.NullPointerException at org.mozilla.gecko.GeckoApp$21.run(GeckoApp.java:1833) # noqa + + `logcat` should be a list of strings. + + If `test_name` is set it will be used as the test name in log output. If not set the + filename of the calling function will be used. + + If `quiet` is set, no PROCESS-CRASH message will be printed to stdout if a + crash is detected. + + Returns True if a fatal Java exception was found, False otherwise. + """ + + # try to get the caller's filename if no test name is given + if test_name is None: + try: + test_name = os.path.basename(sys._getframe(1).f_code.co_filename) + except: + test_name = "unknown" + + found_exception = False + + for i, line in enumerate(logcat): + # Logs will be of form: + # + # 01-30 20:15:41.937 E/GeckoAppShell( 1703): >>> REPORTING UNCAUGHT EXCEPTION FROM THREAD 9 ("GeckoBackgroundThread") # noqa + # 01-30 20:15:41.937 E/GeckoAppShell( 1703): java.lang.NullPointerException + # 01-30 20:15:41.937 E/GeckoAppShell( 1703): at org.mozilla.gecko.GeckoApp$21.run(GeckoApp.java:1833) # noqa + # 01-30 20:15:41.937 E/GeckoAppShell( 1703): at android.os.Handler.handleCallback(Handler.java:587) # noqa + if "REPORTING UNCAUGHT EXCEPTION" in line: + # Strip away the date, time, logcat tag and pid from the next two lines and + # concatenate the remainder to form a concise summary of the exception. + found_exception = True + if len(logcat) >= i + 3: + logre = re.compile(r".*\): \t?(.*)") + m = logre.search(logcat[i + 1]) + if m and m.group(1): + exception_type = m.group(1) + m = logre.search(logcat[i + 2]) + if m and m.group(1): + exception_location = m.group(1) + if not quiet: + print "PROCESS-CRASH | %s | java-exception %s %s" % (test_name, + exception_type, + exception_location) + else: + print "Automation Error: java exception in logcat at line " \ + "%d of %d: %s" % (i, len(logcat), line) + break + + return found_exception + +if mozinfo.isWin: + import ctypes + import uuid + + kernel32 = ctypes.windll.kernel32 + OpenProcess = kernel32.OpenProcess + CloseHandle = kernel32.CloseHandle + + def write_minidump(pid, dump_directory, utility_path): + """ + Write a minidump for a process. + + :param pid: PID of the process to write a minidump for. + :param dump_directory: Directory in which to write the minidump. + """ + PROCESS_QUERY_INFORMATION = 0x0400 + PROCESS_VM_READ = 0x0010 + GENERIC_READ = 0x80000000 + GENERIC_WRITE = 0x40000000 + CREATE_ALWAYS = 2 + FILE_ATTRIBUTE_NORMAL = 0x80 + INVALID_HANDLE_VALUE = -1 + + file_name = os.path.join(dump_directory, + str(uuid.uuid4()) + ".dmp") + + if (mozinfo.info['bits'] != ctypes.sizeof(ctypes.c_voidp) * 8 and + utility_path): + # We're not going to be able to write a minidump with ctypes if our + # python process was compiled for a different architecture than + # firefox, so we invoke the minidumpwriter utility program. + + log = get_logger() + minidumpwriter = os.path.normpath(os.path.join(utility_path, + "minidumpwriter.exe")) + log.info("Using %s to write a dump to %s for [%d]" % + (minidumpwriter, file_name, pid)) + if not os.path.exists(minidumpwriter): + log.error("minidumpwriter not found in %s" % utility_path) + return + + if isinstance(file_name, unicode): + # Convert to a byte string before sending to the shell. + file_name = file_name.encode(sys.getfilesystemencoding()) + + status = subprocess.Popen([minidumpwriter, str(pid), file_name]).wait() + if status: + log.error("minidumpwriter exited with status: %d" % status) + return + + proc_handle = OpenProcess(PROCESS_QUERY_INFORMATION | PROCESS_VM_READ, + 0, pid) + if not proc_handle: + return + + if not isinstance(file_name, unicode): + # Convert to unicode explicitly so our path will be valid as input + # to CreateFileW + file_name = unicode(file_name, sys.getfilesystemencoding()) + + file_handle = kernel32.CreateFileW(file_name, + GENERIC_READ | GENERIC_WRITE, + 0, + None, + CREATE_ALWAYS, + FILE_ATTRIBUTE_NORMAL, + None) + if file_handle != INVALID_HANDLE_VALUE: + ctypes.windll.dbghelp.MiniDumpWriteDump(proc_handle, + pid, + file_handle, + # Dump type - MiniDumpNormal + 0, + # Exception parameter + None, + # User stream parameter + None, + # Callback parameter + None) + CloseHandle(file_handle) + CloseHandle(proc_handle) + + def kill_pid(pid): + """ + Terminate a process with extreme prejudice. + + :param pid: PID of the process to terminate. + """ + PROCESS_TERMINATE = 0x0001 + handle = OpenProcess(PROCESS_TERMINATE, 0, pid) + if handle: + kernel32.TerminateProcess(handle, 1) + CloseHandle(handle) +else: + def kill_pid(pid): + """ + Terminate a process with extreme prejudice. + + :param pid: PID of the process to terminate. + """ + os.kill(pid, signal.SIGKILL) + + +def kill_and_get_minidump(pid, dump_directory, utility_path=None): + """ + Attempt to kill a process and leave behind a minidump describing its + execution state. + + :param pid: The PID of the process to kill. + :param dump_directory: The directory where a minidump should be written on + Windows, where the dump will be written from outside the process. + + On Windows a dump will be written using the MiniDumpWriteDump function + from DbgHelp.dll. On Linux and OS X the process will be sent a SIGABRT + signal to trigger minidump writing via a Breakpad signal handler. On other + platforms the process will simply be killed via SIGKILL. + + If the process is hung in such a way that it cannot respond to SIGABRT + it may still be running after this function returns. In that case it + is the caller's responsibility to deal with killing it. + """ + needs_killing = True + if mozinfo.isWin: + write_minidump(pid, dump_directory, utility_path) + elif mozinfo.isLinux or mozinfo.isMac: + os.kill(pid, signal.SIGABRT) + needs_killing = False + if needs_killing: + kill_pid(pid) + + +def cleanup_pending_crash_reports(): + """ + Delete any pending crash reports. + + The presence of pending crash reports may be reported by the browser, + affecting test results; it is best to ensure that these are removed + before starting any browser tests. + + Firefox stores pending crash reports in "/Crash Reports". + If the browser is not running, it cannot provide , so this + code tries to anticipate its value. + + See dom/system/OSFileConstants.cpp for platform variations of . + """ + if mozinfo.isWin: + location = os.path.expanduser("~\\AppData\\Roaming\\Mozilla\\Firefox\\Crash Reports") + elif mozinfo.isMac: + location = os.path.expanduser("~/Library/Application Support/firefox/Crash Reports") + else: + location = os.path.expanduser("~/.mozilla/firefox/Crash Reports") + logger = get_logger() + if os.path.exists(location): + try: + mozfile.remove(location) + logger.info("Removed pending crash reports at '%s'" % location) + except: + pass + + +if __name__ == '__main__': + import argparse + parser = argparse.ArgumentParser() + parser.add_argument('--stackwalk-binary', '-b') + parser.add_argument('--dump-save-path', '-o') + parser.add_argument('--test-name', '-n') + parser.add_argument('dump_directory') + parser.add_argument('symbols_path') + args = parser.parse_args() + + check_for_crashes(args.dump_directory, args.symbols_path, + stackwalk_binary=args.stackwalk_binary, + dump_save_path=args.dump_save_path, + test_name=args.test_name) diff --git a/testing/mozbase/mozcrash/setup.py b/testing/mozbase/mozcrash/setup.py new file mode 100644 index 000000000..da5ffa19b --- /dev/null +++ b/testing/mozbase/mozcrash/setup.py @@ -0,0 +1,29 @@ +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this file, +# You can obtain one at http://mozilla.org/MPL/2.0/. + +from setuptools import setup + +PACKAGE_NAME = 'mozcrash' +PACKAGE_VERSION = '1.0' + +# dependencies +deps = ['mozfile >= 1.0', + 'mozlog >= 3.0'] + +setup(name=PACKAGE_NAME, + version=PACKAGE_VERSION, + description="Library for printing stack traces from minidumps " + "left behind by crashed processes", + long_description="see http://mozbase.readthedocs.org/", + classifiers=[], # Get strings from http://pypi.python.org/pypi?%3Aaction=list_classifiers + keywords='mozilla', + author='Mozilla Automation and Tools team', + author_email='tools@lists.mozilla.org', + url='https://wiki.mozilla.org/Auto-tools/Projects/Mozbase', + license='MPL', + packages=['mozcrash'], + include_package_data=True, + zip_safe=False, + install_requires=deps, + ) diff --git a/testing/mozbase/mozcrash/tests/manifest.ini b/testing/mozbase/mozcrash/tests/manifest.ini new file mode 100644 index 000000000..528fdea7b --- /dev/null +++ b/testing/mozbase/mozcrash/tests/manifest.ini @@ -0,0 +1 @@ +[test.py] diff --git a/testing/mozbase/mozcrash/tests/test.py b/testing/mozbase/mozcrash/tests/test.py new file mode 100644 index 000000000..8f6b14f50 --- /dev/null +++ b/testing/mozbase/mozcrash/tests/test.py @@ -0,0 +1,241 @@ +#!/usr/bin/env python +# +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this file, +# You can obtain one at http://mozilla.org/MPL/2.0/. + +import os +import unittest +import subprocess +import tempfile +import shutil +import urlparse +import zipfile +import StringIO +import mozcrash +import mozhttpd +import mozlog.unstructured as mozlog + +# Make logs go away +log = mozlog.getLogger("mozcrash", handler=mozlog.FileHandler(os.devnull)) + + +def popen_factory(stdouts): + """ + Generate a class that can mock subprocess.Popen. |stdouts| is an iterable that + should return an iterable for the stdout of each process in turn. + """ + class mock_popen(object): + + def __init__(self, args, *args_rest, **kwargs): + self.stdout = stdouts.next() + self.returncode = 0 + + def wait(self): + return 0 + + def communicate(self): + return (self.stdout.next(), "") + + return mock_popen + + +class TestCrash(unittest.TestCase): + + def setUp(self): + self.tempdir = tempfile.mkdtemp() + # a fake file to use as a stackwalk binary + self.stackwalk = os.path.join(self.tempdir, "stackwalk") + open(self.stackwalk, "w").write("fake binary") + self._subprocess_popen = subprocess.Popen + subprocess.Popen = popen_factory(self.next_mock_stdout()) + self.stdouts = [] + + def tearDown(self): + subprocess.Popen = self._subprocess_popen + shutil.rmtree(self.tempdir) + + def next_mock_stdout(self): + if not self.stdouts: + yield iter([]) + for s in self.stdouts: + yield iter(s) + + def test_nodumps(self): + """ + Test that check_for_crashes returns False if no dumps are present. + """ + self.stdouts.append(["this is some output"]) + self.assertFalse(mozcrash.check_for_crashes(self.tempdir, + symbols_path='symbols_path', + stackwalk_binary=self.stackwalk, + quiet=True)) + + def test_simple(self): + """ + Test that check_for_crashes returns True if a dump is present. + """ + open(os.path.join(self.tempdir, "test.dmp"), "w").write("foo") + self.stdouts.append(["this is some output"]) + self.assert_(mozcrash.check_for_crashes(self.tempdir, + symbols_path='symbols_path', + stackwalk_binary=self.stackwalk, + quiet=True)) + + def test_stackwalk_envvar(self): + """ + Test that check_for_crashes uses the MINIDUMP_STACKWALK environment var. + """ + open(os.path.join(self.tempdir, "test.dmp"), "w").write("foo") + self.stdouts.append(["this is some output"]) + os.environ['MINIDUMP_STACKWALK'] = self.stackwalk + self.assert_(mozcrash.check_for_crashes(self.tempdir, + symbols_path='symbols_path', + quiet=True)) + del os.environ['MINIDUMP_STACKWALK'] + + def test_save_path(self): + """ + Test that dump_save_path works. + """ + open(os.path.join(self.tempdir, "test.dmp"), "w").write("foo") + open(os.path.join(self.tempdir, "test.extra"), "w").write("bar") + save_path = os.path.join(self.tempdir, "saved") + os.mkdir(save_path) + self.stdouts.append(["this is some output"]) + self.assert_(mozcrash.check_for_crashes(self.tempdir, + symbols_path='symbols_path', + stackwalk_binary=self.stackwalk, + dump_save_path=save_path, + quiet=True)) + self.assert_(os.path.isfile(os.path.join(save_path, "test.dmp"))) + self.assert_(os.path.isfile(os.path.join(save_path, "test.extra"))) + + def test_save_path_not_present(self): + """ + Test that dump_save_path works when the directory doesn't exist. + """ + open(os.path.join(self.tempdir, "test.dmp"), "w").write("foo") + open(os.path.join(self.tempdir, "test.extra"), "w").write("bar") + save_path = os.path.join(self.tempdir, "saved") + self.stdouts.append(["this is some output"]) + self.assert_(mozcrash.check_for_crashes(self.tempdir, + symbols_path='symbols_path', + stackwalk_binary=self.stackwalk, + dump_save_path=save_path, + quiet=True)) + self.assert_(os.path.isfile(os.path.join(save_path, "test.dmp"))) + self.assert_(os.path.isfile(os.path.join(save_path, "test.extra"))) + + def test_save_path_isfile(self): + """ + Test that dump_save_path works when the directory doesn't exist, + but a file with the same name exists. + """ + open(os.path.join(self.tempdir, "test.dmp"), "w").write("foo") + open(os.path.join(self.tempdir, "test.extra"), "w").write("bar") + save_path = os.path.join(self.tempdir, "saved") + open(save_path, "w").write("junk") + self.stdouts.append(["this is some output"]) + self.assert_(mozcrash.check_for_crashes(self.tempdir, + symbols_path='symbols_path', + stackwalk_binary=self.stackwalk, + dump_save_path=save_path, + quiet=True)) + self.assert_(os.path.isfile(os.path.join(save_path, "test.dmp"))) + self.assert_(os.path.isfile(os.path.join(save_path, "test.extra"))) + + def test_save_path_envvar(self): + """ + Test that the MINDUMP_SAVE_PATH environment variable works. + """ + open(os.path.join(self.tempdir, "test.dmp"), "w").write("foo") + open(os.path.join(self.tempdir, "test.extra"), "w").write("bar") + save_path = os.path.join(self.tempdir, "saved") + os.mkdir(save_path) + self.stdouts.append(["this is some output"]) + os.environ['MINIDUMP_SAVE_PATH'] = save_path + self.assert_(mozcrash.check_for_crashes(self.tempdir, + symbols_path='symbols_path', + stackwalk_binary=self.stackwalk, + quiet=True)) + del os.environ['MINIDUMP_SAVE_PATH'] + self.assert_(os.path.isfile(os.path.join(save_path, "test.dmp"))) + self.assert_(os.path.isfile(os.path.join(save_path, "test.extra"))) + + def test_symbol_path_not_present(self): + open(os.path.join(self.tempdir, "test.dmp"), "w").write("foo") + self.stdouts.append(["this is some output"]) + self.assert_(mozcrash.check_for_crashes(self.tempdir, + symbols_path=None, + stackwalk_binary=self.stackwalk, + quiet=True)) + + def test_symbol_path_url(self): + """ + Test that passing a URL as symbols_path correctly fetches the URL. + """ + open(os.path.join(self.tempdir, "test.dmp"), "w").write("foo") + self.stdouts.append(["this is some output"]) + + def make_zipfile(): + data = StringIO.StringIO() + z = zipfile.ZipFile(data, 'w') + z.writestr("symbols.txt", "abc/xyz") + z.close() + return data.getvalue() + + def get_symbols(req): + headers = {} + return (200, headers, make_zipfile()) + httpd = mozhttpd.MozHttpd(port=0, + urlhandlers=[{'method': 'GET', + 'path': '/symbols', + 'function': get_symbols}]) + httpd.start() + symbol_url = urlparse.urlunsplit(('http', '%s:%d' % httpd.httpd.server_address, + '/symbols', '', '')) + self.assert_(mozcrash.check_for_crashes(self.tempdir, + symbol_url, + stackwalk_binary=self.stackwalk, + quiet=True)) + + +class TestJavaException(unittest.TestCase): + + def setUp(self): + self.test_log = [ + "01-30 20:15:41.937 E/GeckoAppShell( 1703): >>> " + "REPORTING UNCAUGHT EXCEPTION FROM THREAD 9 (\"GeckoBackgroundThread\")", + "01-30 20:15:41.937 E/GeckoAppShell( 1703): java.lang.NullPointerException", + "01-30 20:15:41.937 E/GeckoAppShell( 1703):" + " at org.mozilla.gecko.GeckoApp$21.run(GeckoApp.java:1833)", + "01-30 20:15:41.937 E/GeckoAppShell( 1703):" + " at android.os.Handler.handleCallback(Handler.java:587)"] + + def test_uncaught_exception(self): + """ + Test for an exception which should be caught + """ + self.assert_(mozcrash.check_for_java_exception(self.test_log, quiet=True)) + + def test_truncated_exception(self): + """ + Test for an exception which should be caught which + was truncated + """ + truncated_log = list(self.test_log) + truncated_log[0], truncated_log[1] = truncated_log[1], truncated_log[0] + self.assert_(mozcrash.check_for_java_exception(truncated_log, quiet=True)) + + def test_unchecked_exception(self): + """ + Test for an exception which should not be caught + """ + passable_log = list(self.test_log) + passable_log[0] = "01-30 20:15:41.937 E/GeckoAppShell( 1703):" \ + " >>> NOT-SO-BAD EXCEPTION FROM THREAD 9 (\"GeckoBackgroundThread\")" + self.assert_(not mozcrash.check_for_java_exception(passable_log, quiet=True)) + +if __name__ == '__main__': + unittest.main() diff --git a/testing/mozbase/mozdebug/mozdebug/__init__.py b/testing/mozbase/mozdebug/mozdebug/__init__.py new file mode 100644 index 000000000..3450d755c --- /dev/null +++ b/testing/mozbase/mozdebug/mozdebug/__init__.py @@ -0,0 +1,31 @@ +# flake8: noqa +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this file, +# You can obtain one at http://mozilla.org/MPL/2.0/. + +""" +This module contains a set of function to gather information about the +debugging capabilities of the platform. It allows to look for a specific +debugger or to query the system for a compatible/default debugger. + +The following simple example looks for the default debugger on the +current platform and launches a debugger process with the correct +debugger-specific arguments: + +:: + + import mozdebug + + debugger = mozdebug.get_default_debugger_name() + debuggerInfo = mozdebug.get_debugger_info(debugger) + + debuggeePath = "toDebug" + + processArgs = [self.debuggerInfo.path] + self.debuggerInfo.args + processArgs.append(debuggeePath) + + run_process(args, ...) + +""" + +from mozdebug import * diff --git a/testing/mozbase/mozdebug/mozdebug/mozdebug.py b/testing/mozbase/mozdebug/mozdebug/mozdebug.py new file mode 100755 index 000000000..5777a0001 --- /dev/null +++ b/testing/mozbase/mozdebug/mozdebug/mozdebug.py @@ -0,0 +1,291 @@ +#!/usr/bin/env python + +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this file, +# You can obtain one at http://mozilla.org/MPL/2.0/. + +import os +import mozinfo +from collections import namedtuple +from distutils.spawn import find_executable +from subprocess import check_output + +__all__ = ['get_debugger_info', + 'get_default_debugger_name', + 'DebuggerSearch', + 'get_default_valgrind_args'] + +''' +Map of debugging programs to information about them, like default arguments +and whether or not they are interactive. + +To add support for a new debugger, simply add the relative entry in +_DEBUGGER_INFO and optionally update the _DEBUGGER_PRIORITIES. +''' +_DEBUGGER_INFO = { + # gdb requires that you supply the '--args' flag in order to pass arguments + # after the executable name to the executable. + 'gdb': { + 'interactive': True, + 'args': ['-q', '--args'] + }, + + 'cgdb': { + 'interactive': True, + 'args': ['-q', '--args'] + }, + + 'lldb': { + 'interactive': True, + 'args': ['--'], + 'requiresEscapedArgs': True + }, + + # Visual Studio Debugger Support. + 'devenv.exe': { + 'interactive': True, + 'args': ['-debugexe'] + }, + + # Visual C++ Express Debugger Support. + 'wdexpress.exe': { + 'interactive': True, + 'args': ['-debugexe'] + }, + + # Windows Development Kit super-debugger. + 'windbg.exe': { + 'interactive': True, + }, +} + +# Maps each OS platform to the preferred debugger programs found in _DEBUGGER_INFO. +_DEBUGGER_PRIORITIES = { + 'win': ['devenv.exe', 'wdexpress.exe'], + 'linux': ['gdb', 'cgdb', 'lldb'], + 'mac': ['lldb', 'gdb'], + 'android': ['gdb'], + 'unknown': ['gdb'] +} + + +def _windbg_installation_paths(): + programFilesSuffixes = ['', ' (x86)'] + programFiles = "C:/Program Files" + # Try the most recent versions first. + windowsKitsVersions = ['10', '8.1', '8'] + + for suffix in programFilesSuffixes: + windowsKitsPrefix = os.path.join(programFiles + suffix, + 'Windows Kits') + for version in windowsKitsVersions: + yield os.path.join(windowsKitsPrefix, version, + 'Debuggers', 'x86', 'windbg.exe') + + +def get_debugger_path(debugger): + ''' + Get the full path of the debugger. + + :param debugger: The name of the debugger. + ''' + + if mozinfo.os == 'mac' and debugger == 'lldb': + # On newer OSX versions System Integrity Protections prevents us from + # setting certain env vars for a process such as DYLD_LIBRARY_PATH if + # it's in a protected directory such as /usr/bin. This is the case for + # lldb, so we try to find an instance under the Xcode install instead. + + # Attempt to use the xcrun util to find the path. + try: + path = check_output(['xcrun', '--find', 'lldb']).strip() + if path: + return path + except: + # Just default to find_executable instead. + pass + + return find_executable(debugger) + + +def get_debugger_info(debugger, debuggerArgs=None, debuggerInteractive=False): + ''' + Get the information about the requested debugger. + + Returns a dictionary containing the |path| of the debugger executable, + if it will run in |interactive| mode, its arguments and whether it needs + to escape arguments it passes to the debugged program (|requiresEscapedArgs|). + If the debugger cannot be found in the system, returns |None|. + + :param debugger: The name of the debugger. + :param debuggerArgs: If specified, it's the arguments to pass to the debugger, + as a string. Any debugger-specific separator arguments are appended after these + arguments. + :param debuggerInteractive: If specified, forces the debugger to be interactive. + ''' + + debuggerPath = None + + if debugger: + # Append '.exe' to the debugger on Windows if it's not present, + # so things like '--debugger=devenv' work. + if (os.name == 'nt' + and not debugger.lower().endswith('.exe')): + debugger += '.exe' + + debuggerPath = get_debugger_path(debugger) + + if not debuggerPath: + # windbg is not installed with the standard set of tools, and it's + # entirely possible that the user hasn't added the install location to + # PATH, so we have to be a little more clever than normal to locate it. + # Just try to look for it in the standard installed location(s). + if debugger == 'windbg.exe': + for candidate in _windbg_installation_paths(): + if os.path.exists(candidate): + debuggerPath = candidate + break + else: + if os.path.exists(debugger): + debuggerPath = debugger + + if not debuggerPath: + print 'Error: Could not find debugger %s.' % debugger + return None + + debuggerName = os.path.basename(debuggerPath).lower() + + def get_debugger_info(type, default): + if debuggerName in _DEBUGGER_INFO and type in _DEBUGGER_INFO[debuggerName]: + return _DEBUGGER_INFO[debuggerName][type] + return default + + # Define a namedtuple to access the debugger information from the outside world. + DebuggerInfo = namedtuple( + 'DebuggerInfo', + ['path', 'interactive', 'args', 'requiresEscapedArgs'] + ) + + debugger_arguments = [] + + if debuggerArgs: + # Append the provided debugger arguments at the end of the arguments list. + debugger_arguments += debuggerArgs.split() + + debugger_arguments += get_debugger_info('args', []) + + # Override the default debugger interactive mode if needed. + debugger_interactive = get_debugger_info('interactive', False) + if debuggerInteractive: + debugger_interactive = debuggerInteractive + + d = DebuggerInfo( + debuggerPath, + debugger_interactive, + debugger_arguments, + get_debugger_info('requiresEscapedArgs', False) + ) + + return d + +# Defines the search policies to use in get_default_debugger_name. + + +class DebuggerSearch: + OnlyFirst = 1 + KeepLooking = 2 + + +def get_default_debugger_name(search=DebuggerSearch.OnlyFirst): + ''' + Get the debugger name for the default debugger on current platform. + + :param search: If specified, stops looking for the debugger if the + default one is not found (|DebuggerSearch.OnlyFirst|) or keeps + looking for other compatible debuggers (|DebuggerSearch.KeepLooking|). + ''' + + mozinfo.find_and_update_from_json() + os = mozinfo.info['os'] + + # Find out which debuggers are preferred for use on this platform. + debuggerPriorities = _DEBUGGER_PRIORITIES[os if os in _DEBUGGER_PRIORITIES else 'unknown'] + + # Finally get the debugger information. + for debuggerName in debuggerPriorities: + debuggerPath = find_executable(debuggerName) + if debuggerPath: + return debuggerName + elif not search == DebuggerSearch.KeepLooking: + return None + + return None + +# Defines default values for Valgrind flags. +# +# --smc-check=all-non-file is required to deal with code generation and +# patching by the various JITS. Note that this is only necessary on +# x86 and x86_64, but not on ARM. This flag is only necessary for +# Valgrind versions prior to 3.11. +# +# --vex-iropt-register-updates=allregs-at-mem-access is required so that +# Valgrind generates correct register values whenever there is a +# segfault that is caught and handled. In particular OdinMonkey +# requires this. More recent Valgrinds (3.11 and later) provide +# --px-default=allregs-at-mem-access and +# --px-file-backed=unwindregs-at-mem-access +# which provide a significantly cheaper alternative, by restricting the +# precise exception behaviour to JIT generated code only. +# +# --trace-children=yes is required to get Valgrind to follow into +# content and other child processes. The resulting output can be +# difficult to make sense of, and --child-silent-after-fork=yes +# helps by causing Valgrind to be silent for the child in the period +# after fork() but before its subsequent exec(). +# +# --trace-children-skip lists processes that we are not interested +# in tracing into. +# +# --leak-check=full requests full stack traces for all leaked blocks +# detected at process exit. +# +# --show-possibly-lost=no requests blocks for which only an interior +# pointer was found to be considered not leaked. +# +# +# TODO: pass in the user supplied args for V (--valgrind-args=) and +# use this to detect if a different tool has been selected. If so +# adjust tool-specific args appropriately. +# +# TODO: pass in the path to the Valgrind to be used (--valgrind=), and +# check what flags it accepts. Possible args that might be beneficial: +# +# --num-transtab-sectors=24 [reduces re-jitting overheads in long runs] +# --px-default=allregs-at-mem-access +# --px-file-backed=unwindregs-at-mem-access +# [these reduce PX overheads as described above] +# + + +def get_default_valgrind_args(): + return (['--fair-sched=yes', + '--smc-check=all-non-file', + '--vex-iropt-register-updates=allregs-at-mem-access', + '--trace-children=yes', + '--child-silent-after-fork=yes', + ('--trace-children-skip=' + + '/usr/bin/hg,/bin/rm,*/bin/certutil,*/bin/pk12util,' + + '*/bin/ssltunnel,*/bin/uname,*/bin/which,*/bin/ps,' + + '*/bin/grep,*/bin/java'), + ] + + get_default_valgrind_tool_specific_args()) + +# The default tool is Memcheck. Feeding these arguments to a different +# Valgrind tool will cause it to fail at startup, so don't do that! + + +def get_default_valgrind_tool_specific_args(): + return ['--partial-loads-ok=yes', + '--leak-check=full', + '--show-possibly-lost=no', + ] diff --git a/testing/mozbase/mozdebug/setup.py b/testing/mozbase/mozdebug/setup.py new file mode 100644 index 000000000..1f4e5329b --- /dev/null +++ b/testing/mozbase/mozdebug/setup.py @@ -0,0 +1,27 @@ +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this file, +# You can obtain one at http://mozilla.org/MPL/2.0/. + +from setuptools import setup + +PACKAGE_VERSION = '0.1' + +setup(name='mozdebug', + version=PACKAGE_VERSION, + description="Utilities for running applications under native code debuggers " + "intended for use in Mozilla testing", + long_description="see http://mozbase.readthedocs.org/", + classifiers=[], # Get strings from http://pypi.python.org/pypi?%3Aaction=list_classifiers + keywords='mozilla', + author='Mozilla Automation and Testing Team', + author_email='tools@lists.mozilla.org', + url='https://wiki.mozilla.org/Auto-tools/Projects/Mozbase', + license='MPL', + packages=['mozdebug'], + include_package_data=True, + zip_safe=False, + install_requires=['mozinfo'], + entry_points=""" + # -*- Entry points: -*- + """, + ) diff --git a/testing/mozbase/mozdevice/adb_tests/test_device_running_adb_as_root.py b/testing/mozbase/mozdevice/adb_tests/test_device_running_adb_as_root.py new file mode 100644 index 000000000..2b223bb04 --- /dev/null +++ b/testing/mozbase/mozdevice/adb_tests/test_device_running_adb_as_root.py @@ -0,0 +1,48 @@ +""" + This test is to test devices that adbd does not get started as root. + Specifically devices that have ro.secure == 1 and ro.debuggable == 1 + + Running this test case requires various reboots which makes it a + very slow test case to run. +""" +import unittest +import sys + +from mozdevice import DeviceManagerADB + + +class TestFileOperations(unittest.TestCase): + + def setUp(self): + dm = DeviceManagerADB() + dm.reboot(wait=True) + + def test_run_adb_as_root_parameter(self): + dm = DeviceManagerADB() + self.assertTrue(dm.processInfo("adbd")[2] != "root") + dm = DeviceManagerADB(runAdbAsRoot=True) + self.assertTrue(dm.processInfo("adbd")[2] == "root") + + def test_after_reboot_adb_runs_as_root(self): + dm = DeviceManagerADB(runAdbAsRoot=True) + self.assertTrue(dm.processInfo("adbd")[2] == "root") + dm.reboot(wait=True) + self.assertTrue(dm.processInfo("adbd")[2] == "root") + + def tearDown(self): + dm = DeviceManagerADB() + dm.reboot() + +if __name__ == "__main__": + dm = DeviceManagerADB() + if not dm.devices(): + print "There are no connected adb devices" + sys.exit(1) + else: + if not (int(dm._runCmd(["shell", "getprop", "ro.secure"]).output[0]) and + int(dm._runCmd(["shell", "getprop", "ro.debuggable"]).output[0])): + print "This test case is meant for devices with devices that start " \ + "adbd as non-root and allows for adbd to be restarted as root." + sys.exit(1) + + unittest.main() diff --git a/testing/mozbase/mozdevice/adb_tests/test_devicemanagerADB.py b/testing/mozbase/mozdevice/adb_tests/test_devicemanagerADB.py new file mode 100644 index 000000000..495e449a4 --- /dev/null +++ b/testing/mozbase/mozdevice/adb_tests/test_devicemanagerADB.py @@ -0,0 +1,219 @@ +""" + Info: + This tests DeviceManagerADB with a real device + + Requirements: + - You must have a device connected + - It should be listed under 'adb devices' + + Notes: + - Not all functions have been covered. + In particular, functions from the parent class + - No testing of properties is done + - The test case are very simple and it could be + done with deeper inspection of the return values + + Author(s): + - Armen Zambrano + + Functions that are not being tested: + - launchProcess - DEPRECATED + - getIP + - recordLogcat + - saveScreenshot + - validateDir + - mkDirs + - getDeviceRoot + - shellCheckOutput + - processExist + + I assume these functions are only useful for Android + - getAppRoot() + - updateApp() + - uninstallApp() + - uninstallAppAndReboot() +""" + +import os +import re +import socket +import sys +import tempfile +import unittest +from StringIO import StringIO + +from mozdevice import DeviceManagerADB, DMError + + +def find_mount_permissions(dm, mount_path): + for mount_point in dm._runCmd(["shell", "mount"]).output: + if mount_point.find(mount_path) > 0: + return re.search('(ro|rw)(?=,)', mount_point).group(0) + + +class DeviceManagerADBTestCase(unittest.TestCase): + tempLocalDir = "tempDir" + tempLocalFile = os.path.join(tempLocalDir, "tempfile.txt") + tempRemoteDir = None + tempRemoteFile = None + tempRemoteSystemFile = None + + def setUp(self): + self.assertTrue(find_mount_permissions(self.dm, "/system"), "ro") + + self.assertTrue(os.path.exists(self.tempLocalDir)) + self.assertTrue(os.path.exists(self.tempLocalFile)) + + if self.dm.fileExists(self.tempRemoteFile): + self.dm.removeFile(self.tempRemoteFile) + self.assertFalse(self.dm.fileExists(self.tempRemoteFile)) + + if self.dm.fileExists(self.tempRemoteSystemFile): + self.dm.removeFile(self.tempRemoteSystemFile) + + self.assertTrue(self.dm.dirExists(self.tempRemoteDir)) + + @classmethod + def setUpClass(self): + self.dm = DeviceManagerADB() + if not os.path.exists(self.tempLocalDir): + os.mkdir(self.tempLocalDir) + if not os.path.exists(self.tempLocalFile): + # Create empty file + open(self.tempLocalFile, 'w').close() + self.tempRemoteDir = self.dm.getTempDir() + self.tempRemoteFile = os.path.join(self.tempRemoteDir, + os.path.basename(self.tempLocalFile)) + self.tempRemoteSystemFile = \ + os.path.join("/system", os.path.basename(self.tempLocalFile)) + + @classmethod + def tearDownClass(self): + os.remove(self.tempLocalFile) + os.rmdir(self.tempLocalDir) + if self.dm.dirExists(self.tempRemoteDir): + # self.tempRemoteFile will get deleted with it + self.dm.removeDir(self.tempRemoteDir) + if self.dm.fileExists(self.tempRemoteSystemFile): + self.dm.removeFile(self.tempRemoteSystemFile) + + +class TestFileOperations(DeviceManagerADBTestCase): + + def test_make_and_remove_directory(self): + dir1 = os.path.join(self.tempRemoteDir, "dir1") + self.assertFalse(self.dm.dirExists(dir1)) + self.dm.mkDir(dir1) + self.assertTrue(self.dm.dirExists(dir1)) + self.dm.removeDir(dir1) + self.assertFalse(self.dm.dirExists(dir1)) + + def test_push_and_remove_file(self): + self.dm.pushFile(self.tempLocalFile, self.tempRemoteFile) + self.assertTrue(self.dm.fileExists(self.tempRemoteFile)) + self.dm.removeFile(self.tempRemoteFile) + self.assertFalse(self.dm.fileExists(self.tempRemoteFile)) + + def test_push_and_pull_file(self): + self.dm.pushFile(self.tempLocalFile, self.tempRemoteFile) + self.assertTrue(self.dm.fileExists(self.tempRemoteFile)) + self.assertFalse(os.path.exists("pulled.txt")) + self.dm.getFile(self.tempRemoteFile, "pulled.txt") + self.assertTrue(os.path.exists("pulled.txt")) + os.remove("pulled.txt") + + def test_push_and_pull_directory_and_list_files(self): + self.dm.removeDir(self.tempRemoteDir) + self.assertFalse(self.dm.dirExists(self.tempRemoteDir)) + self.dm.pushDir(self.tempLocalDir, self.tempRemoteDir) + self.assertTrue(self.dm.dirExists(self.tempRemoteDir)) + response = self.dm.listFiles(self.tempRemoteDir) + # The local dir that was pushed contains the tempLocalFile + self.assertIn(os.path.basename(self.tempLocalFile), response) + # Create a temp dir to pull to + temp_dir = tempfile.mkdtemp() + self.assertTrue(os.path.exists(temp_dir)) + self.dm.getDirectory(self.tempRemoteDir, temp_dir) + self.assertTrue(os.path.exists(self.tempLocalFile)) + + def test_move_and_remove_directories(self): + dir1 = os.path.join(self.tempRemoteDir, "dir1") + dir2 = os.path.join(self.tempRemoteDir, "dir2") + + self.assertFalse(self.dm.dirExists(dir1)) + self.dm.mkDir(dir1) + self.assertTrue(self.dm.dirExists(dir1)) + + self.assertFalse(self.dm.dirExists(dir2)) + self.dm.moveTree(dir1, dir2) + self.assertTrue(self.dm.dirExists(dir2)) + + self.dm.removeDir(dir1) + self.dm.removeDir(dir2) + self.assertFalse(self.dm.dirExists(dir1)) + self.assertFalse(self.dm.dirExists(dir2)) + + def test_push_and_remove_system_file(self): + out = StringIO() + self.assertTrue(find_mount_permissions(self.dm, "/system") == "ro") + self.assertFalse(self.dm.fileExists(self.tempRemoteSystemFile)) + self.assertRaises(DMError, self.dm.pushFile, self.tempLocalFile, self.tempRemoteSystemFile) + self.dm.shell(['mount', '-w', '-o', 'remount', '/system'], out) + self.assertTrue(find_mount_permissions(self.dm, "/system") == "rw") + self.assertFalse(self.dm.fileExists(self.tempRemoteSystemFile)) + self.dm.pushFile(self.tempLocalFile, self.tempRemoteSystemFile) + self.assertTrue(self.dm.fileExists(self.tempRemoteSystemFile)) + self.dm.removeFile(self.tempRemoteSystemFile) + self.assertFalse(self.dm.fileExists(self.tempRemoteSystemFile)) + self.dm.shell(['mount', '-r', '-o', 'remount', '/system'], out) + out.close() + self.assertTrue(find_mount_permissions(self.dm, "/system") == "ro") + + +class TestOther(DeviceManagerADBTestCase): + + def test_get_list_of_processes(self): + self.assertEquals(type(self.dm.getProcessList()), list) + + def test_get_current_time(self): + self.assertEquals(type(self.dm.getCurrentTime()), int) + + def test_get_info(self): + self.assertEquals(type(self.dm.getInfo()), dict) + + def test_list_devices(self): + self.assertEquals(len(list(self.dm.devices())), 1) + + def test_shell(self): + out = StringIO() + self.dm.shell(["echo", "$COMPANY", ";", "pwd"], out, + env={"COMPANY": "Mozilla"}, cwd="/", timeout=4, root=True) + output = str(out.getvalue()).rstrip().splitlines() + out.close() + self.assertEquals(output, ['Mozilla', '/']) + + def test_port_forwarding(self): + s = socket.socket(socket.AF_INET, socket.SOCK_STREAM) + s.bind(("", 0)) + port = s.getsockname()[1] + s.close() + # If successful then no exception is raised + self.dm.forward("tcp:%s" % port, "tcp:2828") + + def test_port_forwarding_error(self): + self.assertRaises(DMError, self.dm.forward, "", "") + + +if __name__ == '__main__': + dm = DeviceManagerADB() + if not dm.devices(): + print "There are no connected adb devices" + sys.exit(1) + + if find_mount_permissions(dm, "/system") == "rw": + print "We've found out that /system is mounted as 'rw'. This is because the command " \ + "'adb remount' has been run before running this test case. Please reboot the device " \ + "and try again." + sys.exit(1) + + unittest.main() diff --git a/testing/mozbase/mozdevice/mozdevice/Zeroconf.py b/testing/mozbase/mozdevice/mozdevice/Zeroconf.py new file mode 100755 index 000000000..54a5d5359 --- /dev/null +++ b/testing/mozbase/mozdevice/mozdevice/Zeroconf.py @@ -0,0 +1,1560 @@ +""" Multicast DNS Service Discovery for Python, v0.12 + Copyright (C) 2003, Paul Scott-Murphy + + This module provides a framework for the use of DNS Service Discovery + using IP multicast. It has been tested against the JRendezvous + implementation from StrangeBerry, + and against the mDNSResponder from Mac OS X 10.3.8. + + This library is free software; you can redistribute it and/or + modify it under the terms of the GNU Lesser General Public + License as published by the Free Software Foundation; either + version 2.1 of the License, or (at your option) any later version. + + This library is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + Lesser General Public License for more details. + + You should have received a copy of the GNU Lesser General Public + License along with this library; if not, write to the Free Software + Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA + +""" + +"""0.12 update - allow selection of binding interface + typo fix - Thanks A. M. Kuchlingi + removed all use of word 'Rendezvous' - this is an API change""" + +"""0.11 update - correction to comments for addListener method + support for new record types seen from OS X + - IPv6 address + - hostinfo + ignore unknown DNS record types + fixes to name decoding + works alongside other processes using port 5353 (e.g. on Mac OS X) + tested against Mac OS X 10.3.2's mDNSResponder + corrections to removal of list entries for service browser""" + +"""0.10 update - Jonathon Paisley contributed these corrections: + always multicast replies, even when query is unicast + correct a pointer encoding problem + can now write records in any order + traceback shown on failure + better TXT record parsing + server is now separate from name + can cancel a service browser + + modified some unit tests to accommodate these changes""" + +"""0.09 update - remove all records on service unregistration + fix DOS security problem with readName""" + +"""0.08 update - changed licensing to LGPL""" + +"""0.07 update - faster shutdown on engine + pointer encoding of outgoing names + ServiceBrowser now works + new unit tests""" + +"""0.06 update - small improvements with unit tests + added defined exception types + new style objects + fixed hostname/interface problem + fixed socket timeout problem + fixed addServiceListener() typo bug + using select() for socket reads + tested on Debian unstable with Python 2.2.2""" + +"""0.05 update - ensure case insensitivty on domain names + support for unicast DNS queries""" + +"""0.04 update - added some unit tests + added __ne__ adjuncts where required + ensure names end in '.local.' + timeout on receiving socket for clean shutdown""" + +__author__ = "Paul Scott-Murphy" +__email__ = "paul at scott dash murphy dot com" +__version__ = "0.12" + +import string +import time +import struct +import socket +import threading +import select +import traceback + +__all__ = ["Zeroconf", "ServiceInfo", "ServiceBrowser"] + +# hook for threads + +globals()['_GLOBAL_DONE'] = 0 + +# Some timing constants + +_UNREGISTER_TIME = 125 +_CHECK_TIME = 175 +_REGISTER_TIME = 225 +_LISTENER_TIME = 200 +_BROWSER_TIME = 500 + +# Some DNS constants + +_MDNS_ADDR = '224.0.0.251' +_MDNS_PORT = 5353; +_DNS_PORT = 53; +_DNS_TTL = 60 * 60; # one hour default TTL + +_MAX_MSG_TYPICAL = 1460 # unused +_MAX_MSG_ABSOLUTE = 8972 + +_FLAGS_QR_MASK = 0x8000 # query response mask +_FLAGS_QR_QUERY = 0x0000 # query +_FLAGS_QR_RESPONSE = 0x8000 # response + +_FLAGS_AA = 0x0400 # Authorative answer +_FLAGS_TC = 0x0200 # Truncated +_FLAGS_RD = 0x0100 # Recursion desired +_FLAGS_RA = 0x8000 # Recursion available + +_FLAGS_Z = 0x0040 # Zero +_FLAGS_AD = 0x0020 # Authentic data +_FLAGS_CD = 0x0010 # Checking disabled + +_CLASS_IN = 1 +_CLASS_CS = 2 +_CLASS_CH = 3 +_CLASS_HS = 4 +_CLASS_NONE = 254 +_CLASS_ANY = 255 +_CLASS_MASK = 0x7FFF +_CLASS_UNIQUE = 0x8000 + +_TYPE_A = 1 +_TYPE_NS = 2 +_TYPE_MD = 3 +_TYPE_MF = 4 +_TYPE_CNAME = 5 +_TYPE_SOA = 6 +_TYPE_MB = 7 +_TYPE_MG = 8 +_TYPE_MR = 9 +_TYPE_NULL = 10 +_TYPE_WKS = 11 +_TYPE_PTR = 12 +_TYPE_HINFO = 13 +_TYPE_MINFO = 14 +_TYPE_MX = 15 +_TYPE_TXT = 16 +_TYPE_AAAA = 28 +_TYPE_SRV = 33 +_TYPE_ANY = 255 + +# Mapping constants to names + +_CLASSES = { _CLASS_IN : "in", + _CLASS_CS : "cs", + _CLASS_CH : "ch", + _CLASS_HS : "hs", + _CLASS_NONE : "none", + _CLASS_ANY : "any" } + +_TYPES = { _TYPE_A : "a", + _TYPE_NS : "ns", + _TYPE_MD : "md", + _TYPE_MF : "mf", + _TYPE_CNAME : "cname", + _TYPE_SOA : "soa", + _TYPE_MB : "mb", + _TYPE_MG : "mg", + _TYPE_MR : "mr", + _TYPE_NULL : "null", + _TYPE_WKS : "wks", + _TYPE_PTR : "ptr", + _TYPE_HINFO : "hinfo", + _TYPE_MINFO : "minfo", + _TYPE_MX : "mx", + _TYPE_TXT : "txt", + _TYPE_AAAA : "quada", + _TYPE_SRV : "srv", + _TYPE_ANY : "any" } + +# utility functions + +def currentTimeMillis(): + """Current system time in milliseconds""" + return time.time() * 1000 + +# Exceptions + +class NonLocalNameException(Exception): + pass + +class NonUniqueNameException(Exception): + pass + +class NamePartTooLongException(Exception): + pass + +class AbstractMethodException(Exception): + pass + +class BadTypeInNameException(Exception): + pass + +# implementation classes + +class DNSEntry(object): + """A DNS entry""" + + def __init__(self, name, type, clazz): + self.key = string.lower(name) + self.name = name + self.type = type + self.clazz = clazz & _CLASS_MASK + self.unique = (clazz & _CLASS_UNIQUE) != 0 + + def __eq__(self, other): + """Equality test on name, type, and class""" + if isinstance(other, DNSEntry): + return self.name == other.name and self.type == other.type and self.clazz == other.clazz + return 0 + + def __ne__(self, other): + """Non-equality test""" + return not self.__eq__(other) + + def getClazz(self, clazz): + """Class accessor""" + try: + return _CLASSES[clazz] + except: + return "?(%s)" % (clazz) + + def getType(self, type): + """Type accessor""" + try: + return _TYPES[type] + except: + return "?(%s)" % (type) + + def toString(self, hdr, other): + """String representation with additional information""" + result = "%s[%s,%s" % (hdr, self.getType(self.type), self.getClazz(self.clazz)) + if self.unique: + result += "-unique," + else: + result += "," + result += self.name + if other is not None: + result += ",%s]" % (other) + else: + result += "]" + return result + +class DNSQuestion(DNSEntry): + """A DNS question entry""" + + def __init__(self, name, type, clazz): + if not name.endswith(".local."): + raise NonLocalNameException + DNSEntry.__init__(self, name, type, clazz) + + def answeredBy(self, rec): + """Returns true if the question is answered by the record""" + return self.clazz == rec.clazz and (self.type == rec.type or self.type == _TYPE_ANY) and self.name == rec.name + + def __repr__(self): + """String representation""" + return DNSEntry.toString(self, "question", None) + + +class DNSRecord(DNSEntry): + """A DNS record - like a DNS entry, but has a TTL""" + + def __init__(self, name, type, clazz, ttl): + DNSEntry.__init__(self, name, type, clazz) + self.ttl = ttl + self.created = currentTimeMillis() + + def __eq__(self, other): + """Tests equality as per DNSRecord""" + if isinstance(other, DNSRecord): + return DNSEntry.__eq__(self, other) + return 0 + + def suppressedBy(self, msg): + """Returns true if any answer in a message can suffice for the + information held in this record.""" + for record in msg.answers: + if self.suppressedByAnswer(record): + return 1 + return 0 + + def suppressedByAnswer(self, other): + """Returns true if another record has same name, type and class, + and if its TTL is at least half of this record's.""" + if self == other and other.ttl > (self.ttl / 2): + return 1 + return 0 + + def getExpirationTime(self, percent): + """Returns the time at which this record will have expired + by a certain percentage.""" + return self.created + (percent * self.ttl * 10) + + def getRemainingTTL(self, now): + """Returns the remaining TTL in seconds.""" + return max(0, (self.getExpirationTime(100) - now) / 1000) + + def isExpired(self, now): + """Returns true if this record has expired.""" + return self.getExpirationTime(100) <= now + + def isStale(self, now): + """Returns true if this record is at least half way expired.""" + return self.getExpirationTime(50) <= now + + def resetTTL(self, other): + """Sets this record's TTL and created time to that of + another record.""" + self.created = other.created + self.ttl = other.ttl + + def write(self, out): + """Abstract method""" + raise AbstractMethodException + + def toString(self, other): + """String representation with addtional information""" + arg = "%s/%s,%s" % (self.ttl, self.getRemainingTTL(currentTimeMillis()), other) + return DNSEntry.toString(self, "record", arg) + +class DNSAddress(DNSRecord): + """A DNS address record""" + + def __init__(self, name, type, clazz, ttl, address): + DNSRecord.__init__(self, name, type, clazz, ttl) + self.address = address + + def write(self, out): + """Used in constructing an outgoing packet""" + out.writeString(self.address, len(self.address)) + + def __eq__(self, other): + """Tests equality on address""" + if isinstance(other, DNSAddress): + return self.address == other.address + return 0 + + def __repr__(self): + """String representation""" + try: + return socket.inet_ntoa(self.address) + except: + return self.address + +class DNSHinfo(DNSRecord): + """A DNS host information record""" + + def __init__(self, name, type, clazz, ttl, cpu, os): + DNSRecord.__init__(self, name, type, clazz, ttl) + self.cpu = cpu + self.os = os + + def write(self, out): + """Used in constructing an outgoing packet""" + out.writeString(self.cpu, len(self.cpu)) + out.writeString(self.os, len(self.os)) + + def __eq__(self, other): + """Tests equality on cpu and os""" + if isinstance(other, DNSHinfo): + return self.cpu == other.cpu and self.os == other.os + return 0 + + def __repr__(self): + """String representation""" + return self.cpu + " " + self.os + +class DNSPointer(DNSRecord): + """A DNS pointer record""" + + def __init__(self, name, type, clazz, ttl, alias): + DNSRecord.__init__(self, name, type, clazz, ttl) + self.alias = alias + + def write(self, out): + """Used in constructing an outgoing packet""" + out.writeName(self.alias) + + def __eq__(self, other): + """Tests equality on alias""" + if isinstance(other, DNSPointer): + return self.alias == other.alias + return 0 + + def __repr__(self): + """String representation""" + return self.toString(self.alias) + +class DNSText(DNSRecord): + """A DNS text record""" + + def __init__(self, name, type, clazz, ttl, text): + DNSRecord.__init__(self, name, type, clazz, ttl) + self.text = text + + def write(self, out): + """Used in constructing an outgoing packet""" + out.writeString(self.text, len(self.text)) + + def __eq__(self, other): + """Tests equality on text""" + if isinstance(other, DNSText): + return self.text == other.text + return 0 + + def __repr__(self): + """String representation""" + if len(self.text) > 10: + return self.toString(self.text[:7] + "...") + else: + return self.toString(self.text) + +class DNSService(DNSRecord): + """A DNS service record""" + + def __init__(self, name, type, clazz, ttl, priority, weight, port, server): + DNSRecord.__init__(self, name, type, clazz, ttl) + self.priority = priority + self.weight = weight + self.port = port + self.server = server + + def write(self, out): + """Used in constructing an outgoing packet""" + out.writeShort(self.priority) + out.writeShort(self.weight) + out.writeShort(self.port) + out.writeName(self.server) + + def __eq__(self, other): + """Tests equality on priority, weight, port and server""" + if isinstance(other, DNSService): + return self.priority == other.priority and self.weight == other.weight and self.port == other.port and self.server == other.server + return 0 + + def __repr__(self): + """String representation""" + return self.toString("%s:%s" % (self.server, self.port)) + +class DNSIncoming(object): + """Object representation of an incoming DNS packet""" + + def __init__(self, data): + """Constructor from string holding bytes of packet""" + self.offset = 0 + self.data = data + self.questions = [] + self.answers = [] + self.numQuestions = 0 + self.numAnswers = 0 + self.numAuthorities = 0 + self.numAdditionals = 0 + + self.readHeader() + self.readQuestions() + self.readOthers() + + def readHeader(self): + """Reads header portion of packet""" + format = '!HHHHHH' + length = struct.calcsize(format) + info = struct.unpack(format, self.data[self.offset:self.offset+length]) + self.offset += length + + self.id = info[0] + self.flags = info[1] + self.numQuestions = info[2] + self.numAnswers = info[3] + self.numAuthorities = info[4] + self.numAdditionals = info[5] + + def readQuestions(self): + """Reads questions section of packet""" + format = '!HH' + length = struct.calcsize(format) + for i in range(0, self.numQuestions): + name = self.readName() + info = struct.unpack(format, self.data[self.offset:self.offset+length]) + self.offset += length + + question = DNSQuestion(name, info[0], info[1]) + self.questions.append(question) + + def readInt(self): + """Reads an integer from the packet""" + format = '!I' + length = struct.calcsize(format) + info = struct.unpack(format, self.data[self.offset:self.offset+length]) + self.offset += length + return info[0] + + def readCharacterString(self): + """Reads a character string from the packet""" + length = ord(self.data[self.offset]) + self.offset += 1 + return self.readString(length) + + def readString(self, len): + """Reads a string of a given length from the packet""" + format = '!' + str(len) + 's' + length = struct.calcsize(format) + info = struct.unpack(format, self.data[self.offset:self.offset+length]) + self.offset += length + return info[0] + + def readUnsignedShort(self): + """Reads an unsigned short from the packet""" + format = '!H' + length = struct.calcsize(format) + info = struct.unpack(format, self.data[self.offset:self.offset+length]) + self.offset += length + return info[0] + + def readOthers(self): + """Reads the answers, authorities and additionals section of the packet""" + format = '!HHiH' + length = struct.calcsize(format) + n = self.numAnswers + self.numAuthorities + self.numAdditionals + for i in range(0, n): + domain = self.readName() + info = struct.unpack(format, self.data[self.offset:self.offset+length]) + self.offset += length + + rec = None + if info[0] == _TYPE_A: + rec = DNSAddress(domain, info[0], info[1], info[2], self.readString(4)) + elif info[0] == _TYPE_CNAME or info[0] == _TYPE_PTR: + rec = DNSPointer(domain, info[0], info[1], info[2], self.readName()) + elif info[0] == _TYPE_TXT: + rec = DNSText(domain, info[0], info[1], info[2], self.readString(info[3])) + elif info[0] == _TYPE_SRV: + rec = DNSService(domain, info[0], info[1], info[2], self.readUnsignedShort(), self.readUnsignedShort(), self.readUnsignedShort(), self.readName()) + elif info[0] == _TYPE_HINFO: + rec = DNSHinfo(domain, info[0], info[1], info[2], self.readCharacterString(), self.readCharacterString()) + elif info[0] == _TYPE_AAAA: + rec = DNSAddress(domain, info[0], info[1], info[2], self.readString(16)) + else: + # Try to ignore types we don't know about + # this may mean the rest of the name is + # unable to be parsed, and may show errors + # so this is left for debugging. New types + # encountered need to be parsed properly. + # + #print "UNKNOWN TYPE = " + str(info[0]) + #raise BadTypeInNameException + pass + + if rec is not None: + self.answers.append(rec) + + def isQuery(self): + """Returns true if this is a query""" + return (self.flags & _FLAGS_QR_MASK) == _FLAGS_QR_QUERY + + def isResponse(self): + """Returns true if this is a response""" + return (self.flags & _FLAGS_QR_MASK) == _FLAGS_QR_RESPONSE + + def readUTF(self, offset, len): + """Reads a UTF-8 string of a given length from the packet""" + result = self.data[offset:offset+len].decode('utf-8') + return result + + def readName(self): + """Reads a domain name from the packet""" + result = '' + off = self.offset + next = -1 + first = off + + while 1: + len = ord(self.data[off]) + off += 1 + if len == 0: + break + t = len & 0xC0 + if t == 0x00: + result = ''.join((result, self.readUTF(off, len) + '.')) + off += len + elif t == 0xC0: + if next < 0: + next = off + 1 + off = ((len & 0x3F) << 8) | ord(self.data[off]) + if off >= first: + raise "Bad domain name (circular) at " + str(off) + first = off + else: + raise "Bad domain name at " + str(off) + + if next >= 0: + self.offset = next + else: + self.offset = off + + return result + + +class DNSOutgoing(object): + """Object representation of an outgoing packet""" + + def __init__(self, flags, multicast = 1): + self.finished = 0 + self.id = 0 + self.multicast = multicast + self.flags = flags + self.names = {} + self.data = [] + self.size = 12 + + self.questions = [] + self.answers = [] + self.authorities = [] + self.additionals = [] + + def addQuestion(self, record): + """Adds a question""" + self.questions.append(record) + + def addAnswer(self, inp, record): + """Adds an answer""" + if not record.suppressedBy(inp): + self.addAnswerAtTime(record, 0) + + def addAnswerAtTime(self, record, now): + """Adds an answer if if does not expire by a certain time""" + if record is not None: + if now == 0 or not record.isExpired(now): + self.answers.append((record, now)) + + def addAuthorativeAnswer(self, record): + """Adds an authoritative answer""" + self.authorities.append(record) + + def addAdditionalAnswer(self, record): + """Adds an additional answer""" + self.additionals.append(record) + + def writeByte(self, value): + """Writes a single byte to the packet""" + format = '!c' + self.data.append(struct.pack(format, chr(value))) + self.size += 1 + + def insertShort(self, index, value): + """Inserts an unsigned short in a certain position in the packet""" + format = '!H' + self.data.insert(index, struct.pack(format, value)) + self.size += 2 + + def writeShort(self, value): + """Writes an unsigned short to the packet""" + format = '!H' + self.data.append(struct.pack(format, value)) + self.size += 2 + + def writeInt(self, value): + """Writes an unsigned integer to the packet""" + format = '!I' + self.data.append(struct.pack(format, value)) + self.size += 4 + + def writeString(self, value, length): + """Writes a string to the packet""" + format = '!' + str(length) + 's' + self.data.append(struct.pack(format, value)) + self.size += length + + def writeUTF(self, s): + """Writes a UTF-8 string of a given length to the packet""" + utfstr = s.encode('utf-8') + length = len(utfstr) + if length > 64: + raise NamePartTooLongException + self.writeByte(length) + self.writeString(utfstr, length) + + def writeName(self, name): + """Writes a domain name to the packet""" + + try: + # Find existing instance of this name in packet + # + index = self.names[name] + except KeyError: + # No record of this name already, so write it + # out as normal, recording the location of the name + # for future pointers to it. + # + self.names[name] = self.size + parts = name.split('.') + if parts[-1] == '': + parts = parts[:-1] + for part in parts: + self.writeUTF(part) + self.writeByte(0) + return + + # An index was found, so write a pointer to it + # + self.writeByte((index >> 8) | 0xC0) + self.writeByte(index) + + def writeQuestion(self, question): + """Writes a question to the packet""" + self.writeName(question.name) + self.writeShort(question.type) + self.writeShort(question.clazz) + + def writeRecord(self, record, now): + """Writes a record (answer, authoritative answer, additional) to + the packet""" + self.writeName(record.name) + self.writeShort(record.type) + if record.unique and self.multicast: + self.writeShort(record.clazz | _CLASS_UNIQUE) + else: + self.writeShort(record.clazz) + if now == 0: + self.writeInt(record.ttl) + else: + self.writeInt(record.getRemainingTTL(now)) + index = len(self.data) + # Adjust size for the short we will write before this record + # + self.size += 2 + record.write(self) + self.size -= 2 + + length = len(''.join(self.data[index:])) + self.insertShort(index, length) # Here is the short we adjusted for + + def packet(self): + """Returns a string containing the packet's bytes + + No further parts should be added to the packet once this + is done.""" + if not self.finished: + self.finished = 1 + for question in self.questions: + self.writeQuestion(question) + for answer, time in self.answers: + self.writeRecord(answer, time) + for authority in self.authorities: + self.writeRecord(authority, 0) + for additional in self.additionals: + self.writeRecord(additional, 0) + + self.insertShort(0, len(self.additionals)) + self.insertShort(0, len(self.authorities)) + self.insertShort(0, len(self.answers)) + self.insertShort(0, len(self.questions)) + self.insertShort(0, self.flags) + if self.multicast: + self.insertShort(0, 0) + else: + self.insertShort(0, self.id) + return ''.join(self.data) + + +class DNSCache(object): + """A cache of DNS entries""" + + def __init__(self): + self.cache = {} + + def add(self, entry): + """Adds an entry""" + try: + list = self.cache[entry.key] + except: + list = self.cache[entry.key] = [] + list.append(entry) + + def remove(self, entry): + """Removes an entry""" + try: + list = self.cache[entry.key] + list.remove(entry) + except: + pass + + def get(self, entry): + """Gets an entry by key. Will return None if there is no + matching entry.""" + try: + list = self.cache[entry.key] + return list[list.index(entry)] + except: + return None + + def getByDetails(self, name, type, clazz): + """Gets an entry by details. Will return None if there is + no matching entry.""" + entry = DNSEntry(name, type, clazz) + return self.get(entry) + + def entriesWithName(self, name): + """Returns a list of entries whose key matches the name.""" + try: + return self.cache[name] + except: + return [] + + def entries(self): + """Returns a list of all entries""" + def add(x, y): return x+y + try: + return reduce(add, self.cache.values()) + except: + return [] + + +class Engine(threading.Thread): + """An engine wraps read access to sockets, allowing objects that + need to receive data from sockets to be called back when the + sockets are ready. + + A reader needs a handle_read() method, which is called when the socket + it is interested in is ready for reading. + + Writers are not implemented here, because we only send short + packets. + """ + + def __init__(self, zeroconf): + threading.Thread.__init__(self) + self.zeroconf = zeroconf + self.readers = {} # maps socket to reader + self.timeout = 5 + self.condition = threading.Condition() + self.daemon = True + self.start() + + def run(self): + while not globals()['_GLOBAL_DONE']: + rs = self.getReaders() + if len(rs) == 0: + # No sockets to manage, but we wait for the timeout + # or addition of a socket + # + self.condition.acquire() + self.condition.wait(self.timeout) + self.condition.release() + else: + try: + rr, wr, er = select.select(rs, [], [], self.timeout) + for socket in rr: + try: + self.readers[socket].handle_read() + except: + # Ignore errors that occur on shutdown + pass + except: + pass + + def getReaders(self): + result = [] + self.condition.acquire() + result = self.readers.keys() + self.condition.release() + return result + + def addReader(self, reader, socket): + self.condition.acquire() + self.readers[socket] = reader + self.condition.notify() + self.condition.release() + + def delReader(self, socket): + self.condition.acquire() + del(self.readers[socket]) + self.condition.notify() + self.condition.release() + + def notify(self): + self.condition.acquire() + self.condition.notify() + self.condition.release() + +class Listener(object): + """A Listener is used by this module to listen on the multicast + group to which DNS messages are sent, allowing the implementation + to cache information as it arrives. + + It requires registration with an Engine object in order to have + the read() method called when a socket is availble for reading.""" + + def __init__(self, zeroconf): + self.zeroconf = zeroconf + self.zeroconf.engine.addReader(self, self.zeroconf.socket) + + def handle_read(self): + data, (addr, port) = self.zeroconf.socket.recvfrom(_MAX_MSG_ABSOLUTE) + self.data = data + msg = DNSIncoming(data) + if msg.isQuery(): + # Always multicast responses + # + if port == _MDNS_PORT: + self.zeroconf.handleQuery(msg, _MDNS_ADDR, _MDNS_PORT) + # If it's not a multicast query, reply via unicast + # and multicast + # + elif port == _DNS_PORT: + self.zeroconf.handleQuery(msg, addr, port) + self.zeroconf.handleQuery(msg, _MDNS_ADDR, _MDNS_PORT) + else: + self.zeroconf.handleResponse(msg) + + +class Reaper(threading.Thread): + """A Reaper is used by this module to remove cache entries that + have expired.""" + + def __init__(self, zeroconf): + threading.Thread.__init__(self) + self.daemon = True + self.zeroconf = zeroconf + self.start() + + def run(self): + while 1: + self.zeroconf.wait(10 * 1000) + if globals()['_GLOBAL_DONE']: + return + now = currentTimeMillis() + for record in self.zeroconf.cache.entries(): + if record.isExpired(now): + self.zeroconf.updateRecord(now, record) + self.zeroconf.cache.remove(record) + + +class ServiceBrowser(threading.Thread): + """Used to browse for a service of a specific type. + + The listener object will have its addService() and + removeService() methods called when this browser + discovers changes in the services availability.""" + + def __init__(self, zeroconf, type, listener): + """Creates a browser for a specific type""" + threading.Thread.__init__(self) + self.zeroconf = zeroconf + self.type = type + self.listener = listener + self.services = {} + self.nextTime = currentTimeMillis() + self.delay = _BROWSER_TIME + self.list = [] + self.daemon = True + + self.done = 0 + + self.zeroconf.addListener(self, DNSQuestion(self.type, _TYPE_PTR, _CLASS_IN)) + self.start() + + def updateRecord(self, zeroconf, now, record): + """Callback invoked by Zeroconf when new information arrives. + + Updates information required by browser in the Zeroconf cache.""" + if record.type == _TYPE_PTR and record.name == self.type: + expired = record.isExpired(now) + try: + oldrecord = self.services[record.alias.lower()] + if not expired: + oldrecord.resetTTL(record) + else: + del(self.services[record.alias.lower()]) + callback = lambda x: self.listener.removeService(x, self.type, record.alias) + self.list.append(callback) + return + except: + if not expired: + self.services[record.alias.lower()] = record + callback = lambda x: self.listener.addService(x, self.type, record.alias) + self.list.append(callback) + + expires = record.getExpirationTime(75) + if expires < self.nextTime: + self.nextTime = expires + + def cancel(self): + self.done = 1 + self.zeroconf.notifyAll() + + def run(self): + while 1: + event = None + now = currentTimeMillis() + if len(self.list) == 0 and self.nextTime > now: + self.zeroconf.wait(self.nextTime - now) + if globals()['_GLOBAL_DONE'] or self.done: + return + now = currentTimeMillis() + + if self.nextTime <= now: + out = DNSOutgoing(_FLAGS_QR_QUERY) + out.addQuestion(DNSQuestion(self.type, _TYPE_PTR, _CLASS_IN)) + for record in self.services.values(): + if not record.isExpired(now): + out.addAnswerAtTime(record, now) + self.zeroconf.send(out) + self.nextTime = now + self.delay + self.delay = min(20 * 1000, self.delay * 2) + + if len(self.list) > 0: + event = self.list.pop(0) + + if event is not None: + event(self.zeroconf) + + +class ServiceInfo(object): + """Service information""" + + def __init__(self, type, name, address=None, port=None, weight=0, priority=0, properties=None, server=None): + """Create a service description. + + type: fully qualified service type name + name: fully qualified service name + address: IP address as unsigned short, network byte order + port: port that the service runs on + weight: weight of the service + priority: priority of the service + properties: dictionary of properties (or a string holding the bytes for the text field) + server: fully qualified name for service host (defaults to name)""" + + if not name.endswith(type): + raise BadTypeInNameException + self.type = type + self.name = name + self.address = address + self.port = port + self.weight = weight + self.priority = priority + if server: + self.server = server + else: + self.server = name + self.setProperties(properties) + + def setProperties(self, properties): + """Sets properties and text of this info from a dictionary""" + if isinstance(properties, dict): + self.properties = properties + list = [] + result = '' + for key in properties: + value = properties[key] + if value is None: + suffix = ''.encode('utf-8') + elif isinstance(value, str): + suffix = value.encode('utf-8') + elif isinstance(value, int): + if value: + suffix = 'true' + else: + suffix = 'false' + else: + suffix = ''.encode('utf-8') + list.append('='.join((key, suffix))) + for item in list: + result = ''.join((result, struct.pack('!c', chr(len(item))), item)) + self.text = result + else: + self.text = properties + + def setText(self, text): + """Sets properties and text given a text field""" + self.text = text + try: + result = {} + end = len(text) + index = 0 + strs = [] + while index < end: + length = ord(text[index]) + index += 1 + strs.append(text[index:index+length]) + index += length + + for s in strs: + eindex = s.find('=') + if eindex == -1: + # No equals sign at all + key = s + value = 0 + else: + key = s[:eindex] + value = s[eindex+1:] + if value == 'true': + value = 1 + elif value == 'false' or not value: + value = 0 + + # Only update non-existent properties + if key and result.get(key) == None: + result[key] = value + + self.properties = result + except: + traceback.print_exc() + self.properties = None + + def getType(self): + """Type accessor""" + return self.type + + def getName(self): + """Name accessor""" + if self.type is not None and self.name.endswith("." + self.type): + return self.name[:len(self.name) - len(self.type) - 1] + return self.name + + def getAddress(self): + """Address accessor""" + return self.address + + def getPort(self): + """Port accessor""" + return self.port + + def getPriority(self): + """Pirority accessor""" + return self.priority + + def getWeight(self): + """Weight accessor""" + return self.weight + + def getProperties(self): + """Properties accessor""" + return self.properties + + def getText(self): + """Text accessor""" + return self.text + + def getServer(self): + """Server accessor""" + return self.server + + def updateRecord(self, zeroconf, now, record): + """Updates service information from a DNS record""" + if record is not None and not record.isExpired(now): + if record.type == _TYPE_A: + if record.name == self.name: + self.address = record.address + elif record.type == _TYPE_SRV: + if record.name == self.name: + self.server = record.server + self.port = record.port + self.weight = record.weight + self.priority = record.priority + self.address = None + self.updateRecord(zeroconf, now, zeroconf.cache.getByDetails(self.server, _TYPE_A, _CLASS_IN)) + elif record.type == _TYPE_TXT: + if record.name == self.name: + self.setText(record.text) + + def request(self, zeroconf, timeout): + """Returns true if the service could be discovered on the + network, and updates this object with details discovered. + """ + now = currentTimeMillis() + delay = _LISTENER_TIME + next = now + delay + last = now + timeout + result = 0 + try: + zeroconf.addListener(self, DNSQuestion(self.name, _TYPE_ANY, _CLASS_IN)) + while self.server is None or self.address is None or self.text is None: + if last <= now: + return 0 + if next <= now: + out = DNSOutgoing(_FLAGS_QR_QUERY) + out.addQuestion(DNSQuestion(self.name, _TYPE_SRV, _CLASS_IN)) + out.addAnswerAtTime(zeroconf.cache.getByDetails(self.name, _TYPE_SRV, _CLASS_IN), now) + out.addQuestion(DNSQuestion(self.name, _TYPE_TXT, _CLASS_IN)) + out.addAnswerAtTime(zeroconf.cache.getByDetails(self.name, _TYPE_TXT, _CLASS_IN), now) + if self.server is not None: + out.addQuestion(DNSQuestion(self.server, _TYPE_A, _CLASS_IN)) + out.addAnswerAtTime(zeroconf.cache.getByDetails(self.server, _TYPE_A, _CLASS_IN), now) + zeroconf.send(out) + next = now + delay + delay = delay * 2 + + zeroconf.wait(min(next, last) - now) + now = currentTimeMillis() + result = 1 + finally: + zeroconf.removeListener(self) + + return result + + def __eq__(self, other): + """Tests equality of service name""" + if isinstance(other, ServiceInfo): + return other.name == self.name + return 0 + + def __ne__(self, other): + """Non-equality test""" + return not self.__eq__(other) + + def __repr__(self): + """String representation""" + result = "service[%s,%s:%s," % (self.name, socket.inet_ntoa(self.getAddress()), self.port) + if self.text is None: + result += "None" + else: + if len(self.text) < 20: + result += self.text + else: + result += self.text[:17] + "..." + result += "]" + return result + + +class Zeroconf(object): + """Implementation of Zeroconf Multicast DNS Service Discovery + + Supports registration, unregistration, queries and browsing. + """ + def __init__(self, bindaddress=None): + """Creates an instance of the Zeroconf class, establishing + multicast communications, listening and reaping threads.""" + globals()['_GLOBAL_DONE'] = 0 + if bindaddress is None: + self.intf = socket.gethostbyname(socket.gethostname()) + else: + self.intf = bindaddress + self.group = ('', _MDNS_PORT) + self.socket = socket.socket(socket.AF_INET, socket.SOCK_DGRAM) + try: + self.socket.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, 1) + self.socket.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEPORT, 1) + except: + # SO_REUSEADDR should be equivalent to SO_REUSEPORT for + # multicast UDP sockets (p 731, "TCP/IP Illustrated, + # Volume 2"), but some BSD-derived systems require + # SO_REUSEPORT to be specified explicity. Also, not all + # versions of Python have SO_REUSEPORT available. So + # if you're on a BSD-based system, and haven't upgraded + # to Python 2.3 yet, you may find this library doesn't + # work as expected. + # + pass + self.socket.setsockopt(socket.SOL_IP, socket.IP_MULTICAST_TTL, 255) + self.socket.setsockopt(socket.SOL_IP, socket.IP_MULTICAST_LOOP, 1) + try: + self.socket.bind(self.group) + except: + # Some versions of linux raise an exception even though + # the SO_REUSE* options have been set, so ignore it + # + pass + self.socket.setsockopt(socket.SOL_IP, socket.IP_MULTICAST_IF, socket.inet_aton(self.intf) + socket.inet_aton('0.0.0.0')) + self.socket.setsockopt(socket.SOL_IP, socket.IP_ADD_MEMBERSHIP, socket.inet_aton(_MDNS_ADDR) + socket.inet_aton('0.0.0.0')) + + self.listeners = [] + self.browsers = [] + self.services = {} + + self.cache = DNSCache() + + self.condition = threading.Condition() + + self.engine = Engine(self) + self.listener = Listener(self) + self.reaper = Reaper(self) + + def isLoopback(self): + return self.intf.startswith("127.0.0.1") + + def isLinklocal(self): + return self.intf.startswith("169.254.") + + def wait(self, timeout): + """Calling thread waits for a given number of milliseconds or + until notified.""" + self.condition.acquire() + self.condition.wait(timeout/1000) + self.condition.release() + + def notifyAll(self): + """Notifies all waiting threads""" + self.condition.acquire() + self.condition.notifyAll() + self.condition.release() + + def getServiceInfo(self, type, name, timeout=3000): + """Returns network's service information for a particular + name and type, or None if no service matches by the timeout, + which defaults to 3 seconds.""" + info = ServiceInfo(type, name) + if info.request(self, timeout): + return info + return None + + def addServiceListener(self, type, listener): + """Adds a listener for a particular service type. This object + will then have its updateRecord method called when information + arrives for that type.""" + self.removeServiceListener(listener) + self.browsers.append(ServiceBrowser(self, type, listener)) + + def removeServiceListener(self, listener): + """Removes a listener from the set that is currently listening.""" + for browser in self.browsers: + if browser.listener == listener: + browser.cancel() + del(browser) + + def registerService(self, info, ttl=_DNS_TTL): + """Registers service information to the network with a default TTL + of 60 seconds. Zeroconf will then respond to requests for + information for that service. The name of the service may be + changed if needed to make it unique on the network.""" + self.checkService(info) + self.services[info.name.lower()] = info + now = currentTimeMillis() + nextTime = now + i = 0 + while i < 3: + if now < nextTime: + self.wait(nextTime - now) + now = currentTimeMillis() + continue + out = DNSOutgoing(_FLAGS_QR_RESPONSE | _FLAGS_AA) + out.addAnswerAtTime(DNSPointer(info.type, _TYPE_PTR, _CLASS_IN, ttl, info.name), 0) + out.addAnswerAtTime(DNSService(info.name, _TYPE_SRV, _CLASS_IN, ttl, info.priority, info.weight, info.port, info.server), 0) + out.addAnswerAtTime(DNSText(info.name, _TYPE_TXT, _CLASS_IN, ttl, info.text), 0) + if info.address: + out.addAnswerAtTime(DNSAddress(info.server, _TYPE_A, _CLASS_IN, ttl, info.address), 0) + self.send(out) + i += 1 + nextTime += _REGISTER_TIME + + def unregisterService(self, info): + """Unregister a service.""" + try: + del(self.services[info.name.lower()]) + except: + pass + now = currentTimeMillis() + nextTime = now + i = 0 + while i < 3: + if now < nextTime: + self.wait(nextTime - now) + now = currentTimeMillis() + continue + out = DNSOutgoing(_FLAGS_QR_RESPONSE | _FLAGS_AA) + out.addAnswerAtTime(DNSPointer(info.type, _TYPE_PTR, _CLASS_IN, 0, info.name), 0) + out.addAnswerAtTime(DNSService(info.name, _TYPE_SRV, _CLASS_IN, 0, info.priority, info.weight, info.port, info.name), 0) + out.addAnswerAtTime(DNSText(info.name, _TYPE_TXT, _CLASS_IN, 0, info.text), 0) + if info.address: + out.addAnswerAtTime(DNSAddress(info.server, _TYPE_A, _CLASS_IN, 0, info.address), 0) + self.send(out) + i += 1 + nextTime += _UNREGISTER_TIME + + def unregisterAllServices(self): + """Unregister all registered services.""" + if len(self.services) > 0: + now = currentTimeMillis() + nextTime = now + i = 0 + while i < 3: + if now < nextTime: + self.wait(nextTime - now) + now = currentTimeMillis() + continue + out = DNSOutgoing(_FLAGS_QR_RESPONSE | _FLAGS_AA) + for info in self.services.values(): + out.addAnswerAtTime(DNSPointer(info.type, _TYPE_PTR, _CLASS_IN, 0, info.name), 0) + out.addAnswerAtTime(DNSService(info.name, _TYPE_SRV, _CLASS_IN, 0, info.priority, info.weight, info.port, info.server), 0) + out.addAnswerAtTime(DNSText(info.name, _TYPE_TXT, _CLASS_IN, 0, info.text), 0) + if info.address: + out.addAnswerAtTime(DNSAddress(info.server, _TYPE_A, _CLASS_IN, 0, info.address), 0) + self.send(out) + i += 1 + nextTime += _UNREGISTER_TIME + + def checkService(self, info): + """Checks the network for a unique service name, modifying the + ServiceInfo passed in if it is not unique.""" + now = currentTimeMillis() + nextTime = now + i = 0 + while i < 3: + for record in self.cache.entriesWithName(info.type): + if record.type == _TYPE_PTR and not record.isExpired(now) and record.alias == info.name: + if (info.name.find('.') < 0): + info.name = info.name + ".[" + info.address + ":" + info.port + "]." + info.type + self.checkService(info) + return + raise NonUniqueNameException + if now < nextTime: + self.wait(nextTime - now) + now = currentTimeMillis() + continue + out = DNSOutgoing(_FLAGS_QR_QUERY | _FLAGS_AA) + self.debug = out + out.addQuestion(DNSQuestion(info.type, _TYPE_PTR, _CLASS_IN)) + out.addAuthorativeAnswer(DNSPointer(info.type, _TYPE_PTR, _CLASS_IN, _DNS_TTL, info.name)) + self.send(out) + i += 1 + nextTime += _CHECK_TIME + + def addListener(self, listener, question): + """Adds a listener for a given question. The listener will have + its updateRecord method called when information is available to + answer the question.""" + now = currentTimeMillis() + self.listeners.append(listener) + if question is not None: + for record in self.cache.entriesWithName(question.name): + if question.answeredBy(record) and not record.isExpired(now): + listener.updateRecord(self, now, record) + self.notifyAll() + + def removeListener(self, listener): + """Removes a listener.""" + try: + self.listeners.remove(listener) + self.notifyAll() + except: + pass + + def updateRecord(self, now, rec): + """Used to notify listeners of new information that has updated + a record.""" + for listener in self.listeners: + listener.updateRecord(self, now, rec) + self.notifyAll() + + def handleResponse(self, msg): + """Deal with incoming response packets. All answers + are held in the cache, and listeners are notified.""" + now = currentTimeMillis() + for record in msg.answers: + expired = record.isExpired(now) + if record in self.cache.entries(): + if expired: + self.cache.remove(record) + else: + entry = self.cache.get(record) + if entry is not None: + entry.resetTTL(record) + record = entry + else: + self.cache.add(record) + + self.updateRecord(now, record) + + def handleQuery(self, msg, addr, port): + """Deal with incoming query packets. Provides a response if + possible.""" + out = None + + # Support unicast client responses + # + if port != _MDNS_PORT: + out = DNSOutgoing(_FLAGS_QR_RESPONSE | _FLAGS_AA, 0) + for question in msg.questions: + out.addQuestion(question) + + for question in msg.questions: + if question.type == _TYPE_PTR: + for service in self.services.values(): + if question.name == service.type: + if out is None: + out = DNSOutgoing(_FLAGS_QR_RESPONSE | _FLAGS_AA) + out.addAnswer(msg, DNSPointer(service.type, _TYPE_PTR, _CLASS_IN, _DNS_TTL, service.name)) + else: + try: + if out is None: + out = DNSOutgoing(_FLAGS_QR_RESPONSE | _FLAGS_AA) + + # Answer A record queries for any service addresses we know + if question.type == _TYPE_A or question.type == _TYPE_ANY: + for service in self.services.values(): + if service.server == question.name.lower(): + out.addAnswer(msg, DNSAddress(question.name, _TYPE_A, _CLASS_IN | _CLASS_UNIQUE, _DNS_TTL, service.address)) + + service = self.services.get(question.name.lower(), None) + if not service: continue + + if question.type == _TYPE_SRV or question.type == _TYPE_ANY: + out.addAnswer(msg, DNSService(question.name, _TYPE_SRV, _CLASS_IN | _CLASS_UNIQUE, _DNS_TTL, service.priority, service.weight, service.port, service.server)) + if question.type == _TYPE_TXT or question.type == _TYPE_ANY: + out.addAnswer(msg, DNSText(question.name, _TYPE_TXT, _CLASS_IN | _CLASS_UNIQUE, _DNS_TTL, service.text)) + if question.type == _TYPE_SRV: + out.addAdditionalAnswer(DNSAddress(service.server, _TYPE_A, _CLASS_IN | _CLASS_UNIQUE, _DNS_TTL, service.address)) + except: + traceback.print_exc() + + if out is not None and out.answers: + out.id = msg.id + self.send(out, addr, port) + + def send(self, out, addr = _MDNS_ADDR, port = _MDNS_PORT): + """Sends an outgoing packet.""" + # This is a quick test to see if we can parse the packets we generate + #temp = DNSIncoming(out.packet()) + try: + bytes_sent = self.socket.sendto(out.packet(), 0, (addr, port)) + except: + # Ignore this, it may be a temporary loss of network connection + pass + + def close(self): + """Ends the background threads, and prevent this instance from + servicing further queries.""" + if globals()['_GLOBAL_DONE'] == 0: + globals()['_GLOBAL_DONE'] = 1 + self.notifyAll() + self.engine.notify() + self.unregisterAllServices() + self.socket.setsockopt(socket.SOL_IP, socket.IP_DROP_MEMBERSHIP, socket.inet_aton(_MDNS_ADDR) + socket.inet_aton('0.0.0.0')) + self.socket.close() + +# Test a few module features, including service registration, service +# query (for Zoe), and service unregistration. + +if __name__ == '__main__': + print "Multicast DNS Service Discovery for Python, version", __version__ + r = Zeroconf() + print "1. Testing registration of a service..." + desc = {'version':'0.10','a':'test value', 'b':'another value'} + info = ServiceInfo("_http._tcp.local.", "My Service Name._http._tcp.local.", socket.inet_aton("127.0.0.1"), 1234, 0, 0, desc) + print " Registering service..." + r.registerService(info) + print " Registration done." + print "2. Testing query of service information..." + print " Getting ZOE service:", str(r.getServiceInfo("_http._tcp.local.", "ZOE._http._tcp.local.")) + print " Query done." + print "3. Testing query of own service..." + print " Getting self:", str(r.getServiceInfo("_http._tcp.local.", "My Service Name._http._tcp.local.")) + print " Query done." + print "4. Testing unregister of service information..." + r.unregisterService(info) + print " Unregister done." + r.close() diff --git a/testing/mozbase/mozdevice/mozdevice/__init__.py b/testing/mozbase/mozdevice/mozdevice/__init__.py new file mode 100644 index 000000000..2493f75db --- /dev/null +++ b/testing/mozbase/mozdevice/mozdevice/__init__.py @@ -0,0 +1,15 @@ +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this file, +# You can obtain one at http://mozilla.org/MPL/2.0/. + +from adb import ADBError, ADBRootError, ADBTimeoutError, ADBProcess, ADBCommand, ADBHost, ADBDevice +from adb_android import ADBAndroid +from adb_b2g import ADBB2G +from devicemanager import DeviceManager, DMError, ZeroconfListener +from devicemanagerADB import DeviceManagerADB +from devicemanagerSUT import DeviceManagerSUT +from droid import DroidADB, DroidSUT, DroidConnectByHWID + +__all__ = ['ADBError', 'ADBRootError', 'ADBTimeoutError', 'ADBProcess', 'ADBCommand', 'ADBHost', + 'ADBDevice', 'ADBAndroid', 'ADBB2G', 'DeviceManager', 'DMError', 'ZeroconfListener', + 'DeviceManagerADB', 'DeviceManagerSUT', 'DroidADB', 'DroidSUT', 'DroidConnectByHWID'] diff --git a/testing/mozbase/mozdevice/mozdevice/adb.py b/testing/mozbase/mozdevice/mozdevice/adb.py new file mode 100644 index 000000000..5958937d9 --- /dev/null +++ b/testing/mozbase/mozdevice/mozdevice/adb.py @@ -0,0 +1,2271 @@ +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this file, +# You can obtain one at http://mozilla.org/MPL/2.0/. + +import os +import posixpath +import re +import shutil +import subprocess +import tempfile +import time +import traceback + +from abc import ABCMeta, abstractmethod +from distutils import dir_util + + +class ADBProcess(object): + """ADBProcess encapsulates the data related to executing the adb process.""" + + def __init__(self, args): + #: command argument argument list. + self.args = args + #: Temporary file handle to be used for stdout. + self.stdout_file = tempfile.TemporaryFile() + #: boolean indicating if the command timed out. + self.timedout = None + #: exitcode of the process. + self.exitcode = None + #: subprocess Process object used to execute the command. + self.proc = subprocess.Popen(args, + stdout=self.stdout_file, + stderr=subprocess.STDOUT) + + @property + def stdout(self): + """Return the contents of stdout.""" + if not self.stdout_file or self.stdout_file.closed: + content = "" + else: + self.stdout_file.seek(0, os.SEEK_SET) + content = self.stdout_file.read().rstrip() + return content + + def __str__(self): + return ('args: %s, exitcode: %s, stdout: %s' % ( + ' '.join(self.args), self.exitcode, self.stdout)) + +# ADBError, ADBRootError, and ADBTimeoutError are treated +# differently in order that unhandled ADBRootErrors and +# ADBTimeoutErrors can be handled distinctly from ADBErrors. + + +class ADBError(Exception): + """ADBError is raised in situations where a command executed on a + device either exited with a non-zero exitcode or when an + unexpected error condition has occurred. Generally, ADBErrors can + be handled and the device can continue to be used. + """ + pass + + +class ADBListDevicesError(ADBError): + """ADBListDevicesError is raised when errors are found listing the + devices, typically not any permissions. + + The devices information is stocked with the *devices* member. + """ + + def __init__(self, msg, devices): + ADBError.__init__(self, msg) + self.devices = devices + + +class ADBRootError(Exception): + """ADBRootError is raised when a shell command is to be executed as + root but the device does not support it. This error is fatal since + there is no recovery possible by the script. You must either root + your device or change your scripts to not require running as root. + """ + pass + + +class ADBTimeoutError(Exception): + """ADBTimeoutError is raised when either a host command or shell + command takes longer than the specified timeout to execute. The + timeout value is set in the ADBCommand constructor and is 300 seconds by + default. This error is typically fatal since the host is having + problems communicating with the device. You may be able to recover + by rebooting, but this is not guaranteed. + + Recovery options are: + + * Killing and restarting the adb server via + :: + + adb kill-server; adb start-server + + * Rebooting the device manually. + * Rebooting the host. + """ + pass + + +class ADBCommand(object): + """ADBCommand provides a basic interface to adb commands + which is used to provide the 'command' methods for the + classes ADBHost and ADBDevice. + + ADBCommand should only be used as the base class for other + classes and should not be instantiated directly. To enforce this + restriction calling ADBCommand's constructor will raise a + NonImplementedError exception. + + :: + + from mozdevice import ADBCommand + + try: + adbcommand = ADBCommand() + except NotImplementedError: + print "ADBCommand can not be instantiated." + """ + + def __init__(self, + adb='adb', + adb_host=None, + adb_port=None, + logger_name='adb', + timeout=300, + verbose=False): + """Initializes the ADBCommand object. + + :param str adb: path to adb executable. Defaults to 'adb'. + :param adb_host: host of the adb server. + :type adb_host: str or None + :param adb_port: port of the adb server. + :type adb_port: integer or None + :param str logger_name: logging logger name. Defaults to 'adb'. + + :raises: * ADBError + * ADBTimeoutError + """ + if self.__class__ == ADBCommand: + raise NotImplementedError + + self._logger = self._get_logger(logger_name) + self._verbose = verbose + self._adb_path = adb + self._adb_host = adb_host + self._adb_port = adb_port + self._timeout = timeout + self._polling_interval = 0.1 + self._adb_version = '' + + self._logger.debug("%s: %s" % (self.__class__.__name__, + self.__dict__)) + + # catch early a missing or non executable adb command + # and get the adb version while we are at it. + try: + output = subprocess.Popen([adb, 'version'], + stdout=subprocess.PIPE, + stderr=subprocess.PIPE).communicate() + re_version = re.compile(r'Android Debug Bridge version (.*)') + self._adb_version = re_version.match(output[0]).group(1) + except Exception as exc: + raise ADBError('%s: %s is not executable.' % (exc, adb)) + + def _get_logger(self, logger_name): + logger = None + try: + import mozlog + logger = mozlog.get_default_logger(logger_name) + except ImportError: + pass + + if logger is None: + import logging + logger = logging.getLogger(logger_name) + return logger + + # Host Command methods + + def command(self, cmds, device_serial=None, timeout=None): + """Executes an adb command on the host. + + :param list cmds: The command and its arguments to be + executed. + :param device_serial: The device's + serial number if the adb command is to be executed against + a specific device. + :type device_serial: str or None + :param timeout: The maximum time in + seconds for any spawned adb process to complete before + throwing an ADBTimeoutError. This timeout is per adb call. The + total time spent may exceed this value. If it is not + specified, the value set in the ADBCommand constructor is used. + :type timeout: integer or None + :returns: :class:`mozdevice.ADBProcess` + + command() provides a low level interface for executing + commands on the host via adb. + + command() executes on the host in such a fashion that stdout + of the adb process is a file handle on the host and + the exit code is available as the exit code of the adb + process. + + The caller provides a list containing commands, as well as a + timeout period in seconds. + + A subprocess is spawned to execute adb with stdout and stderr + directed to a temporary file. If the process takes longer than + the specified timeout, the process is terminated. + + It is the caller's responsibilty to clean up by closing + the stdout temporary file. + """ + args = [self._adb_path] + if self._adb_host: + args.extend(['-H', self._adb_host]) + if self._adb_port: + args.extend(['-P', str(self._adb_port)]) + if device_serial: + args.extend(['-s', device_serial, 'wait-for-device']) + args.extend(cmds) + + adb_process = ADBProcess(args) + + if timeout is None: + timeout = self._timeout + + start_time = time.time() + adb_process.exitcode = adb_process.proc.poll() + while ((time.time() - start_time) <= timeout and + adb_process.exitcode is None): + time.sleep(self._polling_interval) + adb_process.exitcode = adb_process.proc.poll() + if adb_process.exitcode is None: + adb_process.proc.kill() + adb_process.timedout = True + adb_process.exitcode = adb_process.proc.poll() + + adb_process.stdout_file.seek(0, os.SEEK_SET) + + return adb_process + + def command_output(self, cmds, device_serial=None, timeout=None): + """Executes an adb command on the host returning stdout. + + :param list cmds: The command and its arguments to be + executed. + :param device_serial: The device's + serial number if the adb command is to be executed against + a specific device. + :type device_serial: str or None + :param timeout: The maximum time in seconds + for any spawned adb process to complete before throwing + an ADBTimeoutError. + This timeout is per adb call. The total time spent + may exceed this value. If it is not specified, the value + set in the ADBCommand constructor is used. + :type timeout: integer or None + :returns: string - content of stdout. + + :raises: * ADBTimeoutError + * ADBError + """ + adb_process = None + try: + # Need to force the use of the ADBCommand class's command + # since ADBDevice will redefine command and call its + # own version otherwise. + adb_process = ADBCommand.command(self, cmds, + device_serial=device_serial, + timeout=timeout) + if adb_process.timedout: + raise ADBTimeoutError("%s" % adb_process) + elif adb_process.exitcode: + raise ADBError("%s" % adb_process) + output = adb_process.stdout_file.read().rstrip() + if self._verbose: + self._logger.debug('command_output: %s, ' + 'timeout: %s, ' + 'timedout: %s, ' + 'exitcode: %s, output: %s' % + (' '.join(adb_process.args), + timeout, + adb_process.timedout, + adb_process.exitcode, + output)) + + return output + finally: + if adb_process and isinstance(adb_process.stdout_file, file): + adb_process.stdout_file.close() + + +class ADBHost(ADBCommand): + """ADBHost provides a basic interface to adb host commands + which do not target a specific device. + + :: + + from mozdevice import ADBHost + + adbhost = ADBHost() + adbhost.start_server() + """ + + def __init__(self, + adb='adb', + adb_host=None, + adb_port=None, + logger_name='adb', + timeout=300, + verbose=False): + """Initializes the ADBHost object. + + :param str adb: path to adb executable. Defaults to 'adb'. + :param adb_host: host of the adb server. + :type adb_host: str or None + :param adb_port: port of the adb server. + :type adb_port: integer or None + :param str logger_name: logging logger name. Defaults to 'adb'. + + :raises: * ADBError + * ADBTimeoutError + """ + ADBCommand.__init__(self, adb=adb, adb_host=adb_host, + adb_port=adb_port, logger_name=logger_name, + timeout=timeout, verbose=verbose) + + def command(self, cmds, timeout=None): + """Executes an adb command on the host. + + :param list cmds: The command and its arguments to be + executed. + :param timeout: The maximum time in + seconds for any spawned adb process to complete before + throwing an ADBTimeoutError. This timeout is per adb call. The + total time spent may exceed this value. If it is not + specified, the value set in the ADBHost constructor is used. + :type timeout: integer or None + :returns: :class:`mozdevice.ADBProcess` + + command() provides a low level interface for executing + commands on the host via adb. + + command() executes on the host in such a fashion that stdout + of the adb process is a file handle on the host and + the exit code is available as the exit code of the adb + process. + + The caller provides a list containing commands, as well as a + timeout period in seconds. + + A subprocess is spawned to execute adb with stdout and stderr + directed to a temporary file. If the process takes longer than + the specified timeout, the process is terminated. + + It is the caller's responsibilty to clean up by closing + the stdout temporary file. + """ + return ADBCommand.command(self, cmds, timeout=timeout) + + def command_output(self, cmds, timeout=None): + """Executes an adb command on the host returning stdout. + + :param list cmds: The command and its arguments to be + executed. + :param timeout: The maximum time in seconds + for any spawned adb process to complete before throwing + an ADBTimeoutError. + This timeout is per adb call. The total time spent + may exceed this value. If it is not specified, the value + set in the ADBHost constructor is used. + :type timeout: integer or None + :returns: string - content of stdout. + + :raises: * ADBTimeoutError + * ADBError + """ + return ADBCommand.command_output(self, cmds, timeout=timeout) + + def start_server(self, timeout=None): + """Starts the adb server. + + :param timeout: The maximum time in + seconds for any spawned adb process to complete before + throwing an ADBTimeoutError. This timeout is per adb call. The + total time spent may exceed this value. If it is not + specified, the value set in the ADBHost constructor is used. + :type timeout: integer or None + :raises: * ADBTimeoutError + * ADBError + + Attempting to use start_server with any adb_host value other than None + will fail with an ADBError exception. + + You will need to start the server on the remote host via the command: + + .. code-block:: shell + + adb -a fork-server server + + If you wish the remote adb server to restart automatically, you can + enclose the command in a loop as in: + + .. code-block:: shell + + while true; do + adb -a fork-server server + done + """ + self.command_output(["start-server"], timeout=timeout) + + def kill_server(self, timeout=None): + """Kills the adb server. + + :param timeout: The maximum time in + seconds for any spawned adb process to complete before + throwing an ADBTimeoutError. This timeout is per adb call. The + total time spent may exceed this value. If it is not + specified, the value set in the ADBHost constructor is used. + :type timeout: integer or None + :raises: * ADBTimeoutError + * ADBError + """ + self.command_output(["kill-server"], timeout=timeout) + + def devices(self, timeout=None): + """Executes adb devices -l and returns a list of objects describing attached devices. + + :param timeout: The maximum time in + seconds for any spawned adb process to complete before + throwing an ADBTimeoutError. This timeout is per adb call. The + total time spent may exceed this value. If it is not + specified, the value set in the ADBHost constructor is used. + :type timeout: integer or None + :returns: an object contain + :raises: * ADBTimeoutError + * ADBListDevicesError + * ADBError + + The output of adb devices -l :: + + $ adb devices -l + List of devices attached + b313b945 device usb:1-7 product:d2vzw model:SCH_I535 device:d2vzw + + is parsed and placed into an object as in + + [{'device_serial': 'b313b945', 'state': 'device', 'product': 'd2vzw', + 'usb': '1-7', 'device': 'd2vzw', 'model': 'SCH_I535' }] + """ + # b313b945 device usb:1-7 product:d2vzw model:SCH_I535 device:d2vzw + # from Android system/core/adb/transport.c statename() + re_device_info = re.compile( + r"([^\s]+)\s+(offline|bootloader|device|host|recovery|sideload|" + "no permissions|unauthorized|unknown)") + devices = [] + lines = self.command_output(["devices", "-l"], timeout=timeout).split('\n') + for line in lines: + if line == 'List of devices attached ': + continue + match = re_device_info.match(line) + if match: + device = { + 'device_serial': match.group(1), + 'state': match.group(2) + } + remainder = line[match.end(2):].strip() + if remainder: + try: + device.update(dict([j.split(':') + for j in remainder.split(' ')])) + except ValueError: + self._logger.warning('devices: Unable to parse ' + 'remainder for device %s' % line) + devices.append(device) + for device in devices: + if device['state'] == 'no permissions': + raise ADBListDevicesError( + "No permissions to detect devices. You should restart the" + " adb server as root:\n" + "\n# adb kill-server\n# adb start-server\n" + "\nor maybe configure your udev rules.", + devices) + return devices + + +class ADBDevice(ADBCommand): + """ADBDevice is an abstract base class which provides methods which + can be used to interact with the associated Android or B2G based + device. It must be used via one of the concrete implementations in + :class:`ADBAndroid` or :class:`ADBB2G`. + """ + __metaclass__ = ABCMeta + + def __init__(self, + device=None, + adb='adb', + adb_host=None, + adb_port=None, + test_root='', + logger_name='adb', + timeout=300, + verbose=False, + device_ready_retry_wait=20, + device_ready_retry_attempts=3): + """Initializes the ADBDevice object. + + :param device: When a string is passed, it is interpreted as the + device serial number. This form is not compatible with + devices containing a ":" in the serial; in this case + ValueError will be raised. + When a dictionary is passed it must have one or both of + the keys "device_serial" and "usb". This is compatible + with the dictionaries in the list returned by + ADBHost.devices(). If the value of device_serial is a + valid serial not containing a ":" it will be used to + identify the device, otherwise the value of the usb key, + prefixed with "usb:" is used. + If None is passed and there is exactly one device attached + to the host, that device is used. If there is more than one + device attached, ValueError is raised. If no device is + attached the constructor will block until a device is + attached or the timeout is reached. + :type device: dict, str or None + :param adb_host: host of the adb server to connect to. + :type adb_host: str or None + :param adb_port: port of the adb server to connect to. + :type adb_port: integer or None + :param str logger_name: logging logger name. Defaults to 'adb'. + :param integer device_ready_retry_wait: number of seconds to wait + between attempts to check if the device is ready after a + reboot. + :param integer device_ready_retry_attempts: number of attempts when + checking if a device is ready. + + :raises: * ADBError + * ADBTimeoutError + * ValueError + """ + ADBCommand.__init__(self, adb=adb, adb_host=adb_host, + adb_port=adb_port, logger_name=logger_name, + timeout=timeout, verbose=verbose) + self._device_serial = self._get_device_serial(device) + self._initial_test_root = test_root + self._test_root = None + self._device_ready_retry_wait = device_ready_retry_wait + self._device_ready_retry_attempts = device_ready_retry_attempts + self._have_root_shell = False + self._have_su = False + self._have_android_su = False + + # Catch exceptions due to the potential for segfaults + # calling su when using an improperly rooted device. + + # Note this check to see if adbd is running is performed on + # the device in the state it exists in when the ADBDevice is + # initialized. It may be the case that it has been manipulated + # since its last boot and that its current state does not + # match the state the device will have immediately after a + # reboot. For example, if adb root was called manually prior + # to ADBDevice being initialized, then self._have_root_shell + # will not reflect the state of the device after it has been + # rebooted again. Therefore this check will need to be + # performed again after a reboot. + + self._check_adb_root(timeout=timeout) + + uid = 'uid=0' + # Do we have a 'Superuser' sh like su? + try: + if self.shell_output("su -c id", timeout=timeout).find(uid) != -1: + self._have_su = True + self._logger.info("su -c supported") + except ADBError: + self._logger.debug("Check for su -c failed") + + # Do we have Android's su? + try: + if self.shell_output("su 0 id", timeout=timeout).find(uid) != -1: + self._have_android_su = True + self._logger.info("su 0 supported") + except ADBError: + self._logger.debug("Check for su 0 failed") + + self._mkdir_p = None + # Force the use of /system/bin/ls or /system/xbin/ls in case + # there is /sbin/ls which embeds ansi escape codes to colorize + # the output. Detect if we are using busybox ls. We want each + # entry on a single line and we don't want . or .. + if self.shell_bool("/system/bin/ls /data/local/tmp", timeout=timeout): + self._ls = "/system/bin/ls" + elif self.shell_bool("/system/xbin/ls /data/local/tmp", timeout=timeout): + self._ls = "/system/xbin/ls" + else: + raise ADBError("ADBDevice.__init__: ls not found") + try: + self.shell_output("%s -1A /data/local/tmp" % self._ls, timeout=timeout) + self._ls += " -1A" + except ADBError: + self._ls += " -a" + + self._logger.info("%s supported" % self._ls) + + # Do we have cp? + self._have_cp = self.shell_bool("type cp", timeout=timeout) + self._logger.info("Native cp support: %s" % self._have_cp) + + # Do we have chmod -R? + try: + self._chmod_R = False + re_recurse = re.compile(r'[-]R') + chmod_output = self.shell_output("chmod --help", timeout=timeout) + match = re_recurse.search(chmod_output) + if match: + self._chmod_R = True + except (ADBError, ADBTimeoutError) as e: + self._logger.debug('Check chmod -R: %s' % e) + match = re_recurse.search(e.message) + if match: + self._chmod_R = True + self._logger.info("Native chmod -R support: %s" % self._chmod_R) + + self._logger.debug("ADBDevice: %s" % self.__dict__) + + def _get_device_serial(self, device): + if device is None: + devices = ADBHost(adb=self._adb_path, adb_host=self._adb_host, + adb_port=self._adb_port).devices() + if len(devices) > 1: + raise ValueError("ADBDevice called with multiple devices " + "attached and no device specified") + elif len(devices) == 0: + # We could error here, but this way we'll wait-for-device before we next + # run a command, which seems more friendly + return + device = devices[0] + + def is_valid_serial(serial): + return ":" not in serial or serial.startswith("usb:") + + if isinstance(device, (str, unicode)): + # Treat this as a device serial + if not is_valid_serial(device): + raise ValueError("Device serials containing ':' characters are " + "invalid. Pass the output from " + "ADBHost.devices() for the device instead") + return device + + serial = device.get("device_serial") + if serial is not None and is_valid_serial(serial): + return serial + usb = device.get("usb") + if usb is not None: + return "usb:%s" % usb + + raise ValueError("Unable to get device serial") + + def _check_adb_root(self, timeout=None): + self._have_root_shell = False + uid = 'uid=0' + # Is shell already running as root? + try: + if self.shell_output("id", timeout=timeout).find(uid) != -1: + self._have_root_shell = True + self._logger.info("adbd running as root") + except ADBError: + self._logger.debug("Check for root shell failed") + + # Do we need to run adb root to get a root shell? + try: + if (not self._have_root_shell and self.command_output( + ["root"], + timeout=timeout).find("cannot run as root") == -1): + self._have_root_shell = True + self._logger.info("adbd restarted as root") + except ADBError: + self._logger.debug("Check for root adbd failed") + + @staticmethod + def _escape_command_line(cmd): + """Utility function to return escaped and quoted version of command + line. + """ + quoted_cmd = [] + + for arg in cmd: + arg.replace('&', r'\&') + + needs_quoting = False + for char in [' ', '(', ')', '"', '&']: + if arg.find(char) >= 0: + needs_quoting = True + break + if needs_quoting: + arg = "'%s'" % arg + + quoted_cmd.append(arg) + + return " ".join(quoted_cmd) + + @staticmethod + def _get_exitcode(file_obj): + """Get the exitcode from the last line of the file_obj for shell + commands. + """ + file_obj.seek(0, os.SEEK_END) + + line = '' + length = file_obj.tell() + offset = 1 + while length - offset >= 0: + file_obj.seek(-offset, os.SEEK_END) + char = file_obj.read(1) + if not char: + break + if char != '\r' and char != '\n': + line = char + line + elif line: + # we have collected everything up to the beginning of the line + break + offset += 1 + + match = re.match(r'rc=([0-9]+)', line) + if match: + exitcode = int(match.group(1)) + file_obj.seek(-1, os.SEEK_CUR) + file_obj.truncate() + else: + exitcode = None + + return exitcode + + @property + def test_root(self): + """ + The test_root property returns the directory on the device where + temporary test files are stored. + + The first time test_root it is called it determines and caches a value + for the test root on the device. It determines the appropriate test + root by attempting to create a 'dummy' directory on each of a list of + directories and returning the first successful directory as the + test_root value. + + The default list of directories checked by test_root are: + + - /storage/sdcard0/tests + - /storage/sdcard1/tests + - /sdcard/tests + - /mnt/sdcard/tests + - /data/local/tests + + You may override the default list by providing a test_root argument to + the :class:`ADBDevice` constructor which will then be used when + attempting to create the 'dummy' directory. + + :raises: * ADBTimeoutError + * ADBRootError + * ADBError + """ + if self._test_root is not None: + return self._test_root + + if self._initial_test_root: + paths = [self._initial_test_root] + else: + paths = ['/storage/sdcard0/tests', + '/storage/sdcard1/tests', + '/sdcard/tests', + '/mnt/sdcard/tests', + '/data/local/tests'] + + max_attempts = 3 + for attempt in range(1, max_attempts + 1): + for test_root in paths: + self._logger.debug("Setting test root to %s attempt %d of %d" % + (test_root, attempt, max_attempts)) + + if self._try_test_root(test_root): + self._test_root = test_root + return self._test_root + + self._logger.debug('_setup_test_root: ' + 'Attempt %d of %d failed to set test_root to %s' % + (attempt, max_attempts, test_root)) + + if attempt != max_attempts: + time.sleep(20) + + raise ADBError("Unable to set up test root using paths: [%s]" + % ", ".join(paths)) + + def _try_test_root(self, test_root): + base_path, sub_path = posixpath.split(test_root) + if not self.is_dir(base_path): + return False + + try: + dummy_dir = posixpath.join(test_root, 'dummy') + if self.is_dir(dummy_dir): + self.rm(dummy_dir, recursive=True) + self.mkdir(dummy_dir, parents=True) + except ADBError: + self._logger.debug("%s is not writable" % test_root) + return False + + return True + + # Host Command methods + + def command(self, cmds, timeout=None): + """Executes an adb command on the host against the device. + + :param list cmds: The command and its arguments to be + executed. + :param timeout: The maximum time in + seconds for any spawned adb process to complete before + throwing an ADBTimeoutError. This timeout is per adb call. The + total time spent may exceed this value. If it is not + specified, the value set in the ADBDevice constructor is used. + :type timeout: integer or None + :returns: :class:`mozdevice.ADBProcess` + + command() provides a low level interface for executing + commands for a specific device on the host via adb. + + command() executes on the host in such a fashion that stdout + of the adb process are file handles on the host and + the exit code is available as the exit code of the adb + process. + + For executing shell commands on the device, use + ADBDevice.shell(). The caller provides a list containing + commands, as well as a timeout period in seconds. + + A subprocess is spawned to execute adb for the device with + stdout and stderr directed to a temporary file. If the process + takes longer than the specified timeout, the process is + terminated. + + It is the caller's responsibilty to clean up by closing + the stdout temporary file. + """ + + return ADBCommand.command(self, cmds, + device_serial=self._device_serial, + timeout=timeout) + + def command_output(self, cmds, timeout=None): + """Executes an adb command on the host against the device returning + stdout. + + :param list cmds: The command and its arguments to be executed. + :param timeout: The maximum time in seconds + for any spawned adb process to complete before throwing + an ADBTimeoutError. + This timeout is per adb call. The total time spent + may exceed this value. If it is not specified, the value + set in the ADBDevice constructor is used. + :type timeout: integer or None + :returns: string - content of stdout. + + :raises: * ADBTimeoutError + * ADBError + """ + return ADBCommand.command_output(self, cmds, + device_serial=self._device_serial, + timeout=timeout) + + # Port forwarding methods + + def _validate_port(self, port, is_local=True): + """Validate a port forwarding specifier. Raises ValueError on failure. + + :param str port: The port specifier to validate + :param bool is_local: Flag indicating whether the port represents a local port. + """ + prefixes = ["tcp", "localabstract", "localreserved", "localfilesystem", "dev"] + + if not is_local: + prefixes += ["jdwp"] + + parts = port.split(":", 1) + if len(parts) != 2 or parts[0] not in prefixes: + raise ValueError("Invalid forward specifier %s" % port) + + def forward(self, local, remote, allow_rebind=True, timeout=None): + """Forward a local port to a specific port on the device. + + Ports are specified in the form: + tcp: + localabstract: + localreserved: + localfilesystem: + dev: + jdwp: (remote only) + + :param str local: Local port to forward + :param str remote: Remote port to which to forward + :param bool allow_rebind: Don't error if the local port is already forwarded + :param timeout: The maximum time in seconds + for any spawned adb process to complete before throwing + an ADBTimeoutError. If it is not specified, the value + set in the ADBDevice constructor is used. + :type timeout: integer or None + :raises: * ValueError + * ADBTimeoutError + * ADBError + """ + + for port, is_local in [(local, True), (remote, False)]: + self._validate_port(port, is_local=is_local) + + cmd = ["forward", local, remote] + if not allow_rebind: + cmd.insert(1, "--no-rebind") + self.command_output(cmd, timeout=timeout) + + def list_forwards(self, timeout=None): + """Return a list of tuples specifying active forwards + + Return values are of the form (device, local, remote). + + :param timeout: The maximum time in seconds + for any spawned adb process to complete before throwing + an ADBTimeoutError. If it is not specified, the value + set in the ADBDevice constructor is used. + :type timeout: integer or None + :raises: * ADBTimeoutError + * ADBError + """ + forwards = self.command_output(["forward", "--list"], timeout=timeout) + return [tuple(line.split(" ")) for line in forwards.split("\n") if line.strip()] + + def remove_forwards(self, local=None, timeout=None): + """Remove existing port forwards. + + :param local: local port specifier as for ADBDevice.forward. If local + is not specified removes all forwards. + :type local: str or None + :param timeout: The maximum time in seconds + for any spawned adb process to complete before throwing + an ADBTimeoutError. If it is not specified, the value + set in the ADBDevice constructor is used. + :type timeout: integer or None + :raises: * ValueError + * ADBTimeoutError + * ADBError + """ + cmd = ["forward"] + if local is None: + cmd.extend(["--remove-all"]) + else: + self._validate_port(local, is_local=True) + cmd.extend(["--remove", local]) + + self.command_output(cmd, timeout=timeout) + + # Device Shell methods + + def shell(self, cmd, env=None, cwd=None, timeout=None, root=False): + """Executes a shell command on the device. + + :param str cmd: The command to be executed. + :param env: Contains the environment variables and + their values. + :type env: dict or None + :param cwd: The directory from which to execute. + :type cwd: str or None + :param timeout: The maximum time in + seconds for any spawned adb process to complete before + throwing an ADBTimeoutError. This timeout is per adb call. The + total time spent may exceed this value. If it is not + specified, the value set in the ADBDevice constructor is used. + :type timeout: integer or None + :param bool root: Flag specifying if the command should + be executed as root. + :returns: :class:`mozdevice.ADBProcess` + :raises: ADBRootError + + shell() provides a low level interface for executing commands + on the device via adb shell. + + shell() executes on the host in such as fashion that stdout + contains the stdout and stderr of the host abd process + combined with the stdout and stderr of the shell command + on the device. The exit code of shell() is the exit code of + the adb command if it was non-zero or the extracted exit code + from the output of the shell command executed on the + device. + + The caller provides a flag indicating if the command is to be + executed as root, a string for any requested working + directory, a hash defining the environment, a string + containing shell commands, as well as a timeout period in + seconds. + + The command line to be executed is created to set the current + directory, set the required environment variables, optionally + execute the command using su and to output the return code of + the command to stdout. The command list is created as a + command sequence separated by && which will terminate the + command sequence on the first command which returns a non-zero + exit code. + + A subprocess is spawned to execute adb shell for the device + with stdout and stderr directed to a temporary file. If the + process takes longer than the specified timeout, the process + is terminated. The return code is extracted from the stdout + and is then removed from the file. + + It is the caller's responsibilty to clean up by closing + the stdout temporary files. + + """ + if root and not self._have_root_shell: + # If root was requested and we do not already have a root + # shell, then use the appropriate version of su to invoke + # the shell cmd. Prefer Android's su version since it may + # falsely report support for su -c. + if self._have_android_su: + cmd = "su 0 %s" % cmd + elif self._have_su: + cmd = "su -c \"%s\"" % cmd + else: + raise ADBRootError('Can not run command %s as root!' % cmd) + + # prepend cwd and env to command if necessary + if cwd: + cmd = "cd %s && %s" % (cwd, cmd) + if env: + envstr = '&& '.join(map(lambda x: 'export %s=%s' % + (x[0], x[1]), env.iteritems())) + cmd = envstr + "&& " + cmd + cmd += "; echo rc=$?" + + args = [self._adb_path] + if self._adb_host: + args.extend(['-H', self._adb_host]) + if self._adb_port: + args.extend(['-P', str(self._adb_port)]) + if self._device_serial: + args.extend(['-s', self._device_serial]) + args.extend(["wait-for-device", "shell", cmd]) + adb_process = ADBProcess(args) + + if timeout is None: + timeout = self._timeout + + start_time = time.time() + exitcode = adb_process.proc.poll() + while ((time.time() - start_time) <= timeout) and exitcode is None: + time.sleep(self._polling_interval) + exitcode = adb_process.proc.poll() + if exitcode is None: + adb_process.proc.kill() + adb_process.timedout = True + adb_process.exitcode = adb_process.proc.poll() + elif exitcode == 0: + adb_process.exitcode = self._get_exitcode(adb_process.stdout_file) + else: + adb_process.exitcode = exitcode + + adb_process.stdout_file.seek(0, os.SEEK_SET) + + return adb_process + + def shell_bool(self, cmd, env=None, cwd=None, timeout=None, root=False): + """Executes a shell command on the device returning True on success + and False on failure. + + :param str cmd: The command to be executed. + :param env: Contains the environment variables and + their values. + :type env: dict or None + :param cwd: The directory from which to execute. + :type cwd: str or None + :param timeout: The maximum time in + seconds for any spawned adb process to complete before + throwing an ADBTimeoutError. + This timeout is per adb call. The total time spent + may exceed this value. If it is not specified, the value + set in the ADBDevice constructor is used. + :type timeout: integer or None + :param bool root: Flag specifying if the command should + be executed as root. + :returns: boolean + + :raises: * ADBTimeoutError + * ADBRootError + """ + adb_process = None + try: + adb_process = self.shell(cmd, env=env, cwd=cwd, + timeout=timeout, root=root) + if adb_process.timedout: + raise ADBTimeoutError("%s" % adb_process) + return adb_process.exitcode == 0 + finally: + if adb_process: + adb_process.stdout_file.close() + + def shell_output(self, cmd, env=None, cwd=None, timeout=None, root=False): + """Executes an adb shell on the device returning stdout. + + :param str cmd: The command to be executed. + :param env: Contains the environment variables and their values. + :type env: dict or None + :param cwd: The directory from which to execute. + :type cwd: str or None + :param timeout: The maximum time in + seconds for any spawned adb process to complete before + throwing an ADBTimeoutError. This timeout is per + adb call. The total time spent may exceed this + value. If it is not specified, the value set + in the ADBDevice constructor is used. + :type timeout: integer or None + :param bool root: Flag specifying if the command + should be executed as root. + :returns: string - content of stdout. + :raises: * ADBTimeoutError + * ADBRootError + * ADBError + """ + adb_process = None + try: + adb_process = self.shell(cmd, env=env, cwd=cwd, + timeout=timeout, root=root) + if adb_process.timedout: + raise ADBTimeoutError("%s" % adb_process) + elif adb_process.exitcode: + raise ADBError("%s" % adb_process) + output = adb_process.stdout_file.read().rstrip() + if self._verbose: + self._logger.debug('shell_output: %s, ' + 'timeout: %s, ' + 'root: %s, ' + 'timedout: %s, ' + 'exitcode: %s, ' + 'output: %s' % + (' '.join(adb_process.args), + timeout, + root, + adb_process.timedout, + adb_process.exitcode, + output)) + + return output + finally: + if adb_process and isinstance(adb_process.stdout_file, file): + adb_process.stdout_file.close() + + # Informational methods + + def _get_logcat_buffer_args(self, buffers): + valid_buffers = set(['radio', 'main', 'events']) + invalid_buffers = set(buffers).difference(valid_buffers) + if invalid_buffers: + raise ADBError('Invalid logcat buffers %s not in %s ' % ( + list(invalid_buffers), list(valid_buffers))) + args = [] + for b in buffers: + args.extend(['-b', b]) + return args + + def clear_logcat(self, timeout=None, buffers=[]): + """Clears logcat via adb logcat -c. + + :param timeout: The maximum time in + seconds for any spawned adb process to complete before + throwing an ADBTimeoutError. This timeout is per + adb call. The total time spent may exceed this + value. If it is not specified, the value set + in the ADBDevice constructor is used. + :type timeout: integer or None + :param list buffers: Log buffers to clear. Valid buffers are + "radio", "events", and "main". Defaults to "main". + :raises: * ADBTimeoutError + * ADBError + """ + buffers = self._get_logcat_buffer_args(buffers) + cmds = ["logcat", "-c"] + buffers + self.command_output(cmds, timeout=timeout) + self.shell_output("log logcat cleared", timeout=timeout) + + def get_logcat(self, + filter_specs=[ + "dalvikvm:I", + "ConnectivityService:S", + "WifiMonitor:S", + "WifiStateTracker:S", + "wpa_supplicant:S", + "NetworkStateTracker:S"], + format="time", + filter_out_regexps=[], + timeout=None, + buffers=[]): + """Returns the contents of the logcat file as a list of strings. + + :param list filter_specs: Optional logcat messages to + be included. + :param str format: Optional logcat format. + :param list filterOutRexps: Optional logcat messages to be + excluded. + :param timeout: The maximum time in + seconds for any spawned adb process to complete before + throwing an ADBTimeoutError. + This timeout is per adb call. The total time spent + may exceed this value. If it is not specified, the value + set in the ADBDevice constructor is used. + :type timeout: integer or None + :param list buffers: Log buffers to retrieve. Valid buffers are + "radio", "events", and "main". Defaults to "main". + :returns: list of lines logcat output. + :raises: * ADBTimeoutError + * ADBError + """ + buffers = self._get_logcat_buffer_args(buffers) + cmds = ["logcat", "-v", format, "-d"] + buffers + filter_specs + lines = self.command_output(cmds, timeout=timeout).split('\r') + + for regex in filter_out_regexps: + lines = [line for line in lines if not re.search(regex, line)] + + return lines + + def get_prop(self, prop, timeout=None): + """Gets value of a property from the device via adb shell getprop. + + :param str prop: The propery name. + :param timeout: The maximum time in + seconds for any spawned adb process to complete before + throwing an ADBTimeoutError. + This timeout is per adb call. The total time spent + may exceed this value. If it is not specified, the value + set in the ADBDevice constructor is used. + :type timeout: integer or None + :returns: string value of property. + :raises: * ADBTimeoutError + * ADBError + """ + output = self.shell_output('getprop %s' % prop, timeout=timeout) + return output + + def get_state(self, timeout=None): + """Returns the device's state via adb get-state. + + :param timeout: The maximum time in + seconds for any spawned adb process to complete before throwing + an ADBTimeoutError. + This timeout is per adb call. The total time spent + may exceed this value. If it is not specified, the value + set in the ADBDevice constructor is used. + :type timeout: integer or None + :returns: string value of adb get-state. + :raises: * ADBTimeoutError + * ADBError + """ + output = self.command_output(["get-state"], timeout=timeout).strip() + return output + + def get_ip_address(self, interfaces=None, timeout=None): + """Returns the device's ip address, or None if it doesn't have one + + :param interfaces: Interfaces to allow, or None to allow any + non-loopback interface. + :type interfaces: list or None + :param timeout: The maximum time in + seconds for any spawned adb process to complete before throwing + an ADBTimeoutError. + This timeout is per adb call. The total time spent + may exceed this value. If it is not specified, the value + set in the ADBDevice constructor is used. + :type timeout: integer or None + :returns: string ip address of the device or None if it could not + be found. + :raises: * ADBTimeoutError + * ADBError + """ + if not interfaces: + interfaces = ["wlan0", "eth0"] + wifi_interface = self.shell_output('getprop wifi.interface', timeout=timeout) + self._logger.debug('get_ip_address: wifi_interface: %s' % wifi_interface) + if wifi_interface and wifi_interface not in interfaces: + interfaces = interfaces.append(wifi_interface) + + # ifconfig interface + # can return two different formats: + # eth0: ip 192.168.1.139 mask 255.255.255.0 flags [up broadcast running multicast] + # or + # wlan0 Link encap:Ethernet HWaddr 00:9A:CD:B8:39:65 + # inet addr:192.168.1.38 Bcast:192.168.1.255 Mask:255.255.255.0 + # inet6 addr: fe80::29a:cdff:feb8:3965/64 Scope: Link + # UP BROADCAST RUNNING MULTICAST MTU:1500 Metric:1 + # RX packets:180 errors:0 dropped:0 overruns:0 frame:0 + # TX packets:218 errors:0 dropped:0 overruns:0 carrier:0 + # collisions:0 txqueuelen:1000 + # RX bytes:84577 TX bytes:31202 + + re1_ip = re.compile(r'(\w+): ip ([0-9.]+) mask.*') + # re1_ip will match output of the first format + # with group 1 returning the interface and group 2 returing the ip address. + + # re2_interface will match the interface line in the second format + # while re2_ip will match the inet addr line of the second format. + re2_interface = re.compile(r'(\w+)\s+Link') + re2_ip = re.compile(r'\s+inet addr:([0-9.]+)') + + matched_interface = None + matched_ip = None + re_bad_addr = re.compile(r'127.0.0.1|0.0.0.0') + + self._logger.debug('get_ip_address: ifconfig') + for interface in interfaces: + try: + output = self.shell_output('ifconfig %s' % interface, + timeout=timeout) + except ADBError: + output = '' + + for line in output.split("\n"): + if not matched_interface: + match = re1_ip.match(line) + if match: + matched_interface, matched_ip = match.groups() + else: + match = re2_interface.match(line) + if match: + matched_interface = match.group(1) + else: + match = re2_ip.match(line) + if match: + matched_ip = match.group(1) + + if matched_ip: + if not re_bad_addr.match(matched_ip): + self._logger.debug('get_ip_address: found: %s %s' % + (matched_interface, matched_ip)) + return matched_ip + matched_interface = None + matched_ip = None + + self._logger.debug('get_ip_address: netcfg') + # Fall back on netcfg if ifconfig does not work. + # $ adb shell netcfg + # lo UP 127.0.0.1/8 0x00000049 00:00:00:00:00:00 + # dummy0 DOWN 0.0.0.0/0 0x00000082 8e:cd:67:48:b7:c2 + # rmnet0 DOWN 0.0.0.0/0 0x00000000 00:00:00:00:00:00 + # rmnet1 DOWN 0.0.0.0/0 0x00000000 00:00:00:00:00:00 + # rmnet2 DOWN 0.0.0.0/0 0x00000000 00:00:00:00:00:00 + # rmnet3 DOWN 0.0.0.0/0 0x00000000 00:00:00:00:00:00 + # rmnet4 DOWN 0.0.0.0/0 0x00000000 00:00:00:00:00:00 + # rmnet5 DOWN 0.0.0.0/0 0x00000000 00:00:00:00:00:00 + # rmnet6 DOWN 0.0.0.0/0 0x00000000 00:00:00:00:00:00 + # rmnet7 DOWN 0.0.0.0/0 0x00000000 00:00:00:00:00:00 + # sit0 DOWN 0.0.0.0/0 0x00000080 00:00:00:00:00:00 + # vip0 DOWN 0.0.0.0/0 0x00001012 00:01:00:00:00:01 + # wlan0 UP 192.168.1.157/24 0x00001043 38:aa:3c:1c:f6:94 + + re3_netcfg = re.compile(r'(\w+)\s+UP\s+([1-9]\d{0,2}\.\d{1,3}\.\d{1,3}\.\d{1,3})') + try: + output = self.shell_output('netcfg', timeout=timeout) + except ADBError: + output = '' + for line in output.split("\n"): + match = re3_netcfg.search(line) + if match: + matched_interface, matched_ip = match.groups() + if matched_interface == "lo" or re_bad_addr.match(matched_ip): + matched_interface = None + matched_ip = None + elif matched_ip and matched_interface in interfaces: + self._logger.debug('get_ip_address: found: %s %s' % + (matched_interface, matched_ip)) + return matched_ip + self._logger.debug('get_ip_address: not found') + return matched_ip + + # File management methods + + def remount(self, timeout=None): + """Remount /system/ in read/write mode + + :param timeout: The maximum time in + seconds for any spawned adb process to complete before throwing + an ADBTimeoutError. + This timeout is per adb call. The total time spent + may exceed this value. If it is not specified, the value + set in the ADBDevice constructor is used. + :type timeout: integer or None + :raises: * ADBTimeoutError + * ADBError""" + + rv = self.command_output(["remount"], timeout=timeout) + if not rv.startswith("remount succeeded"): + raise ADBError("Unable to remount device") + + def chmod(self, path, recursive=False, mask="777", timeout=None, root=False): + """Recursively changes the permissions of a directory on the + device. + + :param str path: The directory name on the device. + :param bool recursive: Flag specifying if the command should be + executed recursively. + :param str mask: The octal permissions. + :param timeout: The maximum time in + seconds for any spawned adb process to complete before throwing + an ADBTimeoutError. + This timeout is per adb call. The total time spent + may exceed this value. If it is not specified, the value + set in the ADBDevice constructor is used. + :type timeout: integer or None + :param bool root: Flag specifying if the command should + be executed as root. + :raises: * ADBTimeoutError + * ADBRootError + * ADBError + """ + # Note that on some tests such as webappstartup, an error + # occurs during recursive calls to chmod where a "No such file + # or directory" error will occur for the + # /data/data/org.mozilla.fennec/files/mozilla/*.webapp0/lock + # which is a symbolic link to a socket: lock -> + # 127.0.0.1:+. On Linux, chmod -R ignores symbolic + # links but it appear Android's version does not. We ignore + # this type of error, but pass on any other errors that are + # detected. + path = posixpath.normpath(path.strip()) + self._logger.debug('chmod: path=%s, recursive=%s, mask=%s, root=%s' % + (path, recursive, mask, root)) + if not recursive: + self.shell_output("chmod %s %s" % (mask, path), + timeout=timeout, root=root) + return + + if self._chmod_R: + try: + self.shell_output("chmod -R %s %s" % (mask, path), + timeout=timeout, root=root) + except ADBError as e: + if e.message.find('No such file or directory') == -1: + raise + self._logger.warning('chmod -R %s %s: Ignoring Error: %s' % + (mask, path, e.message)) + return + # Obtain a list of the directories and files which match path + # and construct a shell script which explictly calls chmod on + # each of them. + entries = self.ls(path, recursive=recursive, timeout=timeout, + root=root) + tmpf = None + chmodsh = None + try: + tmpf = tempfile.NamedTemporaryFile(delete=False) + for entry in entries: + tmpf.write('chmod %s %s\n' % (mask, entry)) + tmpf.close() + chmodsh = '/data/local/tmp/%s' % os.path.basename(tmpf.name) + self.push(tmpf.name, chmodsh) + self.shell_output('chmod 777 %s' % chmodsh, timeout=timeout, + root=root) + self.shell_output('sh -c %s' % chmodsh, timeout=timeout, + root=root) + finally: + if tmpf: + os.unlink(tmpf.name) + if chmodsh: + self.rm(chmodsh, timeout=timeout, root=root) + + def exists(self, path, timeout=None, root=False): + """Returns True if the path exists on the device. + + :param str path: The directory name on the device. + :param timeout: The maximum time in + seconds for any spawned adb process to complete before + throwing an ADBTimeoutError. + This timeout is per adb call. The total time spent + may exceed this value. If it is not specified, the value + set in the ADBDevice constructor is used. + :type timeout: integer or None + :param bool root: Flag specifying if the command should be + executed as root. + :returns: boolean - True if path exists. + :raises: * ADBTimeoutError + * ADBRootError + """ + path = posixpath.normpath(path) + return self.shell_bool('ls -a %s' % path, timeout=timeout, root=root) + + def is_dir(self, path, timeout=None, root=False): + """Returns True if path is an existing directory on the device. + + :param str path: The path on the device. + :param timeout: The maximum time in + seconds for any spawned adb process to complete before + throwing an ADBTimeoutError. + This timeout is per adb call. The total time spent + may exceed this value. If it is not specified, the value + set in the ADBDevice constructor is used. + :type timeout: integer or None + :param bool root: Flag specifying if the command should + be executed as root. + :returns: boolean - True if path exists on the device and is a + directory. + :raises: * ADBTimeoutError + * ADBRootError + """ + path = posixpath.normpath(path) + return self.shell_bool('ls -a %s/' % path, timeout=timeout, root=root) + + def is_file(self, path, timeout=None, root=False): + """Returns True if path is an existing file on the device. + + :param str path: The file name on the device. + :param timeout: The maximum time in + seconds for any spawned adb process to complete before + throwing an ADBTimeoutError. + This timeout is per adb call. The total time spent + may exceed this value. If it is not specified, the value + set in the ADBDevice constructor is used. + :type timeout: integer or None + :param bool root: Flag specifying if the command should + be executed as root. + :returns: boolean - True if path exists on the device and is a + file. + :raises: * ADBTimeoutError + * ADBRootError + """ + path = posixpath.normpath(path) + return ( + self.exists(path, timeout=timeout, root=root) and + not self.is_dir(path, timeout=timeout, root=root)) + + def list_files(self, path, timeout=None, root=False): + """Return a list of files/directories contained in a directory + on the device. + + :param str path: The directory name on the device. + :param timeout: The maximum time in + seconds for any spawned adb process to complete before + throwing an ADBTimeoutError. + This timeout is per adb call. The total time spent + may exceed this value. If it is not specified, the value + set in the ADBDevice constructor is used. + :type timeout: integer or None + :param bool root: Flag specifying if the command should + be executed as root. + :returns: list of files/directories contained in the directory. + :raises: * ADBTimeoutError + * ADBRootError + """ + path = posixpath.normpath(path.strip()) + data = [] + if self.is_dir(path, timeout=timeout, root=root): + try: + data = self.shell_output("%s %s" % (self._ls, path), + timeout=timeout, + root=root).split('\r\n') + self._logger.debug('list_files: data: %s' % data) + except ADBError: + self._logger.error('Ignoring exception in ADBDevice.list_files\n%s' % + traceback.format_exc()) + data[:] = [item for item in data if item] + self._logger.debug('list_files: %s' % data) + return data + + def ls(self, path, recursive=False, timeout=None, root=False): + """Return a list of matching files/directories on the device. + + The ls method emulates the behavior of the ls shell command. + It differs from the list_files method by supporting wild cards + and returning matches even if the path is not a directory and + by allowing a recursive listing. + + ls /sdcard always returns /sdcard and not the contents of the + sdcard path. The ls method makes the behavior consistent with + others paths by adjusting /sdcard to /sdcard/. Note this is + also the case of other sdcard related paths such as + /storage/emulated/legacy but no adjustment is made in those + cases. + + The ls method works around a Nexus 4 bug which prevents + recursive listing of directories on the sdcard unless the path + ends with "/*" by adjusting sdcard paths ending in "/" to end + with "/*". This adjustment is only made on official Nexus 4 + builds with property ro.product.model "Nexus 4". Note that + this will fail to return any "hidden" files or directories + which begin with ".". + + :param str path: The directory name on the device. + :param bool recursive: Flag specifying if a recursive listing + is to be returned. If recursive is False, the returned + matches will be relative to the path. If recursive is True, + the returned matches will be absolute paths. + :param timeout: The maximum time in + seconds for any spawned adb process to complete before + throwing an ADBTimeoutError. + This timeout is per adb call. The total time spent + may exceed this value. If it is not specified, the value + set in the ADBDevice constructor is used. + :type timeout: integer or None + :param bool root: Flag specifying if the command should + be executed as root. + :returns: list of files/directories contained in the directory. + :raises: * ADBTimeoutError + * ADBRootError + """ + path = posixpath.normpath(path.strip()) + parent = '' + entries = {} + + if path == '/sdcard': + path += '/' + + # Android 2.3 and later all appear to support ls -R however + # Nexus 4 does not perform a recursive search on the sdcard + # unless the path is a directory with * wild card. + if not recursive: + recursive_flag = '' + else: + recursive_flag = '-R' + if path.startswith('/sdcard') and path.endswith('/'): + model = self.shell_output('getprop ro.product.model', + timeout=timeout, + root=root) + if model == 'Nexus 4': + path += '*' + lines = self.shell_output('%s %s %s' % (self._ls, recursive_flag, path), + timeout=timeout, + root=root).split('\r\n') + for line in lines: + line = line.strip() + if not line: + parent = '' + continue + if line.endswith(':'): # This is a directory + parent = line.replace(':', '/') + entry = parent + # Remove earlier entry which is marked as a file. + if parent[:-1] in entries: + del entries[parent[:-1]] + elif parent: + entry = "%s%s" % (parent, line) + else: + entry = line + entries[entry] = 1 + entry_list = entries.keys() + entry_list.sort() + return entry_list + + def mkdir(self, path, parents=False, timeout=None, root=False): + """Create a directory on the device. + + :param str path: The directory name on the device + to be created. + :param bool parents: Flag indicating if the parent directories are + also to be created. Think mkdir -p path. + :param timeout: The maximum time in + seconds for any spawned adb process to complete before + throwing an ADBTimeoutError. + This timeout is per adb call. The total time spent + may exceed this value. If it is not specified, the value + set in the ADBDevice constructor is used. + :type timeout: integer or None + :param bool root: Flag specifying if the command should + be executed as root. + :raises: * ADBTimeoutError + * ADBRootError + * ADBError + """ + path = posixpath.normpath(path) + if parents: + if self._mkdir_p is None or self._mkdir_p: + # Use shell_bool to catch the possible + # non-zero exitcode if -p is not supported. + if self.shell_bool('mkdir -p %s' % path, timeout=timeout, + root=root): + self._mkdir_p = True + return + # mkdir -p is not supported. create the parent + # directories individually. + if not self.is_dir(posixpath.dirname(path), root=root): + parts = path.split('/') + name = "/" + for part in parts[:-1]: + if part != "": + name = posixpath.join(name, part) + if not self.is_dir(name, root=root): + # Use shell_output to allow any non-zero + # exitcode to raise an ADBError. + self.shell_output('mkdir %s' % name, + timeout=timeout, root=root) + + # If parents is True and the directory does exist, we don't + # need to do anything. Otherwise we call mkdir. If the + # directory already exists or if it is a file instead of a + # directory, mkdir will fail and we will raise an ADBError. + if not parents or not self.is_dir(path, root=root): + self.shell_output('mkdir %s' % path, timeout=timeout, root=root) + if not self.is_dir(path, timeout=timeout, root=root): + raise ADBError('mkdir %s Failed' % path) + + def push(self, local, remote, timeout=None): + """Pushes a file or directory to the device. + + :param str local: The name of the local file or + directory name. + :param str remote: The name of the remote file or + directory name. + :param timeout: The maximum time in + seconds for any spawned adb process to complete before + throwing an ADBTimeoutError. + This timeout is per adb call. The total time spent + may exceed this value. If it is not specified, the value + set in the ADBDevice constructor is used. + :type timeout: integer or None + :raises: * ADBTimeoutError + * ADBError + """ + # remove trailing / + local = os.path.normpath(local) + remote = os.path.normpath(remote) + copy_required = False + if self._adb_version >= '1.0.36' and \ + os.path.isdir(local) and self.is_dir(remote): + # See do_sync_push in + # https://android.googlesource.com/platform/system/core/+/master/adb/file_sync_client.cpp + # Work around change in behavior in adb 1.0.36 where if + # the remote destination directory exists, adb push will + # copy the source directory *into* the destination + # directory otherwise it will copy the source directory + # *onto* the destination directory. + # + # If the destination directory does exist, push to its + # parent directory. If the source and destination leaf + # directory names are different, copy the source directory + # to a temporary directory with the same leaf name as the + # destination so that when we push to the parent, the + # source is copied onto the destination directory. + local_name = os.path.basename(local) + remote_name = os.path.basename(remote) + if local_name != remote_name: + copy_required = True + temp_parent = tempfile.mkdtemp() + new_local = os.path.join(temp_parent, remote_name) + dir_util.copy_tree(local, new_local) + local = new_local + remote = '/'.join(remote.rstrip('/').split('/')[:-1]) + try: + self.command_output(["push", local, remote], timeout=timeout) + except: + raise + finally: + if copy_required: + shutil.rmtree(temp_parent) + + def pull(self, remote, local, timeout=None): + """Pulls a file or directory from the device. + + :param str remote: The path of the remote file or + directory. + :param str local: The path of the local file or + directory name. + :param timeout: The maximum time in + seconds for any spawned adb process to complete before + throwing an ADBTimeoutError. + This timeout is per adb call. The total time spent + may exceed this value. If it is not specified, the value + set in the ADBDevice constructor is used. + :type timeout: integer or None + :raises: * ADBTimeoutError + * ADBError + """ + # remove trailing / + local = os.path.normpath(local) + remote = os.path.normpath(remote) + copy_required = False + original_local = local + if self._adb_version >= '1.0.36' and \ + os.path.isdir(local) and self.is_dir(remote): + # See do_sync_pull in + # https://android.googlesource.com/platform/system/core/+/master/adb/file_sync_client.cpp + # Work around change in behavior in adb 1.0.36 where if + # the local destination directory exists, adb pull will + # copy the source directory *into* the destination + # directory otherwise it will copy the source directory + # *onto* the destination directory. + # + # If the destination directory does exist, pull to its + # parent directory. If the source and destination leaf + # directory names are different, pull the source directory + # into a temporary directory and then copy the temporary + # directory onto the destination. + local_name = os.path.basename(local) + remote_name = os.path.basename(remote) + if local_name != remote_name: + copy_required = True + temp_parent = tempfile.mkdtemp() + local = os.path.join(temp_parent, remote_name) + else: + local = '/'.join(local.rstrip('/').split('/')[:-1]) + try: + self.command_output(["pull", remote, local], timeout=timeout) + except: + raise + finally: + if copy_required: + dir_util.copy_tree(local, original_local) + shutil.rmtree(temp_parent) + + def rm(self, path, recursive=False, force=False, timeout=None, root=False): + """Delete files or directories on the device. + + :param str path: The path of the remote file or directory. + :param bool recursive: Flag specifying if the command is + to be applied recursively to the target. Default is False. + :param bool force: Flag which if True will not raise an + error when attempting to delete a non-existent file. Default + is False. + :param timeout: The maximum time in + seconds for any spawned adb process to complete before + throwing an ADBTimeoutError. + This timeout is per adb call. The total time spent + may exceed this value. If it is not specified, the value + set in the ADBDevice constructor is used. + :type timeout: integer or None + :param bool root: Flag specifying if the command should + be executed as root. + :raises: * ADBTimeoutError + * ADBRootError + * ADBError + """ + cmd = "rm" + if recursive: + cmd += " -r" + try: + self.shell_output("%s %s" % (cmd, path), timeout=timeout, root=root) + if self.is_file(path, timeout=timeout, root=root): + raise ADBError('rm("%s") failed to remove file.' % path) + except ADBError as e: + if not force and 'No such file or directory' in e.message: + raise + + def rmdir(self, path, timeout=None, root=False): + """Delete empty directory on the device. + + :param str path: The directory name on the device. + :param timeout: The maximum time in + seconds for any spawned adb process to complete before + throwing an ADBTimeoutError. + This timeout is per adb call. The total time spent + may exceed this value. If it is not specified, the value + set in the ADBDevice constructor is used. + :type timeout: integer or None + :param bool root: Flag specifying if the command should + be executed as root. + :raises: * ADBTimeoutError + * ADBRootError + * ADBError + """ + self.shell_output("rmdir %s" % path, timeout=timeout, root=root) + if self.is_dir(path, timeout=timeout, root=root): + raise ADBError('rmdir("%s") failed to remove directory.' % path) + + # Process management methods + + def get_process_list(self, timeout=None): + """Returns list of tuples (pid, name, user) for running + processes on device. + + :param timeout: The maximum time + in seconds for any spawned adb process to complete before + throwing an ADBTimeoutError. + This timeout is per adb call. The total time spent + may exceed this value. If it is not specified, + the value set in the ADBDevice constructor is used. + :type timeout: integer or None + :returns: list of (pid, name, user) tuples for running processes + on the device. + :raises: * ADBTimeoutError + * ADBError + """ + adb_process = None + try: + adb_process = self.shell("ps", timeout=timeout) + if adb_process.timedout: + raise ADBTimeoutError("%s" % adb_process) + elif adb_process.exitcode: + raise ADBError("%s" % adb_process) + # first line is the headers + header = adb_process.stdout_file.readline() + pid_i = -1 + user_i = -1 + els = header.split() + for i in range(len(els)): + item = els[i].lower() + if item == 'user': + user_i = i + elif item == 'pid': + pid_i = i + if user_i == -1 or pid_i == -1: + self._logger.error('get_process_list: %s' % header) + raise ADBError('get_process_list: Unknown format: %s: %s' % ( + header, adb_process)) + ret = [] + line = adb_process.stdout_file.readline() + while line: + els = line.split() + try: + ret.append([int(els[pid_i]), els[-1], els[user_i]]) + except ValueError: + self._logger.error('get_process_list: %s %s\n%s' % ( + header, line, traceback.format_exc())) + raise ADBError('get_process_list: %s: %s: %s' % ( + header, line, adb_process)) + line = adb_process.stdout_file.readline() + self._logger.debug('get_process_list: %s' % ret) + return ret + finally: + if adb_process and isinstance(adb_process.stdout_file, file): + adb_process.stdout_file.close() + + def kill(self, pids, sig=None, attempts=3, wait=5, + timeout=None, root=False): + """Kills processes on the device given a list of process ids. + + :param list pids: process ids to be killed. + :param sig: signal to be sent to the process. + :type sig: integer or None + :param integer attempts: number of attempts to try to + kill the processes. + :param integer wait: number of seconds to wait after each attempt. + :param timeout: The maximum time in + seconds for any spawned adb process to complete before + throwing an ADBTimeoutError. + This timeout is per adb call. The total time spent + may exceed this value. If it is not specified, the value + set in the ADBDevice constructor is used. + :type timeout: integer or None + :param bool root: Flag specifying if the command should + be executed as root. + :raises: * ADBTimeoutError + * ADBRootError + * ADBError + """ + pid_list = [str(pid) for pid in pids] + for attempt in range(attempts): + args = ["kill"] + if sig: + args.append("-%d" % sig) + args.extend(pid_list) + try: + self.shell_output(' '.join(args), timeout=timeout, root=root) + except ADBError as e: + if 'No such process' not in e.message: + raise + pid_set = set(pid_list) + current_pid_set = set([str(proc[0]) for proc in + self.get_process_list(timeout=timeout)]) + pid_list = list(pid_set.intersection(current_pid_set)) + if not pid_list: + break + self._logger.debug("Attempt %d of %d to kill processes %s failed" % + (attempt + 1, attempts, pid_list)) + time.sleep(wait) + + if pid_list: + raise ADBError('kill: processes %s not killed' % pid_list) + + def pkill(self, appname, sig=None, attempts=3, wait=5, + timeout=None, root=False): + """Kills a processes on the device matching a name. + + :param str appname: The app name of the process to + be killed. Note that only the first 75 characters of the + process name are significant. + :param sig: optional signal to be sent to the process. + :type sig: integer or None + :param integer attempts: number of attempts to try to + kill the processes. + :param integer wait: number of seconds to wait after each attempt. + :param timeout: The maximum time in + seconds for any spawned adb process to complete before + throwing an ADBTimeoutError. + This timeout is per adb call. The total time spent + may exceed this value. If it is not specified, the value + set in the ADBDevice constructor is used. + :type timeout: integer or None + :param bool root: Flag specifying if the command should + be executed as root. + + :raises: * ADBTimeoutError + * ADBRootError + * ADBError + """ + procs = self.get_process_list(timeout=timeout) + # limit the comparion to the first 75 characters due to a + # limitation in processname length in android. + pids = [proc[0] for proc in procs if proc[1] == appname[:75]] + if not pids: + return + + try: + self.kill(pids, sig, attempts=attempts, wait=wait, + timeout=timeout, root=root) + except ADBError as e: + if self.process_exist(appname, timeout=timeout): + raise e + + def process_exist(self, process_name, timeout=None): + """Returns True if process with name process_name is running on + device. + + :param str process_name: The name of the process + to check. Note that only the first 75 characters of the + process name are significant. + :param timeout: The maximum time in + seconds for any spawned adb process to complete before + throwing an ADBTimeoutError. + This timeout is per adb call. The total time spent + may exceed this value. If it is not specified, the value + set in the ADBDevice constructor is used. + :type timeout: integer or None + :returns: boolean - True if process exists. + + :raises: * ADBTimeoutError + * ADBError + """ + if not isinstance(process_name, basestring): + raise ADBError("Process name %s is not a string" % process_name) + + # Filter out extra spaces. + parts = [x for x in process_name.split(' ') if x != ''] + process_name = ' '.join(parts) + + # Filter out the quoted env string if it exists + # ex: '"name=value;name2=value2;etc=..." process args' -> 'process args' + parts = process_name.split('"') + if len(parts) > 2: + process_name = ' '.join(parts[2:]).strip() + + pieces = process_name.split(' ') + parts = pieces[0].split('/') + app = parts[-1] + + proc_list = self.get_process_list(timeout=timeout) + if not proc_list: + return False + + for proc in proc_list: + proc_name = proc[1].split('/')[-1] + # limit the comparion to the first 75 characters due to a + # limitation in processname length in android. + if proc_name == app[:75]: + return True + return False + + def cp(self, source, destination, recursive=False, timeout=None, + root=False): + """Copies a file or directory on the device. + + :param source: string containing the path of the source file or + directory. + :param destination: string containing the path of the destination file + or directory. + :param recursive: optional boolean indicating if a recursive copy is to + be performed. Required if the source is a directory. Defaults to + False. Think cp -R source destination. + :param timeout: optional integer specifying the maximum time in + seconds for any spawned adb process to complete before + throwing an ADBTimeoutError. + This timeout is per adb call. The total time spent + may exceed this value. If it is not specified, the value + set in the ADBDevice constructor is used. + :raises: * ADBTimeoutError + * ADBRootError + * ADBError + """ + source = posixpath.normpath(source) + destination = posixpath.normpath(destination) + if self._have_cp: + r = '-R' if recursive else '' + self.shell_output('cp %s %s %s' % (r, source, destination), + timeout=timeout, root=root) + return + + # Emulate cp behavior depending on if source and destination + # already exists and whether they are a directory or file. + if not self.exists(source, timeout=timeout, root=root): + raise ADBError("cp: can't stat '%s': No such file or directory" % + source) + + if self.is_file(source, timeout=timeout, root=root): + if self.is_dir(destination, timeout=timeout, root=root): + # Copy the source file into the destination directory + destination = posixpath.join(destination, + posixpath.basename(source)) + self.shell_output('dd if=%s of=%s' % (source, destination), + timeout=timeout, root=root) + return + + if self.is_file(destination, timeout=timeout, root=root): + raise ADBError('cp: %s: Not a directory' % destination) + + if not recursive: + raise ADBError("cp: omitting directory '%s'" % source) + + if self.is_dir(destination, timeout=timeout, root=root): + # Copy the source directory into the destination directory. + destination_dir = posixpath.join(destination, + posixpath.basename(source)) + else: + # Copy the contents of the source directory into the + # destination directory. + destination_dir = destination + + try: + # Do not create parent directories since cp does not. + self.mkdir(destination_dir, timeout=timeout, root=root) + except ADBError as e: + if 'File exists' not in e.message: + raise + + for i in self.list_files(source, timeout=timeout, root=root): + self.cp(posixpath.join(source, i), + posixpath.join(destination_dir, i), + recursive=recursive, + timeout=timeout, root=root) + + def mv(self, source, destination, timeout=None, root=False): + """Moves a file or directory on the device. + + :param source: string containing the path of the source file or + directory. + :param destination: string containing the path of the destination file + or directory. + :param timeout: optional integer specifying the maximum time in + seconds for any spawned adb process to complete before + throwing an ADBTimeoutError. + This timeout is per adb call. The total time spent + may exceed this value. If it is not specified, the value + set in the ADBDevice constructor is used. + :raises: * ADBTimeoutError + * ADBRootError + * ADBError + """ + source = posixpath.normpath(source) + destination = posixpath.normpath(destination) + self.shell_output('mv %s %s' % (source, destination), timeout=timeout, + root=root) + + def reboot(self, timeout=None): + """Reboots the device. + + :param timeout: optional integer specifying the maximum time in + seconds for any spawned adb process to complete before + throwing an ADBTimeoutError. + This timeout is per adb call. The total time spent + may exceed this value. If it is not specified, the value + set in the ADB constructor is used. + :raises: * ADBTimeoutError + * ADBError + + reboot() reboots the device, issues an adb wait-for-device in order to + wait for the device to complete rebooting, then calls is_device_ready() + to determine if the device has completed booting. + """ + self.command_output(["reboot"], timeout=timeout) + # command_output automatically inserts a 'wait-for-device' + # argument to adb. Issuing an empty command is the same as adb + # -s wait-for-device. We don't send an explicit + # 'wait-for-device' since that would add duplicate + # 'wait-for-device' arguments which is an error in newer + # versions of adb. + self.command_output([], timeout=timeout) + self._check_adb_root(timeout=timeout) + return self.is_device_ready(timeout=timeout) + + @abstractmethod + def is_device_ready(self, timeout=None): + """Abstract class that returns True if the device is ready. + + :param timeout: optional integer specifying the maximum time in + seconds for any spawned adb process to complete before + throwing an ADBTimeoutError. + This timeout is per adb call. The total time spent + may exceed this value. If it is not specified, the value + set in the ADB constructor is used. + :raises: * ADBTimeoutError + * ADBError + """ + return + + @abstractmethod + def get_battery_percentage(self, timeout=None): + """Abstract class that returns the battery charge as a percentage. + + :param timeout: optional integer specifying the maximum time in + seconds for any spawned adb process to complete before + throwing an ADBTimeoutError. + This timeout is per adb call. The total time spent + may exceed this value. If it is not specified, the value + set in the ADBDevice constructor is used. + :returns: battery charge as a percentage. + :raises: * ADBTimeoutError + * ADBError + """ + return + + def get_info(self, directive=None, timeout=None): + """ + Returns a dictionary of information strings about the device. + + :param directive: information you want to get. Options are: + - `battery` - battery charge as a percentage + - `disk` - total, free, available bytes on disk + - `id` - unique id of the device + - `os` - name of the os + - `process` - list of running processes (same as ps) + - `systime` - system time of the device + - `uptime` - uptime of the device + + If `directive` is `None`, will return all available information + :param timeout: optional integer specifying the maximum time in + seconds for any spawned adb process to complete before + throwing an ADBTimeoutError. + This timeout is per adb call. The total time spent + may exceed this value. If it is not specified, the value + set in the ADB constructor is used. + :raises: * ADBTimeoutError + * ADBError + """ + directives = ['battery', 'disk', 'id', 'os', 'process', 'systime', + 'uptime'] + + if directive in directives: + directives = [directive] + + info = {} + if 'battery' in directives: + info['battery'] = self.get_battery_percentage(timeout=timeout) + if 'disk' in directives: + info['disk'] = self.shell_output('df /data /system /sdcard', + timeout=timeout).splitlines() + if 'id' in directives: + info['id'] = self.command_output(['get-serialno'], timeout=timeout) + if 'os' in directives: + info['os'] = self.shell_output('getprop ro.build.display.id', + timeout=timeout) + if 'process' in directives: + ps = self.shell_output('ps', timeout=timeout) + info['process'] = ps.splitlines() + if 'systime' in directives: + info['systime'] = self.shell_output('date', timeout=timeout) + if 'uptime' in directives: + uptime = self.shell_output('uptime', timeout=timeout) + if uptime: + m = re.match(r'up time: ((\d+) days, )*(\d{2}):(\d{2}):(\d{2})', + uptime) + if m: + uptime = '%d days %d hours %d minutes %d seconds' % tuple( + [int(g or 0) for g in m.groups()[1:]]) + info['uptime'] = uptime + return info diff --git a/testing/mozbase/mozdevice/mozdevice/adb_android.py b/testing/mozbase/mozdevice/mozdevice/adb_android.py new file mode 100644 index 000000000..bf5fffc0e --- /dev/null +++ b/testing/mozbase/mozdevice/mozdevice/adb_android.py @@ -0,0 +1,493 @@ +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this file, +# You can obtain one at http://mozilla.org/MPL/2.0/. + +import os +import re +import time + +from abc import ABCMeta + +import version_codes + +from adb import ADBDevice, ADBError + + +class ADBAndroid(ADBDevice): + """ADBAndroid implements :class:`ADBDevice` providing Android-specific + functionality. + + :: + + from mozdevice import ADBAndroid + + adbdevice = ADBAndroid() + print adbdevice.list_files("/mnt/sdcard") + if adbdevice.process_exist("org.mozilla.fennec"): + print "Fennec is running" + """ + __metaclass__ = ABCMeta + + def __init__(self, + device=None, + adb='adb', + adb_host=None, + adb_port=None, + test_root='', + logger_name='adb', + timeout=300, + verbose=False, + device_ready_retry_wait=20, + device_ready_retry_attempts=3): + """Initializes the ADBAndroid object. + + :param device: When a string is passed, it is interpreted as the + device serial number. This form is not compatible with + devices containing a ":" in the serial; in this case + ValueError will be raised. + When a dictionary is passed it must have one or both of + the keys "device_serial" and "usb". This is compatible + with the dictionaries in the list returned by + ADBHost.devices(). If the value of device_serial is a + valid serial not containing a ":" it will be used to + identify the device, otherwise the value of the usb key, + prefixed with "usb:" is used. + If None is passed and there is exactly one device attached + to the host, that device is used. If there is more than one + device attached, ValueError is raised. If no device is + attached the constructor will block until a device is + attached or the timeout is reached. + :type device: dict, str or None + :param adb_host: host of the adb server to connect to. + :type adb_host: str or None + :param adb_port: port of the adb server to connect to. + :type adb_port: integer or None + :param str logger_name: logging logger name. Defaults to 'adb'. + :param integer device_ready_retry_wait: number of seconds to wait + between attempts to check if the device is ready after a + reboot. + :param integer device_ready_retry_attempts: number of attempts when + checking if a device is ready. + + :raises: * ADBError + * ADBTimeoutError + * ValueError + """ + ADBDevice.__init__(self, device=device, adb=adb, + adb_host=adb_host, adb_port=adb_port, + test_root=test_root, + logger_name=logger_name, timeout=timeout, + verbose=verbose, + device_ready_retry_wait=device_ready_retry_wait, + device_ready_retry_attempts=device_ready_retry_attempts) + # https://source.android.com/devices/tech/security/selinux/index.html + # setenforce + # usage: setenforce [ Enforcing | Permissive | 1 | 0 ] + # getenforce returns either Enforcing or Permissive + + try: + self.selinux = True + if self.shell_output('getenforce', timeout=timeout) != 'Permissive': + self._logger.info('Setting SELinux Permissive Mode') + self.shell_output("setenforce Permissive", timeout=timeout, root=True) + except ADBError: + self.selinux = False + + self.version = int(self.shell_output("getprop ro.build.version.sdk", + timeout=timeout)) + + def reboot(self, timeout=None): + """Reboots the device. + + :param timeout: optional integer specifying the maximum time in + seconds for any spawned adb process to complete before + throwing an ADBTimeoutError. + This timeout is per adb call. The total time spent + may exceed this value. If it is not specified, the value + set in the ADB constructor is used. + :raises: * ADBTimeoutError + * ADBError + + reboot() reboots the device, issues an adb wait-for-device in order to + wait for the device to complete rebooting, then calls is_device_ready() + to determine if the device has completed booting. + + If the device supports running adbd as root, adbd will be + restarted running as root. Then, if the device supports + SELinux, setenforce Permissive will be called to change + SELinux to permissive. This must be done after adbd is + restarted in order for the SELinux Permissive setting to + persist. + + """ + ready = ADBDevice.reboot(self, timeout=timeout) + self._check_adb_root(timeout=timeout) + return ready + + # Informational methods + + def get_battery_percentage(self, timeout=None): + """Returns the battery charge as a percentage. + + :param timeout: The maximum time in + seconds for any spawned adb process to complete before + throwing an ADBTimeoutError. + This timeout is per adb call. The total time spent + may exceed this value. If it is not specified, the value + set in the ADBDevice constructor is used. + :type timeout: integer or None + :returns: battery charge as a percentage. + :raises: * ADBTimeoutError + * ADBError + """ + level = None + scale = None + percentage = 0 + cmd = "dumpsys battery" + re_parameter = re.compile(r'\s+(\w+):\s+(\d+)') + lines = self.shell_output(cmd, timeout=timeout).split('\r') + for line in lines: + match = re_parameter.match(line) + if match: + parameter = match.group(1) + value = match.group(2) + if parameter == 'level': + level = float(value) + elif parameter == 'scale': + scale = float(value) + if parameter is not None and scale is not None: + percentage = 100.0 * level / scale + break + return percentage + + # System control methods + + def is_device_ready(self, timeout=None): + """Checks if a device is ready for testing. + + This method uses the android only package manager to check for + readiness. + + :param timeout: The maximum time + in seconds for any spawned adb process to complete before + throwing an ADBTimeoutError. + This timeout is per adb call. The total time spent + may exceed this value. If it is not specified, the value + set in the ADB constructor is used. + :type timeout: integer or None + :raises: * ADBTimeoutError + * ADBError + """ + # command_output automatically inserts a 'wait-for-device' + # argument to adb. Issuing an empty command is the same as adb + # -s wait-for-device. We don't send an explicit + # 'wait-for-device' since that would add duplicate + # 'wait-for-device' arguments which is an error in newer + # versions of adb. + self.command_output([], timeout=timeout) + pm_error_string = "Error: Could not access the Package Manager" + pm_list_commands = ["packages", "permission-groups", "permissions", + "instrumentation", "features", "libraries"] + ready_path = os.path.join(self.test_root, "ready") + for attempt in range(self._device_ready_retry_attempts): + failure = 'Unknown failure' + success = True + try: + state = self.get_state(timeout=timeout) + if state != 'device': + failure = "Device state: %s" % state + success = False + else: + if (self.selinux and self.shell_output('getenforce', + timeout=timeout) != 'Permissive'): + self._logger.info('Setting SELinux Permissive Mode') + self.shell_output("setenforce Permissive", timeout=timeout, root=True) + if self.is_dir(ready_path, timeout=timeout, root=True): + self.rmdir(ready_path, timeout=timeout, root=True) + self.mkdir(ready_path, timeout=timeout, root=True) + self.rmdir(ready_path, timeout=timeout, root=True) + # Invoke the pm list commands to see if it is up and + # running. + for pm_list_cmd in pm_list_commands: + data = self.shell_output("pm list %s" % pm_list_cmd, + timeout=timeout) + if pm_error_string in data: + failure = data + success = False + break + except ADBError as e: + success = False + failure = e.message + + if not success: + self._logger.debug('Attempt %s of %s device not ready: %s' % ( + attempt + 1, self._device_ready_retry_attempts, + failure)) + time.sleep(self._device_ready_retry_wait) + + return success + + def power_on(self, timeout=None): + """Sets the device's power stayon value. + + :param timeout: The maximum time in + seconds for any spawned adb process to complete before + throwing an ADBTimeoutError. + This timeout is per adb call. The total time spent + may exceed this value. If it is not specified, the value + set in the ADB constructor is used. + :type timeout: integer or None + :raises: * ADBTimeoutError + * ADBError + """ + try: + self.shell_output('svc power stayon true', + timeout=timeout, + root=True) + except ADBError as e: + # Executing this via adb shell errors, but not interactively. + # Any other exitcode is a real error. + if 'exitcode: 137' not in e.message: + raise + self._logger.warning('Unable to set power stayon true: %s' % e) + + # Application management methods + + def install_app(self, apk_path, timeout=None): + """Installs an app on the device. + + :param str apk_path: The apk file name to be installed. + :param timeout: The maximum time in + seconds for any spawned adb process to complete before + throwing an ADBTimeoutError. + This timeout is per adb call. The total time spent + may exceed this value. If it is not specified, the value + set in the ADB constructor is used. + :type timeout: integer or None + :raises: * ADBTimeoutError + * ADBError + """ + cmd = ["install"] + if self.version >= version_codes.M: + cmd.append("-g") + cmd.append(apk_path) + data = self.command_output(cmd, timeout=timeout) + if data.find('Success') == -1: + raise ADBError("install failed for %s. Got: %s" % + (apk_path, data)) + + def is_app_installed(self, app_name, timeout=None): + """Returns True if an app is installed on the device. + + :param str app_name: The name of the app to be checked. + :param timeout: The maximum time in + seconds for any spawned adb process to complete before + throwing an ADBTimeoutError. + This timeout is per adb call. The total time spent + may exceed this value. If it is not specified, the value + set in the ADB constructor is used. + :type timeout: integer or None + :raises: * ADBTimeoutError + * ADBError + """ + pm_error_string = 'Error: Could not access the Package Manager' + data = self.shell_output("pm list package %s" % app_name, timeout=timeout) + if pm_error_string in data: + raise ADBError(pm_error_string) + if app_name not in data: + return False + return True + + def launch_application(self, app_name, activity_name, intent, url=None, + extras=None, wait=True, fail_if_running=True, + timeout=None): + """Launches an Android application + + :param str app_name: Name of application (e.g. `com.android.chrome`) + :param str activity_name: Name of activity to launch (e.g. `.Main`) + :param str intent: Intent to launch application with + :param url: URL to open + :type url: str or None + :param extras: Extra arguments for application. + :type extras: dict or None + :param bool wait: If True, wait for application to start before + returning. + :param bool fail_if_running: Raise an exception if instance of + application is already running. + :param timeout: The maximum time in + seconds for any spawned adb process to complete before + throwing an ADBTimeoutError. + This timeout is per adb call. The total time spent + may exceed this value. If it is not specified, the value + set in the ADB constructor is used. + :type timeout: integer or None + :raises: * ADBTimeoutError + * ADBError + """ + # If fail_if_running is True, we throw an exception here. Only one + # instance of an application can be running at once on Android, + # starting a new instance may not be what we want depending on what + # we want to do + if fail_if_running and self.process_exist(app_name, timeout=timeout): + raise ADBError("Only one instance of an application may be running " + "at once") + + acmd = ["am", "start"] + \ + ["-W" if wait else '', "-n", "%s/%s" % (app_name, activity_name)] + + if intent: + acmd.extend(["-a", intent]) + + if extras: + for (key, val) in extras.iteritems(): + if isinstance(val, int): + extra_type_param = "--ei" + elif isinstance(val, bool): + extra_type_param = "--ez" + else: + extra_type_param = "--es" + acmd.extend([extra_type_param, str(key), str(val)]) + + if url: + acmd.extend(["-d", url]) + + cmd = self._escape_command_line(acmd) + self.shell_output(cmd, timeout=timeout) + + def launch_fennec(self, app_name, intent="android.intent.action.VIEW", + moz_env=None, extra_args=None, url=None, wait=True, + fail_if_running=True, timeout=None): + """Convenience method to launch Fennec on Android with various + debugging arguments + + :param str app_name: Name of fennec application (e.g. + `org.mozilla.fennec`) + :param str intent: Intent to launch application. + :param moz_env: Mozilla specific environment to pass into + application. + :type moz_env: str or None + :param extra_args: Extra arguments to be parsed by fennec. + :type extra_args: str or None + :param url: URL to open + :type url: str or None + :param bool wait: If True, wait for application to start before + returning. + :param bool fail_if_running: Raise an exception if instance of + application is already running. + :param timeout: The maximum time in + seconds for any spawned adb process to complete before + throwing an ADBTimeoutError. + This timeout is per adb call. The total time spent + may exceed this value. If it is not specified, the value + set in the ADB constructor is used. + :type timeout: integer or None + :raises: * ADBTimeoutError + * ADBError + """ + extras = {} + + if moz_env: + # moz_env is expected to be a dictionary of environment variables: + # Fennec itself will set them when launched + for (env_count, (env_key, env_val)) in enumerate(moz_env.iteritems()): + extras["env" + str(env_count)] = env_key + "=" + env_val + + # Additional command line arguments that fennec will read and use (e.g. + # with a custom profile) + if extra_args: + extras['args'] = " ".join(extra_args) + + self.launch_application(app_name, "org.mozilla.gecko.BrowserApp", + intent, url=url, extras=extras, + wait=wait, fail_if_running=fail_if_running, + timeout=timeout) + + def stop_application(self, app_name, timeout=None, root=False): + """Stops the specified application + + For Android 3.0+, we use the "am force-stop" to do this, which + is reliable and does not require root. For earlier versions of + Android, we simply try to manually kill the processes started + by the app repeatedly until none is around any more. This is + less reliable and does require root. + + :param str app_name: Name of application (e.g. `com.android.chrome`) + :param timeout: The maximum time in + seconds for any spawned adb process to complete before + throwing an ADBTimeoutError. + This timeout is per adb call. The total time spent + may exceed this value. If it is not specified, the value + set in the ADB constructor is used. + :type timeout: integer or None + :param bool root: Flag specifying if the command should be + executed as root. + :raises: * ADBTimeoutError + * ADBError + """ + if self.version >= version_codes.HONEYCOMB: + self.shell_output("am force-stop %s" % app_name, + timeout=timeout, root=root) + else: + num_tries = 0 + max_tries = 5 + while self.process_exist(app_name, timeout=timeout): + if num_tries > max_tries: + raise ADBError("Couldn't successfully kill %s after %s " + "tries" % (app_name, max_tries)) + self.pkill(app_name, timeout=timeout, root=root) + num_tries += 1 + + # sleep for a short duration to make sure there are no + # additional processes in the process of being launched + # (this is not 100% guaranteed to work since it is inherently + # racey, but it's the best we can do) + time.sleep(1) + + def uninstall_app(self, app_name, reboot=False, timeout=None): + """Uninstalls an app on the device. + + :param str app_name: The name of the app to be + uninstalled. + :param bool reboot: Flag indicating that the device should + be rebooted after the app is uninstalled. No reboot occurs + if the app is not installed. + :param timeout: The maximum time in + seconds for any spawned adb process to complete before + throwing an ADBTimeoutError. + This timeout is per adb call. The total time spent + may exceed this value. If it is not specified, the value + set in the ADB constructor is used. + :type timeout: integer or None + :raises: * ADBTimeoutError + * ADBError + """ + if self.is_app_installed(app_name, timeout=timeout): + data = self.command_output(["uninstall", app_name], timeout=timeout) + if data.find('Success') == -1: + self._logger.debug('uninstall_app failed: %s' % data) + raise ADBError("uninstall failed for %s. Got: %s" % (app_name, data)) + if reboot: + self.reboot(timeout=timeout) + + def update_app(self, apk_path, timeout=None): + """Updates an app on the device and reboots. + + :param str apk_path: The apk file name to be + updated. + :param timeout: The maximum time in + seconds for any spawned adb process to complete before + throwing an ADBTimeoutError. + This timeout is per adb call. The total time spent + may exceed this value. If it is not specified, the value + set in the ADB constructor is used. + :type timeout: integer or None + :raises: * ADBTimeoutError + * ADBError + """ + cmd = ["install", "-r"] + if self.version >= version_codes.M: + cmd.append("-g") + cmd.append(apk_path) + output = self.command_output(cmd, timeout=timeout) + self.reboot(timeout=timeout) + return output diff --git a/testing/mozbase/mozdevice/mozdevice/adb_b2g.py b/testing/mozbase/mozdevice/mozdevice/adb_b2g.py new file mode 100644 index 000000000..3280e6172 --- /dev/null +++ b/testing/mozbase/mozdevice/mozdevice/adb_b2g.py @@ -0,0 +1,122 @@ +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this file, +# You can obtain one at http://mozilla.org/MPL/2.0/. + +import traceback + +import mozfile + +from adb import ADBDevice, ADBError + + +class ADBB2G(ADBDevice): + """ADBB2G implements :class:`ADBDevice` providing B2G-specific + functionality. + + :: + + from mozdevice import ADBB2G + + adbdevice = ADBB2G() + print adbdevice.list_files("/mnt/sdcard") + if adbdevice.process_exist("b2g"): + print "B2G is running" + """ + + def get_battery_percentage(self, timeout=None): + """Returns the battery charge as a percentage. + + :param timeout: optional integer specifying the maximum time in + seconds for any spawned adb process to complete before + throwing an ADBTimeoutError. + This timeout is per adb call. The total time spent + may exceed this value. If it is not specified, the value + set in the ADBDevice constructor is used. + :returns: battery charge as a percentage. + :raises: * ADBTimeoutError + * ADBError + """ + with mozfile.NamedTemporaryFile() as tf: + self.pull('/sys/class/power_supply/battery/capacity', tf.name, + timeout=timeout) + try: + with open(tf.name) as tf2: + return tf2.read().splitlines()[0] + except Exception as e: + raise ADBError(traceback.format_exception_only( + type(e), e)[0].strip()) + + def get_memory_total(self, timeout=None): + """Returns the total memory available with units. + + :param timeout: optional integer specifying the maximum time in + seconds for any spawned adb process to complete before + throwing an ADBTimeoutError. + This timeout is per adb call. The total time spent + may exceed this value. If it is not specified, the value + set in the ADBDevice constructor is used. + :returns: memory total with units. + :raises: * ADBTimeoutError + * ADBError + """ + meminfo = {} + with mozfile.NamedTemporaryFile() as tf: + self.pull('/proc/meminfo', tf.name, timeout=timeout) + try: + with open(tf.name) as tf2: + for line in tf2.read().splitlines(): + key, value = line.split(':') + meminfo[key] = value.strip() + except Exception as e: + raise ADBError(traceback.format_exception_only( + type(e), e)[0].strip()) + return meminfo['MemTotal'] + + def get_info(self, directive=None, timeout=None): + """ + Returns a dictionary of information strings about the device. + + :param directive: information you want to get. Options are: + - `battery` - battery charge as a percentage + - `disk` - total, free, available bytes on disk + - `id` - unique id of the device + - `memtotal` - total memory available on the device + - `os` - name of the os + - `process` - list of running processes (same as ps) + - `systime` - system time of the device + - `uptime` - uptime of the device + + If `directive` is `None`, will return all available information + :param timeout: optional integer specifying the maximum time in + seconds for any spawned adb process to complete before + throwing an ADBTimeoutError. + This timeout is per adb call. The total time spent + may exceed this value. If it is not specified, the value + set in the ADB constructor is used. + :raises: * ADBTimeoutError + * ADBError + """ + info = super(ADBB2G, self).get_info(directive=directive, + timeout=timeout) + + directives = ['memtotal'] + if directive in directives: + directives = [directive] + + if 'memtotal' in directives: + info['memtotal'] = self.get_memory_total(timeout=timeout) + return info + + def is_device_ready(self, timeout=None): + """Returns True if the device is ready. + + :param timeout: optional integer specifying the maximum time in + seconds for any spawned adb process to complete before + throwing an ADBTimeoutError. + This timeout is per adb call. The total time spent + may exceed this value. If it is not specified, the value + set in the ADB constructor is used. + :raises: * ADBTimeoutError + * ADBError + """ + return self.shell_bool('ls /sbin', timeout=timeout) diff --git a/testing/mozbase/mozdevice/mozdevice/devicemanager.py b/testing/mozbase/mozdevice/mozdevice/devicemanager.py new file mode 100644 index 000000000..de87735ef --- /dev/null +++ b/testing/mozbase/mozdevice/mozdevice/devicemanager.py @@ -0,0 +1,674 @@ +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this file, +# You can obtain one at http://mozilla.org/MPL/2.0/. + +import hashlib +import mozlog +import logging +import os +import posixpath +import re +import struct +import StringIO +import zlib + +from functools import wraps + + +class DMError(Exception): + "generic devicemanager exception." + + def __init__(self, msg='', fatal=False): + self.msg = msg + self.fatal = fatal + + def __str__(self): + return self.msg + + +def abstractmethod(method): + line = method.func_code.co_firstlineno + filename = method.func_code.co_filename + + @wraps(method) + def not_implemented(*args, **kwargs): + raise NotImplementedError('Abstract method %s at File "%s", line %s ' + 'should be implemented by a concrete class' % + (repr(method), filename, line)) + return not_implemented + + +class DeviceManager(object): + """ + Represents a connection to a device. Once an implementation of this class + is successfully instantiated, you may do things like list/copy files to + the device, launch processes on the device, and install or remove + applications from the device. + + Never instantiate this class directly! Instead, instantiate an + implementation of it like DeviceManagerADB or DeviceManagerSUT. + """ + + _logcatNeedsRoot = True + default_timeout = 300 + short_timeout = 30 + + def __init__(self, logLevel=None, deviceRoot=None): + try: + self._logger = mozlog.get_default_logger(component="mozdevice") + if not self._logger: # no global structured logger, fall back to reg logging + self._logger = mozlog.unstructured.getLogger("mozdevice") + if logLevel is not None: + self._logger.setLevel(logLevel) + except AttributeError: + # Structured logging doesn't work on Python 2.6 + self._logger = None + self._logLevel = logLevel + self._remoteIsWin = None + self._isDeviceRootSetup = False + self._deviceRoot = deviceRoot + + def _log(self, data): + """ + This helper function is called by ProcessHandler to log + the output produced by processes + """ + self._logger.debug(data) + + @property + def remoteIsWin(self): + if self._remoteIsWin is None: + self._remoteIsWin = self.getInfo("os")["os"][0] == "windows" + return self._remoteIsWin + + @property + def logLevel(self): + return self._logLevel + + @logLevel.setter + def logLevel_setter(self, newLogLevel): + self._logLevel = newLogLevel + self._logger.setLevel(self._logLevel) + + @property + def debug(self): + self._logger.warning("dm.debug is deprecated. Use logLevel.") + levels = {logging.DEBUG: 5, logging.INFO: 3, logging.WARNING: 2, + logging.ERROR: 1, logging.CRITICAL: 0} + return levels[self.logLevel] + + @debug.setter + def debug_setter(self, newDebug): + self._logger.warning("dm.debug is deprecated. Use logLevel.") + newDebug = 5 if newDebug > 5 else newDebug # truncate >=5 to 5 + levels = {5: logging.DEBUG, 3: logging.INFO, 2: logging.WARNING, + 1: logging.ERROR, 0: logging.CRITICAL} + self.logLevel = levels[newDebug] + + @abstractmethod + def getInfo(self, directive=None): + """ + Returns a dictionary of information strings about the device. + + :param directive: information you want to get. Options are: + + - `os` - name of the os + - `id` - unique id of the device + - `uptime` - uptime of the device + - `uptimemillis` - uptime of the device in milliseconds + (NOT supported on all implementations) + - `systime` - system time of the device + - `screen` - screen resolution + - `memory` - memory stats + - `memtotal` - total memory available on the device, for example 927208 kB + - `process` - list of running processes (same as ps) + - `disk` - total, free, available bytes on disk + - `power` - power status (charge, battery temp) + - `temperature` - device temperature + + If `directive` is `None`, will return all available information + """ + + @abstractmethod + def getCurrentTime(self): + """ + Returns device time in milliseconds since the epoch. + """ + + def getIP(self, interfaces=['eth0', 'wlan0']): + """ + Returns the IP of the device, or None if no connection exists. + """ + for interface in interfaces: + match = re.match(r"%s: ip (\S+)" % interface, + self.shellCheckOutput(['ifconfig', interface], + timeout=self.short_timeout)) + if match: + return match.group(1) + + def recordLogcat(self): + """ + Clears the logcat file making it easier to view specific events. + """ + # TODO: spawn this off in a separate thread/process so we can collect all + # the logcat information + + # Right now this is just clearing the logcat so we can only see what + # happens after this call. + self.shellCheckOutput(['/system/bin/logcat', '-c'], root=self._logcatNeedsRoot, + timeout=self.short_timeout) + + def getLogcat(self, filterSpecs=["dalvikvm:I", "ConnectivityService:S", + "WifiMonitor:S", "WifiStateTracker:S", + "wpa_supplicant:S", "NetworkStateTracker:S"], + format="time", + filterOutRegexps=[]): + """ + Returns the contents of the logcat file as a list of + '\n' terminated strings + """ + cmdline = ["/system/bin/logcat", "-v", format, "-d"] + filterSpecs + output = self.shellCheckOutput(cmdline, + root=self._logcatNeedsRoot, + timeout=self.short_timeout) + lines = output.replace('\r\n', '\n').splitlines(True) + + for regex in filterOutRegexps: + lines = [line for line in lines if not re.search(regex, line)] + + return lines + + def saveScreenshot(self, filename): + """ + Takes a screenshot of what's being display on the device. Uses + "screencap" on newer (Android 3.0+) devices (and some older ones with + the functionality backported). This function also works on B2G. + + Throws an exception on failure. This will always fail on devices + without the screencap utility. + """ + screencap = '/system/bin/screencap' + if not self.fileExists(screencap): + raise DMError("Unable to capture screenshot on device: no screencap utility") + + with open(filename, 'w') as pngfile: + # newer versions of screencap can write directly to a png, but some + # older versions can't + tempScreenshotFile = self.deviceRoot + "/ss-dm.tmp" + self.shellCheckOutput(["sh", "-c", "%s > %s" % + (screencap, tempScreenshotFile)], + root=True) + buf = self.pullFile(tempScreenshotFile) + width = int(struct.unpack("I", buf[0:4])[0]) + height = int(struct.unpack("I", buf[4:8])[0]) + with open(filename, 'w') as pngfile: + pngfile.write(self._writePNG(buf[12:], width, height)) + self.removeFile(tempScreenshotFile) + + @abstractmethod + def pushFile(self, localFilename, remoteFilename, retryLimit=1, createDir=True): + """ + Copies localname from the host to destname on the device. + """ + + @abstractmethod + def pushDir(self, localDirname, remoteDirname, retryLimit=1, timeout=None): + """ + Push local directory from host to remote directory on the device, + """ + + @abstractmethod + def pullFile(self, remoteFilename, offset=None, length=None): + """ + Returns contents of remoteFile using the "pull" command. + + :param remoteFilename: Path to file to pull from remote device. + :param offset: Offset in bytes from which to begin reading (optional) + :param length: Number of bytes to read (optional) + """ + + @abstractmethod + def getFile(self, remoteFilename, localFilename): + """ + Copy file from remote device to local file on host. + """ + + @abstractmethod + def getDirectory(self, remoteDirname, localDirname, checkDir=True): + """ + Copy directory structure from device (remoteDirname) to host (localDirname). + """ + + @abstractmethod + def validateFile(self, remoteFilename, localFilename): + """ + Returns True if a file on the remote device has the same md5 hash as a local one. + """ + + def validateDir(self, localDirname, remoteDirname): + """ + Returns True if remoteDirname on device is same as localDirname on host. + """ + + self._logger.info("validating directory: %s to %s" % (localDirname, remoteDirname)) + for root, dirs, files in os.walk(localDirname): + parts = root.split(localDirname) + for f in files: + remoteRoot = remoteDirname + '/' + parts[1] + remoteRoot = remoteRoot.replace('/', '/') + if (parts[1] == ""): + remoteRoot = remoteDirname + remoteName = remoteRoot + '/' + f + if (self.validateFile(remoteName, os.path.join(root, f)) is not True): + return False + return True + + @abstractmethod + def mkDir(self, remoteDirname): + """ + Creates a single directory on the device file system. + """ + + def mkDirs(self, filename): + """ + Make directory structure on the device. + + WARNING: does not create last part of the path. For example, if asked to + create `/mnt/sdcard/foo/bar/baz`, it will only create `/mnt/sdcard/foo/bar` + """ + filename = posixpath.normpath(filename) + containing = posixpath.dirname(filename) + if not self.dirExists(containing): + parts = filename.split('/') + name = "/" if not self.remoteIsWin else parts.pop(0) + for part in parts[:-1]: + if part != "": + name = posixpath.join(name, part) + self.mkDir(name) # mkDir will check previous existence + + @abstractmethod + def dirExists(self, dirpath): + """ + Returns whether dirpath exists and is a directory on the device file system. + """ + + @abstractmethod + def fileExists(self, filepath): + """ + Return whether filepath exists on the device file system, + regardless of file type. + """ + + @abstractmethod + def listFiles(self, rootdir): + """ + Lists files on the device rootdir. + + Returns array of filenames, ['file1', 'file2', ...] + """ + + @abstractmethod + def removeFile(self, filename): + """ + Removes filename from the device. + """ + + @abstractmethod + def removeDir(self, remoteDirname): + """ + Does a recursive delete of directory on the device: rm -Rf remoteDirname. + """ + + @abstractmethod + def moveTree(self, source, destination): + """ + Does a move of the file or directory on the device. + + :param source: Path to the original file or directory + :param destination: Path to the destination file or directory + """ + + @abstractmethod + def copyTree(self, source, destination): + """ + Does a copy of the file or directory on the device. + + :param source: Path to the original file or directory + :param destination: Path to the destination file or directory + """ + + @abstractmethod + def chmodDir(self, remoteDirname, mask="777"): + """ + Recursively changes file permissions in a directory. + """ + + @property + def deviceRoot(self): + """ + The device root on the device filesystem for putting temporary + testing files. + """ + # derive deviceroot value if not set + if not self._deviceRoot or not self._isDeviceRootSetup: + self._deviceRoot = self._setupDeviceRoot(self._deviceRoot) + self._isDeviceRootSetup = True + + return self._deviceRoot + + @abstractmethod + def _setupDeviceRoot(self): + """ + Sets up and returns a device root location that can be written to by tests. + """ + + def getDeviceRoot(self): + """ + Get the device root on the device filesystem for putting temporary + testing files. + + .. deprecated:: 0.38 + Use the :py:attr:`deviceRoot` property instead. + """ + return self.deviceRoot + + @abstractmethod + def getTempDir(self): + """ + Returns a temporary directory we can use on this device, ensuring + also that it exists. + """ + + @abstractmethod + def shell(self, cmd, outputfile, env=None, cwd=None, timeout=None, root=False): + """ + Executes shell command on device and returns exit code. + + :param cmd: Commandline list to execute + :param outputfile: File to store output + :param env: Environment to pass to exec command + :param cwd: Directory to execute command from + :param timeout: specified in seconds, defaults to 'default_timeout' + :param root: Specifies whether command requires root privileges + """ + + def shellCheckOutput(self, cmd, env=None, cwd=None, timeout=None, root=False): + """ + Executes shell command on device and returns output as a string. Raises if + the return code is non-zero. + + :param cmd: Commandline list to execute + :param env: Environment to pass to exec command + :param cwd: Directory to execute command from + :param timeout: specified in seconds, defaults to 'default_timeout' + :param root: Specifies whether command requires root privileges + :raises: DMError + """ + buf = StringIO.StringIO() + retval = self.shell(cmd, buf, env=env, cwd=cwd, timeout=timeout, root=root) + output = str(buf.getvalue()[0:-1]).rstrip() + buf.close() + if retval != 0: + raise DMError( + "Non-zero return code for command: %s " + "(output: '%s', retval: '%s')" % (cmd, output, retval)) + return output + + @abstractmethod + def getProcessList(self): + """ + Returns array of tuples representing running processes on the device. + + Format of tuples is (processId, processName, userId) + """ + + def processInfo(self, processName): + """ + Returns information on the process with processName. + Information on process is in tuple format: (pid, process path, user) + If a process with the specified name does not exist this function will return None. + """ + if not isinstance(processName, basestring): + raise TypeError("Process name %s is not a string" % processName) + + processInfo = None + + # filter out extra spaces + parts = filter(lambda x: x != '', processName.split(' ')) + processName = ' '.join(parts) + + # filter out the quoted env string if it exists + # ex: '"name=value;name2=value2;etc=..." process args' -> 'process args' + parts = processName.split('"') + if (len(parts) > 2): + processName = ' '.join(parts[2:]).strip() + + pieces = processName.split(' ') + parts = pieces[0].split('/') + app = parts[-1] + + procList = self.getProcessList() + if (procList == []): + return None + + for proc in procList: + procName = proc[1].split('/')[-1] + if (procName == app): + processInfo = proc + break + return processInfo + + def processExist(self, processName): + """ + Returns True if process with name processName is running on device. + """ + processInfo = self.processInfo(processName) + if processInfo: + return processInfo[0] + + @abstractmethod + def killProcess(self, processName, sig=None): + """ + Kills the process named processName. If sig is not None, process is + killed with the specified signal. + + :param processName: path or name of the process to kill + :param sig: signal to pass into the kill command (optional) + """ + + @abstractmethod + def reboot(self, wait=False, ipAddr=None): + """ + Reboots the device. + + :param wait: block on device to come back up before returning + :param ipAddr: if specified, try to make the device connect to this + specific IP address after rebooting (only works with + SUT; if None, we try to determine a reasonable address + ourselves) + """ + + @abstractmethod + def installApp(self, appBundlePath, destPath=None): + """ + Installs an application onto the device. + + :param appBundlePath: path to the application bundle on the device + :param destPath: destination directory of where application should be + installed to (optional) + """ + + @abstractmethod + def uninstallApp(self, appName, installPath=None): + """ + Uninstalls the named application from device and DOES NOT cause a reboot. + + :param appName: the name of the application (e.g org.mozilla.fennec) + :param installPath: the path to where the application was installed (optional) + """ + + @abstractmethod + def uninstallAppAndReboot(self, appName, installPath=None): + """ + Uninstalls the named application from device and causes a reboot. + + :param appName: the name of the application (e.g org.mozilla.fennec) + :param installPath: the path to where the application was installed (optional) + """ + + @abstractmethod + def updateApp(self, appBundlePath, processName=None, destPath=None, + wait=False, ipAddr=None): + """ + Updates the application on the device and reboots. + + :param appBundlePath: path to the application bundle on the device + :param processName: used to end the process if the applicaiton is + currently running (optional) + :param destPath: Destination directory to where the application should + be installed (optional) + :param wait: block on device to come back up before returning + :param ipAddr: if specified, try to make the device connect to this + specific IP address after rebooting (only works with + SUT; if None and wait is True, we try to determine a + reasonable address ourselves) + """ + + @staticmethod + def _writePNG(buf, width, height): + """ + Method for writing a PNG from a buffer, used by getScreenshot on older devices, + """ + # Based on: http://code.activestate.com/recipes/577443-write-a-png-image-in-native-python/ + width_byte_4 = width * 4 + raw_data = b"".join(b'\x00' + buf[span:span + width_byte_4] + for span in range(0, (height - 1) * width * 4, width_byte_4)) + + def png_pack(png_tag, data): + chunk_head = png_tag + data + return struct.pack("!I", len(data)) \ + + chunk_head \ + + struct.pack("!I", 0xFFFFFFFF & zlib.crc32(chunk_head)) + return b"".join([ + b'\x89PNG\r\n\x1a\n', + png_pack(b'IHDR', struct.pack("!2I5B", width, height, 8, 6, 0, 0, 0)), + png_pack(b'IDAT', zlib.compress(raw_data, 9)), + png_pack(b'IEND', b'')]) + + @abstractmethod + def _getRemoteHash(self, filename): + """ + Return the md5 sum of a file on the device. + """ + + @staticmethod + def _getLocalHash(filename): + """ + Return the MD5 sum of a file on the host. + """ + f = open(filename, 'rb') + if f is None: + return None + + try: + mdsum = hashlib.md5() + except: + return None + + while 1: + data = f.read(1024) + if not data: + break + mdsum.update(data) + + f.close() + hexval = mdsum.hexdigest() + return hexval + + @staticmethod + def _escapedCommandLine(cmd): + """ + Utility function to return escaped and quoted version of command line. + """ + quotedCmd = [] + + for arg in cmd: + arg.replace('&', '\&') + + needsQuoting = False + for char in [' ', '(', ')', '"', '&']: + if arg.find(char) >= 0: + needsQuoting = True + break + if needsQuoting: + arg = '\'%s\'' % arg + + quotedCmd.append(arg) + + return " ".join(quotedCmd) + + +def _pop_last_line(file_obj): + """ + Utility function to get the last line from a file (shared between ADB and + SUT device managers). Function also removes it from the file. Intended to + strip off the return code from a shell command. + """ + bytes_from_end = 1 + file_obj.seek(0, 2) + length = file_obj.tell() + 1 + while bytes_from_end < length: + file_obj.seek((-1) * bytes_from_end, 2) + data = file_obj.read() + + if bytes_from_end == length - 1 and len(data) == 0: # no data, return None + return None + + if data[0] == '\n' or bytes_from_end == length - 1: + # found the last line, which should have the return value + if data[0] == '\n': + data = data[1:] + + # truncate off the return code line + file_obj.truncate(length - bytes_from_end) + file_obj.seek(0, 2) + file_obj.write('\0') + + return data + + bytes_from_end += 1 + + return None + + +class ZeroconfListener(object): + + def __init__(self, hwid, evt): + self.hwid = hwid + self.evt = evt + + # Format is 'SUTAgent [hwid:015d2bc2825ff206] [ip:10_242_29_221]._sutagent._tcp.local.' + def addService(self, zeroconf, type, name): + # print "Found _sutagent service broadcast:", name + if not name.startswith("SUTAgent"): + return + + sutname = name.split('.')[0] + m = re.search('\[hwid:([^\]]*)\]', sutname) + if m is None: + return + + hwid = m.group(1) + + m = re.search('\[ip:([0-9_]*)\]', sutname) + if m is None: + return + + ip = m.group(1).replace("_", ".") + + if self.hwid == hwid: + self.ip = ip + self.evt.set() + + def removeService(self, zeroconf, type, name): + pass diff --git a/testing/mozbase/mozdevice/mozdevice/devicemanagerADB.py b/testing/mozbase/mozdevice/mozdevice/devicemanagerADB.py new file mode 100644 index 000000000..74d0a8d23 --- /dev/null +++ b/testing/mozbase/mozdevice/mozdevice/devicemanagerADB.py @@ -0,0 +1,893 @@ +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this file, +# You can obtain one at http://mozilla.org/MPL/2.0/. + +import logging +import re +import os +import tempfile +import time +import traceback + +from distutils import dir_util + +from devicemanager import DeviceManager, DMError +from mozprocess import ProcessHandler +import mozfile +import version_codes + + +class DeviceManagerADB(DeviceManager): + """ + Implementation of DeviceManager interface that uses the Android "adb" + utility to communicate with the device. Normally used to communicate + with a device that is directly connected with the host machine over a USB + port. + """ + + _haveRootShell = None + _haveSu = None + _suModifier = None + _lsModifier = None + _useZip = False + _logcatNeedsRoot = False + _pollingInterval = 0.01 + _packageName = None + _tempDir = None + _adb_version = None + _sdk_version = None + connected = False + + def __init__(self, host=None, port=5555, retryLimit=5, packageName='fennec', + adbPath=None, deviceSerial=None, deviceRoot=None, + logLevel=logging.ERROR, autoconnect=True, runAdbAsRoot=False, + serverHost=None, serverPort=None, **kwargs): + DeviceManager.__init__(self, logLevel=logLevel, + deviceRoot=deviceRoot) + self.host = host + self.port = port + self.retryLimit = retryLimit + + self._serverHost = serverHost + self._serverPort = serverPort + + # the path to adb, or 'adb' to assume that it's on the PATH + self._adbPath = adbPath or 'adb' + + # The serial number of the device to use with adb, used in cases + # where multiple devices are being managed by the same adb instance. + self._deviceSerial = deviceSerial + + # Some devices do no start adb as root, if allowed you can use + # this to reboot adbd on the device as root automatically + self._runAdbAsRoot = runAdbAsRoot + + if packageName == 'fennec': + if os.getenv('USER'): + self._packageName = 'org.mozilla.fennec_' + os.getenv('USER') + else: + self._packageName = 'org.mozilla.fennec_' + elif packageName: + self._packageName = packageName + + # verify that we can run the adb command. can't continue otherwise + self._verifyADB() + + if autoconnect: + self.connect() + + def connect(self): + if not self.connected: + # try to connect to the device over tcp/ip if we have a hostname + if self.host: + self._connectRemoteADB() + + # verify that we can connect to the device. can't continue + self._verifyDevice() + + # Note SDK version + try: + proc = self._runCmd(["shell", "getprop", "ro.build.version.sdk"], + timeout=self.short_timeout) + self._sdk_version = int(proc.output[0]) + except (OSError, ValueError): + self._sdk_version = 0 + self._logger.info("Detected Android sdk %d" % self._sdk_version) + + # Some commands require root to work properly, even with ADB (e.g. + # grabbing APKs out of /data). For these cases, we check whether + # we're running as root. If that isn't true, check for the + # existence of an su binary + self._checkForRoot() + + # can we use zip to speed up some file operations? (currently not + # required) + try: + self._verifyZip() + except DMError: + pass + + def __del__(self): + if self.host: + self._disconnectRemoteADB() + + def shell(self, cmd, outputfile, env=None, cwd=None, timeout=None, root=False): + # FIXME: this function buffers all output of the command into memory, + # always. :( + + # If requested to run as root, check that we can actually do that + if root: + if self._haveRootShell is None and self._haveSu is None: + self._checkForRoot() + if not self._haveRootShell and not self._haveSu: + raise DMError( + "Shell command '%s' requested to run as root but root " + "is not available on this device. Root your device or " + "refactor the test/harness to not require root." % + self._escapedCommandLine(cmd)) + + # Getting the return code is more complex than you'd think because adb + # doesn't actually return the return code from a process, so we have to + # capture the output to get it + if root and self._haveSu: + cmdline = "su %s \"%s\"" % (self._suModifier, + self._escapedCommandLine(cmd)) + else: + cmdline = self._escapedCommandLine(cmd) + cmdline += "; echo $?" + + # prepend cwd and env to command if necessary + if cwd: + cmdline = "cd %s; %s" % (cwd, cmdline) + if env: + envstr = '; '.join(map(lambda x: 'export %s=%s' % (x[0], x[1]), env.iteritems())) + cmdline = envstr + "; " + cmdline + + # all output should be in stdout + args = [self._adbPath] + if self._serverHost is not None: + args.extend(['-H', self._serverHost]) + if self._serverPort is not None: + args.extend(['-P', str(self._serverPort)]) + if self._deviceSerial: + args.extend(['-s', self._deviceSerial]) + args.extend(["shell", cmdline]) + + def _timeout(): + self._logger.error("Timeout exceeded for shell call '%s'" % ' '.join(args)) + + self._logger.debug("shell - command: %s" % ' '.join(args)) + proc = ProcessHandler(args, processOutputLine=self._log, onTimeout=_timeout) + + if not timeout: + # We are asserting that all commands will complete in this time unless + # otherwise specified + timeout = self.default_timeout + + timeout = int(timeout) + proc.run(timeout) + proc.wait() + output = proc.output + + if output: + lastline = output[-1] + if lastline: + m = re.search('([0-9]+)', lastline) + if m: + return_code = m.group(1) + for line in output: + outputfile.write(line + '\n') + outputfile.seek(-2, 2) + outputfile.truncate() # truncate off the return code + return int(return_code) + + return None + + def forward(self, local, remote): + """ + Forward socket connections. + + Forward specs are one of: + tcp: + localabstract: + localreserved: + localfilesystem: + dev: + jdwp: (remote only) + """ + if not self._checkCmd(['forward', local, remote], timeout=self.short_timeout) == 0: + raise DMError("Failed to forward socket connection.") + + def remove_forward(self, local=None): + """ + Turn off forwarding of socket connection. + """ + cmd = ['forward'] + if local is None: + cmd.extend(['--remove-all']) + else: + cmd.extend(['--remove', local]) + if not self._checkCmd(cmd, timeout=self.short_timeout) == 0: + raise DMError("Failed to remove connection forwarding.") + + def remount(self): + "Remounts the /system partition on the device read-write." + return self._checkCmd(['remount'], timeout=self.short_timeout) + + def devices(self): + "Return a list of connected devices as (serial, status) tuples." + proc = self._runCmd(['devices']) + proc.output.pop(0) # ignore first line of output + devices = [] + for line in proc.output: + result = re.match('(.*?)\t(.*)', line) + if result: + devices.append((result.group(1), result.group(2))) + return devices + + def _connectRemoteADB(self): + self._checkCmd(["connect", self.host + ":" + str(self.port)]) + + def _disconnectRemoteADB(self): + self._checkCmd(["disconnect", self.host + ":" + str(self.port)]) + + def pushFile(self, localname, destname, retryLimit=None, createDir=True): + # you might expect us to put the file *in* the directory in this case, + # but that would be different behaviour from devicemanagerSUT. Throw + # an exception so we have the same behaviour between the two + # implementations + retryLimit = retryLimit or self.retryLimit + if self.dirExists(destname): + raise DMError("Attempted to push a file (%s) to a directory (%s)!" % + (localname, destname)) + if not os.access(localname, os.F_OK): + raise DMError("File not found: %s" % localname) + + proc = self._runCmd(["push", os.path.realpath(localname), destname], + retryLimit=retryLimit) + if proc.returncode != 0: + raise DMError("Error pushing file %s -> %s; output: %s" % + (localname, destname, proc.output)) + + def mkDir(self, name): + result = self._runCmd(["shell", "mkdir", name], timeout=self.short_timeout).output + if len(result) and 'read-only file system' in result[0].lower(): + raise DMError("Error creating directory: read only file system") + + def pushDir(self, localDir, remoteDir, retryLimit=None, timeout=None): + # adb "push" accepts a directory as an argument, but if the directory + # contains symbolic links, the links are pushed, rather than the linked + # files; we either zip/unzip or re-copy the directory into a temporary + # one to get around this limitation + retryLimit = retryLimit or self.retryLimit + if self._useZip: + self.removeDir(remoteDir) + self.mkDirs(remoteDir + "/x") + try: + localZip = tempfile.mktemp() + ".zip" + remoteZip = remoteDir + "/adbdmtmp.zip" + proc = ProcessHandler(["zip", "-r", localZip, '.'], cwd=localDir, + processOutputLine=self._log) + proc.run() + proc.wait() + self.pushFile(localZip, remoteZip, retryLimit=retryLimit, createDir=False) + mozfile.remove(localZip) + data = self._runCmd(["shell", "unzip", "-o", remoteZip, + "-d", remoteDir]).output[0] + self._checkCmd(["shell", "rm", remoteZip], + retryLimit=retryLimit, timeout=self.short_timeout) + if re.search("unzip: exiting", data) or re.search("Operation not permitted", data): + raise Exception("unzip failed, or permissions error") + except: + self._logger.warning(traceback.format_exc()) + self._logger.warning("zip/unzip failure: falling back to normal push") + self._useZip = False + self.pushDir(localDir, remoteDir, retryLimit=retryLimit, timeout=timeout) + else: + localDir = os.path.normpath(localDir) + remoteDir = os.path.normpath(remoteDir) + copyRequired = False + if self._adb_version >= '1.0.36' and \ + os.path.isdir(localDir) and self.dirExists(remoteDir): + # See do_sync_push in + # https://android.googlesource.com/platform/system/core/+/master/adb/file_sync_client.cpp + # Work around change in behavior in adb 1.0.36 where if + # the remote destination directory exists, adb push will + # copy the source directory *into* the destination + # directory otherwise it will copy the source directory + # *onto* the destination directory. + # + # If the destination directory does exist, push to its + # parent directory. If the source and destination leaf + # directory names are different, copy the source directory + # to a temporary directory with the same leaf name as the + # destination so that when we push to the parent, the + # source is copied onto the destination directory. + localName = os.path.basename(localDir) + remoteName = os.path.basename(remoteDir) + if localName != remoteName: + copyRequired = True + tempParent = tempfile.mkdtemp() + newLocal = os.path.join(tempParent, remoteName) + dir_util.copy_tree(localDir, newLocal) + localDir = newLocal + remoteDir = '/'.join(remoteDir.rstrip('/').split('/')[:-1]) + try: + self._checkCmd(["push", localDir, remoteDir], + retryLimit=retryLimit, timeout=timeout) + except: + raise + finally: + if copyRequired: + mozfile.remove(tempParent) + + def dirExists(self, remotePath): + self._detectLsModifier() + data = self._runCmd(["shell", "ls", self._lsModifier, remotePath + '/'], + timeout=self.short_timeout).output + + if len(data) == 1: + res = data[0] + if "Not a directory" in res or "No such file or directory" in res: + return False + return True + + def fileExists(self, filepath): + self._detectLsModifier() + data = self._runCmd(["shell", "ls", self._lsModifier, filepath], + timeout=self.short_timeout).output + if len(data) == 1: + foundpath = data[0].decode('utf-8').rstrip() + if foundpath == filepath: + return True + return False + + def removeFile(self, filename): + if self.fileExists(filename): + self._checkCmd(["shell", "rm", filename], timeout=self.short_timeout) + + def removeDir(self, remoteDir): + if self.dirExists(remoteDir): + self._checkCmd(["shell", "rm", "-r", remoteDir], timeout=self.short_timeout) + else: + self.removeFile(remoteDir.strip()) + + def moveTree(self, source, destination): + self._checkCmd(["shell", "mv", source, destination], timeout=self.short_timeout) + + def copyTree(self, source, destination): + self._checkCmd(["shell", "dd", "if=%s" % source, "of=%s" % destination]) + + def listFiles(self, rootdir): + self._detectLsModifier() + data = self._runCmd(["shell", "ls", self._lsModifier, rootdir], + timeout=self.short_timeout).output + data[:] = [item.rstrip('\r\n') for item in data] + if (len(data) == 1): + if (data[0] == rootdir): + return [] + if (data[0].find("No such file or directory") != -1): + return [] + if (data[0].find("Not a directory") != -1): + return [] + if (data[0].find("Permission denied") != -1): + return [] + if (data[0].find("opendir failed") != -1): + return [] + if (data[0].find("Device or resource busy") != -1): + return [] + return data + + def getProcessList(self): + ret = [] + p = self._runCmd(["shell", "ps"], timeout=self.short_timeout) + if not p or not p.output or len(p.output) < 1: + return ret + # first line is the headers + p.output.pop(0) + for proc in p.output: + els = proc.split() + # We need to figure out if this is "user pid name" or + # "pid user vsz stat command" + if els[1].isdigit(): + ret.append(list([int(els[1]), els[len(els) - 1], els[0]])) + else: + ret.append(list([int(els[0]), els[len(els) - 1], els[1]])) + return ret + + def fireProcess(self, appname, failIfRunning=False): + """ + Starts a process + + returns: pid + + DEPRECATED: Use shell() or launchApplication() for new code + """ + # strip out env vars + parts = appname.split('"') + if (len(parts) > 2): + parts = parts[2:] + return self.launchProcess(parts, failIfRunning) + + def launchProcess(self, cmd, outputFile="process.txt", cwd='', env='', failIfRunning=False): + """ + Launches a process, redirecting output to standard out + + WARNING: Does not work how you expect on Android! The application's + own output will be flushed elsewhere. + + DEPRECATED: Use shell() or launchApplication() for new code + """ + if cmd[0] == "am": + self._checkCmd(["shell"] + cmd) + return outputFile + + acmd = ["-W"] + cmd = ' '.join(cmd).strip() + i = cmd.find(" ") + # SUT identifies the URL by looking for :\\ -- another strategy to consider + re_url = re.compile('^[http|file|chrome|about].*') + last = cmd.rfind(" ") + uri = "" + args = "" + if re_url.match(cmd[last:].strip()): + args = cmd[i:last].strip() + uri = cmd[last:].strip() + else: + args = cmd[i:].strip() + acmd.append("-n") + acmd.append(cmd[0:i] + "/org.mozilla.gecko.BrowserApp") + if args != "": + acmd.append("--es") + acmd.append("args") + acmd.append(args) + if env != '' and env is not None: + envCnt = 0 + # env is expected to be a dict of environment variables + for envkey, envval in env.iteritems(): + acmd.append("--es") + acmd.append("env" + str(envCnt)) + acmd.append(envkey + "=" + envval) + envCnt += 1 + if uri != "": + acmd.append("-d") + acmd.append(uri) + + acmd = ["shell", ' '.join(map(lambda x: '"' + x + '"', ["am", "start"] + acmd))] + self._logger.info(acmd) + self._checkCmd(acmd) + return outputFile + + def killProcess(self, appname, sig=None): + shell_args = ["shell"] + if self._sdk_version >= version_codes.N: + # Bug 1334613 - force use of root + if self._haveRootShell is None and self._haveSu is None: + self._checkForRoot() + if not self._haveRootShell and not self._haveSu: + raise DMError( + "killProcess '%s' requested to run as root but root " + "is not available on this device. Root your device or " + "refactor the test/harness to not require root." % + appname) + if not self._haveRootShell: + shell_args.extend(["su", self._suModifier]) + + procs = self.getProcessList() + for (pid, name, user) in procs: + if name == appname: + args = list(shell_args) + args.append("kill") + if sig: + args.append("-%d" % sig) + args.append(str(pid)) + p = self._runCmd(args, timeout=self.short_timeout) + if p.returncode != 0 and len(p.output) > 0 and \ + 'No such process' not in p.output[0]: + raise DMError("Error killing process " + "'%s': %s" % (appname, p.output)) + + def _runPull(self, remoteFile, localFile): + """ + Pulls remoteFile from device to host + """ + try: + self._runCmd(["pull", remoteFile, localFile]) + except (OSError, ValueError): + raise DMError("Error pulling remote file '%s' to '%s'" % (remoteFile, localFile)) + + def pullFile(self, remoteFile, offset=None, length=None): + # TODO: add debug flags and allow for printing stdout + with mozfile.NamedTemporaryFile() as tf: + self._runPull(remoteFile, tf.name) + # we need to reopen the file to get the written contents + with open(tf.name) as tf2: + # ADB pull does not support offset and length, but we can + # instead read only the requested portion of the local file + if offset is not None and length is not None: + tf2.seek(offset) + return tf2.read(length) + elif offset is not None: + tf2.seek(offset) + return tf2.read() + else: + return tf2.read() + + def getFile(self, remoteFile, localFile): + self._runPull(remoteFile, localFile) + + def getDirectory(self, remoteDir, localDir, checkDir=True): + localDir = os.path.normpath(localDir) + remoteDir = os.path.normpath(remoteDir) + copyRequired = False + originalLocal = localDir + if self._adb_version >= '1.0.36' and \ + os.path.isdir(localDir) and self.dirExists(remoteDir): + # See do_sync_pull in + # https://android.googlesource.com/platform/system/core/+/master/adb/file_sync_client.cpp + # Work around change in behavior in adb 1.0.36 where if + # the local destination directory exists, adb pull will + # copy the source directory *into* the destination + # directory otherwise it will copy the source directory + # *onto* the destination directory. + # + # If the destination directory does exist, pull to its + # parent directory. If the source and destination leaf + # directory names are different, pull the source directory + # into a temporary directory and then copy the temporary + # directory onto the destination. + localName = os.path.basename(localDir) + remoteName = os.path.basename(remoteDir) + if localName != remoteName: + copyRequired = True + tempParent = tempfile.mkdtemp() + localDir = os.path.join(tempParent, remoteName) + else: + localDir = '/'.join(localDir.rstrip('/').split('/')[:-1]) + self._runCmd(["pull", remoteDir, localDir]).wait() + if copyRequired: + dir_util.copy_tree(localDir, originalLocal) + mozfile.remove(tempParent) + + def validateFile(self, remoteFile, localFile): + md5Remote = self._getRemoteHash(remoteFile) + md5Local = self._getLocalHash(localFile) + if md5Remote is None or md5Local is None: + return None + return md5Remote == md5Local + + def _getRemoteHash(self, remoteFile): + """ + Return the md5 sum of a file on the device + """ + with tempfile.NamedTemporaryFile() as f: + self._runPull(remoteFile, f.name) + + return self._getLocalHash(f.name) + + def _setupDeviceRoot(self, deviceRoot): + # user-specified device root, create it and return it + if deviceRoot: + self.mkDir(deviceRoot) + return deviceRoot + + # we must determine the device root ourselves + paths = [('/storage/sdcard0', 'tests'), + ('/storage/sdcard1', 'tests'), + ('/storage/sdcard', 'tests'), + ('/mnt/sdcard', 'tests'), + ('/sdcard', 'tests'), + ('/data/local', 'tests')] + for (basePath, subPath) in paths: + if self.dirExists(basePath): + root = os.path.join(basePath, subPath) + try: + self.mkDir(root) + return root + except: + pass + + raise DMError("Unable to set up device root using paths: [%s]" + % ", ".join(["'%s'" % os.path.join(b, s) for b, s in paths])) + + def getTempDir(self): + # Cache result to speed up operations depending + # on the temporary directory. + if not self._tempDir: + self._tempDir = "%s/tmp" % self.deviceRoot + self.mkDir(self._tempDir) + + return self._tempDir + + def reboot(self, wait=False, **kwargs): + self._checkCmd(["reboot"]) + if wait: + self._checkCmd(["wait-for-device"]) + if self._runAdbAsRoot: + self._adb_root() + self._checkCmd(["shell", "ls", "/sbin"], timeout=self.short_timeout) + + def updateApp(self, appBundlePath, **kwargs): + return self._runCmd(["install", "-r", appBundlePath]).output + + def getCurrentTime(self): + timestr = str(self._runCmd(["shell", "date", "+%s"], timeout=self.short_timeout).output[0]) + if (not timestr or not timestr.isdigit()): + raise DMError("Unable to get current time using date (got: '%s')" % timestr) + return int(timestr) * 1000 + + def getInfo(self, directive=None): + directive = directive or "all" + ret = {} + if directive == "id" or directive == "all": + ret["id"] = self._runCmd(["get-serialno"], timeout=self.short_timeout).output[0] + if directive == "os" or directive == "all": + ret["os"] = self.shellCheckOutput( + ["getprop", "ro.build.display.id"], timeout=self.short_timeout) + if directive == "uptime" or directive == "all": + uptime = self.shellCheckOutput(["uptime"], timeout=self.short_timeout) + if not uptime: + raise DMError("error getting uptime") + m = re.match("up time: ((\d+) days, )*(\d{2}):(\d{2}):(\d{2})", uptime) + if m: + uptime = "%d days %d hours %d minutes %d seconds" % tuple( + [int(g or 0) for g in m.groups()[1:]]) + ret["uptime"] = uptime + if directive == "process" or directive == "all": + data = self.shellCheckOutput(["ps"], timeout=self.short_timeout) + ret["process"] = data.split('\n') + if directive == "systime" or directive == "all": + ret["systime"] = self.shellCheckOutput(["date"], timeout=self.short_timeout) + if directive == "memtotal" or directive == "all": + meminfo = {} + for line in self.pullFile("/proc/meminfo").splitlines(): + key, value = line.split(":") + meminfo[key] = value.strip() + ret["memtotal"] = meminfo["MemTotal"] + if directive == "disk" or directive == "all": + data = self.shellCheckOutput( + ["df", "/data", "/system", "/sdcard"], timeout=self.short_timeout) + ret["disk"] = data.split('\n') + self._logger.debug("getInfo: %s" % ret) + return ret + + def uninstallApp(self, appName, installPath=None): + status = self._runCmd(["uninstall", appName]).output[0].strip() + if status != 'Success': + raise DMError("uninstall failed for %s. Got: %s" % (appName, status)) + + def uninstallAppAndReboot(self, appName, installPath=None): + self.uninstallApp(appName) + self.reboot() + + def _runCmd(self, args, timeout=None, retryLimit=None): + """ + Runs a command using adb + If timeout is specified, the process is killed after seconds. + + returns: instance of ProcessHandler + """ + retryLimit = retryLimit or self.retryLimit + finalArgs = [self._adbPath] + if self._serverHost is not None: + finalArgs.extend(['-H', self._serverHost]) + if self._serverPort is not None: + finalArgs.extend(['-P', str(self._serverPort)]) + if self._deviceSerial: + finalArgs.extend(['-s', self._deviceSerial]) + finalArgs.extend(args) + self._logger.debug("_runCmd - command: %s" % ' '.join(finalArgs)) + if not timeout: + timeout = self.default_timeout + + def _timeout(): + self._logger.error("Timeout exceeded for _runCmd call '%s'" % ' '.join(finalArgs)) + + retries = 0 + while retries < retryLimit: + proc = ProcessHandler(finalArgs, storeOutput=True, + processOutputLine=self._log, onTimeout=_timeout) + proc.run(timeout=timeout) + proc.returncode = proc.wait() + if proc.returncode is None: + proc.kill() + retries += 1 + else: + return proc + + # timeout is specified in seconds, and if no timeout is given, + # we will run until we hit the default_timeout specified in the __init__ + def _checkCmd(self, args, timeout=None, retryLimit=None): + """ + Runs a command using adb and waits for the command to finish. + If timeout is specified, the process is killed after seconds. + + returns: returncode from process + """ + retryLimit = retryLimit or self.retryLimit + finalArgs = [self._adbPath] + if self._serverHost is not None: + finalArgs.extend(['-H', self._serverHost]) + if self._serverPort is not None: + finalArgs.extend(['-P', str(self._serverPort)]) + if self._deviceSerial: + finalArgs.extend(['-s', self._deviceSerial]) + finalArgs.extend(args) + self._logger.debug("_checkCmd - command: %s" % ' '.join(finalArgs)) + if not timeout: + # We are asserting that all commands will complete in this + # time unless otherwise specified + timeout = self.default_timeout + + def _timeout(): + self._logger.error("Timeout exceeded for _checkCmd call '%s'" % ' '.join(finalArgs)) + + timeout = int(timeout) + retries = 0 + while retries < retryLimit: + proc = ProcessHandler(finalArgs, processOutputLine=self._log, onTimeout=_timeout) + proc.run(timeout=timeout) + ret_code = proc.wait() + if ret_code is None: + proc.kill() + retries += 1 + else: + return ret_code + + raise DMError("Timeout exceeded for _checkCmd call after %d retries." % retries) + + def chmodDir(self, remoteDir, mask="777"): + if (self.dirExists(remoteDir)): + if '/sdcard' in remoteDir: + self._logger.debug("chmod %s -- skipped (/sdcard)" % remoteDir) + else: + files = self.listFiles(remoteDir.strip()) + for f in files: + remoteEntry = remoteDir.strip() + "/" + f.strip() + if (self.dirExists(remoteEntry)): + self.chmodDir(remoteEntry) + else: + self._checkCmd(["shell", "chmod", mask, remoteEntry], + timeout=self.short_timeout) + self._logger.info("chmod %s" % remoteEntry) + self._checkCmd(["shell", "chmod", mask, remoteDir], timeout=self.short_timeout) + self._logger.debug("chmod %s" % remoteDir) + else: + self._checkCmd(["shell", "chmod", mask, remoteDir.strip()], timeout=self.short_timeout) + self._logger.debug("chmod %s" % remoteDir.strip()) + + def _verifyADB(self): + """ + Check to see if adb itself can be executed. + """ + if self._adbPath != 'adb': + if not os.access(self._adbPath, os.X_OK): + raise DMError("invalid adb path, or adb not executable: %s" % self._adbPath) + + try: + re_version = re.compile(r'Android Debug Bridge version (.*)') + proc = self._runCmd(["version"], timeout=self.short_timeout) + self._adb_version = re_version.match(proc.output[0]).group(1) + self._logger.info("Detected adb %s" % self._adb_version) + except os.error as err: + raise DMError( + "unable to execute ADB (%s): ensure Android SDK is installed " + "and adb is in your $PATH" % err) + + def _verifyDevice(self): + # If there is a device serial number, see if adb is connected to it + if self._deviceSerial: + deviceStatus = None + for line in self._runCmd(["devices"]).output: + m = re.match('(.+)?\s+(.+)$', line) + if m: + if self._deviceSerial == m.group(1): + deviceStatus = m.group(2) + if deviceStatus is None: + raise DMError("device not found: %s" % self._deviceSerial) + elif deviceStatus != "device": + raise DMError("bad status for device %s: %s" % (self._deviceSerial, deviceStatus)) + + # Check to see if we can connect to device and run a simple command + if not self._checkCmd(["shell", "echo"], timeout=self.short_timeout) == 0: + raise DMError("unable to connect to device") + + def _checkForRoot(self): + self._haveRootShell = False + self._haveSu = False + # If requested to attempt to run adbd as root, do so before + # checking whether adbs is running as root. + if self._runAdbAsRoot: + self._adb_root() + + # Check whether we _are_ root by default (some development boards work + # this way, this is also the result of some relatively rare rooting + # techniques) + proc = self._runCmd(["shell", "id"], timeout=self.short_timeout) + if proc.output and 'uid=0(root)' in proc.output[0]: + self._haveRootShell = True + # if this returns true, we don't care about su + return + + # if root shell is not available, check if 'su' can be used to gain + # root + def su_id(su_modifier, timeout): + proc = self._runCmd(["shell", "su", su_modifier, "id"], + timeout=timeout) + + # wait for response for maximum of 15 seconds, in case su + # prompts for a password or triggers the Android SuperUser + # prompt + start_time = time.time() + retcode = None + while (time.time() - start_time) <= 15 and retcode is None: + retcode = proc.poll() + if retcode is None: # still not terminated, kill + proc.kill() + + if proc.output and 'uid=0(root)' in proc.output[0]: + return True + return False + + if su_id('0', self.short_timeout): + self._haveSu = True + self._suModifier = '0' + elif su_id('-c', self.short_timeout): + self._haveSu = True + self._suModifier = '-c' + + def _isUnzipAvailable(self): + data = self._runCmd(["shell", "unzip"]).output + for line in data: + if (re.search('Usage', line)): + return True + return False + + def _isLocalZipAvailable(self): + def _noOutput(line): + # suppress output from zip ProcessHandler + pass + try: + proc = ProcessHandler(["zip", "-?"], storeOutput=False, processOutputLine=_noOutput) + proc.run() + proc.wait() + except: + return False + return True + + def _verifyZip(self): + # If "zip" can be run locally, and "unzip" can be run remotely, then pushDir + # can use these to push just one file per directory -- a significant + # optimization for large directories. + self._useZip = False + if (self._isUnzipAvailable() and self._isLocalZipAvailable()): + self._logger.info("will use zip to push directories") + self._useZip = True + else: + raise DMError("zip not available") + + def _adb_root(self): + """ Some devices require us to reboot adbd as root. + This function takes care of it. + """ + if self.processInfo("adbd")[2] != "root": + self._checkCmd(["root"]) + self._checkCmd(["wait-for-device"]) + if self.processInfo("adbd")[2] != "root": + raise DMError("We tried rebooting adbd as root, however, it failed.") + + def _detectLsModifier(self): + if self._lsModifier is None: + # Check if busybox -1A is required in order to get one + # file per line. + output = self._runCmd(["shell", "ls", "-1A", "/"], + timeout=self.short_timeout).output + output = ' '.join(output) + if 'error: device not found' in output: + raise DMError(output) + if "Unknown option '-1'. Aborting." in output: + self._lsModifier = "-a" + elif "No such file or directory" in output: + self._lsModifier = "-a" + else: + self._lsModifier = "-1A" diff --git a/testing/mozbase/mozdevice/mozdevice/devicemanagerSUT.py b/testing/mozbase/mozdevice/mozdevice/devicemanagerSUT.py new file mode 100644 index 000000000..7816a3fdd --- /dev/null +++ b/testing/mozbase/mozdevice/mozdevice/devicemanagerSUT.py @@ -0,0 +1,975 @@ +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this file, +# You can obtain one at http://mozilla.org/MPL/2.0/. + +import datetime +import logging +import moznetwork +import select +import socket +import time +import os +import re +import posixpath +import subprocess +import StringIO +from devicemanager import DeviceManager, DMError, _pop_last_line +import errno +from distutils.version import StrictVersion + + +class DeviceManagerSUT(DeviceManager): + """ + Implementation of DeviceManager interface that speaks to a device over + TCP/IP using the "system under test" protocol. A software agent such as + Negatus (http://github.com/mozilla/Negatus) or the Mozilla Android SUTAgent + app must be present and listening for connections for this to work. + """ + + _base_prompt = '$>' + _base_prompt_re = '\$\>' + _prompt_sep = '\x00' + _prompt_regex = '.*(' + _base_prompt_re + _prompt_sep + ')' + _agentErrorRE = re.compile('^##AGENT-WARNING##\ ?(.*)') + + reboot_timeout = 600 + reboot_settling_time = 60 + + def __init__(self, host, port=20701, retryLimit=5, deviceRoot=None, + logLevel=logging.ERROR, **kwargs): + DeviceManager.__init__(self, logLevel=logLevel, + deviceRoot=deviceRoot) + self.host = host + self.port = port + self.retryLimit = retryLimit + self._sock = None + self._everConnected = False + + # Get version + verstring = self._runCmds([{'cmd': 'ver'}]) + ver_re = re.match('(\S+) Version (\S+)', verstring) + self.agentProductName = ver_re.group(1) + self.agentVersion = ver_re.group(2) + + def _cmdNeedsResponse(self, cmd): + """ Not all commands need a response from the agent: + * rebt obviously doesn't get a response + * uninstall performs a reboot to ensure starting in a clean state and + so also doesn't look for a response + """ + noResponseCmds = [re.compile('^rebt'), + re.compile('^uninst .*$'), + re.compile('^pull .*$')] + + for c in noResponseCmds: + if (c.match(cmd)): + return False + + # If the command is not in our list, then it gets a response + return True + + def _stripPrompt(self, data): + """ + take a data blob and strip instances of the prompt '$>\x00' + """ + promptre = re.compile(self._prompt_regex + '.*') + retVal = [] + lines = data.split('\n') + for line in lines: + foundPrompt = False + try: + while (promptre.match(line)): + foundPrompt = True + pieces = line.split(self._prompt_sep) + index = pieces.index('$>') + pieces.pop(index) + line = self._prompt_sep.join(pieces) + except(ValueError): + pass + + # we don't want to append lines that are blank after stripping the + # prompt (those are basically "prompts") + if not foundPrompt or line: + retVal.append(line) + + return '\n'.join(retVal) + + def _shouldCmdCloseSocket(self, cmd): + """ + Some commands need to close the socket after they are sent: + * rebt + * uninst + * quit + """ + socketClosingCmds = [re.compile('^quit.*'), + re.compile('^rebt.*'), + re.compile('^uninst .*$')] + + for c in socketClosingCmds: + if (c.match(cmd)): + return True + return False + + def _sendCmds(self, cmdlist, outputfile, timeout=None, retryLimit=None): + """ + Wrapper for _doCmds that loops up to retryLimit iterations + """ + # this allows us to move the retry logic outside of the _doCmds() to make it + # easier for debugging in the future. + # note that since cmdlist is a list of commands, they will all be retried if + # one fails. this is necessary in particular for pushFile(), where we don't want + # to accidentally send extra data if a failure occurs during data transmission. + + retryLimit = retryLimit or self.retryLimit + retries = 0 + while retries < retryLimit: + try: + self._doCmds(cmdlist, outputfile, timeout) + return + except DMError as err: + # re-raise error if it's fatal (i.e. the device got the command but + # couldn't execute it). retry otherwise + if err.fatal: + raise err + self._logger.debug(err) + retries += 1 + # if we lost the connection or failed to establish one, wait a bit + if retries < retryLimit and not self._sock: + sleep_time = 5 * retries + self._logger.info('Could not connect; sleeping for %d seconds.' % sleep_time) + time.sleep(sleep_time) + + raise DMError("Remote Device Error: unable to connect to %s after %s attempts" % + (self.host, retryLimit)) + + def _runCmds(self, cmdlist, timeout=None, retryLimit=None): + """ + Similar to _sendCmds, but just returns any output as a string instead of + writing to a file + """ + retryLimit = retryLimit or self.retryLimit + outputfile = StringIO.StringIO() + self._sendCmds(cmdlist, outputfile, timeout, retryLimit=retryLimit) + outputfile.seek(0) + return outputfile.read() + + def _doCmds(self, cmdlist, outputfile, timeout): + promptre = re.compile(self._prompt_regex + '$') + shouldCloseSocket = False + + if not timeout: + # We are asserting that all commands will complete in this time unless + # otherwise specified + timeout = self.default_timeout + + if not self._sock: + try: + if self._everConnected: + self._logger.info("reconnecting socket") + self._sock = socket.socket(socket.AF_INET, socket.SOCK_STREAM) + except socket.error as msg: + self._sock = None + raise DMError("Automation Error: unable to create socket: " + str(msg)) + + try: + self._sock.settimeout(float(timeout)) + self._sock.connect((self.host, int(self.port))) + self._everConnected = True + except socket.error as msg: + self._sock = None + raise DMError("Remote Device Error: Unable to connect socket: " + str(msg)) + + # consume prompt + try: + self._sock.recv(1024) + except socket.error as msg: + self._sock.close() + self._sock = None + raise DMError( + "Remote Device Error: Did not get prompt after connecting: " + str(msg), + fatal=True) + + # future recv() timeouts are handled by select() calls + self._sock.settimeout(None) + + for cmd in cmdlist: + cmdline = '%s\r\n' % cmd['cmd'] + + try: + sent = self._sock.send(cmdline) + if sent != len(cmdline): + raise DMError("Remote Device Error: our cmd was %s bytes and we " + "only sent %s" % (len(cmdline), sent)) + if cmd.get('data'): + totalsent = 0 + while totalsent < len(cmd['data']): + sent = self._sock.send(cmd['data'][totalsent:]) + self._logger.debug("sent %s bytes of data payload" % sent) + if sent == 0: + raise DMError("Socket connection broken when sending data") + totalsent += sent + + self._logger.debug("sent cmd: %s" % cmd['cmd']) + except socket.error as msg: + self._sock.close() + self._sock = None + self._logger.error("Remote Device Error: Error sending data" + " to socket. cmd=%s; err=%s" % (cmd['cmd'], msg)) + return False + + # Check if the command should close the socket + shouldCloseSocket = self._shouldCmdCloseSocket(cmd['cmd']) + + # Handle responses from commands + if self._cmdNeedsResponse(cmd['cmd']): + foundPrompt = False + data = "" + timer = 0 + select_timeout = 1 + commandFailed = False + + while not foundPrompt: + socketClosed = False + errStr = '' + temp = '' + self._logger.debug("recv'ing...") + + # Get our response + try: + # Wait up to a second for socket to become ready for reading... + if select.select([self._sock], [], [], select_timeout)[0]: + temp = self._sock.recv(1024) + self._logger.debug(u"response: %s" % temp.decode('utf8', 'replace')) + timer = 0 + if not temp: + socketClosed = True + errStr = 'connection closed' + timer += select_timeout + if timer > timeout: + self._sock.close() + self._sock = None + raise DMError("Automation Error: Timeout in command %s" % + cmd['cmd'], fatal=True) + except socket.error as err: + socketClosed = True + errStr = str(err) + # This error shows up with we have our tegra rebooted. + if err[0] == errno.ECONNRESET: + errStr += ' - possible reboot' + + if socketClosed: + self._sock.close() + self._sock = None + raise DMError( + "Automation Error: Error receiving data from socket. " + "cmd=%s; err=%s" % (cmd, errStr)) + + data += temp + + # If something goes wrong in the agent it will send back a string that + # starts with '##AGENT-WARNING##' + if not commandFailed: + errorMatch = self._agentErrorRE.match(data) + if errorMatch: + # We still need to consume the prompt, so raise an error after + # draining the rest of the buffer. + commandFailed = True + + for line in data.splitlines(): + if promptre.match(line): + foundPrompt = True + data = self._stripPrompt(data) + break + + # periodically flush data to output file to make sure it doesn't get + # too big/unwieldly + if len(data) > 1024: + outputfile.write(data[0:1024]) + data = data[1024:] + + if commandFailed: + raise DMError("Automation Error: Error processing command '%s'; err='%s'" % + (cmd['cmd'], errorMatch.group(1)), fatal=True) + + # Write any remaining data to outputfile + outputfile.write(data) + + if shouldCloseSocket: + try: + self._sock.close() + self._sock = None + except: + self._sock = None + raise DMError("Automation Error: Error closing socket") + + def _setupDeviceRoot(self, deviceRoot): + if not deviceRoot: + deviceRoot = "%s/tests" % self._runCmds( + [{'cmd': 'testroot'}]).strip() + self.mkDir(deviceRoot) + + return deviceRoot + + def shell(self, cmd, outputfile, env=None, cwd=None, timeout=None, root=False): + cmdline = self._escapedCommandLine(cmd) + if env: + cmdline = '%s %s' % (self._formatEnvString(env), cmdline) + + # execcwd/execcwdsu currently unsupported in Negatus; see bug 824127. + if cwd and self.agentProductName == 'SUTAgentNegatus': + raise DMError("Negatus does not support execcwd/execcwdsu") + + haveExecSu = (self.agentProductName == 'SUTAgentNegatus' or + StrictVersion(self.agentVersion) >= StrictVersion('1.13')) + + # Depending on agent version we send one of the following commands here: + # * exec (run as normal user) + # * execsu (run as privileged user) + # * execcwd (run as normal user from specified directory) + # * execcwdsu (run as privileged user from specified directory) + + cmd = "exec" + if cwd: + cmd += "cwd" + if root and haveExecSu: + cmd += "su" + + if cwd: + self._sendCmds([{'cmd': '%s %s %s' % (cmd, cwd, cmdline)}], outputfile, timeout) + else: + if (not root) or haveExecSu: + self._sendCmds([{'cmd': '%s %s' % (cmd, cmdline)}], outputfile, timeout) + else: + # need to manually inject su -c for backwards compatibility (this may + # not work on ICS or above!!) + # (FIXME: this backwards compatibility code is really ugly and should + # be deprecated at some point in the future) + self._sendCmds([{'cmd': '%s su -c "%s"' % (cmd, cmdline)}], outputfile, + timeout) + + # dig through the output to get the return code + lastline = _pop_last_line(outputfile) + if lastline: + m = re.search('return code \[([0-9]+)\]', lastline) + if m: + return int(m.group(1)) + + # woops, we couldn't find an end of line/return value + raise DMError( + "Automation Error: Error finding end of line/return value when running '%s'" % cmdline) + + def pushFile(self, localname, destname, retryLimit=None, createDir=True): + retryLimit = retryLimit or self.retryLimit + if createDir: + self.mkDirs(destname) + + try: + filesize = os.path.getsize(localname) + with open(localname, 'rb') as f: + remoteHash = self._runCmds([{'cmd': 'push ' + destname + ' ' + str(filesize), + 'data': f.read()}], retryLimit=retryLimit).strip() + except OSError: + raise DMError("DeviceManager: Error reading file to push") + + self._logger.debug("push returned: %s" % remoteHash) + + localHash = self._getLocalHash(localname) + + if localHash != remoteHash: + raise DMError("Automation Error: Push File failed to Validate! (localhash: %s, " + "remotehash: %s)" % (localHash, remoteHash)) + + def mkDir(self, name): + if not self.dirExists(name): + self._runCmds([{'cmd': 'mkdr ' + name}]) + + def pushDir(self, localDir, remoteDir, retryLimit=None, timeout=None): + retryLimit = retryLimit or self.retryLimit + self._logger.info("pushing directory: %s to %s" % (localDir, remoteDir)) + + existentDirectories = [] + for root, dirs, files in os.walk(localDir, followlinks=True): + _, subpath = root.split(localDir) + subpath = subpath.lstrip('/') + remoteRoot = posixpath.join(remoteDir, subpath) + for f in files: + remoteName = posixpath.join(remoteRoot, f) + + if subpath == "": + remoteRoot = remoteDir + + parent = os.path.dirname(remoteName) + if parent not in existentDirectories: + self.mkDirs(remoteName) + existentDirectories.append(parent) + + self.pushFile(os.path.join(root, f), remoteName, + retryLimit=retryLimit, createDir=False) + + def dirExists(self, remotePath): + ret = self._runCmds([{'cmd': 'isdir ' + remotePath}]).strip() + if not ret: + raise DMError('Automation Error: DeviceManager isdir returned null') + + return ret == 'TRUE' + + def fileExists(self, filepath): + # Because we always have / style paths we make this a lot easier with some + # assumptions + filepath = posixpath.normpath(filepath) + # / should always exist but we can use this to check for things like + # having access to the filesystem + if filepath == '/': + return self.dirExists(filepath) + (containingpath, filename) = posixpath.split(filepath) + return filename in self.listFiles(containingpath) + + def listFiles(self, rootdir): + rootdir = posixpath.normpath(rootdir) + if not self.dirExists(rootdir): + return [] + data = self._runCmds([{'cmd': 'cd ' + rootdir}, {'cmd': 'ls'}]) + + files = filter(lambda x: x, data.splitlines()) + if len(files) == 1 and files[0] == '': + # special case on the agent: empty directories return just the + # string "" + return [] + return files + + def removeFile(self, filename): + self._logger.info("removing file: " + filename) + if self.fileExists(filename): + self._runCmds([{'cmd': 'rm ' + filename}]) + + def removeDir(self, remoteDir): + if self.dirExists(remoteDir): + self._runCmds([{'cmd': 'rmdr ' + remoteDir}]) + + def moveTree(self, source, destination): + self._runCmds([{'cmd': 'mv %s %s' % (source, destination)}]) + + def copyTree(self, source, destination): + self._runCmds([{'cmd': 'dd if=%s of=%s' % (source, destination)}]) + + def getProcessList(self): + data = self._runCmds([{'cmd': 'ps'}]) + + processTuples = [] + for line in data.splitlines(): + if line: + pidproc = line.strip().split() + try: + if (len(pidproc) == 2): + processTuples += [[pidproc[0], pidproc[1]]] + elif (len(pidproc) == 3): + # android returns + processTuples += [[int(pidproc[1]), pidproc[2], int(pidproc[0])]] + else: + # unexpected format + raise ValueError + except ValueError: + self._logger.error("Unable to parse process list (bug 805969)") + self._logger.error("Line: %s\nFull output of process list:\n%s" % (line, data)) + raise DMError("Invalid process line: %s" % line) + + return processTuples + + def fireProcess(self, appname, failIfRunning=False, maxWaitTime=30): + """ + Starts a process + + returns: pid + + DEPRECATED: Use shell() or launchApplication() for new code + """ + if not appname: + raise DMError("Automation Error: fireProcess called with no command to run") + + self._logger.info("FIRE PROC: '%s'" % appname) + + if (self.processExist(appname) is None): + self._logger.warning("process %s appears to be running already\n" % appname) + if (failIfRunning): + raise DMError("Automation Error: Process is already running") + + self._runCmds([{'cmd': 'exec ' + appname}]) + + # The 'exec' command may wait for the process to start and end, so checking + # for the process here may result in process = None. + # The normal case is to launch the process and return right away + # There is one case with robotium (am instrument) where exec returns at the end + pid = None + waited = 0 + while pid is None and waited < maxWaitTime: + pid = self.processExist(appname) + if pid: + break + time.sleep(1) + waited += 1 + + self._logger.debug("got pid: %s for process: %s" % (pid, appname)) + return pid + + def launchProcess(self, cmd, outputFile="process.txt", cwd='', env='', failIfRunning=False): + """ + Launches a process, redirecting output to standard out + + Returns output filename + + WARNING: Does not work how you expect on Android! The application's + own output will be flushed elsewhere. + + DEPRECATED: Use shell() or launchApplication() for new code + """ + if not cmd: + self._logger.warning("launchProcess called without command to run") + return None + + if cmd[0] == 'am' and hasattr(self, '_getExtraAmStartArgs'): + cmd = cmd[:2] + self._getExtraAmStartArgs() + cmd[2:] + + cmdline = subprocess.list2cmdline(cmd) + if outputFile == "process.txt" or outputFile is None: + outputFile += "%s/process.txt" % self.deviceRoot + cmdline += " > " + outputFile + + # Prepend our env to the command + cmdline = '%s %s' % (self._formatEnvString(env), cmdline) + + # fireProcess may trigger an exception, but we won't handle it + if cmd[0] == "am": + # Robocop tests spawn "am instrument". sutAgent's exec ensures that + # am has started before returning, so there is no point in having + # fireProcess wait for it to start. Also, since "am" does not show + # up in the process list while the test is running, waiting for it + # in fireProcess is difficult. + self.fireProcess(cmdline, failIfRunning, 0) + else: + self.fireProcess(cmdline, failIfRunning) + return outputFile + + def killProcess(self, appname, sig=None): + if sig: + pid = self.processExist(appname) + if pid and pid > 0: + try: + self.shellCheckOutput(['kill', '-%d' % sig, str(pid)], + root=True) + except DMError as err: + self._logger.warning("unable to kill -%d %s (pid %s)" % + (sig, appname, str(pid))) + self._logger.debug(err) + raise err + else: + self._logger.warning("unable to kill -%d %s -- not running?" % + (sig, appname)) + else: + retries = 0 + while retries < self.retryLimit: + try: + if self.processExist(appname): + self._runCmds([{'cmd': 'kill ' + appname}]) + return + except DMError as err: + retries += 1 + self._logger.warning("try %d of %d failed to kill %s" % + (retries, self.retryLimit, appname)) + self._logger.debug(err) + if retries >= self.retryLimit: + raise err + + def getTempDir(self): + return self._runCmds([{'cmd': 'tmpd'}]).strip() + + def pullFile(self, remoteFile, offset=None, length=None): + # The "pull" command is different from other commands in that DeviceManager + # has to read a certain number of bytes instead of just reading to the + # next prompt. This is more robust than the "cat" command, which will be + # confused if the prompt string exists within the file being catted. + # However it means we can't use the response-handling logic in sendCMD(). + + def err(error_msg): + err_str = 'DeviceManager: pull unsuccessful: %s' % error_msg + self._logger.error(err_str) + self._sock = None + raise DMError(err_str) + + # FIXME: We could possibly move these socket-reading functions up to + # the class level if we wanted to refactor sendCMD(). For now they are + # only used to pull files. + + def uread(to_recv, error_msg): + """ unbuffered read """ + try: + data = "" + if select.select([self._sock], [], [], self.default_timeout)[0]: + data = self._sock.recv(to_recv) + if not data: + # timed out waiting for response or error response + err(error_msg) + + return data + except: + err(error_msg) + + def read_until_char(c, buf, error_msg): + """ read until 'c' is found; buffer rest """ + while c not in buf: + data = uread(1024, error_msg) + buf += data + return buf.partition(c) + + def read_exact(total_to_recv, buf, error_msg): + """ read exact number of 'total_to_recv' bytes """ + while len(buf) < total_to_recv: + to_recv = min(total_to_recv - len(buf), 1024) + data = uread(to_recv, error_msg) + buf += data + return buf + + prompt = self._base_prompt + self._prompt_sep + buf = '' + + # expected return value: + # ,\n + # or, if error, + # ,-1\n + + # just send the command first, we read the response inline below + if offset is not None and length is not None: + cmd = 'pull %s %d %d' % (remoteFile, offset, length) + elif offset is not None: + cmd = 'pull %s %d' % (remoteFile, offset) + else: + cmd = 'pull %s' % remoteFile + + self._runCmds([{'cmd': cmd}]) + + # read metadata; buffer the rest + metadata, sep, buf = read_until_char('\n', buf, 'could not find metadata') + if not metadata: + return None + self._logger.debug('metadata: %s' % metadata) + + filename, sep, filesizestr = metadata.partition(',') + if sep == '': + err('could not find file size in returned metadata') + try: + filesize = int(filesizestr) + except ValueError: + err('invalid file size in returned metadata') + + if filesize == -1: + # read error message + error_str, sep, buf = read_until_char('\n', buf, 'could not find error message') + if not error_str: + err("blank error message") + # prompt should follow + read_exact(len(prompt), buf, 'could not find prompt') + # failures are expected, so don't use "Remote Device Error" or we'll RETRY + raise DMError("DeviceManager: pulling file '%s' unsuccessful: %s" % + (remoteFile, error_str)) + + # read file data + total_to_recv = filesize + len(prompt) + buf = read_exact(total_to_recv, buf, 'could not get all file data') + if buf[-len(prompt):] != prompt: + err('no prompt found after file data--DeviceManager may be out of sync with agent') + return buf + return buf[:-len(prompt)] + + def getFile(self, remoteFile, localFile): + data = self.pullFile(remoteFile) + + fhandle = open(localFile, 'wb') + fhandle.write(data) + fhandle.close() + if not self.validateFile(remoteFile, localFile): + raise DMError("Automation Error: Failed to validate file when downloading %s" % + remoteFile) + + def getDirectory(self, remoteDir, localDir, checkDir=True): + self._logger.info("getting files in '%s'" % remoteDir) + if checkDir and not self.dirExists(remoteDir): + raise DMError("Automation Error: Error getting directory: %s not a directory" % + remoteDir) + + filelist = self.listFiles(remoteDir) + self._logger.debug(filelist) + if not os.path.exists(localDir): + os.makedirs(localDir) + + for f in filelist: + if f == '.' or f == '..': + continue + remotePath = remoteDir + '/' + f + localPath = os.path.join(localDir, f) + if self.dirExists(remotePath): + self.getDirectory(remotePath, localPath, False) + else: + self.getFile(remotePath, localPath) + + def validateFile(self, remoteFile, localFile): + remoteHash = self._getRemoteHash(remoteFile) + localHash = self._getLocalHash(localFile) + + if (remoteHash is None): + return False + + if (remoteHash == localHash): + return True + + return False + + def _getRemoteHash(self, filename): + data = self._runCmds([{'cmd': 'hash ' + filename}]).strip() + self._logger.debug("remote hash returned: '%s'" % data) + return data + + def unpackFile(self, filePath, destDir=None): + """ + Unzips a bundle to a location on the device + + If destDir is not specified, the bundle is extracted in the same directory + """ + # if no destDir is passed in just set it to filePath's folder + if not destDir: + destDir = posixpath.dirname(filePath) + + if destDir[-1] != '/': + destDir += '/' + + self._runCmds([{'cmd': 'unzp %s %s' % (filePath, destDir)}]) + + def _getRebootServerSocket(self, ipAddr): + serverSocket = socket.socket(socket.AF_INET, socket.SOCK_STREAM) + serverSocket.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, 1) + serverSocket.settimeout(60.0) + serverSocket.bind((ipAddr, 0)) + serverSocket.listen(1) + self._logger.debug('Created reboot callback server at %s:%d' % + serverSocket.getsockname()) + return serverSocket + + def _waitForRebootPing(self, serverSocket): + conn = None + data = None + startTime = datetime.datetime.now() + waitTime = datetime.timedelta(seconds=self.reboot_timeout) + while not data and datetime.datetime.now() - startTime < waitTime: + self._logger.info("Waiting for reboot callback ping from device...") + try: + if not conn: + conn, _ = serverSocket.accept() + # Receiving any data is good enough. + data = conn.recv(1024) + if data: + self._logger.info("Received reboot callback ping from device!") + conn.sendall('OK') + conn.close() + except socket.timeout: + pass + except socket.error as e: + if e.errno != errno.EAGAIN and e.errno != errno.EWOULDBLOCK: + raise + + if not data: + raise DMError('Timed out waiting for reboot callback.') + + self._logger.info("Sleeping for %s seconds to wait for device " + "to 'settle'" % self.reboot_settling_time) + time.sleep(self.reboot_settling_time) + + def reboot(self, ipAddr=None, port=30000, wait=False): + # port ^^^ is here for backwards compatibility only, we now + # determine a port automatically and safely + wait = (wait or ipAddr) + + cmd = 'rebt' + + self._logger.info("Rebooting device") + + # if we're waiting, create a listening server and pass information on + # it to the device before rebooting (we do this instead of just polling + # to make sure the device actually rebooted -- yes, there are probably + # simpler ways of doing this like polling uptime, but this is what we're + # doing for now) + if wait: + if not ipAddr: + ipAddr = moznetwork.get_ip() + serverSocket = self._getRebootServerSocket(ipAddr) + # The update.info command tells the SUTAgent to send a TCP message + # after restarting. + destname = '/data/data/com.mozilla.SUTAgentAndroid/files/update.info' + data = "%s,%s\rrebooting\r" % serverSocket.getsockname() + self._runCmds([{'cmd': 'push %s %s' % (destname, len(data)), + 'data': data}]) + cmd += " %s %s" % serverSocket.getsockname() + + # actually reboot device + self._runCmds([{'cmd': cmd}]) + # if we're waiting, wait for a callback ping from the agent before + # continuing (and throw an exception if we don't receive said ping) + if wait: + self._waitForRebootPing(serverSocket) + + def getInfo(self, directive=None): + data = None + result = {} + collapseSpaces = re.compile(' +') + + directives = ['os', 'id', 'uptime', 'uptimemillis', 'systime', 'screen', + 'rotation', 'memory', 'process', 'disk', 'power', 'sutuserinfo', + 'temperature'] + if (directive in directives): + directives = [directive] + + for d in directives: + data = self._runCmds([{'cmd': 'info ' + d}]) + + data = collapseSpaces.sub(' ', data) + result[d] = data.split('\n') + + # Get rid of any 0 length members of the arrays + for k, v in result.iteritems(): + result[k] = filter(lambda x: x != '', result[k]) + + # Format the process output + if 'process' in result: + proclist = [] + for l in result['process']: + if l: + proclist.append(l.split('\t')) + result['process'] = proclist + + self._logger.debug("results: %s" % result) + return result + + def installApp(self, appBundlePath, destPath=None): + cmd = 'inst ' + appBundlePath + if destPath: + cmd += ' ' + destPath + + data = self._runCmds([{'cmd': cmd}]) + + if 'installation complete [0]' not in data: + raise DMError("Remove Device Error: Error installing app. Error message: %s" % data) + + def uninstallApp(self, appName, installPath=None): + cmd = 'uninstall ' + appName + if installPath: + cmd += ' ' + installPath + data = self._runCmds([{'cmd': cmd}]) + + status = data.split('\n')[0].strip() + self._logger.debug("uninstallApp: '%s'" % status) + if status == 'Success': + return + raise DMError("Remote Device Error: uninstall failed for %s" % appName) + + def uninstallAppAndReboot(self, appName, installPath=None): + cmd = 'uninst ' + appName + if installPath: + cmd += ' ' + installPath + data = self._runCmds([{'cmd': cmd}]) + + self._logger.debug("uninstallAppAndReboot: %s" % data) + return + + def updateApp(self, appBundlePath, processName=None, destPath=None, + ipAddr=None, port=30000, wait=False): + # port ^^^ is here for backwards compatibility only, we now + # determine a port automatically and safely + wait = (wait or ipAddr) + + cmd = 'updt ' + if processName is None: + # Then we pass '' for processName + cmd += "'' " + appBundlePath + else: + cmd += processName + ' ' + appBundlePath + + if destPath: + cmd += " " + destPath + + if wait: + if not ipAddr: + ipAddr = moznetwork.get_ip() + serverSocket = self._getRebootServerSocket(ipAddr) + cmd += " %s %s" % serverSocket.getsockname() + + self._logger.debug("updateApp using command: " % cmd) + + self._runCmds([{'cmd': cmd}]) + + if wait: + self._waitForRebootPing(serverSocket) + + def getCurrentTime(self): + return int(self._runCmds([{'cmd': 'clok'}]).strip()) + + def _formatEnvString(self, env): + """ + Returns a properly formatted env string for the agent. + + Input - env, which is either None, '', or a dict + Output - a quoted string of the form: '"envvar1=val1,envvar2=val2..."' + If env is None or '' return '' (empty quoted string) + """ + if (env is None or env == ''): + return '' + + retVal = '"%s"' % ','.join(map(lambda x: '%s=%s' % (x[0], x[1]), env.iteritems())) + if (retVal == '""'): + return '' + + return retVal + + def adjustResolution(self, width=1680, height=1050, type='hdmi'): + """ + Adjust the screen resolution on the device, REBOOT REQUIRED + + NOTE: this only works on a tegra ATM + + supported resolutions: 640x480, 800x600, 1024x768, 1152x864, 1200x1024, 1440x900, + 1680x1050, 1920x1080 + """ + if self.getInfo('os')['os'][0].split()[0] != 'harmony-eng': + self._logger.warning("unable to adjust screen resolution on non Tegra device") + return False + + results = self.getInfo('screen') + parts = results['screen'][0].split(':') + self._logger.debug("we have a current resolution of %s, %s" % + (parts[1].split()[0], parts[2].split()[0])) + + # verify screen type is valid, and set it to the proper value + # (https://bugzilla.mozilla.org/show_bug.cgi?id=632895#c4) + screentype = -1 + if (type == 'hdmi'): + screentype = 5 + elif (type == 'vga' or type == 'crt'): + screentype = 3 + else: + return False + + # verify we have numbers + if not (isinstance(width, int) and isinstance(height, int)): + return False + + if (width < 100 or width > 9999): + return False + + if (height < 100 or height > 9999): + return False + + self._logger.debug("adjusting screen resolution to %s, %s and rebooting" % (width, height)) + + self._runCmds( + [{'cmd': "exec setprop persist.tegra.dpy%s.mode.width %s" % (screentype, width)}]) + self._runCmds( + [{'cmd': "exec setprop persist.tegra.dpy%s.mode.height %s" % (screentype, height)}]) + + def chmodDir(self, remoteDir, **kwargs): + self._runCmds([{'cmd': "chmod " + remoteDir}]) diff --git a/testing/mozbase/mozdevice/mozdevice/dmcli.py b/testing/mozbase/mozdevice/mozdevice/dmcli.py new file mode 100644 index 000000000..7ba65e842 --- /dev/null +++ b/testing/mozbase/mozdevice/mozdevice/dmcli.py @@ -0,0 +1,382 @@ +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this file, +# You can obtain one at http://mozilla.org/MPL/2.0/. + +""" +Command-line client to control a device +""" + +import errno +import logging +import os +import posixpath +import StringIO +import sys +import mozdevice +import mozlog +import argparse + + +class DMCli(object): + + def __init__(self): + self.commands = {'deviceroot': {'function': self.deviceroot, + 'help': 'get device root directory for storing temporary ' + 'files'}, + 'install': {'function': self.install, + 'args': [{'name': 'file'}], + 'help': 'push this package file to the device' + ' and install it'}, + 'uninstall': {'function': self.uninstall, + 'args': [{'name': 'packagename'}], + 'help': 'uninstall the named app from the device'}, + 'killapp': {'function': self.kill, + 'args': [{'name': 'process_name', 'nargs': '*'}], + 'help': 'kills any processes with name(s) on device'}, + 'launchapp': {'function': self.launchapp, + 'args': [{'name': 'appname'}, + {'name': 'activity_name'}, + {'name': '--intent', + 'action': 'store', + 'default': 'android.intent.action.VIEW'}, + {'name': '--url', + 'action': 'store'}, + {'name': '--no-fail-if-running', + 'action': 'store_true', + 'help': 'Don\'t fail if application is' + ' already running'} + ], + 'help': 'launches application on device'}, + 'listapps': {'function': self.listapps, + 'help': 'list applications on device'}, + 'push': {'function': self.push, + 'args': [{'name': 'local_file'}, + {'name': 'remote_file'} + ], + 'help': 'copy file/dir to device'}, + 'pull': {'function': self.pull, + 'args': [{'name': 'local_file'}, + {'name': 'remote_file', 'nargs': '?'}], + 'help': 'copy file/dir from device'}, + 'shell': {'function': self.shell, + 'args': [{'name': 'command', 'nargs': argparse.REMAINDER}, + {'name': '--root', 'action': 'store_true', + 'help': 'Run command as root'}], + 'help': 'run shell command on device'}, + 'info': {'function': self.getinfo, + 'args': [{'name': 'directive', 'nargs': '?'}], + 'help': 'get information on specified ' + 'aspect of the device (if no argument ' + 'given, print all available information)' + }, + 'ps': {'function': self.processlist, + 'help': 'get information on running processes on device' + }, + 'logcat': {'function': self.logcat, + 'help': 'get logcat from device' + }, + 'ls': {'function': self.listfiles, + 'args': [{'name': 'remote_dir'}], + 'help': 'list files on device' + }, + 'rm': {'function': self.removefile, + 'args': [{'name': 'remote_file'}], + 'help': 'remove file from device' + }, + 'isdir': {'function': self.isdir, + 'args': [{'name': 'remote_dir'}], + 'help': 'print if remote file is a directory' + }, + 'mkdir': {'function': self.mkdir, + 'args': [{'name': 'remote_dir'}], + 'help': 'makes a directory on device' + }, + 'rmdir': {'function': self.rmdir, + 'args': [{'name': 'remote_dir'}], + 'help': 'recursively remove directory from device' + }, + 'screencap': {'function': self.screencap, + 'args': [{'name': 'png_file'}], + 'help': 'capture screenshot of device in action' + }, + 'sutver': {'function': self.sutver, + 'help': 'SUTAgent\'s product name and version (SUT only)' + }, + 'clearlogcat': {'function': self.clearlogcat, + 'help': 'clear the logcat' + }, + 'reboot': {'function': self.reboot, + 'help': 'reboot the device', + 'args': [{'name': '--wait', + 'action': 'store_true', + 'help': 'Wait for device to come back up' + ' before exiting'}] + + }, + 'isfile': {'function': self.isfile, + 'args': [{'name': 'remote_file'}], + 'help': 'check whether a file exists on the device' + }, + 'launchfennec': {'function': self.launchfennec, + 'args': [{'name': 'appname'}, + {'name': '--intent', 'action': 'store', + 'default': 'android.intent.action.VIEW'}, + {'name': '--url', 'action': 'store'}, + {'name': '--extra-args', 'action': 'store'}, + {'name': '--mozenv', 'action': 'store', + 'help': 'Gecko environment variables to set' + ' in "KEY1=VAL1 KEY2=VAL2" format'}, + {'name': '--no-fail-if-running', + 'action': 'store_true', + 'help': 'Don\'t fail if application is ' + 'already running'} + ], + 'help': 'launch fennec' + }, + 'getip': {'function': self.getip, + 'args': [{'name': 'interface', 'nargs': '*'}], + 'help': 'get the ip address of the device' + } + } + + self.parser = argparse.ArgumentParser() + self.add_options(self.parser) + self.add_commands(self.parser) + mozlog.commandline.add_logging_group(self.parser) + + def run(self, args=sys.argv[1:]): + args = self.parser.parse_args() + + mozlog.commandline.setup_logging( + 'mozdevice', args, {'mach': sys.stdout}) + + if args.dmtype == "sut" and not args.host and not args.hwid: + self.parser.error("Must specify device ip in TEST_DEVICE or " + "with --host option with SUT") + + self.dm = self.getDevice(dmtype=args.dmtype, hwid=args.hwid, + host=args.host, port=args.port, + verbose=args.verbose) + + ret = args.func(args) + if ret is None: + ret = 0 + + sys.exit(ret) + + def add_options(self, parser): + parser.add_argument("-v", "--verbose", action="store_true", + help="Verbose output from DeviceManager", + default=bool(os.environ.get('VERBOSE'))) + parser.add_argument("--host", action="store", + help="Device hostname (only if using TCP/IP, " + "defaults to TEST_DEVICE environment " + "variable if present)", + default=os.environ.get('TEST_DEVICE')) + parser.add_argument("-p", "--port", action="store", + type=int, + help="Custom device port (if using SUTAgent or " + "adb-over-tcp)", default=None) + parser.add_argument("-m", "--dmtype", action="store", + help="DeviceManager type (adb or sut, defaults " + "to DM_TRANS environment variable, if " + "present, or adb)", + default=os.environ.get('DM_TRANS', 'adb')) + parser.add_argument("-d", "--hwid", action="store", + help="HWID", default=None) + parser.add_argument("--package-name", action="store", + help="Packagename (if using DeviceManagerADB)", + default=None) + + def add_commands(self, parser): + subparsers = parser.add_subparsers(title="Commands", metavar="") + for (commandname, commandprops) in sorted(self.commands.iteritems()): + subparser = subparsers.add_parser(commandname, help=commandprops['help']) + if commandprops.get('args'): + for arg in commandprops['args']: + # this is more elegant but doesn't work in python 2.6 + # (which we still use on tbpl @ mozilla where we install + # this package) + # kwargs = { k: v for k,v in arg.items() if k is not 'name' } + kwargs = {} + for (k, v) in arg.items(): + if k is not 'name': + kwargs[k] = v + subparser.add_argument(arg['name'], **kwargs) + subparser.set_defaults(func=commandprops['function']) + + def getDevice(self, dmtype="adb", hwid=None, host=None, port=None, + packagename=None, verbose=False): + ''' + Returns a device with the specified parameters + ''' + logLevel = logging.ERROR + if verbose: + logLevel = logging.DEBUG + + if hwid: + return mozdevice.DroidConnectByHWID(hwid, logLevel=logLevel) + + if dmtype == "adb": + if host and not port: + port = 5555 + return mozdevice.DroidADB(packageName=packagename, + host=host, port=port, + logLevel=logLevel) + elif dmtype == "sut": + if not host: + self.parser.error("Must specify host with SUT!") + if not port: + port = 20701 + return mozdevice.DroidSUT(host=host, port=port, + logLevel=logLevel) + else: + self.parser.error("Unknown device manager type: %s" % type) + + def deviceroot(self, args): + print self.dm.deviceRoot + + def push(self, args): + (src, dest) = (args.local_file, args.remote_file) + if os.path.isdir(src): + self.dm.pushDir(src, dest) + else: + dest_is_dir = dest[-1] == '/' or self.dm.dirExists(dest) + dest = posixpath.normpath(dest) + if dest_is_dir: + dest = posixpath.join(dest, os.path.basename(src)) + self.dm.pushFile(src, dest) + + def pull(self, args): + (src, dest) = (args.local_file, args.remote_file) + if not self.dm.fileExists(src): + print 'No such file or directory' + return + if not dest: + dest = posixpath.basename(src) + if self.dm.dirExists(src): + self.dm.getDirectory(src, dest) + else: + self.dm.getFile(src, dest) + + def install(self, args): + basename = os.path.basename(args.file) + app_path_on_device = posixpath.join(self.dm.deviceRoot, + basename) + self.dm.pushFile(args.file, app_path_on_device) + self.dm.installApp(app_path_on_device) + + def uninstall(self, args): + self.dm.uninstallApp(args.packagename) + + def launchapp(self, args): + self.dm.launchApplication(args.appname, args.activity_name, + args.intent, url=args.url, + failIfRunning=(not args.no_fail_if_running)) + + def listapps(self, args): + for app in self.dm.getInstalledApps(): + print app + + def stopapp(self, args): + self.dm.stopApplication(args.appname) + + def kill(self, args): + for name in args.process_name: + self.dm.killProcess(name) + + def shell(self, args): + buf = StringIO.StringIO() + self.dm.shell(args.command, buf, root=args.root) + print str(buf.getvalue()[0:-1]).rstrip() + + def getinfo(self, args): + info = self.dm.getInfo(directive=args.directive) + for (infokey, infoitem) in sorted(info.iteritems()): + if infokey == "process": + pass # skip process list: get that through ps + elif args.directive is None: + print "%s: %s" % (infokey.upper(), infoitem) + else: + print infoitem + + def logcat(self, args): + print ''.join(self.dm.getLogcat()) + + def clearlogcat(self, args): + self.dm.recordLogcat() + + def reboot(self, args): + self.dm.reboot(wait=args.wait) + + def processlist(self, args): + pslist = self.dm.getProcessList() + for ps in pslist: + print " ".join(str(i) for i in ps) + + def listfiles(self, args): + filelist = self.dm.listFiles(args.remote_dir) + for file in filelist: + print file + + def removefile(self, args): + self.dm.removeFile(args.remote_file) + + def isdir(self, args): + if self.dm.dirExists(args.remote_dir): + print "TRUE" + return + + print "FALSE" + return errno.ENOTDIR + + def mkdir(self, args): + self.dm.mkDir(args.remote_dir) + + def rmdir(self, args): + self.dm.removeDir(args.remote_dir) + + def screencap(self, args): + self.dm.saveScreenshot(args.png_file) + + def sutver(self, args): + if args.dmtype == 'sut': + print '%s Version %s' % (self.dm.agentProductName, + self.dm.agentVersion) + else: + print 'Must use SUT transport to get SUT version.' + + def isfile(self, args): + if self.dm.fileExists(args.remote_file): + print "TRUE" + return + print "FALSE" + return errno.ENOENT + + def launchfennec(self, args): + mozEnv = None + if args.mozenv: + mozEnv = {} + keyvals = args.mozenv.split() + for keyval in keyvals: + (key, _, val) = keyval.partition("=") + mozEnv[key] = val + self.dm.launchFennec(args.appname, intent=args.intent, + mozEnv=mozEnv, + extraArgs=args.extra_args, url=args.url, + failIfRunning=(not args.no_fail_if_running)) + + def getip(self, args): + if args.interface: + print(self.dm.getIP(args.interface)) + else: + print(self.dm.getIP()) + + +def cli(args=sys.argv[1:]): + # process the command line + cli = DMCli() + cli.run(args) + +if __name__ == '__main__': + cli() diff --git a/testing/mozbase/mozdevice/mozdevice/droid.py b/testing/mozbase/mozdevice/mozdevice/droid.py new file mode 100644 index 000000000..f06a619c4 --- /dev/null +++ b/testing/mozbase/mozdevice/mozdevice/droid.py @@ -0,0 +1,263 @@ +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this file, +# You can obtain one at http://mozilla.org/MPL/2.0/. + +import StringIO +import moznetwork +import re +import threading +import time + +import version_codes + +from Zeroconf import Zeroconf, ServiceBrowser +from devicemanager import ZeroconfListener +from devicemanagerADB import DeviceManagerADB +from devicemanagerSUT import DeviceManagerSUT +from devicemanager import DMError + + +class DroidMixin(object): + """Mixin to extend DeviceManager with Android-specific functionality""" + + _stopApplicationNeedsRoot = True + + def _getExtraAmStartArgs(self): + return [] + + def launchApplication(self, appName, activityName, intent, url=None, + extras=None, wait=True, failIfRunning=True): + """ + Launches an Android application + + :param appName: Name of application (e.g. `com.android.chrome`) + :param activityName: Name of activity to launch (e.g. `.Main`) + :param intent: Intent to launch application with + :param url: URL to open + :param extras: Dictionary of extra arguments to launch application with + :param wait: If True, wait for application to start before returning + :param failIfRunning: Raise an exception if instance of application is already running + """ + + # If failIfRunning is True, we throw an exception here. Only one + # instance of an application can be running at once on Android, + # starting a new instance may not be what we want depending on what + # we want to do + if failIfRunning and self.processExist(appName): + raise DMError("Only one instance of an application may be running " + "at once") + + acmd = ["am", "start"] + self._getExtraAmStartArgs() + \ + ["-W" if wait else '', "-n", "%s/%s" % (appName, activityName)] + + if intent: + acmd.extend(["-a", intent]) + + if extras: + for (key, val) in extras.iteritems(): + if type(val) is int: + extraTypeParam = "--ei" + elif type(val) is bool: + extraTypeParam = "--ez" + else: + extraTypeParam = "--es" + acmd.extend([extraTypeParam, str(key), str(val)]) + + if url: + acmd.extend(["-d", url]) + + # shell output not that interesting and debugging logs should already + # show what's going on here... so just create an empty memory buffer + # and ignore (except on error) + shellOutput = StringIO.StringIO() + if self.shell(acmd, shellOutput) == 0: + return + + shellOutput.seek(0) + raise DMError("Unable to launch application (shell output: '%s')" % shellOutput.read()) + + def launchFennec(self, appName, intent="android.intent.action.VIEW", + mozEnv=None, extraArgs=None, url=None, wait=True, + failIfRunning=True): + """ + Convenience method to launch Fennec on Android with various debugging + arguments + + :param appName: Name of fennec application (e.g. `org.mozilla.fennec`) + :param intent: Intent to launch application with + :param mozEnv: Mozilla specific environment to pass into application + :param extraArgs: Extra arguments to be parsed by fennec + :param url: URL to open + :param wait: If True, wait for application to start before returning + :param failIfRunning: Raise an exception if instance of application is already running + """ + extras = {} + + if mozEnv: + # mozEnv is expected to be a dictionary of environment variables: Fennec + # itself will set them when launched + for (envCnt, (envkey, envval)) in enumerate(mozEnv.iteritems()): + extras["env" + str(envCnt)] = envkey + "=" + envval + + # Additional command line arguments that fennec will read and use (e.g. + # with a custom profile) + if extraArgs: + extras['args'] = " ".join(extraArgs) + + self.launchApplication(appName, "org.mozilla.gecko.BrowserApp", intent, url=url, + extras=extras, + wait=wait, failIfRunning=failIfRunning) + + def getInstalledApps(self): + """ + Lists applications installed on this Android device + + Returns a list of application names in the form [ 'org.mozilla.fennec', ... ] + """ + output = self.shellCheckOutput(["pm", "list", "packages", "-f"]) + apps = [] + for line in output.splitlines(): + # lines are of form: package:/system/app/qik-tmo.apk=com.qiktmobile.android + apps.append(line.split('=')[1]) + + return apps + + def stopApplication(self, appName): + """ + Stops the specified application + + For Android 3.0+, we use the "am force-stop" to do this, which is + reliable and does not require root. For earlier versions of Android, + we simply try to manually kill the processes started by the app + repeatedly until none is around any more. This is less reliable and + does require root. + + :param appName: Name of application (e.g. `com.android.chrome`) + """ + version = self.shellCheckOutput(["getprop", "ro.build.version.sdk"]) + if int(version) >= version_codes.HONEYCOMB: + self.shellCheckOutput(["am", "force-stop", appName], + root=self._stopApplicationNeedsRoot) + else: + num_tries = 0 + max_tries = 5 + while self.processExist(appName): + if num_tries > max_tries: + raise DMError("Couldn't successfully kill %s after %s " + "tries" % (appName, max_tries)) + self.killProcess(appName) + num_tries += 1 + + # sleep for a short duration to make sure there are no + # additional processes in the process of being launched + # (this is not 100% guaranteed to work since it is inherently + # racey, but it's the best we can do) + time.sleep(1) + + +class DroidADB(DeviceManagerADB, DroidMixin): + + _stopApplicationNeedsRoot = False + + def getTopActivity(self): + package = None + data = None + try: + data = self.shellCheckOutput( + ["dumpsys", "window", "windows"], timeout=self.short_timeout) + except: + # dumpsys seems to intermittently fail (seen on 4.3 emulator), producing + # no output. + return "" + # "dumpsys window windows" produces many lines of input. The top/foreground + # activity is indicated by something like: + # mFocusedApp=AppWindowToken{483e6db0 token=HistoryRecord{484dcad8 com.mozilla.SUTAgentAndroid/.SUTAgentAndroid}} # noqa + # or, on other devices: + # FocusedApplication: name='AppWindowToken{41a65340 token=ActivityRecord{418fbd68 org.mozilla.fennec_mozdev/org.mozilla.gecko.BrowserApp}}', dispatchingTimeout=5000.000ms # noqa + # Extract this line, ending in the forward slash: + m = re.search('mFocusedApp(.+)/', data) + if not m: + m = re.search('FocusedApplication(.+)/', data) + if m: + line = m.group(0) + # Extract package name: string of non-whitespace ending in forward slash + m = re.search('(\S+)/$', line) + if m: + package = m.group(1) + if not package: + # On some Android 4.4 devices, when the home screen is displayed, + # dumpsys reports "mFocusedApp=null". Guard against this case and + # others where the focused app can not be determined by returning + # an empty string -- same as sutagent. + package = "" + return package + + def getAppRoot(self, packageName): + """ + Returns the root directory for the specified android application + """ + # relying on convention + return '/data/data/%s' % packageName + + +class DroidSUT(DeviceManagerSUT, DroidMixin): + + def _getExtraAmStartArgs(self): + # in versions of android in jellybean and beyond, the agent may run as + # a different process than the one that started the app. In this case, + # we need to get back the original user serial number and then pass + # that to the 'am start' command line + if not hasattr(self, '_userSerial'): + infoDict = self.getInfo(directive="sutuserinfo") + if infoDict.get('sutuserinfo') and \ + len(infoDict['sutuserinfo']) > 0: + userSerialString = infoDict['sutuserinfo'][0] + # user serial always an integer, see: + # http://developer.android.com/reference/android/os/UserManager.html#getSerialNumberForUser%28android.os.UserHandle%29 + m = re.match('User Serial:([0-9]+)', userSerialString) + if m: + self._userSerial = m.group(1) + else: + self._userSerial = None + else: + self._userSerial = None + + if self._userSerial is not None: + return ["--user", self._userSerial] + + return [] + + def getTopActivity(self): + return self._runCmds([{'cmd': "activity"}]).strip() + + def getAppRoot(self, packageName): + return self._runCmds([{'cmd': 'getapproot %s' % packageName}]).strip() + + +def DroidConnectByHWID(hwid, timeout=30, **kwargs): + """Try to connect to the given device by waiting for it to show up using + mDNS with the given timeout.""" + zc = Zeroconf(moznetwork.get_ip()) + + evt = threading.Event() + listener = ZeroconfListener(hwid, evt) + sb = ServiceBrowser(zc, "_sutagent._tcp.local.", listener) + foundIP = None + if evt.wait(timeout): + # we found the hwid + foundIP = listener.ip + sb.cancel() + zc.close() + + if foundIP is not None: + return DroidSUT(foundIP, **kwargs) + print "Connected via SUT to %s [at %s]" % (hwid, foundIP) + + # try connecting via adb + try: + sut = DroidADB(deviceSerial=hwid, **kwargs) + except: + return None + + print "Connected via ADB to %s" % (hwid) + return sut diff --git a/testing/mozbase/mozdevice/mozdevice/sutini.py b/testing/mozbase/mozdevice/mozdevice/sutini.py new file mode 100644 index 000000000..7dd5e54c4 --- /dev/null +++ b/testing/mozbase/mozdevice/mozdevice/sutini.py @@ -0,0 +1,126 @@ +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this file, +# You can obtain one at http://mozilla.org/MPL/2.0/. + +import ConfigParser +import StringIO +import os +import sys +import tempfile + +from mozdevice.droid import DroidSUT +from mozdevice.devicemanager import DMError + +USAGE = '%s ' +INI_PATH_JAVA = '/data/data/com.mozilla.SUTAgentAndroid/files/SUTAgent.ini' +INI_PATH_NEGATUS = '/data/local/SUTAgent.ini' +SCHEMA = {'Registration Server': (('IPAddr', ''), + ('PORT', '28001'), + ('HARDWARE', ''), + ('POOL', '')), + 'Network Settings': (('SSID', ''), + ('AUTH', ''), + ('ENCR', ''), + ('EAP', ''))} + + +def get_cfg(d, ini_path): + cfg = ConfigParser.RawConfigParser() + try: + cfg.readfp(StringIO.StringIO(d.pullFile(ini_path)), 'SUTAgent.ini') + except DMError: + # assume this is due to a missing file... + pass + return cfg + + +def put_cfg(d, cfg, ini_path): + print 'Writing modified SUTAgent.ini...' + t = tempfile.NamedTemporaryFile(delete=False) + cfg.write(t) + t.close() + try: + d.pushFile(t.name, ini_path) + except DMError, e: + print e + else: + print 'Done.' + finally: + os.unlink(t.name) + + +def set_opt(cfg, s, o, dflt): + prompt = ' %s' % o + try: + curval = cfg.get(s, o) + except (ConfigParser.NoSectionError, ConfigParser.NoOptionError): + curval = '' + if curval: + dflt = curval + prompt += ': ' + if dflt: + prompt += '[%s] ' % dflt + newval = raw_input(prompt) + if not newval: + newval = dflt + if newval == curval: + return False + cfg.set(s, o, newval) + return True + + +def bool_query(prompt, dflt): + while True: + i = raw_input('%s [%s] ' % (prompt, 'y' if dflt else 'n')).lower() + if not i or i[0] in ('y', 'n'): + break + print 'Enter y or n.' + return (not i and dflt) or (i and i[0] == 'y') + + +def edit_sect(cfg, sect, opts): + changed_vals = False + if bool_query('Edit section %s?' % sect, False): + if not cfg.has_section(sect): + cfg.add_section(sect) + print '%s settings:' % sect + for opt, dflt in opts: + changed_vals |= set_opt(cfg, sect, opt, dflt) + print + else: + if cfg.has_section(sect) and bool_query('Delete section %s?' % sect, + False): + cfg.remove_section(sect) + changed_vals = True + return changed_vals + + +def main(): + try: + host = sys.argv[1] + except IndexError: + print USAGE % sys.argv[0] + sys.exit(1) + try: + d = DroidSUT(host, retryLimit=1) + except DMError, e: + print e + sys.exit(1) + # check if using Negatus and change path accordingly + ini_path = INI_PATH_JAVA + if 'Negatus' in d.agentProductName: + ini_path = INI_PATH_NEGATUS + cfg = get_cfg(d, ini_path) + if not cfg.sections(): + print 'Empty or missing ini file.' + changed_vals = False + for sect, opts in SCHEMA.iteritems(): + changed_vals |= edit_sect(cfg, sect, opts) + if changed_vals: + put_cfg(d, cfg, ini_path) + else: + print 'No changes.' + + +if __name__ == '__main__': + main() diff --git a/testing/mozbase/mozdevice/mozdevice/version_codes.py b/testing/mozbase/mozdevice/mozdevice/version_codes.py new file mode 100644 index 000000000..6602d837a --- /dev/null +++ b/testing/mozbase/mozdevice/mozdevice/version_codes.py @@ -0,0 +1,61 @@ +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this file, +# You can obtain one at http://mozilla.org/MPL/2.0/. + +""" +VERSION CODES of the android releases. + +See http://developer.android.com/reference/android/os/Build.VERSION_CODES.html. +""" + +# Magic version number for a current development build, which has +# not yet turned into an official release. +CUR_DEVELOPMENT = 10000 + +# October 2008: The original, first, version of Android +BASE = 1 +# February 2009: First Android update, officially called 1.1 +BASE_1_1 = 2 +# May 2009: Android 1.5 +CUPCAKE = 3 +# September 2009: Android 1.6 +DONUT = 4 +# November 2009: Android 2.0 +ECLAIR = 5 +# December 2009: Android 2.0.1 +ECLAIR_0_1 = 6 +# January 2010: Android 2.1 +ECLAIR_MR1 = 7 +# June 2010: Android 2.2 +FROYO = 8 +# November 2010: Android 2.3 +GINGERBREAD = 9 +# February 2011: Android 2.3.3 +GINGERBREAD_MR1 = 10 +# February 2011: Android 3.0 +HONEYCOMB = 11 +# May 2011: Android 3.1 +HONEYCOMB_MR1 = 12 +# June 2011: Android 3.2 +HONEYCOMB_MR2 = 13 +# October 2011: Android 4.0 +ICE_CREAM_SANDWICH = 14 +# December 2011: Android 4.0.3 +ICE_CREAM_SANDWICH_MR1 = 15 +# June 2012: Android 4.1 +JELLY_BEAN = 16 +# November 2012: Android 4.2 +JELLY_BEAN_MR1 = 17 +# July 2013: Android 4.3 +JELLY_BEAN_MR2 = 18 +# October 2013: Android 4.4 +KITKAT = 19 +# Android 4.4W +KITKAT_WATCH = 20 +# Lollilop +LOLLIPOP = 21 +LOLLIPOP_MR1 = 22 +# M +M = 23 +# N +N = 24 diff --git a/testing/mozbase/mozdevice/setup.py b/testing/mozbase/mozdevice/setup.py new file mode 100644 index 000000000..477227f28 --- /dev/null +++ b/testing/mozbase/mozdevice/setup.py @@ -0,0 +1,36 @@ +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this file, +# You can obtain one at http://mozilla.org/MPL/2.0/. + +from setuptools import setup + +PACKAGE_NAME = 'mozdevice' +PACKAGE_VERSION = '0.48' + +deps = ['mozfile >= 1.0', + 'mozlog >= 3.0', + 'moznetwork >= 0.24', + 'mozprocess >= 0.19', + ] + +setup(name=PACKAGE_NAME, + version=PACKAGE_VERSION, + description="Mozilla-authored device management", + long_description="see http://mozbase.readthedocs.org/", + classifiers=[], # Get strings from http://pypi.python.org/pypi?%3Aaction=list_classifiers + keywords='', + author='Mozilla Automation and Testing Team', + author_email='tools@lists.mozilla.org', + url='https://wiki.mozilla.org/Auto-tools/Projects/Mozbase', + license='MPL', + packages=['mozdevice'], + include_package_data=True, + zip_safe=False, + install_requires=deps, + entry_points=""" + # -*- Entry points: -*- + [console_scripts] + dm = mozdevice.dmcli:cli + sutini = mozdevice.sutini:main + """, + ) diff --git a/testing/mozbase/mozdevice/sut_tests/README.md b/testing/mozbase/mozdevice/sut_tests/README.md new file mode 100644 index 000000000..ffc100f45 --- /dev/null +++ b/testing/mozbase/mozdevice/sut_tests/README.md @@ -0,0 +1,15 @@ +# SUT Agent tests + +* In order to run these tests you need to have a phone running SUT Agent +connected. + +* Make sure you can reach the device's TCP 20700 and 20701 ports. Doing +*adb forward tcp:20700 tcp:20700 && adb forward tcp:20701 tcp:20701* will +forward your localhost 20700 and 20701 ports to the ones on the device. + +* You might need some common tools like cp. Use the `setup-tools.sh` script +to install them. It requires `$ADB` to point to the `adb` binary on the system. + +* Make sure the SUTAgent on the device is running. + +* Run: python runtests.py diff --git a/testing/mozbase/mozdevice/sut_tests/dmunit.py b/testing/mozbase/mozdevice/sut_tests/dmunit.py new file mode 100644 index 000000000..b4a3d0b9b --- /dev/null +++ b/testing/mozbase/mozdevice/sut_tests/dmunit.py @@ -0,0 +1,55 @@ +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, You can obtain one at http://mozilla.org/MPL/2.0/. + +import logging +import types +import unittest + +from mozdevice import devicemanager +from mozdevice import devicemanagerSUT + +ip = '' +port = 0 +heartbeat_port = 0 +log_level = logging.ERROR + + +class DeviceManagerTestCase(unittest.TestCase): + """DeviceManager tests should subclass this. + """ + + """Set to False in your derived class if this test + should not be run on the Python agent. + """ + runs_on_test_device = True + + def _setUp(self): + """ Override this if you want set-up code in your test.""" + return + + def setUp(self): + self.dm = devicemanagerSUT.DeviceManagerSUT(host=ip, port=port, + logLevel=log_level) + self.dmerror = devicemanager.DMError + self._setUp() + + +class DeviceManagerTestLoader(unittest.TestLoader): + + def __init__(self, isTestDevice=False): + self.isTestDevice = isTestDevice + + def loadTestsFromModuleName(self, module_name): + """Loads tests from modules unless the SUT is a test device and + the test case has runs_on_test_device set to False + """ + tests = [] + module = __import__(module_name) + for name in dir(module): + obj = getattr(module, name) + if (isinstance(obj, (type, types.ClassType)) and + issubclass(obj, unittest.TestCase)) and \ + (not self.isTestDevice or obj.runs_on_test_device): + tests.append(self.loadTestsFromTestCase(obj)) + return self.suiteClass(tests) diff --git a/testing/mozbase/mozdevice/sut_tests/genfiles.py b/testing/mozbase/mozdevice/sut_tests/genfiles.py new file mode 100644 index 000000000..5ab8d6349 --- /dev/null +++ b/testing/mozbase/mozdevice/sut_tests/genfiles.py @@ -0,0 +1,85 @@ +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, You can obtain one at http://mozilla.org/MPL/2.0/. + + +from random import randint +from zipfile import ZipFile +import os +import shutil + + +def gen_binary_file(path, size): + with open(path, 'wb') as f: + for i in xrange(size): + byte = '%c' % randint(0, 255) + f.write(byte) + + +def gen_zip(path, files, stripped_prefix=''): + with ZipFile(path, 'w') as z: + for f in files: + new_name = f.replace(stripped_prefix, '') + z.write(f, new_name) + + +def mkdir(path, *args): + try: + os.mkdir(path, *args) + except OSError: + pass + + +def gen_folder_structure(): + root = 'test-files' + prefix = os.path.join(root, 'push2') + mkdir(prefix) + + gen_binary_file(os.path.join(prefix, 'file4.bin'), 59036) + mkdir(os.path.join(prefix, 'sub1')) + shutil.copyfile(os.path.join(root, 'mytext.txt'), + os.path.join(prefix, 'sub1', 'file1.txt')) + mkdir(os.path.join(prefix, 'sub1', 'sub1.1')) + shutil.copyfile(os.path.join(root, 'mytext.txt'), + os.path.join(prefix, 'sub1', 'sub1.1', 'file2.txt')) + mkdir(os.path.join(prefix, 'sub2')) + shutil.copyfile(os.path.join(root, 'mytext.txt'), + os.path.join(prefix, 'sub2', 'file3.txt')) + + +def gen_test_files(): + gen_folder_structure() + flist = [ + os.path.join('test-files', 'push2'), + os.path.join('test-files', 'push2', 'file4.bin'), + os.path.join('test-files', 'push2', 'sub1'), + os.path.join('test-files', 'push2', 'sub1', 'file1.txt'), + os.path.join('test-files', 'push2', 'sub1', 'sub1.1'), + os.path.join('test-files', 'push2', 'sub1', 'sub1.1', 'file2.txt'), + os.path.join('test-files', 'push2', 'sub2'), + os.path.join('test-files', 'push2', 'sub2', 'file3.txt') + ] + gen_zip(os.path.join('test-files', 'mybinary.zip'), + flist, stripped_prefix=('test-files' + os.path.sep)) + gen_zip(os.path.join('test-files', 'mytext.zip'), + [os.path.join('test-files', 'mytext.txt')]) + + +def clean_test_files(): + ds = [os.path.join('test-files', d) for d in ('push1', 'push2')] + for d in ds: + try: + shutil.rmtree(d) + except OSError: + pass + + fs = [os.path.join('test-files', f) for f in ('mybinary.zip', 'mytext.zip')] + for f in fs: + try: + os.remove(f) + except OSError: + pass + + +if __name__ == '__main__': + gen_test_files() diff --git a/testing/mozbase/mozdevice/sut_tests/runtests.py b/testing/mozbase/mozdevice/sut_tests/runtests.py new file mode 100644 index 000000000..fffc306e3 --- /dev/null +++ b/testing/mozbase/mozdevice/sut_tests/runtests.py @@ -0,0 +1,96 @@ +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, You can obtain one at http://mozilla.org/MPL/2.0/. + +from optparse import OptionParser +import logging +import os +import re +import sys +import unittest + +import dmunit +import genfiles + + +def main(ip, port, heartbeat_port, scripts, directory, isTestDevice, verbose): + dmunit.ip = ip + dmunit.port = port + dmunit.heartbeat_port = heartbeat_port + if verbose: + dmunit.log_level = logging.DEBUG + + suite = unittest.TestSuite() + + genfiles.gen_test_files() + + if scripts: + # Ensure the user didn't include the .py on the name of the test file + # (and get rid of it if they did) + scripts = map(lambda x: x.split('.')[0], scripts) + else: + # Go through the directory and pick up everything + # named test_*.py and run it + testfile = re.compile('^test_.*\.py$') + files = os.listdir(directory) + + for f in files: + if testfile.match(f): + scripts.append(f.split('.')[0]) + + testLoader = dmunit.DeviceManagerTestLoader(isTestDevice) + for s in scripts: + suite.addTest(testLoader.loadTestsFromModuleName(s)) + unittest.TextTestRunner(verbosity=2).run(suite) + + genfiles.clean_test_files() + + +if __name__ == "__main__": + + default_ip = '127.0.0.1' + default_port = 20701 + + env_ip, _, env_port = os.getenv('TEST_DEVICE', '').partition(':') + if env_port: + try: + env_port = int(env_port) + except ValueError: + print >> sys.stderr, "Port in TEST_DEVICE should be an integer." + sys.exit(1) + + # Deal with the options + parser = OptionParser() + parser.add_option("--ip", action="store", type="string", dest="ip", + help="IP address for device running SUTAgent, defaults " + "to what's provided in $TEST_DEVICE or 127.0.0.1", + default=(env_ip or default_ip)) + + parser.add_option("--port", action="store", type="int", dest="port", + help="Port of SUTAgent on device, defaults to " + "what's provided in $TEST_DEVICE or 20701", + default=(env_port or default_port)) + + parser.add_option("--heartbeat", action="store", type="int", + dest="heartbeat_port", help="Port for heartbeat/data " + "channel, defaults to 20700", default=20700) + + parser.add_option("--script", action="append", type="string", + dest="scripts", help="Name of test script to run, " + "can be specified multiple times", default=[]) + + parser.add_option("--directory", action="store", type="string", dest="dir", + help="Directory to look for tests in, defaults to " + "current directory", default=os.getcwd()) + + parser.add_option("--testDevice", action="store_true", dest="isTestDevice", + help="Specifies that the device is a local test agent", + default=False) + + parser.add_option("-v", "--verbose", action="store_true", dest="verbose", + help="Verbose DeviceManager output", default=False) + + (options, args) = parser.parse_args() + + main(options.ip, options.port, options.heartbeat_port, options.scripts, + options.dir, options.isTestDevice, options.verbose) diff --git a/testing/mozbase/mozdevice/sut_tests/setup-tools.sh b/testing/mozbase/mozdevice/sut_tests/setup-tools.sh new file mode 100755 index 000000000..e50a71d0b --- /dev/null +++ b/testing/mozbase/mozdevice/sut_tests/setup-tools.sh @@ -0,0 +1,10 @@ +#!/bin/sh + +if [ ! -f busybox-armv6l ] +then + wget http://busybox.net/downloads/binaries/1.19.0/busybox-armv6l +fi +$ADB remount +$ADB push busybox-armv6l /system/bin/busybox + +$ADB shell 'cd /system/bin; chmod 555 busybox; for x in `./busybox --list`; do ln -s ./busybox $x; done' diff --git a/testing/mozbase/mozdevice/sut_tests/test-files/mytext.txt b/testing/mozbase/mozdevice/sut_tests/test-files/mytext.txt new file mode 100644 index 000000000..74cb65fa8 --- /dev/null +++ b/testing/mozbase/mozdevice/sut_tests/test-files/mytext.txt @@ -0,0 +1,177 @@ +this is a file with 71K bytes of text in it +Lorem ipsum dolor sit amet, consectetur adipiscing elit. Nunc placerat, mi sit amet laoreet sollicitudin, neque urna bibendum eros, nec adipiscing tellus ipsum id risus. Sed aliquam ligula nec nibh sollicitudin venenatis. Praesent faucibus tortor vel felis egestas pellentesque. Cras viverra, dui viverra vulputate ornare, eros nunc volutpat nisl, sed sodales turpis orci quis diam. Donec eu sem mi. Mauris dictum blandit mauris quis ultricies. Sed faucibus erat vel velit viverra adipiscing. Donec placerat mattis venenatis. Suspendisse placerat sagittis risus et dapibus. Vivamus diam nisi, elementum ac mollis nec, porta ut sapien. Curabitur ac dolor ligula, vel sollicitudin sapien. Nullam blandit ligula nisl. Proin faucibus, ipsum sit amet molestie tincidunt, tellus neque accumsan lectus, a congue felis odio eu nunc. Pellentesque mauris sapien, varius ut scelerisque et, dictum sed magna. In faucibus tristique erat, a malesuada justo tincidunt sed. + +Morbi quis iaculis elit. Praesent nec diam mi, eu auctor neque. Phasellus fringilla turpis a metus imperdiet laoreet et ut augue. Mauris imperdiet scelerisque arcu quis sollicitudin. Nulla mauris dui, ultricies at vulputate quis, pharetra in erat. Donec mollis ipsum quis purus fermentum commodo. Nunc nec orci sem, quis rhoncus mauris. Sed iaculis tempus quam, non consectetur nisl tincidunt vitae. Nulla aliquam sodales auctor. Donec placerat venenatis facilisis. In sollicitudin arcu tincidunt lorem molestie bibendum. Phasellus rutrum ante vitae lorem iaculis eget porta odio pretium. + +Duis id mauris ante, eget ullamcorper justo. Integer vitae felis nisi, eget blandit tortor. Vivamus ligula odio, adipiscing sit amet tincidunt id, pretium sed massa. Suspendisse massa felis, viverra non adipiscing quis, dictum eget metus. In porta, tortor a imperdiet sodales, nulla mi mollis ipsum, quis venenatis nunc ipsum sit amet libero. Aenean sed leo eros. Curabitur varius egestas tempor. Nullam vitae convallis nunc. Phasellus molestie volutpat purus ut commodo. Phasellus eget lacus sem. Maecenas ligula magna, lacinia mollis molestie vitae, fringilla ac turpis. Sed ut nunc id nunc fringilla consectetur at et neque. + +Aliquam erat volutpat. Nullam lacinia, neque id luctus consectetur, nisl justo porta justo, eu scelerisque ligula ligula sed purus. Cras faucibus porttitor nisi at vulputate. Integer iaculis urna ut sapien iaculis ac malesuada quam congue. Mauris volutpat tristique est, vitae vehicula nisi imperdiet tincidunt. Curabitur semper, tellus sed cursus placerat, mi nulla dapibus odio, quis adipiscing arcu eros eu quam. Nullam fermentum dictum tellus non pretium. Sed dignissim enim a odio varius pellentesque. Nullam at lacinia mi. Nam et sem non risus suscipit pharetra vel et nisl. Cras porta lorem quis diam tempus nec dapibus velit sodales. Suspendisse laoreet hendrerit fringilla. + +Phasellus velit quam, malesuada eget rhoncus in, hendrerit sed nibh. Quisque nisl erat, pulvinar vitae condimentum sed, vehicula sit amet elit. Nulla eget mauris est, vel lacinia eros. Maecenas feugiat tortor ac nulla porta bibendum. Phasellus commodo ultrices rhoncus. Ut nec lacus in mauris semper congue. Vivamus rhoncus dolor a nulla accumsan semper. Donec vestibulum dictum blandit. Donec lobortis, purus a cursus faucibus, enim nisl fermentum odio, sed sagittis odio quam quis elit. Sed eget varius augue. Quisque a erat dolor, sit amet porttitor eros. Curabitur libero orci, dignissim vel egestas ut, laoreet sit amet augue. Curabitur porta consectetur felis. Etiam sit amet enim dolor, quis lacinia libero. Nunc vel vulputate turpis. Nulla elit nunc, dignissim sed hendrerit vitae, laoreet et urna. Donec massa est, porta eget lobortis sed, dictum vel arcu. Curabitur nec sem neque. + +Cum sociis natoque penatibus et magnis dis parturient montes, nascetur ridiculus mus. In ipsum risus, blandit ac porta non, imperdiet ac erat. Sed libero nisi, gravida quis dignissim vel, mattis quis sem. Ut pretium vulputate augue, a varius mi vehicula at. Ut cursus interdum lobortis. Duis ac sagittis lacus. Suspendisse pulvinar feugiat mi id vestibulum. Integer aliquet augue vitae augue tincidunt pharetra. Duis interdum nunc pellentesque nisl malesuada volutpat. Nam molestie pulvinar felis, quis volutpat urna commodo in. Donec sed adipiscing risus. Mauris nec orci ac eros lacinia euismod sed sed dui. Mauris vel est eget mi bibendum venenatis nec id enim. Lorem ipsum dolor sit amet, consectetur adipiscing elit. Maecenas ac orci varius mi aliquam sodales. Cras dapibus lorem et erat tincidunt non consectetur risus commodo. Aenean tincidunt varius orci eu placerat. Sed in euismod justo. + +Pellentesque auctor porta magna, vitae volutpat est pharetra id. Phasellus at mi nibh, vitae eleifend mi. Sed egestas orci lacus. Mauris suscipit nunc non diam mattis rutrum. Etiam pretium, mi et ultricies molestie, ante nibh posuere dolor, a fermentum diam massa eget purus. Aliquam erat volutpat. Nam accumsan dapibus quam, vitae dictum est bibendum ut. Sed at vehicula mi. Phasellus vitae ipsum a quam cursus euismod sit amet et turpis. Nam ultricies molestie massa, a consectetur ipsum aliquet sit amet. Pellentesque non orci mauris. Suspendisse congue venenatis est convallis laoreet. Aenean nulla est, bibendum id adipiscing quis, fermentum quis nisi. Nam lectus ante, sodales sodales ultrices a, vehicula ac ligula. Phasellus feugiat tempor lectus, id interdum turpis mollis eu. Suspendisse potenti. Sed euismod tempus ipsum, et iaculis felis consequat sed. Mauris bibendum, eros a semper pharetra, nunc urna commodo lacus, quis placerat dui urna semper libero. Mauris turpis metus, mattis id dignissim eget, sollicitudin nec lacus. + +Donec massa dui, laoreet dignissim interdum sit amet, semper vel ligula. Maecenas ut eros est, quis hendrerit purus. In sit amet mattis quam. Curabitur sit amet turpis ac ipsum gravida pulvinar sit amet ut libero. Pellentesque habitant morbi tristique senectus et netus et malesuada fames ac turpis egestas. Vestibulum bibendum massa eu nisi fermentum varius. Mauris sollicitudin ultrices nunc, eget facilisis est imperdiet sit amet. Nam elementum magna eget nisi commodo tincidunt. Aliquam erat volutpat. Curabitur in mauris nunc, at eleifend lectus. Integer tincidunt vestibulum lectus, ut porttitor magna dapibus a. Vivamus erat massa, pretium sed tincidunt ac, tincidunt hendrerit ligula. Praesent purus eros, euismod at commodo eu, bibendum eu turpis. + +Sed tempor ultrices tortor, et imperdiet est porttitor a. Vestibulum sodales mauris sed urna pellentesque eleifend. Ut euismod tristique nulla eu fermentum. Ut eu dui non purus varius mollis in vel enim. Maecenas ut congue nulla. Suspendisse ultrices sollicitudin molestie. Aliquam vel pulvinar metus. Nulla varius adipiscing metus, ac commodo ante dapibus ac. Phasellus sit amet ligula sed elit scelerisque molestie sit amet ac quam. Class aptent taciti sociosqu ad litora torquent per conubia nostra, per inceptos himenaeos. Pellentesque sollicitudin libero a quam rutrum egestas ac quis arcu. Etiam mattis massa vel erat mattis ut elementum diam cursus. Fusce bibendum lorem in erat auctor posuere. Ut non mi sed neque sodales vulputate. Donec lacinia, lacus nec hendrerit luctus, dolor nisi dignissim turpis, at rhoncus dui nisi nec elit. Integer laoreet, justo ut pellentesque iaculis, diam turpis scelerisque quam, sit amet semper purus lacus at erat. Sed sollicitudin consectetur eros at ultricies. + +Nam in dolor massa. Vivamus semper, quam sed bibendum pellentesque, lectus purus auctor dui, eget mollis tellus urna luctus nisi. Duis felis tellus, dapibus sed sollicitudin commodo, ornare id metus. Aliquam rhoncus pulvinar elit sit amet fermentum. Curabitur ut ligula augue, nec rhoncus orci. Proin ipsum elit, tristique semper rhoncus sit amet, ultrices vel orci. Integer mattis hendrerit blandit. Curabitur tempor quam eget nunc rutrum nec porta elit elementum. Morbi at accumsan libero. Etiam vestibulum facilisis augue vitae feugiat. Vivamus in quam arcu, vel ornare purus. Pellentesque non augue sit amet metus imperdiet accumsan. Suspendisse condimentum vulputate congue. Vestibulum ante ipsum primis in faucibus orci luctus et ultrices posuere cubilia Curae; Donec consequat enim ac est iaculis dictum. Vivamus rhoncus, urna sit amet tempor ornare, nulla sem eleifend mi, eu pretium justo sapien a nulla. Nulla facilisi. Class aptent taciti sociosqu ad litora torquent per conubia nostra, per inceptos himenaeos. Aenean sed mattis turpis. + +Nulla in magna scelerisque sem imperdiet tempus. Aenean adipiscing pretium sem, eget vulputate turpis pretium vitae. Etiam id enim a mauris faucibus facilisis faucibus vel enim. Phasellus blandit mi nec nibh rhoncus nec sollicitudin mi semper. Maecenas euismod dui sit amet dolor dictum dignissim. Mauris ac quam urna, quis posuere lacus. Sed velit elit, dapibus hendrerit sagittis at, pulvinar ac velit. Quisque in nulla vel massa posuere feugiat sed quis enim. Donec erat eros, adipiscing at fringilla sed, ornare id nisl. Duis eleifend consectetur tincidunt. Donec enim augue, mollis sed commodo mattis, luctus ac libero. Vestibulum erat ante, lacinia ac porttitor quis, vulputate et ligula. Nunc nisl orci, eleifend et laoreet eu, egestas et est. Nulla nulla purus, euismod nec porttitor quis, volutpat id diam. Nunc ut nisl eget orci venenatis mattis. In eget nisi nibh. Integer erat mauris, interdum nec mattis in, pulvinar vitae orci. Duis dictum tortor in elit aliquet commodo. Vestibulum venenatis auctor faucibus. Nulla adipiscing nisi eu lectus ornare ultrices. + +Curabitur placerat ante a odio dapibus placerat. Praesent ante quam, rutrum quis dignissim vulputate, dignissim vitae elit. Curabitur et nibh ante. Sed luctus bibendum pulvinar. Ut vel justo eros. Maecenas faucibus ornare consequat. Mauris non interdum elit. Mauris tortor magna, tempor quis rutrum ac, congue ut sem. Cum sociis natoque penatibus et magnis dis parturient montes, nascetur ridiculus mus. Sed semper interdum quam eu semper. Vestibulum ante ipsum primis in faucibus orci luctus et ultrices posuere cubilia Curae; Mauris enim velit, mattis at dictum eget, ornare vel erat. Quisque non tincidunt lectus. Vestibulum auctor scelerisque erat eget adipiscing. Mauris ac metus purus, sit amet dignissim felis. + +Curabitur vitae quam sagittis massa aliquet facilisis id tempor justo. Aenean vulputate libero nec odio porta in rhoncus massa interdum. Maecenas consectetur suscipit consectetur. Proin a mauris sit amet ante sollicitudin auctor id ac libero. Vivamus hendrerit porta augue, ac pretium nibh cursus at. Aliquam varius nulla porta quam pellentesque scelerisque eget a felis. Maecenas elit quam, tempor vel dignissim nec, aliquam ac justo. Curabitur scelerisque cursus orci, sit amet scelerisque dolor consectetur vel. Integer tellus tortor, laoreet laoreet consequat id, vehicula nec neque. Sed sit amet ante sed magna faucibus luctus et vel nisi. + +Curabitur placerat viverra urna et auctor. Proin ac lacus urna, vitae sagittis erat. In ut tellus ipsum, rutrum auctor orci. Sed dolor nibh, laoreet egestas egestas non, eleifend eu lectus. Aenean lorem leo, rhoncus sit amet fermentum in, porta vel leo. Pellentesque habitant morbi tristique senectus et netus et malesuada fames ac turpis egestas. Praesent lorem orci, congue nec consectetur eu, ullamcorper non nulla. Duis sed augue libero. Suspendisse potenti. Nunc id neque massa. + +Etiam odio magna, congue ut tristique non, dignissim nec est. Sed id purus velit. Vivamus dui dui, rutrum sit amet imperdiet non, pharetra cursus ante. Curabitur aliquet dapibus massa, non molestie orci aliquet tincidunt. Aenean in varius risus. Nullam faucibus sapien odio. Integer id est erat. Nam iaculis purus a ipsum sagittis in vestibulum lectus pulvinar. Nulla ultricies nisi a nibh gravida eget vestibulum tellus auctor. Suspendisse ut dolor elementum mi iaculis dignissim eu eleifend tellus. Sed pretium mi ligula. Integer vitae sem sit amet nunc dignissim rutrum nec eleifend felis. Aenean blandit fermentum lectus quis dignissim. Cum sociis natoque penatibus et magnis dis parturient montes, nascetur ridiculus mus. Duis congue sem a est accumsan sit amet facilisis erat dapibus. Mauris id lectus ipsum. Sed velit metus, ultrices rhoncus porta non, consectetur id ligula. + +Fusce eu odio volutpat sem pellentesque laoreet. Integer a justo ante, sed elementum elit. Donec sed mattis arcu. Vivamus imperdiet sodales ante, eget tincidunt turpis imperdiet et. Donec mi ante, tincidunt nec adipiscing sit amet, sodales vel arcu. Cras eu libero arcu. Class aptent taciti sociosqu ad litora torquent per conubia nostra, per inceptos himenaeos. Aliquam ac dui justo. Nam nisi ipsum, dignissim id fermentum at, accumsan ut quam. Quisque non est quis nibh iaculis gravida nec id velit. Cras elementum tincidunt mattis. Mauris odio erat, sodales ut egestas nec, semper eget enim. Mauris quis tincidunt quam. + +Nullam vestibulum ligula imperdiet nunc tincidunt feugiat imperdiet neque sodales. Praesent lacinia sollicitudin pulvinar. Donec ipsum augue, interdum et commodo vitae, lobortis nec ipsum. Nulla ac diam sed ipsum venenatis malesuada at eu odio. Vivamus in urna sed sapien mollis convallis eget eu massa. Proin viverra dolor vitae sem porta vulputate. Donec velit leo, ullamcorper dictum rhoncus in, euismod et ante. Morbi at leo venenatis felis faucibus elementum a a elit. Integer aliquet tempor neque ac bibendum. In fermentum commodo faucibus. In hac habitasse platea dictumst. Nam pulvinar gravida metus in rhoncus. Praesent lobortis ornare libero quis faucibus. Donec a erat ligula. Praesent quis sapien sit amet urna adipiscing sagittis. + +Praesent eget libero sed massa ornare congue eget eu lorem. Nunc porta magna ut massa dignissim ultricies. Duis eu arcu quis purus consequat egestas vitae a ipsum. In nunc sapien, venenatis et commodo sollicitudin, facilisis rhoncus risus. Nullam aliquam, orci eu vestibulum sagittis, nulla risus dictum dui, non luctus diam arcu in massa. Maecenas risus lacus, adipiscing sed laoreet sed, ornare sit amet quam. Nam convallis euismod sagittis. Fusce justo mauris, laoreet lobortis gravida semper, tincidunt pellentesque nisl. Sed sit amet turpis in nisi molestie sagittis eget sit amet nulla. Donec eget semper mauris. Aenean nec odio a nibh faucibus dapibus. Donec imperdiet tortor non elit congue varius. Morbi libero enim, tincidunt at bibendum vitae, dapibus ac ante. Proin eu metus quis turpis bibendum molestie. Nulla malesuada magna quis ante mollis ultrices. Suspendisse vel nibh at risus porttitor mattis. Nulla laoreet consequat viverra. Ut scelerisque faucibus mauris sed vestibulum. In pulvinar massa in magna dapibus ullamcorper. Quisque in ante sapien, nec ullamcorper tortor. + +Etiam in ipsum urna, eu feugiat nibh. In sed eros ligula, eget interdum lorem. Cras ut malesuada purus. Suspendisse vel odio quam. Vivamus eu rutrum quam. Integer nec luctus est. Mauris aliquam est ac neque convallis placerat. Sed massa ante, sagittis a tincidunt semper, interdum eget mauris. Sed a ligula sed justo facilisis sagittis vel eu ipsum. Quisque aliquam vestibulum nisl quis commodo. + +Morbi id rutrum mi. Curabitur a est quis mauris accumsan egestas a vulputate urna. Nunc eleifend lacus non lacus tincidunt vitae commodo odio mattis. Cras accumsan blandit odio, vitae mattis est egestas eget. Integer condimentum sem in lectus euismod consectetur. Donec est lectus, posuere sit amet ornare non, ullamcorper vel dolor. Vestibulum luctus consectetur scelerisque. Duis suscipit congue mi id venenatis. Quisque eu mauris venenatis dolor condimentum gravida a a leo. Aenean et massa est. Sed arcu ligula, sagittis in luctus in, condimentum a nisl. In placerat interdum felis, eu luctus dolor rutrum sed. Nam commodo, urna a adipiscing scelerisque, turpis arcu adipiscing metus, at blandit nulla elit quis sapien. Quisque sodales tincidunt odio, quis sodales erat bibendum condimentum. Ut semper dolor in ipsum tincidunt convallis. Phasellus molestie nulla id ipsum semper ultrices. Class aptent taciti sociosqu ad litora torquent per conubia nostra, per inceptos himenaeos. Mauris aliquam semper neque at sagittis. Curabitur luctus tristique facilisis. Donec scelerisque ante non tortor fringilla eleifend non in felis. + +Maecenas nec ipsum eget odio ornare egestas non non tortor. Vestibulum elementum ultrices ipsum, nec elementum augue dapibus vitae. Fusce hendrerit erat eget libero porttitor sit amet venenatis neque mollis. Donec lorem quam, egestas sed rutrum pharetra, ultrices quis quam. Phasellus iaculis risus eget leo suscipit eu consectetur libero bibendum. Nulla euismod, est sit amet tristique tincidunt, nisi turpis sagittis justo, ornare elementum nibh turpis at ipsum. Mauris id velit risus, in lacinia libero. Integer at urna eu sapien luctus sollicitudin. Vestibulum vitae varius est. Curabitur eget quam urna, cursus egestas orci. + +Sed eu felis nisi. Nullam nisi lacus, imperdiet sed accumsan sed, pretium ac dolor. Curabitur feugiat tristique velit, id fermentum velit blandit lobortis. Phasellus ac arcu vel lacus ultricies aliquet. Morbi aliquet pulvinar convallis. Lorem ipsum dolor sit amet, consectetur adipiscing elit. Proin tincidunt commodo tortor, vitae semper velit consequat ac. Suspendisse ac sollicitudin elit. Pellentesque habitant morbi tristique senectus et netus et malesuada fames ac turpis egestas. Quisque imperdiet, mi sed ultrices ullamcorper, eros justo malesuada urna, ac dapibus turpis leo sed sem. Nulla commodo consectetur libero a scelerisque. Maecenas in tortor sem, vitae rhoncus magna. Nulla nec nisl nisl, eget iaculis felis. Phasellus placerat consectetur erat, non porta tellus egestas nec. Praesent gravida pharetra arcu. Nullam bibendum congue eleifend. + +Nam risus dolor, mollis in suscipit vel, egestas eget augue. Donec et nulla mi. Vestibulum nunc mauris, volutpat eget lacinia ut, consequat non justo. Etiam bibendum elit quis ipsum volutpat sit amet convallis erat feugiat. Class aptent taciti sociosqu ad litora torquent per conubia nostra, per inceptos himenaeos. Sed non turpis elit, in laoreet sapien. Quisque ac elit id odio luctus pharetra. Phasellus sit amet est nec orci vestibulum varius. Cras ut justo a velit accumsan scelerisque. Proin lacus odio, convallis in semper egestas, ullamcorper sit amet erat. Proin ornare mollis pharetra. Phasellus convallis, sapien a placerat scelerisque, magna ante lobortis massa, ut semper nibh turpis a nibh. + +Vestibulum risus mauris, auctor eu aliquam quis, pretium vel massa. Nunc imperdiet magna quis nisi facilisis euismod. Nunc aliquam, felis quis mollis aliquam, mi arcu commodo eros, sit amet convallis nunc magna non magna. Suspendisse accumsan tortor non metus convallis pharetra. In vitae mi sed leo ornare viverra. Donec a massa at sem euismod scelerisque id a sapien. Nam nec purus purus, quis lacinia sem. Sed laoreet erat quis tortor feugiat at mattis lacus sollicitudin. In hac habitasse platea dictumst. Vivamus tristique rhoncus eros a hendrerit. Etiam semper dapibus tortor, quis porta purus ullamcorper eget. In iaculis elit ut neque varius at consequat tellus accumsan. + +Praesent ut ipsum nec nulla consequat laoreet. Quisque viverra rutrum bibendum. Vivamus vitae bibendum augue. Class aptent taciti sociosqu ad litora torquent per conubia nostra, per inceptos himenaeos. Nulla hendrerit condimentum lacinia. Donec sed bibendum lectus. Ut venenatis tincidunt neque et fermentum. Mauris fermentum, est at molestie luctus, nunc lorem sodales dolor, ut facilisis massa risus ut sem. Vestibulum nec nisi sed lacus imperdiet ornare. Duis sed lobortis nisi. In urna ipsum, posuere fringilla adipiscing eu, euismod a purus. Proin bibendum feugiat adipiscing. Morbi neque turpis, ullamcorper at feugiat ac, condimentum ut ante. Proin eget orci mauris, nec congue dolor. + +Sed quis dolor massa, sed fermentum eros. Fusce et scelerisque tortor. Donec bibendum vestibulum neque, id tristique leo eleifend non. Ut vel lacinia orci. Etiam lacus erat, varius viverra accumsan sit amet, imperdiet at sapien. Pellentesque habitant morbi tristique senectus et netus et malesuada fames ac turpis egestas. Vestibulum erat lacus, hendrerit consectetur vulputate id, mattis eu nunc. Morbi lacinia bibendum eros, sit amet luctus nisl lobortis lobortis. Nullam sit amet nisl vel justo ornare bibendum eu quis nunc. Morbi faucibus dictum quam, sed suscipit est auctor ac. Sed egestas ultricies sem a pharetra. Phasellus sagittis ornare lorem eu aliquam. Praesent vitae lectus ut dui consectetur varius. Cum sociis natoque penatibus et magnis dis parturient montes, nascetur ridiculus mus. Nam ac orci id metus tincidunt congue vel ut mi. Nunc auctor tristique enim quis sodales. + +Aenean quis tempor libero. In sed quam purus. Nam in velit erat. Ut ullamcorper nunc ut nibh facilisis non imperdiet enim interdum. Praesent et mi nulla, quis facilisis lacus. Nulla luctus, velit vulputate egestas aliquet, arcu dolor vulputate tellus, eu auctor ipsum tortor ut lorem. In sed nulla auctor elit adipiscing laoreet. Mauris id pretium velit. Vestibulum aliquet bibendum laoreet. Duis convallis, leo vitae tincidunt fringilla, massa eros porta lorem, a convallis sem massa sit amet libero. Nam ligula leo, porta non hendrerit a, luctus pellentesque tortor. Nulla fermentum mi lacinia est pellentesque sed rhoncus nisl tristique. Curabitur venenatis neque id magna egestas eget dictum nisi volutpat. + +Ut sagittis fringilla arcu, ut condimentum metus tempor et. Duis elit neque, varius quis consectetur et, vulputate egestas odio. Curabitur molestie congue nibh, pulvinar tincidunt elit tempus ut. Quisque nec magna lacus. Quisque eu justo lacus. Maecenas tempus porttitor consequat. Ut vulputate lacinia tempus. Praesent dignissim iaculis orci ac euismod. Proin porttitor lorem auctor erat placerat quis tincidunt tellus posuere. Nam ultrices sapien ultrices urna aliquet convallis. Aenean auctor fringilla vestibulum. + +Proin eros nisl, viverra placerat eleifend a, facilisis et augue. Duis commodo tincidunt molestie. Nullam malesuada ligula eget libero tincidunt viverra. Ut euismod sem in turpis posuere rhoncus. Donec luctus, eros quis ultricies eleifend, lacus ligula porttitor magna, sit amet lobortis enim turpis non orci. Nunc odio nisi, luctus id euismod non, hendrerit quis dolor. Proin tristique sem semper massa porttitor fringilla. Curabitur a felis tellus. Donec tempus, libero at ornare commodo, risus sapien venenatis mi, sit amet fringilla diam enim at arcu. Suspendisse potenti. Phasellus auctor, lorem sed pulvinar ornare, eros nunc tincidunt dui, semper interdum lorem purus nec turpis. Sed egestas, orci non varius dapibus, nulla felis rutrum tortor, a vehicula nisi magna et magna. Donec aliquam rhoncus arcu ac volutpat. + +Quisque leo risus, egestas eu posuere eget, malesuada quis erat. Donec vel nisi quis erat vestibulum consectetur. Donec mi mi, dictum vel posuere ac, pharetra non justo. Vivamus rhoncus mollis odio, eu fermentum turpis blandit a. Pellentesque ornare consequat odio, non sodales massa sollicitudin ac. Vestibulum euismod nisi non augue commodo vitae laoreet justo tempor. Vestibulum at arcu ac elit tincidunt vehicula pretium eget magna. Nullam non eros eros. Morbi sed diam ut leo viverra gravida a sit amet sem. Duis ultricies tellus in nisi vulputate rhoncus. Praesent molestie eros et ligula sodales ut euismod arcu egestas. Cras ullamcorper dapibus erat id luctus. Maecenas pretium rutrum mauris, ac rhoncus lacus commodo eu. Duis ut diam quis neque accumsan laoreet in eu tellus. Curabitur sit amet ligula nibh. Vestibulum vitae semper leo. Sed volutpat turpis dictum justo luctus quis gravida tortor volutpat. Proin velit dolor, tempor quis iaculis eu, congue vitae nisl. Vestibulum porttitor, risus id consequat suscipit, ipsum leo luctus tellus, sed sollicitudin nulla orci eget arcu. + +Fusce et urna sed erat porttitor condimentum convallis et ipsum. Integer sagittis arcu sit amet dolor interdum eu tincidunt sapien sodales. Sed ut elementum ipsum. Aliquam erat volutpat. Fusce vel enim velit. Duis sit amet gravida quam. Sed iaculis aliquet erat sed semper. Sed in ipsum nisi. Suspendisse blandit urna ac lectus congue hendrerit. Donec sapien enim, auctor quis suscipit id, interdum a nunc. Etiam erat velit, hendrerit eget tincidunt ut, pellentesque in lectus. Integer vitae lacus eget est tempor dapibus. Duis in velit augue. In accumsan ipsum eu nibh commodo id consequat lectus condimentum. Integer volutpat condimentum posuere. + +Proin at turpis sapien, vel bibendum odio. Etiam scelerisque, nulla vel dapibus dapibus, neque nunc fringilla libero, nec malesuada elit erat eget turpis. Pellentesque habitant morbi tristique senectus et netus et malesuada fames ac turpis egestas. Donec eu mi nisi. Mauris quis dolor libero. Etiam non libero mauris. Nam posuere tortor vel dolor aliquam eu porttitor nisi convallis. Sed eu ante nec diam hendrerit aliquet. Suspendisse fermentum, augue ut lobortis viverra, turpis mi tristique felis, a facilisis est nisi vitae nisl. Nunc sit amet semper tortor. Duis et enim in nulla aliquet fermentum. Etiam ultrices facilisis justo, quis molestie enim convallis ut. Donec congue, eros quis rhoncus interdum, nisl orci porta nisl, posuere tincidunt est tellus nec magna. Suspendisse interdum, lorem nec dictum dignissim, justo dui imperdiet felis, laoreet ultricies lacus elit eu libero. Sed quis urna nec nisi condimentum tristique pulvinar id orci. Vivamus a leo nec libero hendrerit imperdiet. Sed gravida interdum urna, ac dictum odio dictum id. Vestibulum vel varius dolor. + +Nulla consequat condimentum eros nec mollis. Donec eget ornare eros. Etiam consequat accumsan aliquet. Quisque non leo nibh. Mauris convallis congue hendrerit. Aliquam nec augue at risus ornare viverra at id felis. Nullam ac turpis ut nisl semper rhoncus quis sit amet justo. Aliquam laoreet arcu vitae odio consequat condimentum. Aliquam erat volutpat. Sed consectetur ipsum nec justo tempor ullamcorper. Donec ac sapien lectus. Suspendisse ut velit eget massa dapibus tincidunt vel eget enim. Etiam quis quam vel lectus tincidunt viverra eget eget risus. + +Nulla pulvinar, odio eu hendrerit egestas, nisl nunc gravida mi, non adipiscing tortor mauris a lectus. Sed sapien mi, porttitor vel consectetur ut, viverra ut ipsum. Duis id velit vel ipsum vestibulum sodales. Nunc lorem mi, mollis nec malesuada nec, ornare faucibus nunc. Vestibulum gravida pulvinar eros quis blandit. Nulla facilisi. Curabitur consectetur condimentum justo sed faucibus. Vestibulum neque urna, tincidunt in adipiscing a, interdum a orci. Lorem ipsum dolor sit amet, consectetur adipiscing elit. Donec libero neque, fringilla quis bibendum vel, tincidunt eget metus. Integer tristique, lectus quis rhoncus iaculis, enim dui adipiscing massa, sit amet blandit risus orci eu magna. Fusce ultricies tellus quis massa tempus at laoreet turpis dapibus. Donec sit amet massa viverra purus tincidunt scelerisque. Nunc ut leo nec tellus imperdiet vulputate tincidunt sed nisi. Suspendisse potenti. Sed a nisi nunc. Ut tortor quam, vestibulum et ultrices id, mattis non lacus. + +Nullam tincidunt quam quis erat rutrum eget tempor diam vestibulum. Morbi dapibus, quam sed placerat blandit, mi enim dictum nulla, sit amet sollicitudin lectus ante eu sem. Lorem ipsum dolor sit amet, consectetur adipiscing elit. Sed in elit id nisi aliquet mollis. Cras non lorem risus. Ut libero elit, ornare id placerat ut, sodales at lectus. Nunc orci turpis, tempus vitae pellentesque id, sodales et sem. Aliquam erat volutpat. Sed sit amet tellus condimentum magna cursus consectetur non sed arcu. Vivamus in consectetur massa. Aliquam vitae nibh nec lacus volutpat sodales. Quisque est arcu, porttitor a pharetra ac, laoreet nec nibh. Nunc ullamcorper adipiscing libero a dictum. Vivamus vulputate egestas arcu non viverra. Phasellus eget libero in ipsum fringilla dapibus. Quisque vehicula rhoncus lorem vel dictum. Sed molestie lorem ac tellus ultrices a varius dui faucibus. Integer quis quam libero. Sed fringilla aliquet lacus, non porttitor erat ultricies eu. + +Fusce bibendum euismod porta. Praesent libero nunc, dapibus ac aliquam fringilla, ornare quis eros. Vivamus tincidunt arcu vitae felis varius nec facilisis elit fermentum. In quis quam eget mauris porta faucibus. Fusce nec erat eu lectus pellentesque tempus. Morbi a justo a ante pulvinar ultricies ac tincidunt turpis. Etiam malesuada ultrices nibh quis bibendum. Quisque lacus dui, mattis id lobortis sit amet, fermentum id nisl. Donec fermentum nisi ac metus consectetur semper. Duis condimentum ipsum sit amet arcu adipiscing cursus. Nulla vulputate risus vel elit adipiscing sed pretium mauris venenatis. Vestibulum tincidunt, sapien at dapibus rutrum, urna nisi sollicitudin orci, ut condimentum lectus tellus ut lacus. Sed in nisl et urna placerat vestibulum. Ut fringilla suscipit iaculis. In in eros eget neque suscipit mollis quis ut libero. Pellentesque hendrerit consectetur tellus. Nulla a purus ut dolor volutpat ultrices. + +Pellentesque at laoreet libero. Quisque pretium tempus placerat. Proin egestas rhoncus est, eu vehicula justo gravida eu. Sed sem velit, sodales tincidunt gravida vitae, rhoncus vel neque. Proin quis quam ut turpis rhoncus suscipit quis vitae tellus. Phasellus non scelerisque nisl. Vestibulum lectus odio, tristique vitae rhoncus id, dapibus vitae magna. Vestibulum aliquet magna in turpis eleifend in dapibus augue lacinia. Ut risus mi, dictum at mollis eu, feugiat a massa. Nam in velit urna. Aliquam imperdiet porta eros a suscipit. Nullam ante quam, congue ut lacinia vel, laoreet vitae felis. Mauris commodo ultricies lobortis. Donec id varius augue. Vivamus convallis, nulla eget aliquam varius, ligula quam rhoncus augue, vel rutrum diam odio in felis. Nulla facilisi. Duis pretium magna nulla, id pretium mi. + +Sed elit odio, semper non semper vel, dapibus eu metus. Ut quis nibh vel leo laoreet egestas vitae id odio. Nunc nec egestas nisl. Vivamus tristique pulvinar leo ullamcorper convallis. Praesent elementum condimentum consectetur. Etiam dui nisi, convallis vel fringilla ac, dignissim vel velit. Fusce magna quam, malesuada at vehicula quis, luctus vel tortor. Vivamus viverra consectetur velit, quis bibendum dolor hendrerit nec. Mauris pretium laoreet eleifend. Donec in ligula a enim fringilla pellentesque vitae sed magna. Integer vitae odio et arcu tempor molestie. + +In lacus quam, placerat nec accumsan ut, faucibus eget tellus. Maecenas cursus risus enim. Pellentesque quis lorem orci, id dictum velit. Cum sociis natoque penatibus et magnis dis parturient montes, nascetur ridiculus mus. Sed justo arcu, tristique vitae mollis id, dictum non enim. Proin gravida fringilla est eu elementum. Donec ac nulla sapien, et volutpat lectus. Mauris eget quam vel dolor aliquet pretium eu nec dolor. Phasellus auctor nunc ut risus aliquet eu consequat urna rutrum. Integer porta lacus vel metus hendrerit sed fermentum arcu hendrerit. Morbi nibh arcu, tristique ut hendrerit in, rhoncus eget elit. + +Morbi tincidunt lectus ut metus aliquam adipiscing. Phasellus eros purus, laoreet non rhoncus nec, aliquet sed justo. Vestibulum ante ipsum primis in faucibus orci luctus et ultrices posuere cubilia Curae; Quisque leo nunc, feugiat ut consequat in, condimentum sit amet urna. Nam et dui sed orci pellentesque feugiat. Aliquam erat volutpat. Aliquam rhoncus sollicitudin orci. Ut blandit dignissim est, a dapibus erat tincidunt vel. Fusce dignissim vehicula lorem non suscipit. Vivamus gravida accumsan est nec consectetur. Etiam congue diam non nisi ornare semper. Maecenas pretium vestibulum velit. Suspendisse at tincidunt quam. In vitae sagittis est. Duis convallis sollicitudin nunc quis posuere. Quisque et augue eget metus commodo pulvinar. Pellentesque et velit eget massa scelerisque sagittis. Aenean tortor magna, auctor sed sodales et, vestibulum sit amet leo. Vestibulum id ligula vel nisi faucibus cursus. + +Quisque hendrerit, lorem vel ultricies adipiscing, massa ligula consectetur odio, eu eleifend sem eros varius magna. Mauris metus arcu, hendrerit et fringilla sit amet, vehicula vel leo. Pellentesque eu tellus in nulla sollicitudin tempus. Sed dapibus cursus facilisis. Cras id lectus turpis, et iaculis felis. Nulla dignissim dui non sem posuere posuere. Ut id arcu sit amet quam tristique malesuada. Curabitur ut posuere urna. Vivamus aliquet pretium leo, id sollicitudin nulla tempor eget. Aliquam commodo enim lacus, quis hendrerit lacus. Praesent tortor felis, semper vel aliquet eget, aliquet a ante. Nullam ullamcorper arcu nibh, a facilisis neque. Nunc rutrum posuere sagittis. Donec eleifend aliquam vulputate. Curabitur eget dapibus ipsum. Pellentesque habitant morbi tristique senectus et netus et malesuada fames ac turpis egestas. Cras mollis laoreet nunc, ut suscipit tellus laoreet semper. + +Phasellus libero enim, malesuada ut rutrum a, sollicitudin sed elit. Ut suscipit imperdiet nibh, vel gravida mauris fringilla non. Pellentesque sagittis libero id nulla adipiscing vitae iaculis justo consequat. In hac habitasse platea dictumst. Sed venenatis cursus est, et iaculis nisi convallis vel. Etiam non elementum mi. Etiam semper faucibus orci. Nullam tincidunt, lorem commodo sodales placerat, est velit interdum nulla, ut rutrum lectus massa malesuada elit. Cum sociis natoque penatibus et magnis dis parturient montes, nascetur ridiculus mus. Mauris faucibus odio vel tellus ornare vitae lacinia libero lobortis. Class aptent taciti sociosqu ad litora torquent per conubia nostra, per inceptos himenaeos. Cras porttitor, tortor vel adipiscing ornare, nunc elit lobortis nisl, eu vehicula sapien purus id diam. Sed blandit bibendum facilisis. Pellentesque ornare auctor commodo. In et aliquet magna. Fusce molestie sem eget orci semper sollicitudin. Donec placerat tristique urna, a varius velit sagittis eget. Aliquam vitae rutrum orci. Vivamus ac lobortis dui. Integer ornare lobortis sem vel convallis. + +Praesent ornare aliquet arcu, sed lacinia dui convallis quis. Suspendisse nec arcu lectus. Suspendisse potenti. Curabitur scelerisque quam id lacus vehicula ut tristique eros viverra. Mauris et mi ac massa auctor pharetra a eget enim. Sed vel dui sem, ut pulvinar risus. Etiam ac ipsum ipsum, eu venenatis odio. Proin lacinia eleifend risus sed hendrerit. Quisque velit nunc, sodales vitae venenatis vitae, lacinia porta neque. Donec nec vestibulum massa. Duis blandit, sapien in congue pharetra, dolor felis pharetra velit, semper vulputate metus massa ac leo. Etiam dictum neque sed lectus condimentum euismod. Maecenas vel magna ultrices lorem fermentum feugiat. Proin pulvinar ornare libero, aliquet tincidunt neque laoreet vitae. Mauris adipiscing convallis massa, quis pellentesque nulla rhoncus quis. Etiam viverra condimentum commodo. Nulla feugiat molestie ipsum sed pretium. Aenean rhoncus imperdiet urna, quis fringilla justo commodo sed. Aliquam erat volutpat. Morbi sed sem nulla. + +Integer scelerisque leo eu massa porta non tincidunt velit dictum. Ut ac ligula ipsum. Phasellus vehicula gravida felis, ac commodo lacus mattis ac. Nam bibendum enim eget diam mattis pharetra. Suspendisse malesuada arcu lacus. Nulla elementum arcu a nulla aliquam eu vestibulum dui pulvinar. Duis a facilisis risus. Nam ac dui nibh, eu porttitor mauris. Integer sollicitudin egestas dui, mollis laoreet mauris molestie ac. Aliquam egestas auctor neque, vitae aliquet dolor tincidunt blandit. Suspendisse laoreet orci at augue dapibus suscipit. In hac habitasse platea dictumst. Phasellus egestas ornare sem ac tincidunt. Suspendisse condimentum sem non augue tincidunt vulputate. Mauris cursus quam vel tortor dapibus eu ultricies mauris viverra. Nulla elit dolor, placerat sit amet facilisis non, fringilla in felis. + +Proin consequat diam non quam accumsan faucibus. Sed malesuada, dui quis placerat sagittis, sapien libero molestie libero, a sodales tortor neque non elit. Nulla et sodales ante. Donec tempor, tortor ut congue pulvinar, mi elit tempus risus, a pharetra libero quam a augue. Nulla facilisi. Quisque feugiat tortor a arcu dictum tincidunt. Nulla tincidunt tincidunt tortor, ac suscipit eros bibendum pharetra. Ut dignissim sollicitudin massa, et porttitor ligula vulputate a. Integer condimentum dapibus diam in tempor. Pellentesque molestie fringilla rhoncus. Donec eget laoreet libero. Suspendisse vulputate sapien eu sapien faucibus egestas. + +Integer nec erat dui, at eleifend arcu. Cras mauris est, cursus vel euismod sed, suscipit quis lorem. Donec neque sem, laoreet suscipit scelerisque a, volutpat at lectus. Pellentesque non felis erat, sed pulvinar nisl. In congue sollicitudin metus sodales convallis. Fusce venenatis risus ut velit adipiscing vestibulum eu sed augue. Proin metus turpis, sodales at faucibus vel, fringilla sodales ligula. Sed fringilla magna sed diam lacinia adipiscing. Maecenas nibh nibh, consequat vel malesuada sed, vestibulum nec felis. Quisque tempus lobortis dui ut euismod. Nulla facilisi. Ut adipiscing purus quis purus pellentesque eu viverra nunc placerat. Nullam nec dignissim diam. Fusce non dignissim massa. Donec condimentum, orci iaculis vulputate elementum, lectus nunc luctus augue, sit amet suscipit nulla odio at massa. Aliquam eu turpis nec massa feugiat condimentum a ac lectus. Nunc lectus ligula, feugiat vel bibendum et, tempor quis mi. Curabitur molestie, urna quis fringilla consequat, ipsum erat sodales turpis, ut laoreet velit risus vitae libero. + +Aliquam erat volutpat. Suspendisse tincidunt accumsan eros in posuere. Morbi non ullamcorper augue. Lorem ipsum dolor sit amet, consectetur adipiscing elit. Donec dui felis, feugiat semper venenatis vitae, lobortis nec ipsum. In lacinia mauris in massa ornare vel congue ipsum lacinia. Maecenas rhoncus vulputate enim, ut porttitor purus gravida id. Nunc urna ligula, pulvinar eu lacinia nec, scelerisque at nibh. Nam accumsan leo est. Pellentesque congue fermentum nisl ac semper. Sed eget blandit urna. Nullam interdum, risus id hendrerit ultrices, turpis erat vestibulum turpis, quis vehicula mauris sem sit amet est. Mauris et lorem metus, id rutrum nisl. Donec blandit dapibus neque, hendrerit fringilla diam tempus sed. Integer vestibulum, felis quis pulvinar adipiscing, ipsum risus convallis lorem, ac pulvinar lacus nunc sed felis. + +Vestibulum vitae tristique orci. Lorem ipsum dolor sit amet, consectetur adipiscing elit. Aliquam erat volutpat. Maecenas non eros quis nulla adipiscing rutrum eu at mi. Suspendisse laoreet nulla vitae nunc venenatis vitae adipiscing felis pharetra. Integer viverra vehicula risus, vitae dictum massa tempor a. Sed id leo neque, nec consectetur tellus. Donec fermentum eros vitae magna vulputate ac volutpat ligula suscipit. Curabitur mi orci, molestie tristique bibendum egestas, blandit vel arcu. Sed molestie ullamcorper nisl nec dignissim. Fusce consectetur suscipit mauris at ullamcorper. In massa diam, feugiat in euismod id, tincidunt id libero. Donec adipiscing, tellus id vehicula hendrerit, justo mauris sagittis odio, eget placerat felis ante et enim. + +Curabitur posuere fermentum arcu id fringilla. Maecenas et purus ipsum. Maecenas auctor, velit a ullamcorper eleifend, arcu tellus adipiscing turpis, ac malesuada ante lorem eu massa. Aenean libero velit, mollis sed imperdiet in, fringilla eu lectus. Cras ullamcorper lobortis massa non volutpat. Nunc sapien lorem, posuere posuere mattis at, rutrum et dolor. Vivamus dignissim consequat nisi in viverra. Maecenas nec diam quis urna ultrices rutrum feugiat quis urna. Cras sed leo mauris. Vestibulum vitae odio ut nunc posuere lobortis. Ut felis eros, posuere at porttitor sit amet, tincidunt in justo. Nullam turpis magna, egestas ac sodales ut, cursus in eros. Cum sociis natoque penatibus et magnis dis parturient montes, nascetur ridiculus mus. Phasellus bibendum nulla nec augue vehicula laoreet. Morbi vitae nunc ac lorem pharetra pellentesque sit amet ut sapien. Maecenas placerat nunc ultricies felis consequat varius. Duis scelerisque ultrices dolor in commodo. Sed sagittis, enim quis pulvinar volutpat, lectus ante tempor arcu, id fringilla velit risus id nibh. Vivamus ac risus ut lorem dapibus ullamcorper. + +Vestibulum blandit lacus mattis eros cursus hendrerit. Quisque nibh arcu, condimentum ut imperdiet eget, interdum sed magna. Cras sem mauris, sagittis at dapibus sit amet, vulputate et felis. Suspendisse gravida tincidunt pellentesque. Fusce aliquet, augue eu porttitor ultricies, diam quam lacinia eros, sed consectetur diam dui ut augue. Phasellus turpis diam, hendrerit faucibus convallis et, rhoncus ac mauris. Vivamus vel turpis id arcu mattis imperdiet a nec enim. Ut ultricies mauris at sapien sollicitudin pharetra. Donec dignissim, metus ut condimentum semper, sapien elit pulvinar nisi, id placerat est orci iaculis lectus. Suspendisse quis sem a libero placerat placerat. Etiam ligula nisi, mattis vitae faucibus nec, malesuada et leo. + +Fusce mollis venenatis vehicula. Maecenas sit amet tortor mi, et dapibus leo. In ullamcorper dignissim lorem nec interdum. Sed nisl arcu, aliquet vel facilisis sed, rhoncus at quam. Nunc et posuere arcu. Nam faucibus blandit mi ac lacinia. Nullam ultrices tellus a turpis tincidunt sit amet convallis lacus posuere. Proin vitae orci vel justo tempus consequat sed mollis elit. Integer pellentesque bibendum nunc, et gravida mi auctor et. In vitae arcu eros. Class aptent taciti sociosqu ad litora torquent per conubia nostra, per inceptos himenaeos. Lorem ipsum dolor sit amet, consectetur adipiscing elit. Duis vestibulum nulla bibendum justo fermentum suscipit. Cras lobortis lobortis vulputate. Donec risus nisl, sagittis vel congue vel, adipiscing ac augue. Curabitur at diam quis nisl fermentum luctus non ut nisi. Nulla sed justo urna, non viverra ante. Suspendisse congue, sem non convallis fringilla, est nisl varius nunc, id laoreet nisl neque in elit. Fusce posuere euismod mattis. + +Curabitur a massa vitae lectus laoreet eleifend. Lorem ipsum dolor sit amet, consectetur adipiscing elit. Nunc malesuada turpis vehicula velit placerat ut venenatis eros ultricies. Sed nulla ligula, pretium vestibulum sagittis sed, ornare at sem. Fusce ultrices, nibh non rhoncus semper, massa tortor eleifend ante, ac mollis odio arcu ac velit. Aenean quis augue eget lectus sollicitudin accumsan. Curabitur non tortor eros, eu condimentum nulla. Phasellus at sapien ac nibh pretium condimentum. Nulla rhoncus eros vel lorem ultricies dignissim. Donec tempor, risus in mollis pretium, justo urna fermentum mi, id varius ipsum ipsum quis felis. Mauris mollis diam quis lacus laoreet sit amet ultrices felis hendrerit. Nam ac dui nisi. Cras vel risus turpis. + +Vivamus eleifend sapien pulvinar libero blandit ullamcorper. Morbi vitae nisl eros, sit amet porttitor erat. Donec varius velit eu tellus feugiat a tempor nunc pellentesque. Morbi sed est libero. Nulla in turpis molestie orci posuere interdum vel vel erat. Curabitur tempus eros id sem scelerisque euismod. Pellentesque varius egestas metus, id cursus massa condimentum non. Donec sagittis ultricies lacus, sit amet iaculis magna bibendum vel. Nulla cursus velit vitae neque ultricies id bibendum dui eleifend. Pellentesque porttitor rutrum interdum. Fusce nulla mi, elementum vitae sagittis id, luctus id urna. + +Proin nec ornare magna. Morbi euismod sapien dolor, sed consectetur nisl. In erat dui, tristique ut fringilla sit amet, imperdiet eu sem. Quisque tristique augue sodales nunc malesuada nec varius lectus laoreet. In hac habitasse platea dictumst. Vestibulum a dolor leo, ut interdum lectus. Etiam eu tortor augue, nec tristique metus. Maecenas gravida mauris a ligula vulputate consequat. Suspendisse potenti. Proin id quam magna. Etiam at ipsum augue. Nam tincidunt bibendum mi, ac vehicula tellus pretium eu. Vivamus consectetur risus id enim aliquam et laoreet tortor lacinia. Phasellus interdum dapibus orci eu imperdiet. Nulla egestas, ipsum non rhoncus suscipit, tellus purus porttitor elit, et tempus arcu odio ut justo. Etiam id lorem sed velit sagittis consequat. Duis diam sem, scelerisque in mollis non, tempor eu elit. Nullam molestie blandit dapibus. Nullam interdum laoreet iaculis. + +Cras sagittis luctus risus vel placerat. In in justo eget nisi pellentesque varius ut quis mi. Cras eleifend leo ultricies metus auctor accumsan. Class aptent taciti sociosqu ad litora torquent per conubia nostra, per inceptos himenaeos. Etiam vel velit orci, nec tristique metus. Aliquam viverra leo sit amet leo viverra tincidunt. Praesent pellentesque nisl vehicula leo fringilla blandit. Duis dignissim tincidunt placerat. Quisque ornare pellentesque nisi, a tempor odio laoreet sed. Ut dapibus dolor cursus arcu suscipit in facilisis nunc ornare. Suspendisse consectetur pulvinar tellus eget rutrum. Aenean sagittis egestas diam, sit amet posuere lorem euismod vitae. + +Morbi sit amet leo metus, non vehicula ligula. Mauris nec sem sit amet ipsum feugiat sodales id vitae risus. Nullam viverra nisi at erat vestibulum dictum. Morbi et nulla magna. Proin a augue neque, sit amet tristique orci. Suspendisse ornare lorem sodales augue vehicula nec varius turpis hendrerit. Praesent nulla augue, euismod ut pretium id, luctus vel mauris. Morbi eu elit eu augue scelerisque gravida. Sed porta tortor a magna mattis volutpat. Nullam vitae tellus quam, et rhoncus dolor. Nulla ultrices nunc nec mauris mattis in blandit nibh placerat. Nam velit arcu, ultrices a imperdiet eget, pulvinar vel augue. Sed at sapien magna. Nullam accumsan nulla in nulla bibendum molestie sollicitudin lorem faucibus. In nisl tortor, tincidunt ac molestie non, commodo ut dolor. Nullam non nunc enim. Mauris ultrices, dui nec scelerisque hendrerit, erat orci feugiat eros, sed elementum ligula ipsum at velit. + +Donec sit amet nisi at est aliquam euismod in eget justo. Ut justo turpis, lobortis quis accumsan sit amet, suscipit non lorem. Duis pulvinar lorem at magna porttitor tristique. Duis tortor mauris, auctor sit amet feugiat in, luctus et risus. Nunc lacinia, arcu id convallis lobortis, nibh sapien scelerisque dui, ac volutpat ante tellus nec odio. In euismod viverra nibh non fringilla. Nunc non nisl risus, at interdum nunc. Phasellus porta tempus aliquam. Cras massa tellus, aliquet a dignissim sed, posuere nec massa. Vestibulum et nisi nulla. Donec ut nisi ante. Sed ac justo eu ligula varius hendrerit a sed justo. Fusce ornare eleifend nisl, at condimentum arcu lobortis ut. Mauris neque felis, viverra ut dignissim dignissim, faucibus et lectus. Aenean laoreet tristique massa id congue. + +Mauris accumsan elit quis augue consectetur faucibus. Donec blandit, libero in tincidunt volutpat, purus est gravida eros, ut accumsan orci felis eu purus. Nam est nibh, tincidunt ut faucibus quis, consequat at est. Fusce nec diam ligula. Morbi eu ipsum purus, non semper neque. Maecenas in lacus arcu, vel imperdiet turpis. Curabitur eget nunc velit, in consequat nulla. Donec magna tortor, faucibus vitae hendrerit ac, pretium sed ipsum. Etiam pulvinar cursus enim facilisis consectetur. Maecenas pretium pellentesque nulla, nec viverra risus placerat sed. Nam rutrum justo id augue venenatis ut feugiat risus ultricies. Sed vitae risus nec velit rutrum faucibus at vel orci. Ut feugiat mi eu dui condimentum sit amet suscipit ligula imperdiet. Sed eu bibendum augue. Ut sit amet pulvinar libero. Duis luctus urna tincidunt purus porta euismod. + +Suspendisse ullamcorper mi congue lacus volutpat aliquam. Nam pharetra vestibulum enim. Aliquam erat volutpat. Ut convallis consequat neque. Donec commodo vulputate fermentum. Suspendisse potenti. Mauris at nibh ac felis blandit sagittis at sed velit. Morbi fringilla consequat eleifend. Duis lobortis, erat at vulputate posuere, odio diam sodales turpis, ut iaculis tortor leo ac risus. Proin blandit eleifend lacus ac imperdiet. Quisque consequat mollis elementum. Proin hendrerit odio ut orci tempor porta et non enim. + +Nullam luctus sagittis molestie. Proin sollicitudin rhoncus condimentum. Cum sociis natoque penatibus et magnis dis parturient montes, nascetur ridiculus mus. Pellentesque mattis, eros non ultrices sodales, libero sem iaculis dui, tempus porttitor libero nibh a eros. Donec et mauris imperdiet arcu semper luctus aliquet dictum turpis. Sed porttitor scelerisque vehicula. Sed eget metus elit, ac accumsan massa. Nam et diam quis purus rhoncus ultrices. Proin dapibus malesuada metus eu elementum. Aliquam luctus lorem non massa ornare non tincidunt quam ultricies. Vestibulum convallis diam id urna vestibulum aliquet. Nulla facilisi. Vestibulum nec egestas turpis. Cum sociis natoque penatibus et magnis dis parturient montes, nascetur ridiculus mus. Aenean purus elit, vestibulum quis vehicula vel, auctor vel odio. Mauris eu tellus nunc. + +Sed placerat, massa et adipiscing aliquam, massa neque gravida velit, vitae consectetur quam velit non augue. Vivamus et eros metus, et aliquet diam. Aliquam a dui sem. Aenean pretium lacus ut massa faucibus in iaculis sapien pellentesque. Integer odio nibh, condimentum et condimentum vitae, ullamcorper sit amet odio. Fusce vel velit ut diam imperdiet interdum eget at nibh. Suspendisse potenti. Proin vestibulum, ante nec scelerisque volutpat, sapien purus porta ante, at gravida arcu urna consequat dolor. Praesent lorem magna, fringilla quis faucibus id, ultrices sollicitudin risus. Etiam leo lectus, viverra eu laoreet in, sollicitudin eget felis. + +Vestibulum tincidunt enim ac diam commodo id placerat erat lacinia. Duis egestas ante venenatis est ullamcorper viverra. Fusce suscipit eleifend velit quis sollicitudin. Donec felis libero, ullamcorper tincidunt luctus eget, fermentum a risus. Phasellus placerat egestas dui, sit amet aliquet arcu tincidunt sit amet. Suspendisse pharetra pellentesque ante sed egestas. Ut sit amet nibh urna, quis tincidunt arcu. Fusce sed sapien in diam rutrum pretium. Duis eu congue diam. Lorem ipsum dolor sit amet, consectetur adipiscing elit. Sed placerat fermentum ligula eu pretium. Aliquam vitae orci nibh. + +Donec justo nisi, congue non ultrices eget, pharetra sit amet nunc. Etiam ullamcorper massa vel mauris semper posuere viverra nisi aliquet. Nullam mi tortor, feugiat sed viverra eu, congue id mi. Vestibulum nec enim sit amet libero dapibus hendrerit eget ac diam. Sed at massa nisl, a placerat tellus. Donec hendrerit tempus scelerisque. Nulla facilisi. Vivamus nec ipsum nisl, ut tempor mi. Integer ornare augue et orci scelerisque sed condimentum lectus scelerisque. Sed mauris lacus, egestas a laoreet facilisis, venenatis at ipsum. Aenean vel ante sed tortor sodales faucibus. Curabitur quis magna quis quam ultrices luctus vitae ac neque. Vivamus sed tortor et purus adipiscing consectetur hendrerit non eros. Vivamus et tristique erat. Maecenas eu quam nibh, sit amet fermentum ante. Fusce adipiscing congue nulla sodales condimentum. Nulla viverra dapibus enim vel rutrum. Mauris sodales varius metus sed gravida. + +Suspendisse potenti. Aliquam erat volutpat. Integer et diam purus, et semper erat. Proin ornare, lectus ac congue tincidunt, erat sapien ultrices erat, ac sagittis enim nulla faucibus ligula. In malesuada velit eu velit tincidunt et vestibulum nibh auctor. Integer in felis justo. Nullam in lorem lacus, eget sagittis odio. Quisque congue lorem vitae massa laoreet tempor. Quisque congue magna quis eros cursus vel luctus tellus gravida. Vivamus risus nibh, cursus pulvinar porttitor in, accumsan id orci. Donec hendrerit velit vel sem tristique porta. Vestibulum libero elit, aliquam et blandit nec, convallis id sem. + +Cras et odio urna. Sed ut semper metus. In hac habitasse platea dictumst. In hac habitasse platea dictumst. In nec augue eget sapien lacinia porta. Phasellus odio neque, tempus nec commodo at, vehicula ut lacus. Nullam accumsan ultricies placerat. Mauris tincidunt, erat ultrices placerat tincidunt, libero erat tempus nunc, eu consectetur risus est vel mauris. Duis in justo at augue lobortis molestie. Donec ut sem sed orci gravida tristique in at magna. Aliquam pellentesque, justo non mattis egestas, dolor purus aliquam elit, at blandit lectus neque non enim. Fusce sed turpis nisl, quis varius ligula. Proin id enim in neque scelerisque ultrices non id magna. Aenean tortor lectus, viverra eu elementum et, fringilla non arcu. Mauris eget odio eget enim aliquet fringilla. Proin pretium, libero eget dignissim rhoncus, ante sapien accumsan diam, a accumsan nibh neque id dolor. Sed est ante, euismod nec pulvinar sed, faucibus at turpis. Integer fringilla consequat sagittis. Sed elit ipsum, laoreet id viverra in, ornare sed massa. + +Praesent eget ligula quis orci condimentum congue ut id dolor. Ut eleifend, dui eu lacinia luctus, magna tellus consectetur nunc, gravida placerat elit risus sed nibh. Integer tristique ornare nibh, eu cursus ante ultrices et. Etiam vehicula pharetra purus quis aliquam. Lorem ipsum dolor sit amet, consectetur adipiscing elit. Ut at velit mauris. Nulla in ipsum ante, vel sollicitudin quam. Integer a justo ut mi tempor vulputate quis malesuada magna. Ut lacinia ligula nec massa tincidunt at vehicula tortor facilisis. Donec malesuada volutpat adipiscing. Donec iaculis mi at est venenatis consequat. Sed risus sem, accumsan ut dapibus sit amet, laoreet sed mauris. + +Phasellus quis euismod sem. Praesent sit amet odio libero. Proin ullamcorper lectus nec arcu pulvinar vitae commodo nunc porttitor. Sed accumsan tellus et nisl dictum vel ornare neque porttitor. Morbi id egestas massa. Nunc condimentum leo vitae nibh pulvinar facilisis. Nunc elit ligula, commodo sed mollis et, ullamcorper et risus. Curabitur risus justo, viverra vel malesuada quis, convallis vitae tortor. Pellentesque habitant morbi tristique senectus et netus et malesuada fames ac turpis egestas. Suspendisse potenti. Phasellus interdum nulla a est hendrerit quis scelerisque ante convallis. Duis suscipit dolor nec lectus rhoncus vestibulum. Lorem ipsum dolor sit amet, consectetur adipiscing elit. Vestibulum pellentesque pulvinar tellus quis rhoncus. + +Integer et pellentesque lorem. Maecenas blandit laoreet justo, non interdum nulla pellentesque sed. Nullam rutrum justo et nibh varius convallis. Praesent rhoncus eleifend ante vitae venenatis. Suspendisse ullamcorper sem at tortor fermentum suscipit sit amet in libero. Aliquam erat volutpat. Suspendisse egestas accumsan tortor, quis egestas lacus vulputate et. Sed vitae turpis in purus volutpat consectetur. Duis imperdiet nisi non augue iaculis dictum. Cras ut ipsum enim, vitae convallis urna. Sed ornare, lorem ac pellentesque iaculis, urna augue egestas arcu, nec mollis dolor tortor non justo. Cras adipiscing, massa vel tristique dignissim, dolor arcu sollicitudin mauris, eget luctus tortor purus in velit. Aenean suscipit erat et dui sagittis elementum. Mauris elementum, lorem et placerat fringilla, ante enim luctus nisl, id posuere dolor urna vel metus. Proin ligula mi, elementum fermentum rhoncus eget, sagittis at eros. Integer fringilla porta varius. Nullam dignissim semper tempus. + +Curabitur leo nibh, cursus vitae ultrices id, vulputate sit amet arcu. Aenean vitae lectus turpis, et gravida odio. Praesent mattis sagittis diam, ut fermentum justo euismod et. Nam pharetra, nibh non gravida dignissim, ipsum leo malesuada augue, egestas semper ipsum est sed tortor. Sed quis malesuada elit. In hac habitasse platea dictumst. Mauris ornare aliquet purus, scelerisque gravida orci pretium sed. Nunc sed orci massa, vel molestie lectus. Quisque eget adipiscing odio. Donec vestibulum justo dui, quis malesuada urna. Donec pretium tellus eget erat condimentum ornare. Sed sem urna, rutrum nec elementum ac, ornare vel enim. Fusce pellentesque varius ultricies. Suspendisse vulputate consectetur erat, ut pellentesque felis congue sit amet. Maecenas nisi tellus, fringilla a aliquam sit amet, consectetur eget felis. Maecenas nec urna at lacus posuere hendrerit nec sit amet nisl. Proin quis ligula eu mauris volutpat hendrerit. In interdum bibendum ultricies. Cras sit amet neque at felis sodales scelerisque. Etiam et vulputate sem. + +Fusce neque nulla, pharetra sit amet varius eget, aliquam vel tortor. Curabitur a odio velit. Phasellus tempus luctus vulputate. Aenean et libero pulvinar velit aliquet vulputate. In hac habitasse platea dictumst. Etiam at massa urna, eu pulvinar elit. Vestibulum lectus risus, tempor eget cursus ac, fermentum a augue. Maecenas at neque at lacus mollis elementum quis id tellus. Pellentesque ultricies eleifend urna, at blandit augue commodo non. Praesent tincidunt mauris sit amet enim posuere ullamcorper. Pellentesque habitant morbi tristique senectus et netus et malesuada fames ac turpis egestas. In euismod dignissim ligula, et tincidunt justo hendrerit id. + +Donec mi eros, bibendum id suscipit vel, posuere non tortor. Praesent enim urna, posuere at mollis vel, commodo sit amet urna. Vestibulum quis arcu quam. Fusce risus tortor, tempor vel mollis sit amet, ornare et lacus. Quisque sit amet velit justo. Nam vitae erat et nisi ultrices vehicula in quis nunc. Curabitur tristique elit eu nibh fringilla eget adipiscing elit sollicitudin. Nulla nec leo vel enim luctus scelerisque a id augue. Quisque interdum rhoncus elit, consectetur viverra felis fringilla id. Mauris volutpat ultricies nibh quis euismod. Quisque mattis semper purus et aliquet. Praesent vestibulum pulvinar quam, a dictum enim mattis non. Aenean mollis vehicula lorem, vel cursus leo venenatis id. Vivamus dapibus bibendum diam, at ultricies massa interdum et. Nulla lobortis aliquet nisi, non vehicula elit commodo in. Donec commodo, elit vel malesuada suscipit, urna lacus feugiat mi, ac mollis metus enim nec mi. Quisque fermentum, quam quis commodo luctus, quam ligula rhoncus urna, vel molestie ipsum risus ut nulla. Donec mi ligula, pulvinar vel convallis sed, volutpat eu urna. Curabitur a gravida lorem. Quisque sagittis felis ac urna laoreet quis pretium dolor congue. + +Proin vehicula diam id odio laoreet in suscipit quam blandit. Nullam sed ante at augue iaculis dignissim et quis ligula. Integer cursus posuere egestas. Duis turpis lacus, bibendum sit amet hendrerit ut, tincidunt vestibulum ante. Maecenas faucibus velit sit amet erat hendrerit et sodales neque scelerisque. Proin sit amet risus pharetra justo tincidunt accumsan ut posuere urna. In massa odio, viverra et pretium at, lobortis non tellus. Aliquam facilisis eleifend facilisis. Maecenas a risus id ante semper ultricies nec nec quam. Curabitur elementum, arcu ut fermentum luctus, nulla lorem accumsan mauris, vitae elementum felis enim ullamcorper lectus. Nulla facilisi. Etiam at turpis sed turpis viverra posuere. Praesent porta mattis mi id feugiat. + +Fusce commodo sodales erat quis sodales. Vestibulum dolor felis, interdum semper consectetur eu, mollis eget turpis. Integer accumsan elit sit amet libero dapibus eu viverra tortor porttitor. Ut pulvinar mattis tellus, non pulvinar erat dignissim vitae. Donec sagittis tincidunt quam, in auctor est euismod eu. Aenean feugiat luctus dolor at tincidunt. Aenean a mi sed lacus porta dapibus. Pellentesque ligula est, ultricies vitae tincidunt nec, placerat quis ipsum. Morbi dignissim libero sed nunc mollis feugiat. Vivamus mauris ante, venenatis eget sodales pharetra, vestibulum a ipsum. Vestibulum suscipit tempor sem, sagittis bibendum diam vehicula tempor. Proin at imperdiet dui. Mauris et metus quis mauris tincidunt tempor. + +Nunc pulvinar scelerisque magna non lobortis. Pellentesque eget risus mauris, sed suscipit lectus. Nam pharetra magna non urna vehicula rutrum. Duis adipiscing elementum porta. Donec eleifend enim vitae justo ultrices sodales. Nunc facilisis dui nec justo pretium blandit eu in est. Sed turpis lectus, imperdiet ac convallis ut, adipiscing vel mauris. Nulla commodo sollicitudin ante, ut vestibulum leo sollicitudin vitae. Curabitur imperdiet tellus sed tellus tincidunt porta. Vestibulum ante ipsum primis in faucibus orci luctus et ultrices posuere cubilia Curae; Phasellus convallis viverra vulputate. Vestibulum ante ipsum primis in faucibus orci luctus et ultrices posuere cubilia Curae; + +Sed at mi tortor. Morbi ac sapien nisl. Etiam at sollicitudin nisl. Vestibulum vulputate varius tortor. Donec id magna dolor, non molestie sem. Nullam dolor elit, vulputate ac convallis quis, adipiscing id neque. Aliquam at justo justo, non sagittis nunc. Nullam quis sem libero, non suscipit quam. Duis nec ipsum metus. Praesent nec turpis quam, non malesuada nisi. Praesent ultricies suscipit sollicitudin. Pellentesque at massa nec nisl aliquet dapibus eget et dolor. Vestibulum ullamcorper dui sit amet erat imperdiet varius. In porttitor ultrices purus in imperdiet. Maecenas at erat fringilla tellus ultricies placerat id ut nulla. Aliquam tempus condimentum nunc, in molestie erat laoreet et. Phasellus at erat in massa luctus facilisis quis id purus. Duis dui turpis, gravida in aliquet sit amet, condimentum sed magna. Praesent non tellus in nunc aliquam dictum quis a enim. + +Maecenas sed neque velit, ut iaculis neque. Morbi leo arcu, volutpat non sodales ut, volutpat in ligula. Curabitur blandit neque ac arcu lobortis egestas. Nunc id odio ante, in sodales quam. Suspendisse condimentum est et massa bibendum malesuada. Sed fermentum tellus vel lorem dignissim fermentum. Maecenas pretium est sit amet dui congue viverra. Nullam vestibulum accumsan sagittis. Phasellus sit amet justo leo. Pellentesque ut sem lectus, elementum convallis nisl. Pellentesque dictum porttitor nisi, vel feugiat dui interdum nec. Phasellus arcu risus, convallis sit amet sodales in, imperdiet sed lacus. Mauris sed quam sit amet est venenatis sodales. Ut eleifend quam in enim bibendum eu rutrum erat placerat. Nunc faucibus massa ac augue dignissim venenatis. Class aptent taciti sociosqu ad litora torquent per conubia nostra, per inceptos himenaeos. Sed sed commodo urna. Nam eu urna tortor, eu pharetra magna. Pellentesque tortor elit, molestie sit amet rhoncus eget, aliquam a quam. Phasellus vel nunc et sem pellentesque hendrerit. + +Aliquam eu arcu ac felis volutpat scelerisque. Morbi ut dignissim nibh. Nullam convallis, odio a aliquet dignissim, purus leo elementum augue, vitae tristique neque dolor eu nulla. Vivamus sit amet massa a augue lacinia vehicula et vel dolor. Etiam sapien sem, consequat vel vehicula id, pellentesque at augue. Donec est neque, consequat ac convallis in, suscipit sed tortor. Maecenas imperdiet, dolor sit amet congue congue, metus urna suscipit libero, ut congue nisl sapien facilisis est. Nunc eget orci odio, ut aliquam dolor. Fusce nec leo eu enim sollicitudin pharetra in nec sapien. Cras id nisi vitae ipsum semper vehicula. Nunc eu magna ac felis vehicula eleifend vel non felis. + +Vestibulum mattis dapibus mauris varius pretium. Nulla facilisi. Morbi quis euismod turpis. Nunc dignissim molestie consectetur. Quisque a mattis ipsum. Ut viverra leo sed odio faucibus sodales. Sed placerat luctus mattis. Aenean auctor iaculis placerat. Pellentesque lorem dui, pharetra id faucibus eget, iaculis egestas diam. Sed a metus tellus, eu aliquam dolor. Pellentesque eget nunc urna. Ut placerat erat in velit ornare luctus. + +Proin pharetra enim non lectus fringilla eu varius diam fermentum. Etiam tellus quam, sagittis a pellentesque in, tincidunt non ipsum. Vivamus id faucibus metus. Aliquam sodales venenatis massa nec lacinia. Pellentesque a urna a quam accumsan sollicitudin. Donec feugiat ante a urna aliquam ut laoreet neque molestie. Sed metus erat, hendrerit ornare tempus ut, aliquet eget neque. Morbi rutrum, lectus sit amet dictum luctus, ante tellus molestie nunc, non interdum orci velit a lorem. Suspendisse scelerisque augue eu velit placerat ac iaculis est mattis. Mauris lorem quam, molestie vel tempus eget, tincidunt et est. Etiam sit amet risus ac tellus ultrices porta sit amet a nulla. Class aptent taciti sociosqu ad litora torquent per conubia nostra, per inceptos himenaeos. Vestibulum ante ipsum primis in faucibus orci luctus et ultrices posuere cubilia Curae; + +Fusce orci leo, tempor sed fermentum ut, rhoncus et erat. Integer a vulputate diam. Pellentesque luctus ornare varius. Quisque ornare tempus lacus quis porta. Integer consequat vestibulum eleifend. Nulla id eros eget odio eleifend vehicula. Duis ultricies ante eget massa vestibulum suscipit. Nunc et dui mi. Aliquam sit amet nunc neque, ut iaculis lorem. Nunc ornare lacinia mauris sed semper. Donec venenatis mollis urna at posuere. Etiam vestibulum dignissim magna nec hendrerit. Nullam interdum suscipit eros, ac sollicitudin mi semper in. Etiam eget feugiat augue. Etiam id imperdiet enim. Proin sed libero id quam dapibus sollicitudin. Cras suscipit dapibus nisi, quis sagittis dui consectetur vitae. Aenean lobortis congue sapien a pulvinar. + +Lorem ipsum dolor sit amet, consectetur adipiscing elit. Maecenas a diam in nulla porta hendrerit. Suspendisse massa ligula, tristique eu molestie quis, congue ut neque. Nullam vitae libero eget justo feugiat gravida ullamcorper at quam. Aenean eget interdum risus. Aliquam erat volutpat. Morbi odio purus, pharetra at cursus eget, tristique sit amet est. Pellentesque et turpis nisi, vitae vulputate dolor. Quisque odio nunc, condimentum ut mollis eget, laoreet pretium metus. Morbi vel est a nulla ultricies laoreet. Morbi ac ultrices eros. Fusce et pharetra leo. Pellentesque volutpat urna orci, sit amet scelerisque urna. Etiam vel orci mauris. Etiam sit amet lectus id massa elementum accumsan. Ut tincidunt ultricies lorem lacinia tempor. + +Mauris placerat massa at arcu ultricies sit amet malesuada urna sollicitudin. Pellentesque eleifend rhoncus ullamcorper. Fusce malesuada tincidunt lorem vel ullamcorper. Fusce non quam sapien. In hac habitasse platea dictumst. Praesent facilisis feugiat tempus. Quisque dictum placerat odio, vitae tincidunt lorem tincidunt in. Nam molestie, nisl id tempor auctor, erat nunc gravida nisi, nec vulputate tellus turpis tincidunt mi. Maecenas pretium porttitor lectus, vitae volutpat massa rutrum quis. Mauris ac sapien a arcu interdum condimentum ut quis urna. Mauris ligula neque, malesuada non rutrum et, condimentum ac velit. Sed condimentum neque at eros placerat placerat. Sed porttitor nibh non ipsum vehicula auctor commodo velit lobortis. Aliquam auctor elementum elementum. + +Nam aliquam pretium purus vel auctor. Mauris et arcu vel libero adipiscing dictum fermentum sed metus. Mauris dictum elit sed neque pharetra ac facilisis ante volutpat. Ut ut aliquam ligula. Duis vitae tortor nibh. Cum sociis natoque penatibus et magnis dis parturient montes, nascetur ridiculus mus. Morbi varius pulvinar purus id vehicula. Proin sit amet libero at leo varius egestas. Vestibulum posuere porttitor felis, nec lacinia orci rhoncus non. Suspendisse potenti. Maecenas ut tempor felis. Cras at ipsum vitae tellus luctus aliquet. Nulla mauris erat, feugiat et condimentum id, adipiscing sed tellus. Nunc condimentum luctus auctor. In hac habitasse platea dictumst. Cras libero ante, commodo at adipiscing ut, consectetur ut metus. Maecenas eros augue, cursus cursus porta vitae, ullamcorper egestas tortor. Nullam ante felis, viverra in convallis quis, gravida sit amet velit. + +Duis consectetur sagittis enim ut dignissim. Integer ut augue at odio vehicula tincidunt. Nam sapien tortor, euismod et suscipit eu, euismod in tellus. Nam ornare orci ac nulla consequat quis semper risus aliquam. Nunc tristique turpis et lacus venenatis a fermentum odio placerat. Morbi condimentum, enim ac tristique rutrum, sem nisi rhoncus orci, id mollis purus justo ut dui. Nulla facilisi. Suspendisse consectetur odio rhoncus ante porttitor ac eleifend metus suscipit. In porttitor tempus massa quis dictum. Integer in orci nibh. Duis nec risus eu nunc sagittis mattis at vitae nunc. Donec sed mi sed ante fermentum posuere nec a est. Quisque vel massa quam. Pellentesque feugiat massa venenatis risus bibendum sit amet dapibus lectus gravida. Mauris nunc lorem, interdum sit amet pulvinar vitae, euismod id mi. Suspendisse turpis elit, lobortis ac fringilla at, aliquet eget libero. Quisque eleifend ullamcorper pharetra. Fusce vitae eros tortor, sed pulvinar neque. Praesent pretium, felis quis adipiscing laoreet, sapien turpis molestie erat, malesuada pretium urna purus id ante. Aliquam ac massa sit amet sapien scelerisque convallis. + +Quisque eget libero leo. In nec diam vitae metus varius tempus vitae non purus. Phasellus porttitor, lectus vel aliquam tincidunt, nisl odio volutpat diam, nec ultrices elit quam eget lectus. Sed mollis purus at ipsum porta tempus. Sed rhoncus nisi vel magna rhoncus vitae tristique massa tempor. Etiam metus ligula, hendrerit eu accumsan vitae, euismod ac mi. Suspendisse dui turpis, congue ut fringilla et, laoreet eu enim. Pellentesque habitant morbi tristique senectus et netus et malesuada fames ac turpis egestas. Nunc aliquet lorem quis dolor pharetra tempor. Integer molestie varius laoreet. Curabitur ultrices nibh sit amet elit condimentum sit amet sagittis elit venenatis. Aliquam magna nunc, suscipit sed aliquam in, fringilla vel libero. Nunc eget elit risus. Suspendisse imperdiet, magna vel pulvinar sodales, metus velit accumsan mi, sed venenatis erat dolor eget turpis. Proin lacinia tincidunt semper. Fusce vestibulum sodales massa, a dapibus libero lobortis a. Pellentesque augue mauris, posuere sed faucibus eget, molestie at ante. Proin orci nunc, auctor vel auctor vitae, ultricies sit amet lectus. Integer at nunc eget diam tincidunt suscipit vitae et libero. Donec ac quam tortor, in vestibulum leo. + +Praesent laoreet pharetra libero, quis cursus erat tincidunt ac. Vivamus euismod odio vel erat placerat sed vehicula eros rutrum. Sed fermentum, lectus feugiat feugiat dictum, quam sapien commodo tellus, vel ornare urna felis interdum est. Integer condimentum lectus eu nulla lacinia ut porta turpis tempor. Pellentesque quis semper justo. Duis malesuada faucibus condimentum. Vestibulum ante ipsum primis in faucibus orci luctus et ultrices posuere cubilia Curae; Proin condimentum est quis urna pulvinar vehicula. Nam convallis enim non nibh elementum blandit. Curabitur dui urna, aliquam sed posuere eget, porttitor et tortor. Nam vitae velit dignissim quam porttitor congue sed quis massa. Cras sed diam vestibulum nisl pretium rutrum vel at ipsum. In eget euismod sem. Pellentesque vitae sem et augue vehicula pretium sit amet et quam. Proin enim nunc, malesuada vel lobortis non, viverra non leo. Donec eu convallis nibh. Fusce sodales orci nec felis vulputate interdum at in sem. Nulla facilisi. + +Nunc posuere orci sed diam fringilla ullamcorper. Vivamus laoreet condimentum purus sit amet consequat. Donec at tristique ipsum. Donec tincidunt, nisi sit amet commodo sagittis, velit diam eleifend nulla, sed faucibus enim arcu eget nisi. Quisque condimentum laoreet ante vel posuere. Aliquam sit amet massa quis orci placerat posuere ut at velit. Ut eu commodo nisi. Pellentesque ornare quam et lorem facilisis nec venenatis ligula dictum. Aliquam vel arcu diam. Nullam ut elit nec lorem eleifend tincidunt vel sed orci. In vulputate semper felis, id tincidunt neque mollis a. Quisque eu nisi non justo vehicula pellentesque. Maecenas nec sem nibh, dictum sagittis nibh. Cum sociis natoque penatibus et magnis dis parturient montes, nascetur ridiculus mus. In diam purus, commodo eget hendrerit eget, aliquam a sapien. Sed et justo eros. Etiam eget massa urna, non gravida enim. Cras ac ornare ligula. + +Suspendisse potenti. Sed non suscipit arcu. Mauris augue elit, porttitor non hendrerit id, egestas a eros. Nunc id orci magna. Fusce massa urna, gravida et porttitor ac, posuere eget nisl. Proin sed. +Here is the last line there is no return diff --git a/testing/mozbase/mozdevice/sut_tests/test-files/smalltext.txt b/testing/mozbase/mozdevice/sut_tests/test-files/smalltext.txt new file mode 100644 index 000000000..9ec831b83 --- /dev/null +++ b/testing/mozbase/mozdevice/sut_tests/test-files/smalltext.txt @@ -0,0 +1 @@ +this is a short text file diff --git a/testing/mozbase/mozdevice/sut_tests/test-files/test_script.sh b/testing/mozbase/mozdevice/sut_tests/test-files/test_script.sh new file mode 100644 index 000000000..4f56dae89 --- /dev/null +++ b/testing/mozbase/mozdevice/sut_tests/test-files/test_script.sh @@ -0,0 +1 @@ +echo $THE_ANSWER diff --git a/testing/mozbase/mozdevice/sut_tests/test_datachannel.py b/testing/mozbase/mozdevice/sut_tests/test_datachannel.py new file mode 100644 index 000000000..99b71a584 --- /dev/null +++ b/testing/mozbase/mozdevice/sut_tests/test_datachannel.py @@ -0,0 +1,53 @@ +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, You can obtain one at http://mozilla.org/MPL/2.0/. + +import re +import socket +from time import strptime + +from dmunit import DeviceManagerTestCase, heartbeat_port + + +class DataChannelTestCase(DeviceManagerTestCase): + + runs_on_test_device = False + + def runTest(self): + """This tests the heartbeat and the data channel. + """ + ip = self.dm.host + + # Let's connect + self._datasock = socket.socket(socket.AF_INET, socket.SOCK_STREAM) + # Assume 60 seconds between heartbeats + self._datasock.settimeout(float(60 * 2)) + self._datasock.connect((ip, heartbeat_port)) + self._connected = True + + # Let's listen + numbeats = 0 + capturedHeader = False + while numbeats < 3: + data = self._datasock.recv(1024) + print data + self.assertNotEqual(len(data), 0) + + # Check for the header + if not capturedHeader: + m = re.match(r"(.*?) trace output", data) + self.assertNotEqual(m, None, + 'trace output line does not match. The line: ' + str(data)) + capturedHeader = True + + # Check for standard heartbeat messsage + m = re.match(r"(.*?) Thump thump - (.*)", data) + if m is None: + # This isn't an error, it usually means we've obtained some + # unexpected data from the device + continue + + # Ensure it matches our format + mHeartbeatTime = m.group(1) + mHeartbeatTime = strptime(mHeartbeatTime, "%Y%m%d-%H:%M:%S") + numbeats = numbeats + 1 diff --git a/testing/mozbase/mozdevice/sut_tests/test_exec.py b/testing/mozbase/mozdevice/sut_tests/test_exec.py new file mode 100644 index 000000000..e262bd1ad --- /dev/null +++ b/testing/mozbase/mozdevice/sut_tests/test_exec.py @@ -0,0 +1,24 @@ +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, You can obtain one at http://mozilla.org/MPL/2.0/. + +import posixpath +from StringIO import StringIO + +from dmunit import DeviceManagerTestCase + + +class ExecTestCase(DeviceManagerTestCase): + + def runTest(self): + """Simple exec test, does not use env vars.""" + out = StringIO() + filename = posixpath.join(self.dm.deviceRoot, 'test_exec_file') + # Make sure the file was not already there + self.dm.removeFile(filename) + self.dm.shell(['dd', 'if=/dev/zero', 'of=%s' % filename, 'bs=1024', + 'count=1'], out) + # Check that the file has been created + self.assertTrue(self.dm.fileExists(filename)) + # Clean up + self.dm.removeFile(filename) diff --git a/testing/mozbase/mozdevice/sut_tests/test_exec_env.py b/testing/mozbase/mozdevice/sut_tests/test_exec_env.py new file mode 100644 index 000000000..bf7029e7d --- /dev/null +++ b/testing/mozbase/mozdevice/sut_tests/test_exec_env.py @@ -0,0 +1,32 @@ +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, You can obtain one at http://mozilla.org/MPL/2.0/. + +import os +import posixpath +from StringIO import StringIO + +from dmunit import DeviceManagerTestCase + + +class ExecEnvTestCase(DeviceManagerTestCase): + + def runTest(self): + """Exec test with env vars.""" + # Push the file + localfile = os.path.join('test-files', 'test_script.sh') + remotefile = posixpath.join(self.dm.deviceRoot, 'test_script.sh') + self.dm.pushFile(localfile, remotefile) + + # Run the cmd + out = StringIO() + self.dm.shell(['sh', remotefile], out, env={'THE_ANSWER': 42}) + + # Rewind the output file + out.seek(0) + # Make sure first line is 42 + line = out.readline() + self.assertTrue(int(line) == 42) + + # Clean up + self.dm.removeFile(remotefile) diff --git a/testing/mozbase/mozdevice/sut_tests/test_fileExists.py b/testing/mozbase/mozdevice/sut_tests/test_fileExists.py new file mode 100644 index 000000000..27822bc95 --- /dev/null +++ b/testing/mozbase/mozdevice/sut_tests/test_fileExists.py @@ -0,0 +1,37 @@ +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, You can obtain one at http://mozilla.org/MPL/2.0/. + +import tempfile +import posixpath + +from dmunit import DeviceManagerTestCase + + +class FileExistsTestCase(DeviceManagerTestCase): + """This tests the "fileExists" command. + """ + + def testOnRoot(self): + self.assertTrue(self.dm.fileExists('/')) + + def testOnNonexistent(self): + self.assertFalse(self.dm.fileExists('/doesNotExist')) + + def testOnRegularFile(self): + remote_path = posixpath.join(self.dm.deviceRoot, 'testFile') + self.assertFalse(self.dm.fileExists(remote_path)) + with tempfile.NamedTemporaryFile() as f: + self.dm.pushFile(f.name, remote_path) + self.assertTrue(self.dm.fileExists(remote_path)) + self.dm.removeFile(remote_path) + + def testOnDirectory(self): + remote_path = posixpath.join(self.dm.deviceRoot, 'testDir') + remote_path_file = posixpath.join(remote_path, 'testFile') + self.assertFalse(self.dm.fileExists(remote_path)) + with tempfile.NamedTemporaryFile() as f: + self.dm.pushFile(f.name, remote_path_file) + self.assertTrue(self.dm.fileExists(remote_path)) + self.dm.removeFile(remote_path_file) + self.dm.removeDir(remote_path) diff --git a/testing/mozbase/mozdevice/sut_tests/test_getdir.py b/testing/mozbase/mozdevice/sut_tests/test_getdir.py new file mode 100644 index 000000000..00ea8c9ae --- /dev/null +++ b/testing/mozbase/mozdevice/sut_tests/test_getdir.py @@ -0,0 +1,51 @@ +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, You can obtain one at http://mozilla.org/MPL/2.0/. + +import os +import posixpath +import shutil +import tempfile + +from mozdevice.devicemanager import DMError +from dmunit import DeviceManagerTestCase + + +class GetDirectoryTestCase(DeviceManagerTestCase): + + def _setUp(self): + self.localsrcdir = tempfile.mkdtemp() + os.makedirs(os.path.join(self.localsrcdir, 'push1', 'sub.1', 'sub.2')) + path = os.path.join(self.localsrcdir, + 'push1', 'sub.1', 'sub.2', 'testfile') + file(path, 'w').close() + os.makedirs(os.path.join(self.localsrcdir, 'push1', 'emptysub')) + self.localdestdir = tempfile.mkdtemp() + self.expected_filelist = ['emptysub', 'sub.1'] + + def tearDown(self): + shutil.rmtree(self.localsrcdir) + shutil.rmtree(self.localdestdir) + + def runTest(self): + """This tests the getDirectory() function. + """ + testroot = posixpath.join(self.dm.deviceRoot, 'infratest') + self.dm.removeDir(testroot) + self.dm.mkDir(testroot) + self.dm.pushDir( + os.path.join(self.localsrcdir, 'push1'), + posixpath.join(testroot, 'push1')) + # pushDir doesn't copy over empty directories, but we want to make sure + # that they are retrieved correctly. + self.dm.mkDir(posixpath.join(testroot, 'push1', 'emptysub')) + self.dm.getDirectory(posixpath.join(testroot, 'push1'), + os.path.join(self.localdestdir, 'push1')) + self.assertTrue(os.path.exists( + os.path.join(self.localdestdir, + 'push1', 'sub.1', 'sub.2', 'testfile'))) + self.assertTrue(os.path.exists( + os.path.join(self.localdestdir, 'push1', 'emptysub'))) + self.assertRaises(DMError, self.dm.getDirectory, + '/dummy', os.path.join(self.localdestdir, '/none')) + self.assertFalse(os.path.exists(self.localdestdir + '/none')) diff --git a/testing/mozbase/mozdevice/sut_tests/test_info.py b/testing/mozbase/mozdevice/sut_tests/test_info.py new file mode 100644 index 000000000..57bb4fce0 --- /dev/null +++ b/testing/mozbase/mozdevice/sut_tests/test_info.py @@ -0,0 +1,20 @@ +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, You can obtain one at http://mozilla.org/MPL/2.0/. + +from dmunit import DeviceManagerTestCase + + +class InfoTestCase(DeviceManagerTestCase): + + runs_on_test_device = False + + def runTest(self): + """This tests the "info" command. + """ + cmds = ('os', 'id', 'systime', 'uptime', 'screen', 'memory', 'power') + for c in cmds: + data = self.dm.getInfo(c) + print c + str(data) + + # No real good way to verify this. If it doesn't throw, we're ok. diff --git a/testing/mozbase/mozdevice/sut_tests/test_prompt.py b/testing/mozbase/mozdevice/sut_tests/test_prompt.py new file mode 100644 index 000000000..9980ee2ab --- /dev/null +++ b/testing/mozbase/mozdevice/sut_tests/test_prompt.py @@ -0,0 +1,30 @@ +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, You can obtain one at http://mozilla.org/MPL/2.0/. + +import re +import socket + +from dmunit import DeviceManagerTestCase + + +class PromptTestCase(DeviceManagerTestCase): + + def tearDown(self): + if self.sock: + self.sock.close() + + def runTest(self): + """This tests getting a prompt from the device. + """ + self.sock = None + ip = self.dm.host + port = self.dm.port + + promptre = re.compile('.*\$\>\x00') + data = "" + self.sock = socket.socket(socket.AF_INET, socket.SOCK_STREAM) + self.sock.connect((ip, int(port))) + data = self.sock.recv(1024) + print data + self.assertTrue(promptre.match(data)) diff --git a/testing/mozbase/mozdevice/sut_tests/test_ps.py b/testing/mozbase/mozdevice/sut_tests/test_ps.py new file mode 100644 index 000000000..b36e61179 --- /dev/null +++ b/testing/mozbase/mozdevice/sut_tests/test_ps.py @@ -0,0 +1,27 @@ +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, You can obtain one at http://mozilla.org/MPL/2.0/. + +from dmunit import DeviceManagerTestCase + + +class ProcessListTestCase(DeviceManagerTestCase): + + def runTest(self): + """This tests getting a process list from the device. + """ + proclist = self.dm.getProcessList() + + # This returns a process list of the form: + # [[, ], [, ], ...] + # on android the userID is affixed to the process array: + # [[, , ], ...] + + self.assertNotEqual(len(proclist), 0) + + for item in proclist: + self.assertIsInstance(item[0], int) + self.assertIsInstance(item[1], str) + self.assertGreater(len(item[1]), 0) + if len(item) > 2: + self.assertIsInstance(item[2], int) diff --git a/testing/mozbase/mozdevice/sut_tests/test_pull.py b/testing/mozbase/mozdevice/sut_tests/test_pull.py new file mode 100644 index 000000000..753d8ddd5 --- /dev/null +++ b/testing/mozbase/mozdevice/sut_tests/test_pull.py @@ -0,0 +1,34 @@ +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, You can obtain one at http://mozilla.org/MPL/2.0/. + +import hashlib +import os +import posixpath + +from dmunit import DeviceManagerTestCase +from mozdevice.devicemanager import DMError + + +class PullTestCase(DeviceManagerTestCase): + + def runTest(self): + """Tests the "pull" command with a binary file. + """ + orig = hashlib.md5() + new = hashlib.md5() + local_test_file = os.path.join('test-files', 'mybinary.zip') + orig.update(file(local_test_file, 'r').read()) + + testroot = self.dm.deviceRoot + remote_test_file = posixpath.join(testroot, 'mybinary.zip') + self.dm.removeFile(remote_test_file) + self.dm.pushFile(local_test_file, remote_test_file) + new.update(self.dm.pullFile(remote_test_file)) + # Use hexdigest() instead of digest() since values are printed + # if assert fails + self.assertEqual(orig.hexdigest(), new.hexdigest()) + + remote_missing_file = posixpath.join(testroot, 'doesnotexist') + self.dm.removeFile(remote_missing_file) # Just to be sure + self.assertRaises(DMError, self.dm.pullFile, remote_missing_file) diff --git a/testing/mozbase/mozdevice/sut_tests/test_push1.py b/testing/mozbase/mozdevice/sut_tests/test_push1.py new file mode 100644 index 000000000..f457d6cc5 --- /dev/null +++ b/testing/mozbase/mozdevice/sut_tests/test_push1.py @@ -0,0 +1,38 @@ +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, You can obtain one at http://mozilla.org/MPL/2.0/. + +import os +import posixpath + +from dmunit import DeviceManagerTestCase + + +class Push1TestCase(DeviceManagerTestCase): + + def runTest(self): + """This tests copying a directory structure to the device. + """ + dvroot = self.dm.deviceRoot + dvpath = posixpath.join(dvroot, 'infratest') + self.dm.removeDir(dvpath) + self.dm.mkDir(dvpath) + + p1 = os.path.join('test-files', 'push1') + # Set up local stuff + try: + os.rmdir(p1) + except: + pass + + if not os.path.exists(p1): + os.makedirs(os.path.join(p1, 'sub.1', 'sub.2')) + if not os.path.exists(os.path.join(p1, 'sub.1', 'sub.2', 'testfile')): + file(os.path.join(p1, 'sub.1', 'sub.2', 'testfile'), 'w').close() + + self.dm.pushDir(p1, posixpath.join(dvpath, 'push1')) + + self.assertTrue( + self.dm.dirExists(posixpath.join(dvpath, 'push1', 'sub.1'))) + self.assertTrue(self.dm.dirExists( + posixpath.join(dvpath, 'push1', 'sub.1', 'sub.2'))) diff --git a/testing/mozbase/mozdevice/sut_tests/test_push2.py b/testing/mozbase/mozdevice/sut_tests/test_push2.py new file mode 100644 index 000000000..5ccea509e --- /dev/null +++ b/testing/mozbase/mozdevice/sut_tests/test_push2.py @@ -0,0 +1,39 @@ +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, You can obtain one at http://mozilla.org/MPL/2.0/. + +import os +import posixpath + +from dmunit import DeviceManagerTestCase + + +class Push2TestCase(DeviceManagerTestCase): + + def runTest(self): + """This tests copying a directory structure with files to the device. + """ + testroot = posixpath.join(self.dm.deviceRoot, 'infratest') + self.dm.removeDir(testroot) + self.dm.mkDir(testroot) + path = posixpath.join(testroot, 'push2') + self.dm.pushDir(os.path.join('test-files', 'push2'), path) + + # Let's walk the tree and make sure everything is there + # though it's kind of cheesy, we'll use the validate file to compare + # hashes - we use the client side hashing when testing the cat command + # specifically, so that makes this a little less cheesy, I guess. + self.assertTrue( + self.dm.dirExists(posixpath.join(testroot, 'push2', 'sub1'))) + self.assertTrue(self.dm.validateFile( + posixpath.join(testroot, 'push2', 'sub1', 'file1.txt'), + os.path.join('test-files', 'push2', 'sub1', 'file1.txt'))) + self.assertTrue(self.dm.validateFile( + posixpath.join(testroot, 'push2', 'sub1', 'sub1.1', 'file2.txt'), + os.path.join('test-files', 'push2', 'sub1', 'sub1.1', 'file2.txt'))) + self.assertTrue(self.dm.validateFile( + posixpath.join(testroot, 'push2', 'sub2', 'file3.txt'), + os.path.join('test-files', 'push2', 'sub2', 'file3.txt'))) + self.assertTrue(self.dm.validateFile( + posixpath.join(testroot, 'push2', 'file4.bin'), + os.path.join('test-files', 'push2', 'file4.bin'))) diff --git a/testing/mozbase/mozdevice/sut_tests/test_pushbinary.py b/testing/mozbase/mozdevice/sut_tests/test_pushbinary.py new file mode 100644 index 000000000..86809dc1f --- /dev/null +++ b/testing/mozbase/mozdevice/sut_tests/test_pushbinary.py @@ -0,0 +1,19 @@ +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, You can obtain one at http://mozilla.org/MPL/2.0/. + +import os +import posixpath + +from dmunit import DeviceManagerTestCase + + +class PushBinaryTestCase(DeviceManagerTestCase): + + def runTest(self): + """This tests copying a binary file. + """ + testroot = self.dm.deviceRoot + self.dm.removeFile(posixpath.join(testroot, 'mybinary.zip')) + self.dm.pushFile(os.path.join('test-files', 'mybinary.zip'), + posixpath.join(testroot, 'mybinary.zip')) diff --git a/testing/mozbase/mozdevice/sut_tests/test_pushsmalltext.py b/testing/mozbase/mozdevice/sut_tests/test_pushsmalltext.py new file mode 100644 index 000000000..174b3b117 --- /dev/null +++ b/testing/mozbase/mozdevice/sut_tests/test_pushsmalltext.py @@ -0,0 +1,19 @@ +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, You can obtain one at http://mozilla.org/MPL/2.0/. + +import os +import posixpath + +from dmunit import DeviceManagerTestCase + + +class PushSmallTextTestCase(DeviceManagerTestCase): + + def runTest(self): + """This tests copying a small text file. + """ + testroot = self.dm.deviceRoot + self.dm.removeFile(posixpath.join(testroot, 'smalltext.txt')) + self.dm.pushFile(os.path.join('test-files', 'smalltext.txt'), + posixpath.join(testroot, 'smalltext.txt')) diff --git a/testing/mozbase/mozdevice/tests/droidsut_launch.py b/testing/mozbase/mozdevice/tests/droidsut_launch.py new file mode 100644 index 000000000..b9872e096 --- /dev/null +++ b/testing/mozbase/mozdevice/tests/droidsut_launch.py @@ -0,0 +1,36 @@ +from sut import MockAgent +import mozdevice +import logging +import unittest + + +class LaunchTest(unittest.TestCase): + + def test_nouserserial(self): + a = MockAgent(self, commands=[("ps", + "10029 549 com.android.launcher\n" + "10066 1198 com.twitter.android"), + ("info sutuserinfo", ""), + ("exec am start -W -n " + "org.mozilla.fennec/org.mozilla.gecko.BrowserApp -a " + "android.intent.action.VIEW", + "OK\nreturn code [0]")]) + d = mozdevice.DroidSUT("127.0.0.1", port=a.port, logLevel=logging.DEBUG) + d.launchFennec("org.mozilla.fennec") + a.wait() + + def test_userserial(self): + a = MockAgent(self, commands=[("ps", + "10029 549 com.android.launcher\n" + "10066 1198 com.twitter.android"), + ("info sutuserinfo", "User Serial:0"), + ("exec am start --user 0 -W -n " + "org.mozilla.fennec/org.mozilla.gecko.BrowserApp -a " + "android.intent.action.VIEW", + "OK\nreturn code [0]")]) + d = mozdevice.DroidSUT("127.0.0.1", port=a.port, logLevel=logging.DEBUG) + d.launchFennec("org.mozilla.fennec") + a.wait() + +if __name__ == '__main__': + unittest.main() diff --git a/testing/mozbase/mozdevice/tests/manifest.ini b/testing/mozbase/mozdevice/tests/manifest.ini new file mode 100644 index 000000000..63825c85b --- /dev/null +++ b/testing/mozbase/mozdevice/tests/manifest.ini @@ -0,0 +1,23 @@ +[DEFAULT] +skip-if = os == 'win' + +[sut_app.py] +[sut_basic.py] +[sut_chmod.py] +[sut_copytree.py] +[sut_fileExists.py] +[sut_fileMethods.py] +[sut_info.py] +[sut_ip.py] +[sut_kill.py] +[sut_list.py] +[sut_logcat.py] +[sut_mkdir.py] +[sut_movetree.py] +[sut_ps.py] +[sut_push.py] +[sut_pull.py] +[sut_remove.py] +[sut_time.py] +[sut_unpackfile.py] +[droidsut_launch.py] diff --git a/testing/mozbase/mozdevice/tests/sut.py b/testing/mozbase/mozdevice/tests/sut.py new file mode 100644 index 000000000..76a5ed313 --- /dev/null +++ b/testing/mozbase/mozdevice/tests/sut.py @@ -0,0 +1,89 @@ +#!/usr/bin/env python + +# Any copyright is dedicated to the Public Domain. +# http://creativecommons.org/publicdomain/zero/1.0/ + +import datetime +import socket +import time + +from threading import Thread + + +class MockAgent(object): + + MAX_WAIT_TIME_SECONDS = 10 + SOCKET_TIMEOUT_SECONDS = 5 + + def __init__(self, tester, start_commands=None, commands=[]): + if start_commands: + self.commands = start_commands + else: + self.commands = [("ver", "SUTAgentAndroid Version 1.14")] + self.commands = self.commands + commands + + self._sock = socket.socket(socket.AF_INET, socket.SOCK_STREAM) + self._sock.bind(("127.0.0.1", 0)) + self._sock.listen(1) + + self.tester = tester + + self.thread = Thread(target=self._serve_thread) + self.thread.start() + + self.should_stop = False + + @property + def port(self): + return self._sock.getsockname()[1] + + def _serve_thread(self): + conn = None + while self.commands: + if not conn: + conn, addr = self._sock.accept() + conn.settimeout(self.SOCKET_TIMEOUT_SECONDS) + conn.send("$>\x00") + (command, response) = self.commands.pop(0) + data = '' + timeout = datetime.datetime.now() + datetime.timedelta( + seconds=self.MAX_WAIT_TIME_SECONDS) + # The data might come in chunks, particularly if we are expecting + # multiple lines, as with push commands. + while (len(data) < len(command) and + datetime.datetime.now() < timeout): + try: + data += conn.recv(1024) + except socket.timeout: + # We handle timeouts in the main loop. + pass + self.tester.assertEqual(data.strip(), command) + # send response and prompt separately to test for bug 789496 + # FIXME: Improve the mock agent, since overloading the meaning + # of 'response' is getting confusing. + if response is None: # code for "shut down" + conn.shutdown(socket.SHUT_RDWR) + conn.close() + conn = None + elif type(response) is int: # code for "time out" + max_timeout = 15.0 + timeout = 0.0 + interval = 0.1 + while not self.should_stop and timeout < max_timeout: + time.sleep(interval) + timeout += interval + if timeout >= max_timeout: + raise Exception("Maximum timeout reached! This should not " + "happen") + return + else: + # pull is handled specially, as we just pass back the full + # command line + if "pull" in command: + conn.send(response) + else: + conn.send("%s\n" % response) + conn.send("$>\x00") + + def wait(self): + self.thread.join() diff --git a/testing/mozbase/mozdevice/tests/sut_app.py b/testing/mozbase/mozdevice/tests/sut_app.py new file mode 100644 index 000000000..0a5d996ae --- /dev/null +++ b/testing/mozbase/mozdevice/tests/sut_app.py @@ -0,0 +1,20 @@ +#!/usr/bin/env python +import mozdevice +import logging +import unittest +from sut import MockAgent + + +class TestApp(unittest.TestCase): + + def test_getAppRoot(self): + command = [("getapproot org.mozilla.firefox", + "/data/data/org.mozilla.firefox")] + + m = MockAgent(self, commands=command) + d = mozdevice.DroidSUT("127.0.0.1", port=m.port, logLevel=logging.DEBUG) + + self.assertEqual(command[0][1], d.getAppRoot('org.mozilla.firefox')) + +if __name__ == '__main__': + unittest.main() diff --git a/testing/mozbase/mozdevice/tests/sut_basic.py b/testing/mozbase/mozdevice/tests/sut_basic.py new file mode 100644 index 000000000..666d4915c --- /dev/null +++ b/testing/mozbase/mozdevice/tests/sut_basic.py @@ -0,0 +1,73 @@ +from sut import MockAgent +import mozdevice +import logging +import unittest + + +class BasicTest(unittest.TestCase): + + def test_init(self): + """Tests DeviceManager initialization.""" + a = MockAgent(self) + + mozdevice.DroidSUT("127.0.0.1", port=a.port, logLevel=logging.DEBUG) + # all testing done in device's constructor + a.wait() + + def test_init_err(self): + """Tests error handling during initialization.""" + a = MockAgent(self, start_commands=[("ver", "##AGENT-WARNING## No version")]) + self.assertRaises(mozdevice.DMError, + lambda: mozdevice.DroidSUT("127.0.0.1", + port=a.port, + logLevel=logging.DEBUG)) + a.wait() + + def test_timeout_normal(self): + """Tests DeviceManager timeout, normal case.""" + a = MockAgent(self, commands=[("isdir /mnt/sdcard/tests", "TRUE"), + ("cd /mnt/sdcard/tests", ""), + ("ls", "test.txt"), + ("rm /mnt/sdcard/tests/test.txt", + "Removed the file")]) + d = mozdevice.DroidSUT("127.0.0.1", port=a.port, logLevel=logging.DEBUG) + ret = d.removeFile('/mnt/sdcard/tests/test.txt') + self.assertEqual(ret, None) # if we didn't throw an exception, we're ok + a.wait() + + def test_timeout_timeout(self): + """Tests DeviceManager timeout, timeout case.""" + a = MockAgent(self, commands=[("isdir /mnt/sdcard/tests", "TRUE"), + ("cd /mnt/sdcard/tests", ""), + ("ls", "test.txt"), + ("rm /mnt/sdcard/tests/test.txt", 0)]) + d = mozdevice.DroidSUT("127.0.0.1", port=a.port, logLevel=logging.DEBUG) + d.default_timeout = 1 + exceptionThrown = False + try: + d.removeFile('/mnt/sdcard/tests/test.txt') + except mozdevice.DMError: + exceptionThrown = True + self.assertEqual(exceptionThrown, True) + a.should_stop = True + a.wait() + + def test_shell(self): + """Tests shell command""" + for cmd in [("exec foobar", False), ("execsu foobar", True)]: + for retcode in [1, 2]: + a = MockAgent(self, commands=[(cmd[0], + "\nreturn code [%s]" % retcode)]) + d = mozdevice.DroidSUT("127.0.0.1", port=a.port) + exceptionThrown = False + try: + d.shellCheckOutput(["foobar"], root=cmd[1]) + except mozdevice.DMError: + exceptionThrown = True + expectedException = (retcode != 0) + self.assertEqual(exceptionThrown, expectedException) + + a.wait() + +if __name__ == '__main__': + unittest.main() diff --git a/testing/mozbase/mozdevice/tests/sut_chmod.py b/testing/mozbase/mozdevice/tests/sut_chmod.py new file mode 100644 index 000000000..404330c03 --- /dev/null +++ b/testing/mozbase/mozdevice/tests/sut_chmod.py @@ -0,0 +1,22 @@ +#!/usr/bin/env python +import mozdevice +import logging +import unittest +from sut import MockAgent + + +class TestChmod(unittest.TestCase): + + def test_chmod(self): + + command = [('chmod /mnt/sdcard/test', + 'Changing permissions for /storage/emulated/legacy/Test\n' + ' \n' + 'chmod /storage/emulated/legacy/Test ok\n')] + m = MockAgent(self, commands=command) + d = mozdevice.DroidSUT('127.0.0.1', port=m.port, logLevel=logging.DEBUG) + + self.assertEqual(None, d.chmodDir('/mnt/sdcard/test')) + +if __name__ == '__main__': + unittest.main() diff --git a/testing/mozbase/mozdevice/tests/sut_copytree.py b/testing/mozbase/mozdevice/tests/sut_copytree.py new file mode 100644 index 000000000..ec22828d0 --- /dev/null +++ b/testing/mozbase/mozdevice/tests/sut_copytree.py @@ -0,0 +1,67 @@ +#!/usr/bin/env python +# -*- coding: utf-8 -*- + +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this file, +# You can obtain one at http://mozilla.org/MPL/2.0/. + +import mozdevice +import logging +import unittest +from sut import MockAgent + + +class CopyTreeTest(unittest.TestCase): + + def test_copyFile(self): + commands = [('dd if=/mnt/sdcard/tests/test.txt of=/mnt/sdcard/tests/test2.txt', ''), + ('isdir /mnt/sdcard/tests', 'TRUE'), + ('cd /mnt/sdcard/tests', ''), + ('ls', 'test.txt\ntest2.txt')] + + m = MockAgent(self, commands=commands) + d = mozdevice.DroidSUT("127.0.0.1", port=m.port, logLevel=logging.DEBUG) + + self.assertEqual(None, d.copyTree('/mnt/sdcard/tests/test.txt', + '/mnt/sdcard/tests/test2.txt')) + expected = (commands[3][1].strip()).split('\n') + self.assertEqual(expected, d.listFiles('/mnt/sdcard/tests')) + + def test_copyDir(self): + commands = [('dd if=/mnt/sdcard/tests/foo of=/mnt/sdcard/tests/bar', ''), + ('isdir /mnt/sdcard/tests', 'TRUE'), + ('cd /mnt/sdcard/tests', ''), + ('ls', 'foo\nbar')] + + m = MockAgent(self, commands=commands) + d = mozdevice.DroidSUT("127.0.0.1", port=m.port, + logLevel=logging.DEBUG) + + self.assertEqual(None, d.copyTree('/mnt/sdcard/tests/foo', + '/mnt/sdcard/tests/bar')) + expected = (commands[3][1].strip()).split('\n') + self.assertEqual(expected, d.listFiles('/mnt/sdcard/tests')) + + def test_copyNonEmptyDir(self): + commands = [('isdir /mnt/sdcard/tests/foo/bar', 'TRUE'), + ('dd if=/mnt/sdcard/tests/foo of=/mnt/sdcard/tests/foo2', ''), + ('isdir /mnt/sdcard/tests', 'TRUE'), + ('cd /mnt/sdcard/tests', ''), + ('ls', 'foo\nfoo2'), + ('isdir /mnt/sdcard/tests/foo2', 'TRUE'), + ('cd /mnt/sdcard/tests/foo2', ''), + ('ls', 'bar')] + + m = MockAgent(self, commands=commands) + d = mozdevice.DroidSUT("127.0.0.1", port=m.port, + logLevel=logging.DEBUG) + + self.assertTrue(d.dirExists('/mnt/sdcard/tests/foo/bar')) + self.assertEqual(None, d.copyTree('/mnt/sdcard/tests/foo', + '/mnt/sdcard/tests/foo2')) + expected = (commands[4][1].strip()).split('\n') + self.assertEqual(expected, d.listFiles('/mnt/sdcard/tests')) + self.assertTrue(d.fileExists('/mnt/sdcard/tests/foo2/bar')) + +if __name__ == "__main__": + unittest.main() diff --git a/testing/mozbase/mozdevice/tests/sut_fileExists.py b/testing/mozbase/mozdevice/tests/sut_fileExists.py new file mode 100644 index 000000000..702fd2de3 --- /dev/null +++ b/testing/mozbase/mozdevice/tests/sut_fileExists.py @@ -0,0 +1,29 @@ +from sut import MockAgent +import mozdevice +import unittest + + +class FileExistsTest(unittest.TestCase): + + commands = [('isdir /', 'TRUE'), + ('cd /', ''), + ('ls', 'init')] + + def test_onRoot(self): + root_commands = [('isdir /', 'TRUE')] + a = MockAgent(self, commands=root_commands) + d = mozdevice.DroidSUT("127.0.0.1", port=a.port) + self.assertTrue(d.fileExists('/')) + + def test_onNonexistent(self): + a = MockAgent(self, commands=self.commands) + d = mozdevice.DroidSUT("127.0.0.1", port=a.port) + self.assertFalse(d.fileExists('/doesNotExist')) + + def test_onRegularFile(self): + a = MockAgent(self, commands=self.commands) + d = mozdevice.DroidSUT("127.0.0.1", port=a.port) + self.assertTrue(d.fileExists('/init')) + +if __name__ == '__main__': + unittest.main() diff --git a/testing/mozbase/mozdevice/tests/sut_fileMethods.py b/testing/mozbase/mozdevice/tests/sut_fileMethods.py new file mode 100644 index 000000000..142950a81 --- /dev/null +++ b/testing/mozbase/mozdevice/tests/sut_fileMethods.py @@ -0,0 +1,72 @@ +#!/usr/bin/env python + +import hashlib +import mozdevice +import logging +import shutil +import tempfile +import unittest +from sut import MockAgent + + +class TestFileMethods(unittest.TestCase): + """ Class to test misc file methods """ + + content = "What is the answer to the life, universe and everything? 42" + h = hashlib.md5() + h.update(content) + temp_hash = h.hexdigest() + + def test_validateFile(self): + + with tempfile.NamedTemporaryFile() as f: + f.write(self.content) + f.flush() + + # Test Valid Hashes + commands_valid = [("hash /sdcard/test/file", self.temp_hash)] + + m = MockAgent(self, commands=commands_valid) + d = mozdevice.DroidSUT("127.0.0.1", port=m.port, logLevel=logging.DEBUG) + self.assertTrue(d.validateFile('/sdcard/test/file', f.name)) + + # Test invalid hashes + commands_invalid = [("hash /sdcard/test/file", "0this0hash0is0completely0invalid")] + + m = MockAgent(self, commands=commands_invalid) + d = mozdevice.DroidSUT("127.0.0.1", port=m.port, logLevel=logging.DEBUG) + self.assertFalse(d.validateFile('/sdcard/test/file', f.name)) + + def test_getFile(self): + + fname = "/mnt/sdcard/file" + commands = [("pull %s" % fname, "%s,%s\n%s" % (fname, len(self.content), self.content)), + ("hash %s" % fname, self.temp_hash)] + + with tempfile.NamedTemporaryFile() as f: + m = MockAgent(self, commands=commands) + d = mozdevice.DroidSUT("127.0.0.1", port=m.port, logLevel=logging.DEBUG) + # No error means success + self.assertEqual(None, d.getFile(fname, f.name)) + + def test_getDirectory(self): + + fname = "/mnt/sdcard/file" + commands = [("isdir /mnt/sdcard", "TRUE"), + ("isdir /mnt/sdcard", "TRUE"), + ("cd /mnt/sdcard", ""), + ("ls", "file"), + ("isdir %s" % fname, "FALSE"), + ("pull %s" % fname, "%s,%s\n%s" % (fname, len(self.content), self.content)), + ("hash %s" % fname, self.temp_hash)] + + tmpdir = tempfile.mkdtemp() + m = MockAgent(self, commands=commands) + d = mozdevice.DroidSUT("127.0.0.1", port=m.port, logLevel=logging.DEBUG) + self.assertEqual(None, d.getDirectory("/mnt/sdcard", tmpdir)) + + # Cleanup + shutil.rmtree(tmpdir) + +if __name__ == '__main__': + unittest.main() diff --git a/testing/mozbase/mozdevice/tests/sut_info.py b/testing/mozbase/mozdevice/tests/sut_info.py new file mode 100644 index 000000000..93f3d4258 --- /dev/null +++ b/testing/mozbase/mozdevice/tests/sut_info.py @@ -0,0 +1,49 @@ +#!/usr/bin/env python +import mozdevice +import logging +import re +import unittest +from sut import MockAgent + + +class TestGetInfo(unittest.TestCase): + + commands = {'os': ('info os', 'JDQ39'), + 'id': ('info id', '11:22:33:44:55:66'), + 'uptime': ('info uptime', '0 days 0 hours 7 minutes 0 seconds 0 ms'), + 'uptimemillis': ('info uptimemillis', '666'), + 'systime': ('info systime', '2013/04/2 12:42:00:007'), + 'screen': ('info screen', 'X:768 Y:1184'), + 'rotation': ('info rotation', 'ROTATION:0'), + 'memory': ('info memory', 'PA:1351032832, FREE: 878645248'), + 'process': ('info process', '1000 527 system\n' + '10091 3443 org.mozilla.firefox\n' + '10112 3137 com.mozilla.SUTAgentAndroid\n' + '10035 807 com.android.launcher'), + 'disk': ('info disk', '/data: 6084923392 total, 980922368 available\n' + '/system: 867999744 total, 332333056 available\n' + '/mnt/sdcard: 6084923392 total, 980922368 available'), + 'power': ('info power', 'Power status:\n' + ' AC power OFFLINE\n' + ' Battery charge LOW DISCHARGING\n' + ' Remaining charge: 20%\n' + ' Battery Temperature: 25.2 (c)'), + 'sutuserinfo': ('info sutuserinfo', 'User Serial:0'), + 'temperature': ('info temperature', 'Temperature: unknown') + } + + def test_getInfo(self): + + for directive in self.commands.keys(): + m = MockAgent(self, commands=[self.commands[directive]]) + d = mozdevice.DroidSUT('127.0.0.1', port=m.port, logLevel=logging.DEBUG) + + expected = re.sub(r'\ +', ' ', self.commands[directive][1]).split('\n') + # Account for slightly different return format for 'process' + if directive is 'process': + expected = [[x] for x in expected] + + self.assertEqual(d.getInfo(directive=directive)[directive], expected) + +if __name__ == '__main__': + unittest.main() diff --git a/testing/mozbase/mozdevice/tests/sut_ip.py b/testing/mozbase/mozdevice/tests/sut_ip.py new file mode 100644 index 000000000..31428a624 --- /dev/null +++ b/testing/mozbase/mozdevice/tests/sut_ip.py @@ -0,0 +1,37 @@ +#!/usr/bin/env python +import mozdevice +import logging +import unittest +from sut import MockAgent + + +class TestGetIP(unittest.TestCase): + """ class to test IP methods """ + + commands = [('exec ifconfig eth0', 'eth0: ip 192.168.0.1 ' + 'mask 255.255.255.0 flags [up broadcast running multicast]\n' + 'return code [0]'), + ('exec ifconfig wlan0', 'wlan0: ip 10.1.39.126\n' + 'mask 255.255.0.0 flags [up broadcast running multicast]\n' + 'return code [0]'), + ('exec ifconfig fake0', '##AGENT-WARNING## [ifconfig] ' + 'command with arg(s) = [fake0] is currently not implemented.') + ] + + def test_getIP_eth0(self): + m = MockAgent(self, commands=[self.commands[0]]) + d = mozdevice.DroidSUT("127.0.0.1", port=m.port, logLevel=logging.DEBUG) + self.assertEqual('192.168.0.1', d.getIP(interfaces=['eth0'])) + + def test_getIP_wlan0(self): + m = MockAgent(self, commands=[self.commands[1]]) + d = mozdevice.DroidSUT("127.0.0.1", port=m.port, logLevel=logging.DEBUG) + self.assertEqual('10.1.39.126', d.getIP(interfaces=['wlan0'])) + + def test_getIP_error(self): + m = MockAgent(self, commands=[self.commands[2]]) + d = mozdevice.DroidSUT("127.0.0.1", port=m.port, logLevel=logging.DEBUG) + self.assertRaises(mozdevice.DMError, d.getIP, interfaces=['fake0']) + +if __name__ == '__main__': + unittest.main() diff --git a/testing/mozbase/mozdevice/tests/sut_kill.py b/testing/mozbase/mozdevice/tests/sut_kill.py new file mode 100644 index 000000000..fea2f57e0 --- /dev/null +++ b/testing/mozbase/mozdevice/tests/sut_kill.py @@ -0,0 +1,24 @@ +#!/usr/bin/env python + +import mozdevice +import logging +import unittest +from sut import MockAgent + + +class TestKill(unittest.TestCase): + + def test_killprocess(self): + commands = [("ps", "1000 1486 com.android.settings\n" + "10016 420 com.android.location.fused\n" + "10023 335 com.android.systemui\n"), + ("kill com.android.settings", + "Successfully killed com.android.settings\n")] + m = MockAgent(self, commands=commands) + d = mozdevice.DroidSUT("127.0.0.1", port=m.port, logLevel=logging.DEBUG) + # No error raised means success + self.assertEqual(None, d.killProcess("com.android.settings")) + + +if __name__ == '__main__': + unittest.main() diff --git a/testing/mozbase/mozdevice/tests/sut_list.py b/testing/mozbase/mozdevice/tests/sut_list.py new file mode 100644 index 000000000..a319fd725 --- /dev/null +++ b/testing/mozbase/mozdevice/tests/sut_list.py @@ -0,0 +1,22 @@ +#!/usr/bin/env python +import mozdevice +import logging +import unittest +from sut import MockAgent + + +class TestListFiles(unittest.TestCase): + commands = [("isdir /mnt/sdcard", "TRUE"), + ("cd /mnt/sdcard", ""), + ("ls", "Android\nMusic\nPodcasts\nRingtones\nAlarms\n" + "Notifications\nPictures\nMovies\nDownload\nDCIM\n")] + + def test_listFiles(self): + m = MockAgent(self, commands=self.commands) + d = mozdevice.DroidSUT("127.0.0.1", port=m.port, logLevel=logging.DEBUG) + + expected = (self.commands[2][1].strip()).split("\n") + self.assertEqual(expected, d.listFiles("/mnt/sdcard")) + +if __name__ == '__main__': + unittest.main() diff --git a/testing/mozbase/mozdevice/tests/sut_logcat.py b/testing/mozbase/mozdevice/tests/sut_logcat.py new file mode 100644 index 000000000..b4c1a742d --- /dev/null +++ b/testing/mozbase/mozdevice/tests/sut_logcat.py @@ -0,0 +1,52 @@ +#!/usr/bin/env python + +import mozdevice +import logging +import unittest +from sut import MockAgent + + +class TestLogCat(unittest.TestCase): + """ Class to test methods associated with logcat """ + + def test_getLogcat(self): + + logcat_output = ( + "07-17 00:51:10.377 I/SUTAgentAndroid( 2933): onCreate\r\n" + "07-17 00:51:10.457 D/dalvikvm( 2933): GC_CONCURRENT freed 351K, 17% free 2523K/3008K, paused 5ms+2ms, total 38ms\r\n" # noqa + "07-17 00:51:10.497 I/SUTAgentAndroid( 2933): Caught exception creating file in /data/local/tmp: open failed: EACCES (Permission denied)\r\n" # noqa + "07-17 00:51:10.507 E/SUTAgentAndroid( 2933): ERROR: Cannot access world writeable test root\r\n" # noqa + "07-17 00:51:10.547 D/GeckoHealthRec( 3253): Initializing profile cache.\r\n" + "07-17 00:51:10.607 D/GeckoHealthRec( 3253): Looking for /data/data/org.mozilla.fennec/files/mozilla/c09kfhne.default/times.json\r\n" # noqa + "07-17 00:51:10.637 D/GeckoHealthRec( 3253): Using times.json for profile creation time.\r\n" # noqa + "07-17 00:51:10.707 D/GeckoHealthRec( 3253): Incorporating environment: times.json profile creation = 1374026758604\r\n" # noqa + "07-17 00:51:10.507 D/GeckoHealthRec( 3253): Requested prefs.\r\n" + "07-17 06:50:54.907 I/SUTAgentAndroid( 3876): \r\n" + "07-17 06:50:54.907 I/SUTAgentAndroid( 3876): Total Private Dirty Memory 3176 kb\r\n" # noqa + "07-17 06:50:54.907 I/SUTAgentAndroid( 3876): Total Proportional Set Size Memory 5679 kb\r\n" # noqa + "07-17 06:50:54.907 I/SUTAgentAndroid( 3876): Total Shared Dirty Memory 9216 kb\r\n" # noqa + "07-17 06:55:21.627 I/SUTAgentAndroid( 3876): 127.0.0.1 : execsu /system/bin/logcat -v time -d dalvikvm:I " # noqa + "ConnectivityService:S WifiMonitor:S WifiStateTracker:S wpa_supplicant:S NetworkStateTracker:S\r\n" # noqa + "07-17 06:55:21.827 I/dalvikvm-heap( 3876): Grow heap (frag case) to 3.019MB for 102496-byte allocation\r\n" # noqa + "return code [0]") + + inp = ("execsu /system/bin/logcat -v time -d " + "dalvikvm:I ConnectivityService:S WifiMonitor:S " + "WifiStateTracker:S wpa_supplicant:S NetworkStateTracker:S") + + commands = [(inp, logcat_output)] + m = MockAgent(self, commands=commands) + d = mozdevice.DroidSUT("127.0.0.1", port=m.port, logLevel=logging.DEBUG) + self.assertEqual(logcat_output[:-17].replace('\r\n', '\n').splitlines(True), d.getLogcat()) + + def test_recordLogcat(self): + + commands = [("execsu /system/bin/logcat -c", "return code [0]")] + + m = MockAgent(self, commands=commands) + d = mozdevice.DroidSUT("127.0.0.1", port=m.port, logLevel=logging.DEBUG) + # No error raised means success + self.assertEqual(None, d.recordLogcat()) + +if __name__ == '__main__': + unittest.main() diff --git a/testing/mozbase/mozdevice/tests/sut_mkdir.py b/testing/mozbase/mozdevice/tests/sut_mkdir.py new file mode 100644 index 000000000..bacaae324 --- /dev/null +++ b/testing/mozbase/mozdevice/tests/sut_mkdir.py @@ -0,0 +1,78 @@ +# Any copyright is dedicated to the Public Domain. +# http://creativecommons.org/publicdomain/zero/1.0/ + +import mozdevice +import logging +import unittest +from sut import MockAgent + + +class MkDirsTest(unittest.TestCase): + + def test_mkdirs(self): + subTests = [{'cmds': [('isdir /mnt/sdcard/baz/boop', 'FALSE'), + ('info os', 'android'), + ('isdir /mnt', 'TRUE'), + ('isdir /mnt/sdcard', 'TRUE'), + ('isdir /mnt/sdcard/baz', 'FALSE'), + ('mkdr /mnt/sdcard/baz', + '/mnt/sdcard/baz successfully created'), + ('isdir /mnt/sdcard/baz/boop', 'FALSE'), + ('mkdr /mnt/sdcard/baz/boop', + '/mnt/sdcard/baz/boop successfully created')], + 'expectException': False}, + {'cmds': [('isdir /mnt/sdcard/baz/boop', 'FALSE'), + ('info os', 'android'), + ('isdir /mnt', 'TRUE'), + ('isdir /mnt/sdcard', 'TRUE'), + ('isdir /mnt/sdcard/baz', 'FALSE'), + ('mkdr /mnt/sdcard/baz', + "##AGENT-WARNING## " + "Could not create the directory /mnt/sdcard/baz")], + 'expectException': True}, + ] + for subTest in subTests: + a = MockAgent(self, commands=subTest['cmds']) + + exceptionThrown = False + try: + d = mozdevice.DroidSUT('127.0.0.1', port=a.port, + logLevel=logging.DEBUG) + d.mkDirs('/mnt/sdcard/baz/boop/bip') + except mozdevice.DMError: + exceptionThrown = True + self.assertEqual(exceptionThrown, subTest['expectException']) + + a.wait() + + def test_repeated_path_part(self): + """ + Ensure that all dirs are created when last path part also found + earlier in the path (bug 826492). + """ + + cmds = [('isdir /mnt/sdcard/foo', 'FALSE'), + ('info os', 'android'), + ('isdir /mnt', 'TRUE'), + ('isdir /mnt/sdcard', 'TRUE'), + ('isdir /mnt/sdcard/foo', 'FALSE'), + ('mkdr /mnt/sdcard/foo', + '/mnt/sdcard/foo successfully created')] + a = MockAgent(self, commands=cmds) + d = mozdevice.DroidSUT('127.0.0.1', port=a.port, + logLevel=logging.DEBUG) + d.mkDirs('/mnt/sdcard/foo/foo') + a.wait() + + def test_mkdirs_on_root(self): + cmds = [('isdir /', 'TRUE')] + a = MockAgent(self, commands=cmds) + d = mozdevice.DroidSUT('127.0.0.1', port=a.port, + logLevel=logging.DEBUG) + d.mkDirs('/foo') + + a.wait() + + +if __name__ == '__main__': + unittest.main() diff --git a/testing/mozbase/mozdevice/tests/sut_movetree.py b/testing/mozbase/mozdevice/tests/sut_movetree.py new file mode 100644 index 000000000..0e106577c --- /dev/null +++ b/testing/mozbase/mozdevice/tests/sut_movetree.py @@ -0,0 +1,65 @@ +#!/usr/bin/env python +# -*- coding: utf-8 -*- + +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this file, +# You can obtain one at http://mozilla.org/MPL/2.0/. + +import mozdevice +import logging +import unittest +from sut import MockAgent + + +class MoveTreeTest(unittest.TestCase): + + def test_moveFile(self): + commands = [('mv /mnt/sdcard/tests/test.txt /mnt/sdcard/tests/test1.txt', ''), + ('isdir /mnt/sdcard/tests', 'TRUE'), + ('cd /mnt/sdcard/tests', ''), + ('ls', 'test1.txt'), + ('isdir /mnt/sdcard/tests', 'TRUE'), + ('cd /mnt/sdcard/tests', ''), + ('ls', 'test1.txt')] + + m = MockAgent(self, commands=commands) + d = mozdevice.DroidSUT("127.0.0.1", port=m.port, logLevel=logging.DEBUG) + self.assertEqual(None, d.moveTree('/mnt/sdcard/tests/test.txt', + '/mnt/sdcard/tests/test1.txt')) + self.assertFalse(d.fileExists('/mnt/sdcard/tests/test.txt')) + self.assertTrue(d.fileExists('/mnt/sdcard/tests/test1.txt')) + + def test_moveDir(self): + commands = [("mv /mnt/sdcard/tests/foo /mnt/sdcard/tests/bar", ""), + ('isdir /mnt/sdcard/tests', 'TRUE'), + ('cd /mnt/sdcard/tests', ''), + ('ls', 'bar')] + + m = MockAgent(self, commands=commands) + d = mozdevice.DroidSUT("127.0.0.1", port=m.port, logLevel=logging.DEBUG) + self.assertEqual(None, d.moveTree('/mnt/sdcard/tests/foo', + '/mnt/sdcard/tests/bar')) + self.assertTrue(d.fileExists('/mnt/sdcard/tests/bar')) + + def test_moveNonEmptyDir(self): + commands = [('isdir /mnt/sdcard/tests/foo/bar', 'TRUE'), + ('mv /mnt/sdcard/tests/foo /mnt/sdcard/tests/foo2', ''), + ('isdir /mnt/sdcard/tests', 'TRUE'), + ('cd /mnt/sdcard/tests', ''), + ('ls', 'foo2'), + ('isdir /mnt/sdcard/tests/foo2', 'TRUE'), + ('cd /mnt/sdcard/tests/foo2', ''), + ('ls', 'bar')] + + m = MockAgent(self, commands=commands) + d = mozdevice.DroidSUT("127.0.0.1", port=m.port, + logLevel=logging.DEBUG) + + self.assertTrue(d.dirExists('/mnt/sdcard/tests/foo/bar')) + self.assertEqual(None, d.moveTree('/mnt/sdcard/tests/foo', + '/mnt/sdcard/tests/foo2')) + self.assertTrue(d.fileExists('/mnt/sdcard/tests/foo2')) + self.assertTrue(d.fileExists('/mnt/sdcard/tests/foo2/bar')) + +if __name__ == "__main__": + unittest.main() diff --git a/testing/mozbase/mozdevice/tests/sut_ps.py b/testing/mozbase/mozdevice/tests/sut_ps.py new file mode 100644 index 000000000..03f431c1d --- /dev/null +++ b/testing/mozbase/mozdevice/tests/sut_ps.py @@ -0,0 +1,50 @@ +from sut import MockAgent +import mozdevice +import unittest + + +class PsTest(unittest.TestCase): + + pscommands = [('ps', + "10029 549 com.android.launcher\n" + "10066 1198 com.twitter.android")] + + bad_pscommands = [('ps', + "abcdef 549 com.android.launcher\n" + "10066 1198 com.twitter.android")] + + def test_processList(self): + a = MockAgent(self, + commands=self.pscommands) + d = mozdevice.DroidSUT("127.0.0.1", port=a.port) + pslist = d.getProcessList() + self.assertEqual(len(pslist), 2) + self.assertEqual(pslist[0], [549, 'com.android.launcher', 10029]) + self.assertEqual(pslist[1], [1198, 'com.twitter.android', 10066]) + + a.wait() + + def test_badProcessList(self): + a = MockAgent(self, + commands=self.bad_pscommands) + d = mozdevice.DroidSUT("127.0.0.1", port=a.port) + exceptionTriggered = False + try: + d.getProcessList() + except mozdevice.DMError: + exceptionTriggered = True + + self.assertTrue(exceptionTriggered) + + a.wait() + + def test_processExist(self): + for i in [('com.android.launcher', 549), + ('com.fennec.android', None)]: + a = MockAgent(self, commands=self.pscommands) + d = mozdevice.DroidSUT("127.0.0.1", port=a.port) + self.assertEqual(d.processExist(i[0]), i[1]) + a.wait() + +if __name__ == '__main__': + unittest.main() diff --git a/testing/mozbase/mozdevice/tests/sut_pull.py b/testing/mozbase/mozdevice/tests/sut_pull.py new file mode 100644 index 000000000..c9fcae42a --- /dev/null +++ b/testing/mozbase/mozdevice/tests/sut_pull.py @@ -0,0 +1,47 @@ +from sut import MockAgent +import mozdevice +import logging +import unittest + + +class PullTest(unittest.TestCase): + + def test_pull_success(self): + for count in [1, 4, 1024, 2048]: + cheeseburgers = "" + for i in range(count): + cheeseburgers += "cheeseburgers" + + # pull file is kind of gross, make sure we can still execute commands after it's done + remoteName = "/mnt/sdcard/cheeseburgers" + a = MockAgent(self, commands=[("pull %s" % remoteName, + "%s,%s\n%s" % (remoteName, + len(cheeseburgers), + cheeseburgers)), + ("isdir /mnt/sdcard", "TRUE")]) + + d = mozdevice.DroidSUT("127.0.0.1", port=a.port, + logLevel=logging.DEBUG) + pulledData = d.pullFile("/mnt/sdcard/cheeseburgers") + self.assertEqual(pulledData, cheeseburgers) + d.dirExists('/mnt/sdcard') + + def test_pull_failure(self): + + # this test simulates only receiving a few bytes of what we expect + # to be larger file + remoteName = "/mnt/sdcard/cheeseburgers" + a = MockAgent(self, commands=[("pull %s" % remoteName, + "%s,15\n%s" % (remoteName, + "cheeseburgh"))]) + d = mozdevice.DroidSUT("127.0.0.1", port=a.port, + logLevel=logging.DEBUG) + exceptionThrown = False + try: + d.pullFile("/mnt/sdcard/cheeseburgers") + except mozdevice.DMError: + exceptionThrown = True + self.assertTrue(exceptionThrown) + +if __name__ == '__main__': + unittest.main() diff --git a/testing/mozbase/mozdevice/tests/sut_push.py b/testing/mozbase/mozdevice/tests/sut_push.py new file mode 100644 index 000000000..023d5315c --- /dev/null +++ b/testing/mozbase/mozdevice/tests/sut_push.py @@ -0,0 +1,88 @@ +from sut import MockAgent +import mozfile +import mozdevice +import logging +import unittest +import hashlib +import tempfile +import os + + +class PushTest(unittest.TestCase): + + def test_push(self): + pushfile = "1234ABCD" + mdsum = hashlib.md5() + mdsum.update(pushfile) + expectedResponse = mdsum.hexdigest() + + # (good response, no exception), (bad response, exception) + for response in [(expectedResponse, False), ("BADHASH", True)]: + cmd = "push /mnt/sdcard/foobar %s\r\n%s" % (len(pushfile), pushfile) + a = MockAgent(self, commands=[("isdir /mnt/sdcard", "TRUE"), + (cmd, response[0])]) + exceptionThrown = False + with tempfile.NamedTemporaryFile() as f: + try: + f.write(pushfile) + f.flush() + d = mozdevice.DroidSUT("127.0.0.1", port=a.port) + d.pushFile(f.name, '/mnt/sdcard/foobar') + except mozdevice.DMError: + exceptionThrown = True + self.assertEqual(exceptionThrown, response[1]) + a.wait() + + def test_push_dir(self): + pushfile = "1234ABCD" + mdsum = hashlib.md5() + mdsum.update(pushfile) + expectedFileResponse = mdsum.hexdigest() + + tempdir = tempfile.mkdtemp() + self.addCleanup(mozfile.remove, tempdir) + complex_path = os.path.join(tempdir, "baz") + os.mkdir(complex_path) + f = tempfile.NamedTemporaryFile(dir=complex_path) + f.write(pushfile) + f.flush() + + subTests = [{'cmds': [("isdir /mnt/sdcard/baz", "TRUE"), + ("push /mnt/sdcard/baz/%s %s\r\n%s" % + (os.path.basename(f.name), len(pushfile), + pushfile), + expectedFileResponse)], + 'expectException': False}, + {'cmds': [("isdir /mnt/sdcard/baz", "TRUE"), + ("push /mnt/sdcard/baz/%s %s\r\n%s" % + (os.path.basename(f.name), len(pushfile), + pushfile), + "BADHASH")], + 'expectException': True}, + {'cmds': [("isdir /mnt/sdcard/baz", "FALSE"), + ('info os', 'android'), + ("isdir /mnt", "FALSE"), + ("mkdr /mnt", + "##AGENT-WARNING## Could not create the directory /mnt")], + 'expectException': True}, + + ] + + for subTest in subTests: + a = MockAgent(self, commands=subTest['cmds']) + + exceptionThrown = False + try: + d = mozdevice.DroidSUT("127.0.0.1", port=a.port, + logLevel=logging.DEBUG) + d.pushDir(tempdir, "/mnt/sdcard") + except mozdevice.DMError: + exceptionThrown = True + self.assertEqual(exceptionThrown, subTest['expectException']) + + a.wait() + + # FIXME: delete directory when done + +if __name__ == '__main__': + unittest.main() diff --git a/testing/mozbase/mozdevice/tests/sut_remove.py b/testing/mozbase/mozdevice/tests/sut_remove.py new file mode 100644 index 000000000..636190186 --- /dev/null +++ b/testing/mozbase/mozdevice/tests/sut_remove.py @@ -0,0 +1,24 @@ +#!/usr/bin/env python +import mozdevice +import logging +import unittest +from sut import MockAgent + + +class TestRemove(unittest.TestCase): + + def test_removeDir(self): + commands = [("isdir /mnt/sdcard/test", "TRUE"), + ("rmdr /mnt/sdcard/test", "Deleting file(s) from " + "/storage/emulated/legacy/Moztest\n" + " \n" + "Deleting directory " + "/storage/emulated/legacy/Moztest\n")] + + m = MockAgent(self, commands=commands) + d = mozdevice.DroidSUT("127.0.0.1", port=m.port, logLevel=logging.DEBUG) + # No error implies we're all good + self.assertEqual(None, d.removeDir("/mnt/sdcard/test")) + +if __name__ == '__main__': + unittest.main() diff --git a/testing/mozbase/mozdevice/tests/sut_time.py b/testing/mozbase/mozdevice/tests/sut_time.py new file mode 100644 index 000000000..11dc421cb --- /dev/null +++ b/testing/mozbase/mozdevice/tests/sut_time.py @@ -0,0 +1,18 @@ +#!/usr/bin/env python +import mozdevice +import logging +import unittest +from sut import MockAgent + + +class TestGetCurrentTime(unittest.TestCase): + + def test_getCurrentTime(self): + command = [('clok', '1349980200')] + + m = MockAgent(self, commands=command) + d = mozdevice.DroidSUT("127.0.0.1", port=m.port, logLevel=logging.DEBUG) + self.assertEqual(d.getCurrentTime(), int(command[0][1])) + +if __name__ == '__main__': + unittest.main() diff --git a/testing/mozbase/mozdevice/tests/sut_unpackfile.py b/testing/mozbase/mozdevice/tests/sut_unpackfile.py new file mode 100644 index 000000000..1a531fe17 --- /dev/null +++ b/testing/mozbase/mozdevice/tests/sut_unpackfile.py @@ -0,0 +1,23 @@ +#!/usr/bin/env python + +import mozdevice +import logging +import unittest +from sut import MockAgent + + +class TestUnpack(unittest.TestCase): + + def test_unpackFile(self): + + commands = [("unzp /data/test/sample.zip /data/test/", + "Checksum: 653400271\n" + "1 of 1 successfully extracted\n")] + m = MockAgent(self, commands=commands) + d = mozdevice.DroidSUT("127.0.0.1", port=m.port, logLevel=logging.DEBUG) + # No error being thrown imples all is well + self.assertEqual(None, d.unpackFile("/data/test/sample.zip", + "/data/test/")) + +if __name__ == '__main__': + unittest.main() diff --git a/testing/mozbase/mozfile/mozfile/__init__.py b/testing/mozbase/mozfile/mozfile/__init__.py new file mode 100644 index 000000000..a527f0ad6 --- /dev/null +++ b/testing/mozbase/mozfile/mozfile/__init__.py @@ -0,0 +1,8 @@ +# flake8: noqa +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, You can obtain one at http://mozilla.org/MPL/2.0/. + +from __future__ import absolute_import + +from .mozfile import * diff --git a/testing/mozbase/mozfile/mozfile/mozfile.py b/testing/mozbase/mozfile/mozfile/mozfile.py new file mode 100644 index 000000000..94805594e --- /dev/null +++ b/testing/mozbase/mozfile/mozfile/mozfile.py @@ -0,0 +1,449 @@ +# -*- coding: utf-8 -*- + +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, You can obtain one at http://mozilla.org/MPL/2.0/. + +# We don't import all modules at the top for performance reasons. See Bug 1008943 + +from __future__ import absolute_import + +from contextlib import contextmanager +import errno +import os +import stat +import time +import warnings + +__all__ = ['extract_tarball', + 'extract_zip', + 'extract', + 'is_url', + 'load', + 'move', + 'remove', + 'rmtree', + 'tree', + 'NamedTemporaryFile', + 'TemporaryDirectory'] + +# utilities for extracting archives + + +def extract_tarball(src, dest): + """extract a .tar file""" + + import tarfile + + bundle = tarfile.open(src) + namelist = bundle.getnames() + + for name in namelist: + bundle.extract(name, path=dest) + bundle.close() + return namelist + + +def extract_zip(src, dest): + """extract a zip file""" + + import zipfile + + if isinstance(src, zipfile.ZipFile): + bundle = src + else: + try: + bundle = zipfile.ZipFile(src) + except Exception: + print "src: %s" % src + raise + + namelist = bundle.namelist() + + for name in namelist: + filename = os.path.realpath(os.path.join(dest, name)) + if name.endswith('/'): + if not os.path.isdir(filename): + os.makedirs(filename) + else: + path = os.path.dirname(filename) + if not os.path.isdir(path): + os.makedirs(path) + _dest = open(filename, 'wb') + _dest.write(bundle.read(name)) + _dest.close() + mode = bundle.getinfo(name).external_attr >> 16 & 0x1FF + # Only update permissions if attributes are set. Otherwise fallback to the defaults. + if mode: + os.chmod(filename, mode) + bundle.close() + return namelist + + +def extract(src, dest=None): + """ + Takes in a tar or zip file and extracts it to dest + + If dest is not specified, extracts to os.path.dirname(src) + + Returns the list of top level files that were extracted + """ + + import zipfile + import tarfile + + assert os.path.exists(src), "'%s' does not exist" % src + + if dest is None: + dest = os.path.dirname(src) + elif not os.path.isdir(dest): + os.makedirs(dest) + assert not os.path.isfile(dest), "dest cannot be a file" + + if zipfile.is_zipfile(src): + namelist = extract_zip(src, dest) + elif tarfile.is_tarfile(src): + namelist = extract_tarball(src, dest) + else: + raise Exception("mozfile.extract: no archive format found for '%s'" % + src) + + # namelist returns paths with forward slashes even in windows + top_level_files = [os.path.join(dest, name.rstrip('/')) for name in namelist + if len(name.rstrip('/').split('/')) == 1] + + # namelist doesn't include folders, append these to the list + for name in namelist: + index = name.find('/') + if index != -1: + root = os.path.join(dest, name[:index]) + if root not in top_level_files: + top_level_files.append(root) + + return top_level_files + + +# utilities for removal of files and directories + +def rmtree(dir): + """Deprecated wrapper method to remove a directory tree. + + Ensure to update your code to use mozfile.remove() directly + + :param dir: directory to be removed + """ + + warnings.warn("mozfile.rmtree() is deprecated in favor of mozfile.remove()", + PendingDeprecationWarning, stacklevel=2) + return remove(dir) + + +def _call_windows_retry(func, args=(), retry_max=5, retry_delay=0.5): + """ + It's possible to see spurious errors on Windows due to various things + keeping a handle to the directory open (explorer, virus scanners, etc) + So we try a few times if it fails with a known error. + retry_delay is multiplied by the number of failed attempts to increase + the likelihood of success in subsequent attempts. + """ + retry_count = 0 + while True: + try: + func(*args) + except OSError as e: + # Error codes are defined in: + # http://docs.python.org/2/library/errno.html#module-errno + if e.errno not in (errno.EACCES, errno.ENOTEMPTY): + raise + + if retry_count == retry_max: + raise + + retry_count += 1 + + print '%s() failed for "%s". Reason: %s (%s). Retrying...' % \ + (func.__name__, args, e.strerror, e.errno) + time.sleep(retry_count * retry_delay) + else: + # If no exception has been thrown it should be done + break + + +def remove(path): + """Removes the specified file, link, or directory tree. + + This is a replacement for shutil.rmtree that works better under + windows. It does the following things: + + - check path access for the current user before trying to remove + - retry operations on some known errors due to various things keeping + a handle on file paths - like explorer, virus scanners, etc. The + known errors are errno.EACCES and errno.ENOTEMPTY, and it will + retry up to 5 five times with a delay of (failed_attempts * 0.5) seconds + between each attempt. + + Note that no error will be raised if the given path does not exists. + + :param path: path to be removed + """ + + import shutil + + def _call_with_windows_retry(*args, **kwargs): + try: + _call_windows_retry(*args, **kwargs) + except OSError as e: + # The file or directory to be removed doesn't exist anymore + if e.errno != errno.ENOENT: + raise + + def _update_permissions(path): + """Sets specified pemissions depending on filetype""" + if os.path.islink(path): + # Path is a symlink which we don't have to modify + # because it should already have all the needed permissions + return + + stats = os.stat(path) + + if os.path.isfile(path): + mode = stats.st_mode | stat.S_IWUSR + elif os.path.isdir(path): + mode = stats.st_mode | stat.S_IWUSR | stat.S_IXUSR + else: + # Not supported type + return + + _call_with_windows_retry(os.chmod, (path, mode)) + + if not os.path.exists(path): + return + + if os.path.isfile(path) or os.path.islink(path): + # Verify the file or link is read/write for the current user + _update_permissions(path) + _call_with_windows_retry(os.remove, (path,)) + + elif os.path.isdir(path): + # Verify the directory is read/write/execute for the current user + _update_permissions(path) + + # We're ensuring that every nested item has writable permission. + for root, dirs, files in os.walk(path): + for entry in dirs + files: + _update_permissions(os.path.join(root, entry)) + _call_with_windows_retry(shutil.rmtree, (path,)) + + +def move(src, dst): + """ + Move a file or directory path. + + This is a replacement for shutil.move that works better under windows, + retrying operations on some known errors due to various things keeping + a handle on file paths. + """ + import shutil + _call_windows_retry(shutil.move, (src, dst)) + + +def depth(directory): + """returns the integer depth of a directory or path relative to '/' """ + + directory = os.path.abspath(directory) + level = 0 + while True: + directory, remainder = os.path.split(directory) + level += 1 + if not remainder: + break + return level + + +# ASCII delimeters +ascii_delimeters = { + 'vertical_line': '|', + 'item_marker': '+', + 'last_child': '\\' +} + +# unicode delimiters +unicode_delimeters = { + 'vertical_line': '│', + 'item_marker': '├', + 'last_child': '└' +} + + +def tree(directory, + item_marker=unicode_delimeters['item_marker'], + vertical_line=unicode_delimeters['vertical_line'], + last_child=unicode_delimeters['last_child'], + sort_key=lambda x: x.lower()): + """ + display tree directory structure for `directory` + """ + + retval = [] + indent = [] + last = {} + top = depth(directory) + + for dirpath, dirnames, filenames in os.walk(directory, topdown=True): + + abspath = os.path.abspath(dirpath) + basename = os.path.basename(abspath) + parent = os.path.dirname(abspath) + level = depth(abspath) - top + + # sort articles of interest + for resource in (dirnames, filenames): + resource[:] = sorted(resource, key=sort_key) + + if level > len(indent): + indent.append(vertical_line) + indent = indent[:level] + + if dirnames: + files_end = item_marker + last[abspath] = dirnames[-1] + else: + files_end = last_child + + if last.get(parent) == os.path.basename(abspath): + # last directory of parent + dirpath_mark = last_child + indent[-1] = ' ' + elif not indent: + dirpath_mark = '' + else: + dirpath_mark = item_marker + + # append the directory and piece of tree structure + # if the top-level entry directory, print as passed + retval.append('%s%s%s' % (''.join(indent[:-1]), + dirpath_mark, + basename if retval else directory)) + # add the files + if filenames: + last_file = filenames[-1] + retval.extend([('%s%s%s' % (''.join(indent), + files_end if filename == last_file else item_marker, + filename)) + for index, filename in enumerate(filenames)]) + + return '\n'.join(retval) + + +# utilities for temporary resources + +class NamedTemporaryFile(object): + """ + Like tempfile.NamedTemporaryFile except it works on Windows + in the case where you open the created file a second time. + + This behaves very similarly to tempfile.NamedTemporaryFile but may + not behave exactly the same. For example, this function does not + prevent fd inheritance by children. + + Example usage: + + with NamedTemporaryFile() as fh: + fh.write(b'foobar') + + print('Filename: %s' % fh.name) + + see https://bugzilla.mozilla.org/show_bug.cgi?id=821362 + """ + + def __init__(self, mode='w+b', bufsize=-1, suffix='', prefix='tmp', + dir=None, delete=True): + + import tempfile + fd, path = tempfile.mkstemp(suffix, prefix, dir, 't' in mode) + os.close(fd) + + self.file = open(path, mode) + self._path = path + self._delete = delete + self._unlinked = False + + def __getattr__(self, k): + return getattr(self.__dict__['file'], k) + + def __iter__(self): + return self.__dict__['file'] + + def __enter__(self): + self.file.__enter__() + return self + + def __exit__(self, exc, value, tb): + self.file.__exit__(exc, value, tb) + if self.__dict__['_delete']: + os.unlink(self.__dict__['_path']) + self._unlinked = True + + def __del__(self): + if self.__dict__['_unlinked']: + return + self.file.__exit__(None, None, None) + if self.__dict__['_delete']: + os.unlink(self.__dict__['_path']) + + +@contextmanager +def TemporaryDirectory(): + """ + create a temporary directory using tempfile.mkdtemp, and then clean it up. + + Example usage: + with TemporaryDirectory() as tmp: + open(os.path.join(tmp, "a_temp_file"), "w").write("data") + + """ + + import tempfile + import shutil + + tempdir = tempfile.mkdtemp() + try: + yield tempdir + finally: + shutil.rmtree(tempdir) + + +# utilities dealing with URLs + +def is_url(thing): + """ + Return True if thing looks like a URL. + """ + + import urlparse + + parsed = urlparse.urlparse(thing) + if 'scheme' in parsed: + return len(parsed.scheme) >= 2 + else: + return len(parsed[0]) >= 2 + + +def load(resource): + """ + open a file or URL for reading. If the passed resource string is not a URL, + or begins with 'file://', return a ``file``. Otherwise, return the + result of urllib2.urlopen() + """ + + import urllib2 + + # handle file URLs separately due to python stdlib limitations + if resource.startswith('file://'): + resource = resource[len('file://'):] + + if not is_url(resource): + # if no scheme is given, it is a file path + return file(resource) + + return urllib2.urlopen(resource) diff --git a/testing/mozbase/mozfile/setup.py b/testing/mozbase/mozfile/setup.py new file mode 100644 index 000000000..277ff7b52 --- /dev/null +++ b/testing/mozbase/mozfile/setup.py @@ -0,0 +1,25 @@ +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this file, +# You can obtain one at http://mozilla.org/MPL/2.0/. + +from setuptools import setup + +PACKAGE_NAME = 'mozfile' +PACKAGE_VERSION = '1.2' + +setup(name=PACKAGE_NAME, + version=PACKAGE_VERSION, + description="Library of file utilities for use in Mozilla testing", + long_description="see http://mozbase.readthedocs.org/", + classifiers=[], # Get strings from http://pypi.python.org/pypi?%3Aaction=list_classifiers + keywords='mozilla', + author='Mozilla Automation and Tools team', + author_email='tools@lists.mozilla.org', + url='https://wiki.mozilla.org/Auto-tools/Projects/Mozbase', + license='MPL', + packages=['mozfile'], + include_package_data=True, + zip_safe=False, + install_requires=[], + tests_require=['mozhttpd'] + ) diff --git a/testing/mozbase/mozfile/tests/files/missing_file_attributes.zip b/testing/mozbase/mozfile/tests/files/missing_file_attributes.zip new file mode 100644 index 000000000..2b5409e89 Binary files /dev/null and b/testing/mozbase/mozfile/tests/files/missing_file_attributes.zip differ diff --git a/testing/mozbase/mozfile/tests/manifest.ini b/testing/mozbase/mozfile/tests/manifest.ini new file mode 100644 index 000000000..c7889beca --- /dev/null +++ b/testing/mozbase/mozfile/tests/manifest.ini @@ -0,0 +1,6 @@ +[test_extract.py] +[test_load.py] +[test_move_remove.py] +[test_tempdir.py] +[test_tempfile.py] +[test_url.py] diff --git a/testing/mozbase/mozfile/tests/stubs.py b/testing/mozbase/mozfile/tests/stubs.py new file mode 100644 index 000000000..06d79e7af --- /dev/null +++ b/testing/mozbase/mozfile/tests/stubs.py @@ -0,0 +1,37 @@ +import os +import shutil +import tempfile + + +# stub file paths +files = [('foo.txt',), + ('foo', 'bar.txt',), + ('foo', 'bar', 'fleem.txt',), + ('foobar', 'fleem.txt',), + ('bar.txt',), + ('nested_tree', 'bar', 'fleem.txt',), + ('readonly.txt',), + ] + + +def create_stub(): + """create a stub directory""" + + tempdir = tempfile.mkdtemp() + try: + for path in files: + fullpath = os.path.join(tempdir, *path) + dirname = os.path.dirname(fullpath) + if not os.path.exists(dirname): + os.makedirs(dirname) + contents = path[-1] + f = file(fullpath, 'w') + f.write(contents) + f.close() + return tempdir + except Exception: + try: + shutil.rmtree(tempdir) + except: + pass + raise diff --git a/testing/mozbase/mozfile/tests/test_extract.py b/testing/mozbase/mozfile/tests/test_extract.py new file mode 100644 index 000000000..e91f52349 --- /dev/null +++ b/testing/mozbase/mozfile/tests/test_extract.py @@ -0,0 +1,154 @@ +#!/usr/bin/env python + +import os +import shutil +import tarfile +import tempfile +import unittest +import zipfile + +import mozfile + +import stubs + + +class TestExtract(unittest.TestCase): + """test extracting archives""" + + def ensure_directory_contents(self, directory): + """ensure the directory contents match""" + for f in stubs.files: + path = os.path.join(directory, *f) + exists = os.path.exists(path) + if not exists: + print "%s does not exist" % (os.path.join(f)) + self.assertTrue(exists) + if exists: + contents = file(path).read().strip() + self.assertTrue(contents == f[-1]) + + def test_extract_zipfile(self): + """test extracting a zipfile""" + _zipfile = self.create_zip() + self.assertTrue(os.path.exists(_zipfile)) + try: + dest = tempfile.mkdtemp() + try: + mozfile.extract_zip(_zipfile, dest) + self.ensure_directory_contents(dest) + finally: + shutil.rmtree(dest) + finally: + os.remove(_zipfile) + + def test_extract_zipfile_missing_file_attributes(self): + """if files do not have attributes set the default permissions have to be inherited.""" + _zipfile = os.path.join(os.path.dirname(__file__), 'files', 'missing_file_attributes.zip') + self.assertTrue(os.path.exists(_zipfile)) + dest = tempfile.mkdtemp() + try: + # Get the default file permissions for the user + fname = os.path.join(dest, 'foo') + with open(fname, 'w'): + pass + default_stmode = os.stat(fname).st_mode + + files = mozfile.extract_zip(_zipfile, dest) + for filename in files: + self.assertEqual(os.stat(os.path.join(dest, filename)).st_mode, + default_stmode) + finally: + shutil.rmtree(dest) + + def test_extract_tarball(self): + """test extracting a tarball""" + tarball = self.create_tarball() + self.assertTrue(os.path.exists(tarball)) + try: + dest = tempfile.mkdtemp() + try: + mozfile.extract_tarball(tarball, dest) + self.ensure_directory_contents(dest) + finally: + shutil.rmtree(dest) + finally: + os.remove(tarball) + + def test_extract(self): + """test the generalized extract function""" + + # test extracting a tarball + tarball = self.create_tarball() + self.assertTrue(os.path.exists(tarball)) + try: + dest = tempfile.mkdtemp() + try: + mozfile.extract(tarball, dest) + self.ensure_directory_contents(dest) + finally: + shutil.rmtree(dest) + finally: + os.remove(tarball) + + # test extracting a zipfile + _zipfile = self.create_zip() + self.assertTrue(os.path.exists(_zipfile)) + try: + dest = tempfile.mkdtemp() + try: + mozfile.extract_zip(_zipfile, dest) + self.ensure_directory_contents(dest) + finally: + shutil.rmtree(dest) + finally: + os.remove(_zipfile) + + # test extracting some non-archive; this should fail + fd, filename = tempfile.mkstemp() + os.write(fd, 'This is not a zipfile or tarball') + os.close(fd) + exception = None + try: + dest = tempfile.mkdtemp() + mozfile.extract(filename, dest) + except Exception as exception: + pass + finally: + os.remove(filename) + os.rmdir(dest) + self.assertTrue(isinstance(exception, Exception)) + + # utility functions + + def create_tarball(self): + """create a stub tarball for testing""" + tempdir = stubs.create_stub() + filename = tempfile.mktemp(suffix='.tar') + archive = tarfile.TarFile(filename, mode='w') + try: + for path in stubs.files: + archive.add(os.path.join(tempdir, *path), arcname=os.path.join(*path)) + except: + os.remove(archive) + raise + finally: + shutil.rmtree(tempdir) + archive.close() + return filename + + def create_zip(self): + """create a stub zipfile for testing""" + + tempdir = stubs.create_stub() + filename = tempfile.mktemp(suffix='.zip') + archive = zipfile.ZipFile(filename, mode='w') + try: + for path in stubs.files: + archive.write(os.path.join(tempdir, *path), arcname=os.path.join(*path)) + except: + os.remove(filename) + raise + finally: + shutil.rmtree(tempdir) + archive.close() + return filename diff --git a/testing/mozbase/mozfile/tests/test_load.py b/testing/mozbase/mozfile/tests/test_load.py new file mode 100755 index 000000000..13a5b519c --- /dev/null +++ b/testing/mozbase/mozfile/tests/test_load.py @@ -0,0 +1,62 @@ +#!/usr/bin/env python + +""" +tests for mozfile.load +""" + +import mozhttpd +import os +import tempfile +import unittest +from mozfile import load + + +class TestLoad(unittest.TestCase): + """test the load function""" + + def test_http(self): + """test with mozhttpd and a http:// URL""" + + def example(request): + """example request handler""" + body = 'example' + return (200, {'Content-type': 'text/plain', + 'Content-length': len(body) + }, body) + + host = '127.0.0.1' + httpd = mozhttpd.MozHttpd(host=host, + urlhandlers=[{'method': 'GET', + 'path': '.*', + 'function': example}]) + try: + httpd.start(block=False) + content = load(httpd.get_url()).read() + self.assertEqual(content, 'example') + finally: + httpd.stop() + + def test_file_path(self): + """test loading from file path""" + try: + # create a temporary file + tmp = tempfile.NamedTemporaryFile(delete=False) + tmp.write('foo bar') + tmp.close() + + # read the file + contents = file(tmp.name).read() + self.assertEqual(contents, 'foo bar') + + # read the file with load and a file path + self.assertEqual(load(tmp.name).read(), contents) + + # read the file with load and a file URL + self.assertEqual(load('file://%s' % tmp.name).read(), contents) + finally: + # remove the tempfile + if os.path.exists(tmp.name): + os.remove(tmp.name) + +if __name__ == '__main__': + unittest.main() diff --git a/testing/mozbase/mozfile/tests/test_move_remove.py b/testing/mozbase/mozfile/tests/test_move_remove.py new file mode 100644 index 000000000..e9d0cd434 --- /dev/null +++ b/testing/mozbase/mozfile/tests/test_move_remove.py @@ -0,0 +1,232 @@ +#!/usr/bin/env python + +import os +import stat +import shutil +import threading +import time +import unittest +import errno +from contextlib import contextmanager + +import mozfile +import mozinfo + +import stubs + + +def mark_readonly(path): + """Removes all write permissions from given file/directory. + + :param path: path of directory/file of which modes must be changed + """ + mode = os.stat(path)[stat.ST_MODE] + os.chmod(path, mode & ~stat.S_IWUSR & ~stat.S_IWGRP & ~stat.S_IWOTH) + + +class FileOpenCloseThread(threading.Thread): + """Helper thread for asynchronous file handling""" + + def __init__(self, path, delay, delete=False): + threading.Thread.__init__(self) + self.file_opened = threading.Event() + self.delay = delay + self.path = path + self.delete = delete + + def run(self): + with open(self.path): + self.file_opened.set() + time.sleep(self.delay) + if self.delete: + try: + os.remove(self.path) + except: + pass + + +@contextmanager +def wait_file_opened_in_thread(*args, **kwargs): + thread = FileOpenCloseThread(*args, **kwargs) + thread.start() + thread.file_opened.wait() + try: + yield thread + finally: + thread.join() + + +class MozfileRemoveTestCase(unittest.TestCase): + """Test our ability to remove directories and files""" + + def setUp(self): + # Generate a stub + self.tempdir = stubs.create_stub() + + def tearDown(self): + if os.path.isdir(self.tempdir): + shutil.rmtree(self.tempdir) + + def test_remove_directory(self): + """Test the removal of a directory""" + self.assertTrue(os.path.isdir(self.tempdir)) + mozfile.remove(self.tempdir) + self.assertFalse(os.path.exists(self.tempdir)) + + def test_remove_directory_with_open_file(self): + """Test removing a directory with an open file""" + # Open a file in the generated stub + filepath = os.path.join(self.tempdir, *stubs.files[1]) + f = file(filepath, 'w') + f.write('foo-bar') + + # keep file open and then try removing the dir-tree + if mozinfo.isWin: + # On the Windows family WindowsError should be raised. + self.assertRaises(OSError, mozfile.remove, self.tempdir) + self.assertTrue(os.path.exists(self.tempdir)) + else: + # Folder should be deleted on all other platforms + mozfile.remove(self.tempdir) + self.assertFalse(os.path.exists(self.tempdir)) + + def test_remove_closed_file(self): + """Test removing a closed file""" + # Open a file in the generated stub + filepath = os.path.join(self.tempdir, *stubs.files[1]) + with open(filepath, 'w') as f: + f.write('foo-bar') + + # Folder should be deleted on all platforms + mozfile.remove(self.tempdir) + self.assertFalse(os.path.exists(self.tempdir)) + + def test_removing_open_file_with_retry(self): + """Test removing a file in use with retry""" + filepath = os.path.join(self.tempdir, *stubs.files[1]) + + with wait_file_opened_in_thread(filepath, 0.2): + # on windows first attempt will fail, + # and it will be retried until the thread leave the handle + mozfile.remove(filepath) + + # Check deletion was successful + self.assertFalse(os.path.exists(filepath)) + + def test_removing_already_deleted_file_with_retry(self): + """Test removing a meanwhile removed file with retry""" + filepath = os.path.join(self.tempdir, *stubs.files[1]) + + with wait_file_opened_in_thread(filepath, 0.2, True): + # on windows first attempt will fail, and before + # the retry the opened file will be deleted in the thread + mozfile.remove(filepath) + + # Check deletion was successful + self.assertFalse(os.path.exists(filepath)) + + def test_remove_readonly_tree(self): + """Test removing a read-only directory""" + + dirpath = os.path.join(self.tempdir, "nested_tree") + mark_readonly(dirpath) + + # However, mozfile should change write permissions and remove dir. + mozfile.remove(dirpath) + + self.assertFalse(os.path.exists(dirpath)) + + def test_remove_readonly_file(self): + """Test removing read-only files""" + filepath = os.path.join(self.tempdir, *stubs.files[1]) + mark_readonly(filepath) + + # However, mozfile should change write permission and then remove file. + mozfile.remove(filepath) + + self.assertFalse(os.path.exists(filepath)) + + @unittest.skipIf(mozinfo.isWin, "Symlinks are not supported on Windows") + def test_remove_symlink(self): + """Test removing a symlink""" + file_path = os.path.join(self.tempdir, *stubs.files[1]) + symlink_path = os.path.join(self.tempdir, 'symlink') + + os.symlink(file_path, symlink_path) + self.assertTrue(os.path.islink(symlink_path)) + + # The linked folder and files should not be deleted + mozfile.remove(symlink_path) + self.assertFalse(os.path.exists(symlink_path)) + self.assertTrue(os.path.exists(file_path)) + + @unittest.skipIf(mozinfo.isWin, "Symlinks are not supported on Windows") + def test_remove_symlink_in_subfolder(self): + """Test removing a folder with an contained symlink""" + file_path = os.path.join(self.tempdir, *stubs.files[0]) + dir_path = os.path.dirname(os.path.join(self.tempdir, *stubs.files[1])) + symlink_path = os.path.join(dir_path, 'symlink') + + os.symlink(file_path, symlink_path) + self.assertTrue(os.path.islink(symlink_path)) + + # The folder with the contained symlink will be deleted but not the + # original linked file + mozfile.remove(dir_path) + self.assertFalse(os.path.exists(dir_path)) + self.assertFalse(os.path.exists(symlink_path)) + self.assertTrue(os.path.exists(file_path)) + + @unittest.skipIf(mozinfo.isWin or not os.geteuid(), + "Symlinks are not supported on Windows and cannot run test as root") + def test_remove_symlink_for_system_path(self): + """Test removing a symlink which points to a system folder""" + symlink_path = os.path.join(self.tempdir, 'symlink') + + os.symlink(os.path.dirname(self.tempdir), symlink_path) + self.assertTrue(os.path.islink(symlink_path)) + + # The folder with the contained symlink will be deleted but not the + # original linked file + mozfile.remove(symlink_path) + self.assertFalse(os.path.exists(symlink_path)) + + def test_remove_path_that_does_not_exists(self): + not_existing_path = os.path.join(self.tempdir, 'I_do_not_not_exists') + try: + mozfile.remove(not_existing_path) + except OSError as exc: + if exc.errno == errno.ENOENT: + self.fail("removing non existing path must not raise error") + raise + + +class MozFileMoveTestCase(unittest.TestCase): + + def setUp(self): + # Generate a stub + self.tempdir = stubs.create_stub() + self.addCleanup(mozfile.rmtree, self.tempdir) + + def test_move_file(self): + file_path = os.path.join(self.tempdir, *stubs.files[1]) + moved_path = file_path + '.moved' + self.assertTrue(os.path.isfile(file_path)) + self.assertFalse(os.path.exists(moved_path)) + mozfile.move(file_path, moved_path) + self.assertFalse(os.path.exists(file_path)) + self.assertTrue(os.path.isfile(moved_path)) + + def test_move_file_with_retry(self): + file_path = os.path.join(self.tempdir, *stubs.files[1]) + moved_path = file_path + '.moved' + + with wait_file_opened_in_thread(file_path, 0.2): + # first move attempt should fail on windows and be retried + mozfile.move(file_path, moved_path) + self.assertFalse(os.path.exists(file_path)) + self.assertTrue(os.path.isfile(moved_path)) + + +if __name__ == '__main__': + unittest.main() diff --git a/testing/mozbase/mozfile/tests/test_tempdir.py b/testing/mozbase/mozfile/tests/test_tempdir.py new file mode 100644 index 000000000..81f03d095 --- /dev/null +++ b/testing/mozbase/mozfile/tests/test_tempdir.py @@ -0,0 +1,42 @@ +#!/usr/bin/env python + +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, You can obtain one at http://mozilla.org/MPL/2.0/. + +""" +tests for mozfile.TemporaryDirectory +""" + +from mozfile import TemporaryDirectory +import os +import unittest + + +class TestTemporaryDirectory(unittest.TestCase): + + def test_removed(self): + """ensure that a TemporaryDirectory gets removed""" + path = None + with TemporaryDirectory() as tmp: + path = tmp + self.assertTrue(os.path.isdir(tmp)) + tmpfile = os.path.join(tmp, "a_temp_file") + open(tmpfile, "w").write("data") + self.assertTrue(os.path.isfile(tmpfile)) + self.assertFalse(os.path.isdir(path)) + self.assertFalse(os.path.exists(path)) + + def test_exception(self): + """ensure that TemporaryDirectory handles exceptions""" + path = None + with self.assertRaises(Exception): + with TemporaryDirectory() as tmp: + path = tmp + self.assertTrue(os.path.isdir(tmp)) + raise Exception("oops") + self.assertFalse(os.path.isdir(path)) + self.assertFalse(os.path.exists(path)) + +if __name__ == '__main__': + unittest.main() diff --git a/testing/mozbase/mozfile/tests/test_tempfile.py b/testing/mozbase/mozfile/tests/test_tempfile.py new file mode 100644 index 000000000..3c3d26d5d --- /dev/null +++ b/testing/mozbase/mozfile/tests/test_tempfile.py @@ -0,0 +1,102 @@ +#!/usr/bin/env python + +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, You can obtain one at http://mozilla.org/MPL/2.0/. + +""" +tests for mozfile.NamedTemporaryFile +""" + +import mozfile +import os +import unittest + + +class TestNamedTemporaryFile(unittest.TestCase): + """test our fix for NamedTemporaryFile""" + + def test_named_temporary_file(self): + """ Ensure the fix for re-opening a NamedTemporaryFile works + + Refer to https://bugzilla.mozilla.org/show_bug.cgi?id=818777 + and https://bugzilla.mozilla.org/show_bug.cgi?id=821362 + """ + + test_string = "A simple test" + with mozfile.NamedTemporaryFile() as temp: + # Test we can write to file + temp.write(test_string) + # Forced flush, so that we can read later + temp.flush() + + # Test we can open the file again on all platforms + self.assertEqual(open(temp.name).read(), test_string) + + def test_iteration(self): + """ensure the line iterator works""" + + # make a file and write to it + tf = mozfile.NamedTemporaryFile() + notes = ['doe', 'rae', 'mi'] + for note in notes: + tf.write('%s\n' % note) + tf.flush() + + # now read from it + tf.seek(0) + lines = [line.rstrip('\n') for line in tf.readlines()] + self.assertEqual(lines, notes) + + # now read from it iteratively + lines = [] + for line in tf: + lines.append(line.strip()) + self.assertEqual(lines, []) # because we did not seek(0) + tf.seek(0) + lines = [] + for line in tf: + lines.append(line.strip()) + self.assertEqual(lines, notes) + + def test_delete(self): + """ensure ``delete=True/False`` works as expected""" + + # make a deleteable file; ensure it gets cleaned up + path = None + with mozfile.NamedTemporaryFile(delete=True) as tf: + path = tf.name + self.assertTrue(isinstance(path, basestring)) + self.assertFalse(os.path.exists(path)) + + # it is also deleted when __del__ is called + # here we will do so explicitly + tf = mozfile.NamedTemporaryFile(delete=True) + path = tf.name + self.assertTrue(os.path.exists(path)) + del tf + self.assertFalse(os.path.exists(path)) + + # Now the same thing but we won't delete the file + path = None + try: + with mozfile.NamedTemporaryFile(delete=False) as tf: + path = tf.name + self.assertTrue(os.path.exists(path)) + finally: + if path and os.path.exists(path): + os.remove(path) + + path = None + try: + tf = mozfile.NamedTemporaryFile(delete=False) + path = tf.name + self.assertTrue(os.path.exists(path)) + del tf + self.assertTrue(os.path.exists(path)) + finally: + if path and os.path.exists(path): + os.remove(path) + +if __name__ == '__main__': + unittest.main() diff --git a/testing/mozbase/mozfile/tests/test_url.py b/testing/mozbase/mozfile/tests/test_url.py new file mode 100755 index 000000000..7d2b12b39 --- /dev/null +++ b/testing/mozbase/mozfile/tests/test_url.py @@ -0,0 +1,21 @@ +#!/usr/bin/env python + +""" +tests for is_url +""" + +import unittest +from mozfile import is_url + + +class TestIsUrl(unittest.TestCase): + """test the is_url function""" + + def test_is_url(self): + self.assertTrue(is_url('http://mozilla.org')) + self.assertFalse(is_url('/usr/bin/mozilla.org')) + self.assertTrue(is_url('file:///usr/bin/mozilla.org')) + self.assertFalse(is_url('c:\foo\bar')) + +if __name__ == '__main__': + unittest.main() diff --git a/testing/mozbase/mozhttpd/mozhttpd/__init__.py b/testing/mozbase/mozhttpd/mozhttpd/__init__.py new file mode 100644 index 000000000..c15b0d028 --- /dev/null +++ b/testing/mozbase/mozhttpd/mozhttpd/__init__.py @@ -0,0 +1,48 @@ +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this file, +# You can obtain one at http://mozilla.org/MPL/2.0/. + +""" +Mozhttpd is a simple http webserver written in python, designed expressly +for use in automated testing scenarios. It is designed to both serve static +content and provide simple web services. + +The server is based on python standard library modules such as +SimpleHttpServer, urlparse, etc. The ThreadingMixIn is used to +serve each request on a discrete thread. + +Some existing uses of mozhttpd include Peptest_, Eideticker_, and Talos_. + +.. _Peptest: https://github.com/mozilla/peptest/ + +.. _Eideticker: https://github.com/mozilla/eideticker/ + +.. _Talos: http://hg.mozilla.org/build/ + +The following simple example creates a basic HTTP server which serves +content from the current directory, defines a single API endpoint +`/api/resource/` and then serves requests indefinitely: + +:: + + import mozhttpd + + @mozhttpd.handlers.json_response + def resource_get(request, objid): + return (200, { 'id': objid, + 'query': request.query }) + + + httpd = mozhttpd.MozHttpd(port=8080, docroot='.', + urlhandlers = [ { 'method': 'GET', + 'path': '/api/resources/([^/]+)/?', + 'function': resource_get } ]) + print "Serving '%s' at %s:%s" % (httpd.docroot, httpd.host, httpd.port) + httpd.start(block=True) + +""" + +from mozhttpd import MozHttpd, Request, RequestHandler, main +from handlers import json_response + +__all__ = ['MozHttpd', 'Request', 'RequestHandler', 'main', 'json_response'] diff --git a/testing/mozbase/mozhttpd/mozhttpd/handlers.py b/testing/mozbase/mozhttpd/mozhttpd/handlers.py new file mode 100644 index 000000000..1b0a86a40 --- /dev/null +++ b/testing/mozbase/mozhttpd/mozhttpd/handlers.py @@ -0,0 +1,16 @@ +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this file, +# You can obtain one at http://mozilla.org/MPL/2.0/. + +import json + + +def json_response(func): + """ Translates results of 'func' into a JSON response. """ + def wrap(*a, **kw): + (code, data) = func(*a, **kw) + json_data = json.dumps(data) + return (code, {'Content-type': 'application/json', + 'Content-Length': len(json_data)}, json_data) + + return wrap diff --git a/testing/mozbase/mozhttpd/mozhttpd/mozhttpd.py b/testing/mozbase/mozhttpd/mozhttpd/mozhttpd.py new file mode 100755 index 000000000..4ca0847d2 --- /dev/null +++ b/testing/mozbase/mozhttpd/mozhttpd/mozhttpd.py @@ -0,0 +1,330 @@ +#!/usr/bin/env python + +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this file, +# You can obtain one at http://mozilla.org/MPL/2.0/. + +import BaseHTTPServer +import SimpleHTTPServer +import errno +import logging +import threading +import posixpath +import socket +import sys +import os +import urllib +import urlparse +import re +import moznetwork +import time +from SocketServer import ThreadingMixIn + + +class EasyServer(ThreadingMixIn, BaseHTTPServer.HTTPServer): + allow_reuse_address = True + acceptable_errors = (errno.EPIPE, errno.ECONNABORTED) + + def handle_error(self, request, client_address): + error = sys.exc_value + + if ((isinstance(error, socket.error) and + isinstance(error.args, tuple) and + error.args[0] in self.acceptable_errors) + or + (isinstance(error, IOError) and + error.errno in self.acceptable_errors)): + pass # remote hang up before the result is sent + else: + logging.error(error) + + +class Request(object): + """Details of a request.""" + + # attributes from urlsplit that this class also sets + uri_attrs = ('scheme', 'netloc', 'path', 'query', 'fragment') + + def __init__(self, uri, headers, rfile=None): + self.uri = uri + self.headers = headers + parsed = urlparse.urlsplit(uri) + for i, attr in enumerate(self.uri_attrs): + setattr(self, attr, parsed[i]) + try: + body_len = int(self.headers.get('Content-length', 0)) + except ValueError: + body_len = 0 + if body_len and rfile: + self.body = rfile.read(body_len) + else: + self.body = None + + +class RequestHandler(SimpleHTTPServer.SimpleHTTPRequestHandler): + + docroot = os.getcwd() # current working directory at time of import + proxy_host_dirs = False + request_log = [] + log_requests = False + request = None + + def __init__(self, *args, **kwargs): + SimpleHTTPServer.SimpleHTTPRequestHandler.__init__(self, *args, **kwargs) + self.extensions_map['.svg'] = 'image/svg+xml' + + def _try_handler(self, method): + if self.log_requests: + self.request_log.append({'method': method, + 'path': self.request.path, + 'time': time.time()}) + + handlers = [handler for handler in self.urlhandlers + if handler['method'] == method] + for handler in handlers: + m = re.match(handler['path'], self.request.path) + if m: + (response_code, headerdict, data) = \ + handler['function'](self.request, *m.groups()) + self.send_response(response_code) + for (keyword, value) in headerdict.iteritems(): + self.send_header(keyword, value) + self.end_headers() + self.wfile.write(data) + + return True + + return False + + def _find_path(self): + """Find the on-disk path to serve this request from, + using self.path_mappings and self.docroot. + Return (url_path, disk_path).""" + path_components = filter(None, self.request.path.split('/')) + for prefix, disk_path in self.path_mappings.iteritems(): + prefix_components = filter(None, prefix.split('/')) + if len(path_components) < len(prefix_components): + continue + if path_components[:len(prefix_components)] == prefix_components: + return ('/'.join(path_components[len(prefix_components):]), + disk_path) + if self.docroot: + return self.request.path, self.docroot + return None + + def parse_request(self): + retval = SimpleHTTPServer.SimpleHTTPRequestHandler.parse_request(self) + self.request = Request(self.path, self.headers, self.rfile) + return retval + + def do_GET(self): + if not self._try_handler('GET'): + res = self._find_path() + if res: + self.path, self.disk_root = res + # don't include query string and fragment, and prepend + # host directory if required. + if self.request.netloc and self.proxy_host_dirs: + self.path = '/' + self.request.netloc + \ + self.path + SimpleHTTPServer.SimpleHTTPRequestHandler.do_GET(self) + else: + self.send_response(404) + self.end_headers() + self.wfile.write('') + + def do_POST(self): + # if we don't have a match, we always fall through to 404 (this may + # not be "technically" correct if we have a local file at the same + # path as the resource but... meh) + if not self._try_handler('POST'): + self.send_response(404) + self.end_headers() + self.wfile.write('') + + def do_DEL(self): + # if we don't have a match, we always fall through to 404 (this may + # not be "technically" correct if we have a local file at the same + # path as the resource but... meh) + if not self._try_handler('DEL'): + self.send_response(404) + self.end_headers() + self.wfile.write('') + + def translate_path(self, path): + # this is taken from SimpleHTTPRequestHandler.translate_path(), + # except we serve from self.docroot instead of os.getcwd(), and + # parse_request()/do_GET() have already stripped the query string and + # fragment and mangled the path for proxying, if required. + path = posixpath.normpath(urllib.unquote(self.path)) + words = path.split('/') + words = filter(None, words) + path = self.disk_root + for word in words: + drive, word = os.path.splitdrive(word) + head, word = os.path.split(word) + if word in (os.curdir, os.pardir): + continue + path = os.path.join(path, word) + return path + + # I found on my local network that calls to this were timing out + # I believe all of these calls are from log_message + def address_string(self): + return "a.b.c.d" + + # This produces a LOT of noise + def log_message(self, format, *args): + pass + + +class MozHttpd(object): + """ + :param host: Host from which to serve (default 127.0.0.1) + :param port: Port from which to serve (default 8888) + :param docroot: Server root (default os.getcwd()) + :param urlhandlers: Handlers to specify behavior against method and path match (default None) + :param path_mappings: A dict mapping URL prefixes to additional on-disk paths. + :param proxy_host_dirs: Toggle proxy behavior (default False) + :param log_requests: Toggle logging behavior (default False) + + Very basic HTTP server class. Takes a docroot (path on the filesystem) + and a set of urlhandler dictionaries of the form: + + :: + + { + 'method': HTTP method (string): GET, POST, or DEL, + 'path': PATH_INFO (regular expression string), + 'function': function of form fn(arg1, arg2, arg3, ..., request) + } + + and serves HTTP. For each request, MozHttpd will either return a file + off the docroot, or dispatch to a handler function (if both path and + method match). + + Note that one of docroot or urlhandlers may be None (in which case no + local files or handlers, respectively, will be used). If both docroot or + urlhandlers are None then MozHttpd will default to serving just the local + directory. + + MozHttpd also handles proxy requests (i.e. with a full URI on the request + line). By default files are served from docroot according to the request + URI's path component, but if proxy_host_dirs is True, files are served + from //. + + For example, the request "GET http://foo.bar/dir/file.html" would + (assuming no handlers match) serve /dir/file.html if + proxy_host_dirs is False, or /foo.bar/dir/file.html if it is + True. + """ + + def __init__(self, + host="127.0.0.1", + port=0, + docroot=None, + urlhandlers=None, + path_mappings=None, + proxy_host_dirs=False, + log_requests=False): + self.host = host + self.port = int(port) + self.docroot = docroot + if not (urlhandlers or docroot or path_mappings): + self.docroot = os.getcwd() + self.proxy_host_dirs = proxy_host_dirs + self.httpd = None + self.urlhandlers = urlhandlers or [] + self.path_mappings = path_mappings or {} + self.log_requests = log_requests + self.request_log = [] + + class RequestHandlerInstance(RequestHandler): + docroot = self.docroot + urlhandlers = self.urlhandlers + path_mappings = self.path_mappings + proxy_host_dirs = self.proxy_host_dirs + request_log = self.request_log + log_requests = self.log_requests + + self.handler_class = RequestHandlerInstance + + def start(self, block=False): + """ + Starts the server. + + If `block` is True, the call will not return. If `block` is False, the + server will be started on a separate thread that can be terminated by + a call to stop(). + """ + self.httpd = EasyServer((self.host, self.port), self.handler_class) + if block: + self.httpd.serve_forever() + else: + self.server = threading.Thread(target=self.httpd.serve_forever) + self.server.setDaemon(True) # don't hang on exit + self.server.start() + + def stop(self): + """ + Stops the server. + + If the server is not running, this method has no effect. + """ + if self.httpd: + # FIXME: There is no shutdown() method in Python 2.4... + try: + self.httpd.shutdown() + except AttributeError: + pass + self.httpd = None + + def get_url(self, path="/"): + """ + Returns a URL that can be used for accessing the server (e.g. http://192.168.1.3:4321/) + + :param path: Path to append to URL (e.g. if path were /foobar.html you would get a URL like + http://192.168.1.3:4321/foobar.html). Default is `/`. + """ + if not self.httpd: + return None + + return "http://%s:%s%s" % (self.host, self.httpd.server_port, path) + + __del__ = stop + + +def main(args=sys.argv[1:]): + + # parse command line options + from optparse import OptionParser + parser = OptionParser() + parser.add_option('-p', '--port', dest='port', + type="int", default=8888, + help="port to run the server on [DEFAULT: %default]") + parser.add_option('-H', '--host', dest='host', + default='127.0.0.1', + help="host [DEFAULT: %default]") + parser.add_option('-i', '--external-ip', action="store_true", + dest='external_ip', default=False, + help="find and use external ip for host") + parser.add_option('-d', '--docroot', dest='docroot', + default=os.getcwd(), + help="directory to serve files from [DEFAULT: %default]") + options, args = parser.parse_args(args) + if args: + parser.error("mozhttpd does not take any arguments") + + if options.external_ip: + host = moznetwork.get_lan_ip() + else: + host = options.host + + # create the server + server = MozHttpd(host=host, port=options.port, docroot=options.docroot) + + print "Serving '%s' at %s:%s" % (server.docroot, server.host, server.port) + server.start(block=True) + +if __name__ == '__main__': + main() diff --git a/testing/mozbase/mozhttpd/setup.py b/testing/mozbase/mozhttpd/setup.py new file mode 100644 index 000000000..b7799dddd --- /dev/null +++ b/testing/mozbase/mozhttpd/setup.py @@ -0,0 +1,29 @@ +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this file, +# You can obtain one at http://mozilla.org/MPL/2.0/. + +from setuptools import setup + +PACKAGE_VERSION = '0.7' +deps = ['moznetwork >= 0.24'] + +setup(name='mozhttpd', + version=PACKAGE_VERSION, + description="Python webserver intended for use with Mozilla testing", + long_description="see http://mozbase.readthedocs.org/", + classifiers=[], # Get strings from http://pypi.python.org/pypi?%3Aaction=list_classifiers + keywords='mozilla', + author='Mozilla Automation and Testing Team', + author_email='tools@lists.mozilla.org', + url='https://wiki.mozilla.org/Auto-tools/Projects/Mozbase', + license='MPL', + packages=['mozhttpd'], + include_package_data=True, + zip_safe=False, + install_requires=deps, + entry_points=""" + # -*- Entry points: -*- + [console_scripts] + mozhttpd = mozhttpd:main + """, + ) diff --git a/testing/mozbase/mozhttpd/tests/api.py b/testing/mozbase/mozhttpd/tests/api.py new file mode 100644 index 000000000..b785ac5ef --- /dev/null +++ b/testing/mozbase/mozhttpd/tests/api.py @@ -0,0 +1,266 @@ +#!/usr/bin/env python + +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this file, +# You can obtain one at http://mozilla.org/MPL/2.0/. + +import mozfile +import mozhttpd +import urllib2 +import os +import unittest +import json +import tempfile + +here = os.path.dirname(os.path.abspath(__file__)) + + +class ApiTest(unittest.TestCase): + resource_get_called = 0 + resource_post_called = 0 + resource_del_called = 0 + + @mozhttpd.handlers.json_response + def resource_get(self, request, objid): + self.resource_get_called += 1 + return (200, {'called': self.resource_get_called, + 'id': objid, + 'query': request.query}) + + @mozhttpd.handlers.json_response + def resource_post(self, request): + self.resource_post_called += 1 + return (201, {'called': self.resource_post_called, + 'data': json.loads(request.body), + 'query': request.query}) + + @mozhttpd.handlers.json_response + def resource_del(self, request, objid): + self.resource_del_called += 1 + return (200, {'called': self.resource_del_called, + 'id': objid, + 'query': request.query}) + + def get_url(self, path, server_port, querystr): + url = "http://127.0.0.1:%s%s" % (server_port, path) + if querystr: + url += "?%s" % querystr + return url + + def try_get(self, server_port, querystr): + self.resource_get_called = 0 + + f = urllib2.urlopen(self.get_url('/api/resource/1', server_port, querystr)) + try: + self.assertEqual(f.getcode(), 200) + except AttributeError: + pass # python 2.4 + self.assertEqual(json.loads(f.read()), {'called': 1, 'id': str(1), 'query': querystr}) + self.assertEqual(self.resource_get_called, 1) + + def try_post(self, server_port, querystr): + self.resource_post_called = 0 + + postdata = {'hamburgers': '1234'} + try: + f = urllib2.urlopen(self.get_url('/api/resource/', server_port, querystr), + data=json.dumps(postdata)) + except urllib2.HTTPError as e: + # python 2.4 + self.assertEqual(e.code, 201) + body = e.fp.read() + else: + self.assertEqual(f.getcode(), 201) + body = f.read() + self.assertEqual(json.loads(body), {'called': 1, + 'data': postdata, + 'query': querystr}) + self.assertEqual(self.resource_post_called, 1) + + def try_del(self, server_port, querystr): + self.resource_del_called = 0 + + opener = urllib2.build_opener(urllib2.HTTPHandler) + request = urllib2.Request(self.get_url('/api/resource/1', server_port, querystr)) + request.get_method = lambda: 'DEL' + f = opener.open(request) + + try: + self.assertEqual(f.getcode(), 200) + except AttributeError: + pass # python 2.4 + self.assertEqual(json.loads(f.read()), {'called': 1, 'id': str(1), 'query': querystr}) + self.assertEqual(self.resource_del_called, 1) + + def test_api(self): + httpd = mozhttpd.MozHttpd(port=0, + urlhandlers=[{'method': 'GET', + 'path': '/api/resource/([^/]+)/?', + 'function': self.resource_get}, + {'method': 'POST', + 'path': '/api/resource/?', + 'function': self.resource_post}, + {'method': 'DEL', + 'path': '/api/resource/([^/]+)/?', + 'function': self.resource_del} + ]) + httpd.start(block=False) + + server_port = httpd.httpd.server_port + + # GET + self.try_get(server_port, '') + self.try_get(server_port, '?foo=bar') + + # POST + self.try_post(server_port, '') + self.try_post(server_port, '?foo=bar') + + # DEL + self.try_del(server_port, '') + self.try_del(server_port, '?foo=bar') + + # GET: By default we don't serve any files if we just define an API + exception_thrown = False + try: + urllib2.urlopen(self.get_url('/', server_port, None)) + except urllib2.HTTPError as e: + self.assertEqual(e.code, 404) + exception_thrown = True + self.assertTrue(exception_thrown) + + def test_nonexistent_resources(self): + # Create a server with a placeholder handler so we don't fall back + # to serving local files + httpd = mozhttpd.MozHttpd(port=0) + httpd.start(block=False) + server_port = httpd.httpd.server_port + + # GET: Return 404 for non-existent endpoint + exception_thrown = False + try: + urllib2.urlopen(self.get_url('/api/resource/', server_port, None)) + except urllib2.HTTPError as e: + self.assertEqual(e.code, 404) + exception_thrown = True + self.assertTrue(exception_thrown) + + # POST: POST should also return 404 + exception_thrown = False + try: + urllib2.urlopen(self.get_url('/api/resource/', server_port, None), + data=json.dumps({})) + except urllib2.HTTPError as e: + self.assertEqual(e.code, 404) + exception_thrown = True + self.assertTrue(exception_thrown) + + # DEL: DEL should also return 404 + exception_thrown = False + try: + opener = urllib2.build_opener(urllib2.HTTPHandler) + request = urllib2.Request(self.get_url('/api/resource/', server_port, + None)) + request.get_method = lambda: 'DEL' + opener.open(request) + except urllib2.HTTPError: + self.assertEqual(e.code, 404) + exception_thrown = True + self.assertTrue(exception_thrown) + + def test_api_with_docroot(self): + httpd = mozhttpd.MozHttpd(port=0, docroot=here, + urlhandlers=[{'method': 'GET', + 'path': '/api/resource/([^/]+)/?', + 'function': self.resource_get}]) + httpd.start(block=False) + server_port = httpd.httpd.server_port + + # We defined a docroot, so we expect a directory listing + f = urllib2.urlopen(self.get_url('/', server_port, None)) + try: + self.assertEqual(f.getcode(), 200) + except AttributeError: + pass # python 2.4 + self.assertTrue('Directory listing for' in f.read()) + + # Make sure API methods still work + self.try_get(server_port, '') + self.try_get(server_port, '?foo=bar') + + +class ProxyTest(unittest.TestCase): + + def tearDown(self): + # reset proxy opener in case it changed + urllib2.install_opener(None) + + def test_proxy(self): + docroot = tempfile.mkdtemp() + self.addCleanup(mozfile.remove, docroot) + hosts = ('mozilla.com', 'mozilla.org') + unproxied_host = 'notmozilla.org' + + def url(host): return 'http://%s/' % host + + index_filename = 'index.html' + + def index_contents(host): return '%s index' % host + + index = file(os.path.join(docroot, index_filename), 'w') + index.write(index_contents('*')) + index.close() + + httpd = mozhttpd.MozHttpd(port=0, docroot=docroot) + httpd.start(block=False) + server_port = httpd.httpd.server_port + + proxy_support = urllib2.ProxyHandler({'http': 'http://127.0.0.1:%d' % + server_port}) + urllib2.install_opener(urllib2.build_opener(proxy_support)) + + for host in hosts: + f = urllib2.urlopen(url(host)) + try: + self.assertEqual(f.getcode(), 200) + except AttributeError: + pass # python 2.4 + self.assertEqual(f.read(), index_contents('*')) + + httpd.stop() + + # test separate directories per host + + httpd = mozhttpd.MozHttpd(port=0, docroot=docroot, proxy_host_dirs=True) + httpd.start(block=False) + server_port = httpd.httpd.server_port + + proxy_support = urllib2.ProxyHandler({'http': 'http://127.0.0.1:%d' % + server_port}) + urllib2.install_opener(urllib2.build_opener(proxy_support)) + + # set up dirs + for host in hosts: + os.mkdir(os.path.join(docroot, host)) + file(os.path.join(docroot, host, index_filename), 'w') \ + .write(index_contents(host)) + + for host in hosts: + f = urllib2.urlopen(url(host)) + try: + self.assertEqual(f.getcode(), 200) + except AttributeError: + pass # python 2.4 + self.assertEqual(f.read(), index_contents(host)) + + exc = None + try: + urllib2.urlopen(url(unproxied_host)) + except urllib2.HTTPError as e: + exc = e + self.assertNotEqual(exc, None) + self.assertEqual(exc.code, 404) + + +if __name__ == '__main__': + unittest.main() diff --git a/testing/mozbase/mozhttpd/tests/baseurl.py b/testing/mozbase/mozhttpd/tests/baseurl.py new file mode 100644 index 000000000..0e971e6b2 --- /dev/null +++ b/testing/mozbase/mozhttpd/tests/baseurl.py @@ -0,0 +1,19 @@ +import mozhttpd +import unittest + + +class BaseUrlTest(unittest.TestCase): + + def test_base_url(self): + httpd = mozhttpd.MozHttpd(port=0) + self.assertEqual(httpd.get_url(), None) + httpd.start(block=False) + self.assertEqual("http://127.0.0.1:%s/" % httpd.httpd.server_port, + httpd.get_url()) + self.assertEqual("http://127.0.0.1:%s/cheezburgers.html" % + httpd.httpd.server_port, + httpd.get_url(path="/cheezburgers.html")) + httpd.stop() + +if __name__ == '__main__': + unittest.main() diff --git a/testing/mozbase/mozhttpd/tests/basic.py b/testing/mozbase/mozhttpd/tests/basic.py new file mode 100644 index 000000000..8d64b4332 --- /dev/null +++ b/testing/mozbase/mozhttpd/tests/basic.py @@ -0,0 +1,46 @@ +#!/usr/bin/env python + +import mozhttpd +import mozfile +import os +import tempfile +import unittest + + +class TestBasic(unittest.TestCase): + """ Test basic Mozhttpd capabilites """ + + def test_basic(self): + """ Test mozhttpd can serve files """ + + tempdir = tempfile.mkdtemp() + + # sizes is a dict of the form: name -> [size, binary_string, filepath] + sizes = {'small': [128], 'large': [16384]} + + for k in sizes.keys(): + # Generate random binary string + sizes[k].append(os.urandom(sizes[k][0])) + + # Add path of file with binary string to list + fpath = os.path.join(tempdir, k) + sizes[k].append(fpath) + + # Write binary string to file + with open(fpath, 'wb') as f: + f.write(sizes[k][1]) + + server = mozhttpd.MozHttpd(docroot=tempdir) + server.start() + server_url = server.get_url() + + # Retrieve file and check contents matchup + for k in sizes.keys(): + retrieved_content = mozfile.load(server_url + k).read() + self.assertEqual(retrieved_content, sizes[k][1]) + + # Cleanup tempdir and related files + mozfile.rmtree(tempdir) + +if __name__ == '__main__': + unittest.main() diff --git a/testing/mozbase/mozhttpd/tests/filelisting.py b/testing/mozbase/mozhttpd/tests/filelisting.py new file mode 100644 index 000000000..6abea757f --- /dev/null +++ b/testing/mozbase/mozhttpd/tests/filelisting.py @@ -0,0 +1,43 @@ +#!/usr/bin/env python + +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this file, +# You can obtain one at http://mozilla.org/MPL/2.0/. + +import mozhttpd +import urllib2 +import os +import unittest +import re + +here = os.path.dirname(os.path.abspath(__file__)) + + +class FileListingTest(unittest.TestCase): + + def check_filelisting(self, path=''): + filelist = os.listdir(here) + + httpd = mozhttpd.MozHttpd(port=0, docroot=here) + httpd.start(block=False) + f = urllib2.urlopen("http://%s:%s/%s" % ('127.0.0.1', httpd.httpd.server_port, path)) + for line in f.readlines(): + webline = re.sub('\<[a-zA-Z0-9\-\_\.\=\"\'\/\\\%\!\@\#\$\^\&\*\(\) ]*\>', + '', line.strip('\n')).strip('/').strip().strip('@') + + if webline and not webline.startswith("Directory listing for"): + self.assertTrue(webline in filelist, + "File %s in dir listing corresponds to a file" % webline) + filelist.remove(webline) + self.assertFalse( + filelist, "Should have no items in filelist (%s) unaccounted for" % filelist) + + def test_filelist(self): + self.check_filelisting() + + def test_filelist_params(self): + self.check_filelisting('?foo=bar&fleem=&foo=fleem') + + +if __name__ == '__main__': + unittest.main() diff --git a/testing/mozbase/mozhttpd/tests/manifest.ini b/testing/mozbase/mozhttpd/tests/manifest.ini new file mode 100644 index 000000000..3f3d42d9b --- /dev/null +++ b/testing/mozbase/mozhttpd/tests/manifest.ini @@ -0,0 +1,6 @@ +[api.py] +[baseurl.py] +[basic.py] +[filelisting.py] +[paths.py] +[requestlog.py] diff --git a/testing/mozbase/mozhttpd/tests/paths.py b/testing/mozbase/mozhttpd/tests/paths.py new file mode 100644 index 000000000..45ae40144 --- /dev/null +++ b/testing/mozbase/mozhttpd/tests/paths.py @@ -0,0 +1,77 @@ +#!/usr/bin/env python + +# Any copyright is dedicated to the Public Domain. +# http://creativecommons.org/publicdomain/zero/1.0/ + +from mozfile import TemporaryDirectory +import mozhttpd +import os +import unittest +import urllib2 + + +class PathTest(unittest.TestCase): + + def try_get(self, url, expected_contents): + f = urllib2.urlopen(url) + self.assertEqual(f.getcode(), 200) + self.assertEqual(f.read(), expected_contents) + + def try_get_expect_404(self, url): + with self.assertRaises(urllib2.HTTPError) as cm: + urllib2.urlopen(url) + self.assertEqual(404, cm.exception.code) + + def test_basic(self): + """Test that requests to docroot and a path mapping work as expected.""" + with TemporaryDirectory() as d1, TemporaryDirectory() as d2: + open(os.path.join(d1, "test1.txt"), "w").write("test 1 contents") + open(os.path.join(d2, "test2.txt"), "w").write("test 2 contents") + httpd = mozhttpd.MozHttpd(port=0, + docroot=d1, + path_mappings={'/files': d2} + ) + httpd.start(block=False) + self.try_get(httpd.get_url("/test1.txt"), "test 1 contents") + self.try_get(httpd.get_url("/files/test2.txt"), "test 2 contents") + self.try_get_expect_404(httpd.get_url("/files/test2_nope.txt")) + httpd.stop() + + def test_substring_mappings(self): + """Test that a path mapping that's a substring of another works.""" + with TemporaryDirectory() as d1, TemporaryDirectory() as d2: + open(os.path.join(d1, "test1.txt"), "w").write("test 1 contents") + open(os.path.join(d2, "test2.txt"), "w").write("test 2 contents") + httpd = mozhttpd.MozHttpd(port=0, + path_mappings={'/abcxyz': d1, + '/abc': d2, } + ) + httpd.start(block=False) + self.try_get(httpd.get_url("/abcxyz/test1.txt"), "test 1 contents") + self.try_get(httpd.get_url("/abc/test2.txt"), "test 2 contents") + httpd.stop() + + def test_multipart_path_mapping(self): + """Test that a path mapping with multiple directories works.""" + with TemporaryDirectory() as d1: + open(os.path.join(d1, "test1.txt"), "w").write("test 1 contents") + httpd = mozhttpd.MozHttpd(port=0, + path_mappings={'/abc/def/ghi': d1} + ) + httpd.start(block=False) + self.try_get(httpd.get_url("/abc/def/ghi/test1.txt"), "test 1 contents") + self.try_get_expect_404(httpd.get_url("/abc/test1.txt")) + self.try_get_expect_404(httpd.get_url("/abc/def/test1.txt")) + httpd.stop() + + def test_no_docroot(self): + """Test that path mappings with no docroot work.""" + with TemporaryDirectory() as d1: + httpd = mozhttpd.MozHttpd(port=0, + path_mappings={'/foo': d1}) + httpd.start(block=False) + self.try_get_expect_404(httpd.get_url()) + httpd.stop() + +if __name__ == '__main__': + unittest.main() diff --git a/testing/mozbase/mozhttpd/tests/requestlog.py b/testing/mozbase/mozhttpd/tests/requestlog.py new file mode 100644 index 000000000..bf2c59ec3 --- /dev/null +++ b/testing/mozbase/mozhttpd/tests/requestlog.py @@ -0,0 +1,41 @@ +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this file, +# You can obtain one at http://mozilla.org/MPL/2.0/. + +import mozhttpd +import urllib2 +import os +import unittest + +here = os.path.dirname(os.path.abspath(__file__)) + + +class RequestLogTest(unittest.TestCase): + + def check_logging(self, log_requests=False): + + httpd = mozhttpd.MozHttpd(port=0, docroot=here, log_requests=log_requests) + httpd.start(block=False) + url = "http://%s:%s/" % ('127.0.0.1', httpd.httpd.server_port) + f = urllib2.urlopen(url) + f.read() + + return httpd.request_log + + def test_logging_enabled(self): + request_log = self.check_logging(log_requests=True) + + self.assertEqual(len(request_log), 1) + + log_entry = request_log[0] + self.assertEqual(log_entry['method'], 'GET') + self.assertEqual(log_entry['path'], '/') + self.assertEqual(type(log_entry['time']), float) + + def test_logging_disabled(self): + request_log = self.check_logging(log_requests=False) + + self.assertEqual(len(request_log), 0) + +if __name__ == '__main__': + unittest.main() diff --git a/testing/mozbase/mozinfo/mozinfo/__init__.py b/testing/mozbase/mozinfo/mozinfo/__init__.py new file mode 100644 index 000000000..7d0483cb5 --- /dev/null +++ b/testing/mozbase/mozinfo/mozinfo/__init__.py @@ -0,0 +1,60 @@ +# flake8: noqa +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this file, +# You can obtain one at http://mozilla.org/MPL/2.0/. + +from __future__ import absolute_import + +""" +interface to transform introspected system information to a format palatable to +Mozilla + +Module variables: + +.. attribute:: bits + + 32 or 64 + +.. attribute:: isBsd + + Returns ``True`` if the operating system is BSD + +.. attribute:: isLinux + + Returns ``True`` if the operating system is Linux + +.. attribute:: isMac + + Returns ``True`` if the operating system is Mac + +.. attribute:: isWin + + Returns ``True`` if the operating system is Windows + +.. attribute:: os + + Operating system [``'win'``, ``'mac'``, ``'linux'``, ...] + +.. attribute:: processor + + Processor architecture [``'x86'``, ``'x86_64'``, ``'ppc'``, ...] + +.. attribute:: version + + Operating system version string. For windows, the service pack information is also included + +.. attribute:: info + + Returns information identifying the current system. + + * :attr:`bits` + * :attr:`os` + * :attr:`processor` + * :attr:`version` + +""" + +from . import mozinfo +from .mozinfo import * + +__all__ = mozinfo.__all__ diff --git a/testing/mozbase/mozinfo/mozinfo/mozinfo.py b/testing/mozbase/mozinfo/mozinfo/mozinfo.py new file mode 100755 index 000000000..81a30307d --- /dev/null +++ b/testing/mozbase/mozinfo/mozinfo/mozinfo.py @@ -0,0 +1,300 @@ +#!/usr/bin/env python + +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this file, +# You can obtain one at http://mozilla.org/MPL/2.0/. + +# TODO: it might be a good idea of adding a system name (e.g. 'Ubuntu' for +# linux) to the information; I certainly wouldn't want anyone parsing this +# information and having behaviour depend on it + +from __future__ import absolute_import + +import os +import platform +import re +import sys +from .string_version import StringVersion + + +# keep a copy of the os module since updating globals overrides this +_os = os + + +class unknown(object): + """marker class for unknown information""" + + def __nonzero__(self): + return False + + def __str__(self): + return 'UNKNOWN' +unknown = unknown() # singleton + + +def get_windows_version(): + import ctypes + + class OSVERSIONINFOEXW(ctypes.Structure): + _fields_ = [('dwOSVersionInfoSize', ctypes.c_ulong), + ('dwMajorVersion', ctypes.c_ulong), + ('dwMinorVersion', ctypes.c_ulong), + ('dwBuildNumber', ctypes.c_ulong), + ('dwPlatformId', ctypes.c_ulong), + ('szCSDVersion', ctypes.c_wchar * 128), + ('wServicePackMajor', ctypes.c_ushort), + ('wServicePackMinor', ctypes.c_ushort), + ('wSuiteMask', ctypes.c_ushort), + ('wProductType', ctypes.c_byte), + ('wReserved', ctypes.c_byte)] + + os_version = OSVERSIONINFOEXW() + os_version.dwOSVersionInfoSize = ctypes.sizeof(os_version) + retcode = ctypes.windll.Ntdll.RtlGetVersion(ctypes.byref(os_version)) + if retcode != 0: + raise OSError + + return os_version.dwMajorVersion, os_version.dwMinorVersion, os_version.dwBuildNumber + +# get system information +info = {'os': unknown, + 'processor': unknown, + 'version': unknown, + 'os_version': unknown, + 'bits': unknown, + 'has_sandbox': unknown} +(system, node, release, version, machine, processor) = platform.uname() +(bits, linkage) = platform.architecture() + +# get os information and related data +if system in ["Microsoft", "Windows"]: + info['os'] = 'win' + # There is a Python bug on Windows to determine platform values + # http://bugs.python.org/issue7860 + if "PROCESSOR_ARCHITEW6432" in os.environ: + processor = os.environ.get("PROCESSOR_ARCHITEW6432", processor) + else: + processor = os.environ.get('PROCESSOR_ARCHITECTURE', processor) + system = os.environ.get("OS", system).replace('_', ' ') + (major, minor, _, _, service_pack) = os.sys.getwindowsversion() + info['service_pack'] = service_pack + if major >= 6 and minor >= 2: + # On windows >= 8.1 the system call that getwindowsversion uses has + # been frozen to always return the same values. In this case we call + # the RtlGetVersion API directly, which still provides meaningful + # values, at least for now. + major, minor, build_number = get_windows_version() + version = "%d.%d.%d" % (major, minor, build_number) + + os_version = "%d.%d" % (major, minor) +elif system.startswith('MINGW'): + # windows/mingw python build (msys) + info['os'] = 'win' + os_version = version = unknown +elif system == "Linux": + if hasattr(platform, "linux_distribution"): + (distro, os_version, codename) = platform.linux_distribution() + else: + (distro, os_version, codename) = platform.dist() + if not processor: + processor = machine + version = "%s %s" % (distro, os_version) + + # Bug in Python 2's `platform` library: + # It will return a triple of empty strings if the distribution is not supported. + # It works on Python 3. If we don't have an OS version, + # the unit tests fail to run. + if not distro and not os_version and not codename: + distro = 'lfs' + version = release + os_version = release + + info['os'] = 'linux' + info['linux_distro'] = distro +elif system in ['DragonFly', 'FreeBSD', 'NetBSD', 'OpenBSD']: + info['os'] = 'bsd' + version = os_version = sys.platform +elif system == "Darwin": + (release, versioninfo, machine) = platform.mac_ver() + version = "OS X %s" % release + versionNums = release.split('.')[:2] + os_version = "%s.%s" % (versionNums[0], versionNums[1]) + info['os'] = 'mac' +elif sys.platform in ('solaris', 'sunos5'): + info['os'] = 'unix' + os_version = version = sys.platform +else: + os_version = version = unknown + +info['version'] = version +info['os_version'] = StringVersion(os_version) + +# processor type and bits +if processor in ["i386", "i686"]: + if bits == "32bit": + processor = "x86" + elif bits == "64bit": + processor = "x86_64" +elif processor.upper() == "AMD64": + bits = "64bit" + processor = "x86_64" +elif processor == "Power Macintosh": + processor = "ppc" +bits = re.search('(\d+)bit', bits).group(1) +info.update({'processor': processor, + 'bits': int(bits), + }) + +if info['os'] == 'linux': + import ctypes + import errno + PR_SET_SECCOMP = 22 + SECCOMP_MODE_FILTER = 2 + ctypes.CDLL("libc.so.6", use_errno=True).prctl(PR_SET_SECCOMP, SECCOMP_MODE_FILTER, 0) + info['has_sandbox'] = ctypes.get_errno() == errno.EFAULT +else: + info['has_sandbox'] = True + +# standard value of choices, for easy inspection +choices = {'os': ['linux', 'bsd', 'win', 'mac', 'unix'], + 'bits': [32, 64], + 'processor': ['x86', 'x86_64', 'ppc']} + + +def sanitize(info): + """Do some sanitization of input values, primarily + to handle universal Mac builds.""" + if "processor" in info and info["processor"] == "universal-x86-x86_64": + # If we're running on OS X 10.6 or newer, assume 64-bit + if release[:4] >= "10.6": # Note this is a string comparison + info["processor"] = "x86_64" + info["bits"] = 64 + else: + info["processor"] = "x86" + info["bits"] = 32 + +# method for updating information + + +def update(new_info): + """ + Update the info. + + :param new_info: Either a dict containing the new info or a path/url + to a json file containing the new info. + """ + + if isinstance(new_info, basestring): + # lazy import + import mozfile + import json + f = mozfile.load(new_info) + new_info = json.loads(f.read()) + f.close() + + info.update(new_info) + sanitize(info) + globals().update(info) + + # convenience data for os access + for os_name in choices['os']: + globals()['is' + os_name.title()] = info['os'] == os_name + # unix is special + if isLinux or isBsd: # noqa + globals()['isUnix'] = True + + +def find_and_update_from_json(*dirs): + """ + Find a mozinfo.json file, load it, and update the info with the + contents. + + :param dirs: Directories in which to look for the file. They will be + searched after first looking in the root of the objdir + if the current script is being run from a Mozilla objdir. + + Returns the full path to mozinfo.json if it was found, or None otherwise. + """ + # First, see if we're in an objdir + try: + from mozbuild.base import MozbuildObject, BuildEnvironmentNotFoundException + build = MozbuildObject.from_environment() + json_path = _os.path.join(build.topobjdir, "mozinfo.json") + if _os.path.isfile(json_path): + update(json_path) + return json_path + except ImportError: + pass + except BuildEnvironmentNotFoundException: + pass + + for d in dirs: + d = _os.path.abspath(d) + json_path = _os.path.join(d, "mozinfo.json") + if _os.path.isfile(json_path): + update(json_path) + return json_path + + return None + + +def output_to_file(path): + import json + with open(path, 'w') as f: + f.write(json.dumps(info)) + +update({}) + +# exports +__all__ = info.keys() +__all__ += ['is' + os_name.title() for os_name in choices['os']] +__all__ += [ + 'info', + 'unknown', + 'main', + 'choices', + 'update', + 'find_and_update_from_json', + 'output_to_file', + 'StringVersion', +] + + +def main(args=None): + + # parse the command line + from optparse import OptionParser + parser = OptionParser(description=__doc__) + for key in choices: + parser.add_option('--%s' % key, dest=key, + action='store_true', default=False, + help="display choices for %s" % key) + options, args = parser.parse_args() + + # args are JSON blobs to override info + if args: + # lazy import + import json + for arg in args: + if _os.path.exists(arg): + string = file(arg).read() + else: + string = arg + update(json.loads(string)) + + # print out choices if requested + flag = False + for key, value in options.__dict__.items(): + if value is True: + print '%s choices: %s' % (key, ' '.join([str(choice) + for choice in choices[key]])) + flag = True + if flag: + return + + # otherwise, print out all info + for key, value in info.items(): + print '%s: %s' % (key, value) + +if __name__ == '__main__': + main() diff --git a/testing/mozbase/mozinfo/mozinfo/string_version.py b/testing/mozbase/mozinfo/mozinfo/string_version.py new file mode 100644 index 000000000..fd77fa566 --- /dev/null +++ b/testing/mozbase/mozinfo/mozinfo/string_version.py @@ -0,0 +1,43 @@ +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this file, +# You can obtain one at http://mozilla.org/MPL/2.0/. + +from distutils.version import LooseVersion + + +class StringVersion(str): + """ + A string version that can be compared with comparison operators. + """ + + def __init__(self, vstring): + str.__init__(self, vstring) + self.version = LooseVersion(vstring) + + def __repr__(self): + return "StringVersion ('%s')" % self + + def __to_version(self, other): + if not isinstance(other, StringVersion): + other = StringVersion(other) + return other.version + + # rich comparison methods + + def __lt__(self, other): + return self.version < self.__to_version(other) + + def __le__(self, other): + return self.version <= self.__to_version(other) + + def __eq__(self, other): + return self.version == self.__to_version(other) + + def __ne__(self, other): + return self.version != self.__to_version(other) + + def __gt__(self, other): + return self.version > self.__to_version(other) + + def __ge__(self, other): + return self.version >= self.__to_version(other) diff --git a/testing/mozbase/mozinfo/setup.py b/testing/mozbase/mozinfo/setup.py new file mode 100644 index 000000000..3e76b9db4 --- /dev/null +++ b/testing/mozbase/mozinfo/setup.py @@ -0,0 +1,31 @@ +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this file, +# You can obtain one at http://mozilla.org/MPL/2.0/. + +from setuptools import setup + +PACKAGE_VERSION = '0.9' + +# dependencies +deps = ['mozfile >= 0.12'] + +setup(name='mozinfo', + version=PACKAGE_VERSION, + description="Library to get system information for use in Mozilla testing", + long_description="see http://mozbase.readthedocs.org", + classifiers=[], # Get strings from http://pypi.python.org/pypi?%3Aaction=list_classifiers + keywords='mozilla', + author='Mozilla Automation and Testing Team', + author_email='tools@lists.mozilla.org', + url='https://wiki.mozilla.org/Auto-tools/Projects/Mozbase', + license='MPL', + packages=['mozinfo'], + include_package_data=True, + zip_safe=False, + install_requires=deps, + entry_points=""" + # -*- Entry points: -*- + [console_scripts] + mozinfo = mozinfo:main + """, + ) diff --git a/testing/mozbase/mozinfo/tests/manifest.ini b/testing/mozbase/mozinfo/tests/manifest.ini new file mode 100644 index 000000000..528fdea7b --- /dev/null +++ b/testing/mozbase/mozinfo/tests/manifest.ini @@ -0,0 +1 @@ +[test.py] diff --git a/testing/mozbase/mozinfo/tests/test.py b/testing/mozbase/mozinfo/tests/test.py new file mode 100644 index 000000000..b9457cff9 --- /dev/null +++ b/testing/mozbase/mozinfo/tests/test.py @@ -0,0 +1,121 @@ +#!/usr/bin/env python +# +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this file, +# You can obtain one at http://mozilla.org/MPL/2.0/. + +import json +import mock +import os +import shutil +import sys +import tempfile +import unittest +import mozinfo + + +class TestMozinfo(unittest.TestCase): + + def setUp(self): + reload(mozinfo) + self.tempdir = os.path.abspath(tempfile.mkdtemp()) + + # When running from an objdir mozinfo will use a build generated json file + # instead of the ones created for testing. Prevent that from happening. + # See bug 896038 for details. + sys.modules['mozbuild'] = None + + def tearDown(self): + shutil.rmtree(self.tempdir) + del sys.modules['mozbuild'] + + def test_basic(self): + """Test that mozinfo has a few attributes.""" + self.assertNotEqual(mozinfo.os, None) + # should have isFoo == True where os == "foo" + self.assertTrue(getattr(mozinfo, "is" + mozinfo.os[0].upper() + mozinfo.os[1:])) + + def test_update(self): + """Test that mozinfo.update works.""" + mozinfo.update({"foo": 123}) + self.assertEqual(mozinfo.info["foo"], 123) + + def test_update_file(self): + """Test that mozinfo.update can load a JSON file.""" + j = os.path.join(self.tempdir, "mozinfo.json") + with open(j, "w") as f: + f.write(json.dumps({"foo": "xyz"})) + mozinfo.update(j) + self.assertEqual(mozinfo.info["foo"], "xyz") + + def test_update_file_invalid_json(self): + """Test that mozinfo.update handles invalid JSON correctly""" + j = os.path.join(self.tempdir, 'test.json') + with open(j, 'w') as f: + f.write('invalid{"json":') + self.assertRaises(ValueError, mozinfo.update, [j]) + + def test_find_and_update_file(self): + """Test that mozinfo.find_and_update_from_json can + find mozinfo.json in a directory passed to it.""" + j = os.path.join(self.tempdir, "mozinfo.json") + with open(j, "w") as f: + f.write(json.dumps({"foo": "abcdefg"})) + self.assertEqual(mozinfo.find_and_update_from_json(self.tempdir), j) + self.assertEqual(mozinfo.info["foo"], "abcdefg") + + def test_find_and_update_file_invalid_json(self): + """Test that mozinfo.find_and_update_from_json can + handle invalid JSON""" + j = os.path.join(self.tempdir, "mozinfo.json") + with open(j, 'w') as f: + f.write('invalid{"json":') + self.assertRaises(ValueError, mozinfo.find_and_update_from_json, self.tempdir) + + def test_find_and_update_file_mozbuild(self): + """Test that mozinfo.find_and_update_from_json can + find mozinfo.json using the mozbuild module.""" + j = os.path.join(self.tempdir, "mozinfo.json") + with open(j, "w") as f: + f.write(json.dumps({"foo": "123456"})) + m = mock.MagicMock() + # Mock the value of MozbuildObject.from_environment().topobjdir. + m.MozbuildObject.from_environment.return_value.topobjdir = self.tempdir + with mock.patch.dict(sys.modules, {"mozbuild": m, "mozbuild.base": m}): + self.assertEqual(mozinfo.find_and_update_from_json(), j) + self.assertEqual(mozinfo.info["foo"], "123456") + + def test_output_to_file(self): + """Test that mozinfo.output_to_file works.""" + path = os.path.join(self.tempdir, "mozinfo.json") + mozinfo.output_to_file(path) + self.assertEqual(open(path).read(), json.dumps(mozinfo.info)) + + +class TestStringVersion(unittest.TestCase): + + def test_os_version_is_a_StringVersion(self): + self.assertIsInstance(mozinfo.os_version, mozinfo.StringVersion) + + def test_compare_to_string(self): + version = mozinfo.StringVersion('10.10') + + self.assertGreater(version, '10.2') + self.assertGreater('11', version) + self.assertGreaterEqual(version, '10.10') + self.assertGreaterEqual('10.11', version) + self.assertEqual(version, '10.10') + self.assertEqual('10.10', version) + self.assertNotEqual(version, '10.2') + self.assertNotEqual('11', version) + self.assertLess(version, '11.8.5') + self.assertLess('10.2', version) + self.assertLessEqual(version, '11') + self.assertLessEqual('10.10', version) + + def test_to_string(self): + self.assertEqual('10.10', str(mozinfo.StringVersion('10.10'))) + + +if __name__ == '__main__': + unittest.main() diff --git a/testing/mozbase/mozinstall/mozinstall/__init__.py b/testing/mozbase/mozinstall/mozinstall/__init__.py new file mode 100644 index 000000000..5f96b7fac --- /dev/null +++ b/testing/mozbase/mozinstall/mozinstall/__init__.py @@ -0,0 +1,6 @@ +# flake8: noqa +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this file, +# You can obtain one at http://mozilla.org/MPL/2.0/. + +from mozinstall import * diff --git a/testing/mozbase/mozinstall/mozinstall/mozinstall.py b/testing/mozbase/mozinstall/mozinstall/mozinstall.py new file mode 100755 index 000000000..b4c6f95f7 --- /dev/null +++ b/testing/mozbase/mozinstall/mozinstall/mozinstall.py @@ -0,0 +1,342 @@ +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this file, +# You can obtain one at http://mozilla.org/MPL/2.0/. + +from optparse import OptionParser +import os +import shutil +import subprocess +import sys +import tarfile +import time +import zipfile + +import mozfile +import mozinfo + +try: + import pefile + has_pefile = True +except ImportError: + has_pefile = False + +if mozinfo.isMac: + from plistlib import readPlist + + +TIMEOUT_UNINSTALL = 60 + + +class InstallError(Exception): + """Thrown when installation fails. Includes traceback if available.""" + + +class InvalidBinary(Exception): + """Thrown when the binary cannot be found after the installation.""" + + +class InvalidSource(Exception): + """Thrown when the specified source is not a recognized file type. + + Supported types: + Linux: tar.gz, tar.bz2 + Mac: dmg + Windows: zip, exe + + """ + + +class UninstallError(Exception): + """Thrown when uninstallation fails. Includes traceback if available.""" + + +def get_binary(path, app_name): + """Find the binary in the specified path, and return its path. If binary is + not found throw an InvalidBinary exception. + + :param path: Path within to search for the binary + :param app_name: Application binary without file extension to look for + """ + binary = None + + # On OS X we can get the real binary from the app bundle + if mozinfo.isMac: + plist = '%s/Contents/Info.plist' % path + if not os.path.isfile(plist): + raise InvalidBinary('%s/Contents/Info.plist not found' % path) + + binary = os.path.join(path, 'Contents/MacOS/', + readPlist(plist)['CFBundleExecutable']) + + else: + app_name = app_name.lower() + + if mozinfo.isWin: + app_name = app_name + '.exe' + + for root, dirs, files in os.walk(path): + for filename in files: + # os.access evaluates to False for some reason, so not using it + if filename.lower() == app_name: + binary = os.path.realpath(os.path.join(root, filename)) + break + + if not binary: + # The expected binary has not been found. + raise InvalidBinary('"%s" does not contain a valid binary.' % path) + + return binary + + +def install(src, dest): + """Install a zip, exe, tar.gz, tar.bz2 or dmg file, and return the path of + the installation folder. + + :param src: Path to the install file + :param dest: Path to install to (to ensure we do not overwrite any existent + files the folder should not exist yet) + """ + src = os.path.realpath(src) + dest = os.path.realpath(dest) + + if not is_installer(src): + raise InvalidSource(src + ' is not valid installer file.') + + did_we_create = False + if not os.path.exists(dest): + did_we_create = True + os.makedirs(dest) + + trbk = None + try: + install_dir = None + if src.lower().endswith('.dmg'): + install_dir = _install_dmg(src, dest) + elif src.lower().endswith('.exe'): + install_dir = _install_exe(src, dest) + elif zipfile.is_zipfile(src) or tarfile.is_tarfile(src): + install_dir = mozfile.extract(src, dest)[0] + + return install_dir + + except: + cls, exc, trbk = sys.exc_info() + if did_we_create: + try: + # try to uninstall this properly + uninstall(dest) + except: + # uninstall may fail, let's just try to clean the folder + # in this case + try: + mozfile.remove(dest) + except: + pass + if issubclass(cls, Exception): + error = InstallError('Failed to install "%s (%s)"' % (src, str(exc))) + raise InstallError, error, trbk + # any other kind of exception like KeyboardInterrupt is just re-raised. + raise cls, exc, trbk + + finally: + # trbk won't get GC'ed due to circular reference + # http://docs.python.org/library/sys.html#sys.exc_info + del trbk + + +def is_installer(src): + """Tests if the given file is a valid installer package. + + Supported types: + Linux: tar.gz, tar.bz2 + Mac: dmg + Windows: zip, exe + + On Windows pefile will be used to determine if the executable is the + right type, if it is installed on the system. + + :param src: Path to the install file. + """ + src = os.path.realpath(src) + + if not os.path.isfile(src): + return False + + if mozinfo.isLinux: + return tarfile.is_tarfile(src) + elif mozinfo.isMac: + return src.lower().endswith('.dmg') + elif mozinfo.isWin: + if zipfile.is_zipfile(src): + return True + + if os.access(src, os.X_OK) and src.lower().endswith('.exe'): + if has_pefile: + # try to determine if binary is actually a gecko installer + pe_data = pefile.PE(src) + data = {} + for info in getattr(pe_data, 'FileInfo', []): + if info.Key == 'StringFileInfo': + for string in info.StringTable: + data.update(string.entries) + return 'BuildID' not in data + else: + # pefile not available, just assume a proper binary was passed in + return True + + return False + + +def uninstall(install_folder): + """Uninstalls the application in the specified path. If it has been + installed via an installer on Windows, use the uninstaller first. + + :param install_folder: Path of the installation folder + + """ + install_folder = os.path.realpath(install_folder) + assert os.path.isdir(install_folder), \ + 'installation folder "%s" exists.' % install_folder + + # On Windows we have to use the uninstaller. If it's not available fallback + # to the directory removal code + if mozinfo.isWin: + uninstall_folder = '%s\uninstall' % install_folder + log_file = '%s\uninstall.log' % uninstall_folder + + if os.path.isfile(log_file): + trbk = None + try: + cmdArgs = ['%s\uninstall\helper.exe' % install_folder, '/S'] + result = subprocess.call(cmdArgs) + if result is not 0: + raise Exception('Execution of uninstaller failed.') + + # The uninstaller spawns another process so the subprocess call + # returns immediately. We have to wait until the uninstall + # folder has been removed or until we run into a timeout. + end_time = time.time() + TIMEOUT_UNINSTALL + while os.path.exists(uninstall_folder): + time.sleep(1) + + if time.time() > end_time: + raise Exception('Failure removing uninstall folder.') + + except Exception, ex: + cls, exc, trbk = sys.exc_info() + error = UninstallError('Failed to uninstall %s (%s)' % (install_folder, str(ex))) + raise UninstallError, error, trbk + + finally: + # trbk won't get GC'ed due to circular reference + # http://docs.python.org/library/sys.html#sys.exc_info + del trbk + + # Ensure that we remove any trace of the installation. Even the uninstaller + # on Windows leaves files behind we have to explicitely remove. + mozfile.remove(install_folder) + + +def _install_dmg(src, dest): + """Extract a dmg file into the destination folder and return the + application folder. + + src -- DMG image which has to be extracted + dest -- the path to extract to + + """ + try: + proc = subprocess.Popen('hdiutil attach -nobrowse -noautoopen "%s"' % src, + shell=True, + stdout=subprocess.PIPE) + + for data in proc.communicate()[0].split(): + if data.find('/Volumes/') != -1: + appDir = data + break + + for appFile in os.listdir(appDir): + if appFile.endswith('.app'): + appName = appFile + break + + mounted_path = os.path.join(appDir, appName) + + dest = os.path.join(dest, appName) + + # copytree() would fail if dest already exists. + if os.path.exists(dest): + raise InstallError('App bundle "%s" already exists.' % dest) + + shutil.copytree(mounted_path, dest, False) + + finally: + subprocess.call('hdiutil detach %s -quiet' % appDir, + shell=True) + + return dest + + +def _install_exe(src, dest): + """Run the MSI installer to silently install the application into the + destination folder. Return the folder path. + + Arguments: + src -- MSI installer to be executed + dest -- the path to install to + + """ + # The installer doesn't automatically create a sub folder. Lets guess the + # best name from the src file name + filename = os.path.basename(src) + dest = os.path.join(dest, filename.split('.')[0]) + + # possibly gets around UAC in vista (still need to run as administrator) + os.environ['__compat_layer'] = 'RunAsInvoker' + cmd = '"%s" /extractdir=%s' % (src, os.path.realpath(dest)) + + # As long as we support Python 2.4 check_call will not be available. + result = subprocess.call(cmd) + + if result is not 0: + raise Exception('Execution of installer failed.') + + return dest + + +def install_cli(argv=sys.argv[1:]): + parser = OptionParser(usage="usage: %prog [options] installer") + parser.add_option('-d', '--destination', + dest='dest', + default=os.getcwd(), + help='Directory to install application into. ' + '[default: "%default"]') + parser.add_option('--app', dest='app', + default='firefox', + help='Application being installed. [default: %default]') + + (options, args) = parser.parse_args(argv) + if not len(args) == 1: + parser.error('An installer file has to be specified.') + + src = args[0] + + # Run it + if os.path.isdir(src): + binary = get_binary(src, app_name=options.app) + else: + install_path = install(src, options.dest) + binary = get_binary(install_path, app_name=options.app) + + print binary + + +def uninstall_cli(argv=sys.argv[1:]): + parser = OptionParser(usage="usage: %prog install_path") + + (options, args) = parser.parse_args(argv) + if not len(args) == 1: + parser.error('An installation path has to be specified.') + + # Run it + uninstall(argv[0]) diff --git a/testing/mozbase/mozinstall/setup.py b/testing/mozbase/mozinstall/setup.py new file mode 100644 index 000000000..7759f0728 --- /dev/null +++ b/testing/mozbase/mozinstall/setup.py @@ -0,0 +1,53 @@ +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this file, +# You can obtain one at http://mozilla.org/MPL/2.0/. + +import os +from setuptools import setup + +try: + here = os.path.dirname(os.path.abspath(__file__)) + description = file(os.path.join(here, 'README.md')).read() +except IOError: + description = None + +PACKAGE_VERSION = '1.12' + +deps = ['mozinfo >= 0.7', + 'mozfile >= 1.0', + ] + +setup(name='mozInstall', + version=PACKAGE_VERSION, + description="package for installing and uninstalling Mozilla applications", + long_description="see http://mozbase.readthedocs.org/", + # Get strings from http://pypi.python.org/pypi?%3Aaction=list_classifiers + classifiers=['Environment :: Console', + 'Intended Audience :: Developers', + 'License :: OSI Approved :: Mozilla Public License 2.0 (MPL 2.0)', + 'Natural Language :: English', + 'Operating System :: OS Independent', + 'Programming Language :: Python', + 'Topic :: Software Development :: Libraries :: Python Modules', + ], + keywords='mozilla', + author='Mozilla Automation and Tools team', + author_email='tools@lists.mozilla.org', + url='https://wiki.mozilla.org/Auto-tools/Projects/Mozbase', + license='MPL 2.0', + packages=['mozinstall'], + include_package_data=True, + zip_safe=False, + install_requires=deps, + tests_require=['mozprocess >= 0.15', ], + # we have to generate two more executables for those systems that cannot run as Administrator + # and the filename containing "install" triggers the UAC + entry_points=""" + # -*- Entry points: -*- + [console_scripts] + mozinstall = mozinstall:install_cli + mozuninstall = mozinstall:uninstall_cli + moz_add_to_system = mozinstall:install_cli + moz_remove_from_system = mozinstall:uninstall_cli + """, + ) diff --git a/testing/mozbase/mozinstall/tests/Installer-Stubs/firefox.dmg b/testing/mozbase/mozinstall/tests/Installer-Stubs/firefox.dmg new file mode 100644 index 000000000..f7f36f631 Binary files /dev/null and b/testing/mozbase/mozinstall/tests/Installer-Stubs/firefox.dmg differ diff --git a/testing/mozbase/mozinstall/tests/Installer-Stubs/firefox.tar.bz2 b/testing/mozbase/mozinstall/tests/Installer-Stubs/firefox.tar.bz2 new file mode 100644 index 000000000..cb046a0e7 Binary files /dev/null and b/testing/mozbase/mozinstall/tests/Installer-Stubs/firefox.tar.bz2 differ diff --git a/testing/mozbase/mozinstall/tests/Installer-Stubs/firefox.zip b/testing/mozbase/mozinstall/tests/Installer-Stubs/firefox.zip new file mode 100644 index 000000000..7c3f61a5e Binary files /dev/null and b/testing/mozbase/mozinstall/tests/Installer-Stubs/firefox.zip differ diff --git a/testing/mozbase/mozinstall/tests/manifest.ini b/testing/mozbase/mozinstall/tests/manifest.ini new file mode 100644 index 000000000..528fdea7b --- /dev/null +++ b/testing/mozbase/mozinstall/tests/manifest.ini @@ -0,0 +1 @@ +[test.py] diff --git a/testing/mozbase/mozinstall/tests/test.py b/testing/mozbase/mozinstall/tests/test.py new file mode 100644 index 000000000..b4c53bb42 --- /dev/null +++ b/testing/mozbase/mozinstall/tests/test.py @@ -0,0 +1,169 @@ +#!/usr/bin/env python + +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this file, +# You can obtain one at http://mozilla.org/MPL/2.0/. + +import mozinfo +import mozinstall +import mozfile +import os +import tempfile +import unittest + +# Store file location at load time +here = os.path.dirname(os.path.abspath(__file__)) + + +class TestMozInstall(unittest.TestCase): + + @classmethod + def setUpClass(cls): + """ Setting up stub installers """ + cls.dmg = os.path.join(here, 'Installer-Stubs', 'firefox.dmg') + # XXX: We have removed firefox.exe since it is not valid for mozinstall 1.12 and higher + # Bug 1157352 - We should grab a firefox.exe from the build process or download it + cls.exe = os.path.join(here, 'Installer-Stubs', 'firefox.exe') + cls.zipfile = os.path.join(here, 'Installer-Stubs', 'firefox.zip') + cls.bz2 = os.path.join(here, 'Installer-Stubs', 'firefox.tar.bz2') + + def setUp(self): + self.tempdir = tempfile.mkdtemp() + + def tearDown(self): + mozfile.rmtree(self.tempdir) + + @unittest.skipIf(mozinfo.isWin, "Bug 1157352 - We need a new firefox.exe " + "for mozinstall 1.12 and higher.") + def test_get_binary(self): + """ Test mozinstall's get_binary method """ + + if mozinfo.isLinux: + installdir = mozinstall.install(self.bz2, self.tempdir) + binary = os.path.join(installdir, 'firefox') + self.assertEqual(binary, mozinstall.get_binary(installdir, 'firefox')) + + elif mozinfo.isWin: + installdir_exe = mozinstall.install(self.exe, + os.path.join(self.tempdir, 'exe')) + binary_exe = os.path.join(installdir_exe, 'core', 'firefox.exe') + self.assertEqual(binary_exe, mozinstall.get_binary(installdir_exe, + 'firefox')) + + installdir_zip = mozinstall.install(self.zipfile, + os.path.join(self.tempdir, 'zip')) + binary_zip = os.path.join(installdir_zip, 'firefox.exe') + self.assertEqual(binary_zip, mozinstall.get_binary(installdir_zip, + 'firefox')) + + elif mozinfo.isMac: + installdir = mozinstall.install(self.dmg, self.tempdir) + binary = os.path.join(installdir, 'Contents', 'MacOS', 'firefox') + self.assertEqual(binary, mozinstall.get_binary(installdir, 'firefox')) + + def test_get_binary_error(self): + """ Test an InvalidBinary error is raised """ + + tempdir_empty = tempfile.mkdtemp() + self.assertRaises(mozinstall.InvalidBinary, mozinstall.get_binary, + tempdir_empty, 'firefox') + mozfile.rmtree(tempdir_empty) + + @unittest.skipIf(mozinfo.isWin, "Bug 1157352 - We need a new firefox.exe " + "for mozinstall 1.12 and higher.") + def test_is_installer(self): + """ Test we can identify a correct installer """ + + if mozinfo.isLinux: + self.assertTrue(mozinstall.is_installer(self.bz2)) + + if mozinfo.isWin: + # test zip installer + self.assertTrue(mozinstall.is_installer(self.zipfile)) + + # test exe installer + self.assertTrue(mozinstall.is_installer(self.exe)) + + try: + # test stub browser file + # without pefile on the system this test will fail + import pefile # noqa + stub_exe = os.path.join(here, 'build_stub', 'firefox.exe') + self.assertFalse(mozinstall.is_installer(stub_exe)) + except ImportError: + pass + + if mozinfo.isMac: + self.assertTrue(mozinstall.is_installer(self.dmg)) + + def test_invalid_source_error(self): + """ Test InvalidSource error is raised with an incorrect installer """ + + if mozinfo.isLinux: + self.assertRaises(mozinstall.InvalidSource, mozinstall.install, + self.dmg, 'firefox') + + elif mozinfo.isWin: + self.assertRaises(mozinstall.InvalidSource, mozinstall.install, + self.bz2, 'firefox') + + elif mozinfo.isMac: + self.assertRaises(mozinstall.InvalidSource, mozinstall.install, + self.bz2, 'firefox') + + @unittest.skipIf(mozinfo.isWin, "Bug 1157352 - We need a new firefox.exe " + "for mozinstall 1.12 and higher.") + def test_install(self): + """ Test mozinstall's install capability """ + + if mozinfo.isLinux: + installdir = mozinstall.install(self.bz2, self.tempdir) + self.assertEqual(os.path.join(self.tempdir, 'firefox'), installdir) + + elif mozinfo.isWin: + installdir_exe = mozinstall.install(self.exe, + os.path.join(self.tempdir, 'exe')) + self.assertEqual(os.path.join(self.tempdir, 'exe', 'firefox'), + installdir_exe) + + installdir_zip = mozinstall.install(self.zipfile, + os.path.join(self.tempdir, 'zip')) + self.assertEqual(os.path.join(self.tempdir, 'zip', 'firefox'), + installdir_zip) + + elif mozinfo.isMac: + installdir = mozinstall.install(self.dmg, self.tempdir) + self.assertEqual(os.path.join(os.path.realpath(self.tempdir), + 'FirefoxStub.app'), installdir) + + @unittest.skipIf(mozinfo.isWin, "Bug 1157352 - We need a new firefox.exe " + "for mozinstall 1.12 and higher.") + def test_uninstall(self): + """ Test mozinstall's uninstall capabilites """ + # Uninstall after installing + + if mozinfo.isLinux: + installdir = mozinstall.install(self.bz2, self.tempdir) + mozinstall.uninstall(installdir) + self.assertFalse(os.path.exists(installdir)) + + elif mozinfo.isWin: + # Exe installer for Windows + installdir_exe = mozinstall.install(self.exe, + os.path.join(self.tempdir, 'exe')) + mozinstall.uninstall(installdir_exe) + self.assertFalse(os.path.exists(installdir_exe)) + + # Zip installer for Windows + installdir_zip = mozinstall.install(self.zipfile, + os.path.join(self.tempdir, 'zip')) + mozinstall.uninstall(installdir_zip) + self.assertFalse(os.path.exists(installdir_zip)) + + elif mozinfo.isMac: + installdir = mozinstall.install(self.dmg, self.tempdir) + mozinstall.uninstall(installdir) + self.assertFalse(os.path.exists(installdir)) + +if __name__ == '__main__': + unittest.main() diff --git a/testing/mozbase/mozleak/mozleak/__init__.py b/testing/mozbase/mozleak/mozleak/__init__.py new file mode 100644 index 000000000..ce0c084e0 --- /dev/null +++ b/testing/mozbase/mozleak/mozleak/__init__.py @@ -0,0 +1,11 @@ +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this file, +# You can obtain one at http://mozilla.org/MPL/2.0/. + +""" +mozleak is a library for extracting memory leaks from leak logs files. +""" + +from .leaklog import process_leak_log + +__all__ = ['process_leak_log'] diff --git a/testing/mozbase/mozleak/mozleak/leaklog.py b/testing/mozbase/mozleak/mozleak/leaklog.py new file mode 100644 index 000000000..9688974d1 --- /dev/null +++ b/testing/mozbase/mozleak/mozleak/leaklog.py @@ -0,0 +1,205 @@ +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this file, +# You can obtain one at http://mozilla.org/MPL/2.0/. + + +import os +import re + + +def _get_default_logger(): + from mozlog import get_default_logger + log = get_default_logger(component='mozleak') + + if not log: + import logging + log = logging.getLogger(__name__) + return log + + +def process_single_leak_file(leakLogFileName, processType, leakThreshold, + ignoreMissingLeaks, log=None, + stackFixer=None): + """Process a single leak log. + """ + + # | |Per-Inst Leaked| Total Rem| + # 0 |TOTAL | 17 192| 419115886 2| + # 833 |nsTimerImpl | 60 120| 24726 2| + # 930 |Foo | 32 8| 100 1| + lineRe = re.compile(r"^\s*\d+ \|" + r"(?P[^|]+)\|" + r"\s*(?P-?\d+)\s+(?P-?\d+)\s*\|" + r"\s*-?\d+\s+(?P-?\d+)") + # The class name can contain spaces. We remove trailing whitespace later. + + log = log or _get_default_logger() + + processString = "%s process:" % processType + crashedOnPurpose = False + totalBytesLeaked = None + leakedObjectAnalysis = [] + leakedObjectNames = [] + recordLeakedObjects = False + with open(leakLogFileName, "r") as leaks: + for line in leaks: + if line.find("purposefully crash") > -1: + crashedOnPurpose = True + matches = lineRe.match(line) + if not matches: + # eg: the leak table header row + strippedLine = line.rstrip() + log.info(stackFixer(strippedLine) if stackFixer else strippedLine) + continue + name = matches.group("name").rstrip() + size = int(matches.group("size")) + bytesLeaked = int(matches.group("bytesLeaked")) + numLeaked = int(matches.group("numLeaked")) + # Output the raw line from the leak log table if it is the TOTAL row, + # or is for an object row that has been leaked. + if numLeaked != 0 or name == "TOTAL": + log.info(line.rstrip()) + # Analyse the leak log, but output later or it will interrupt the + # leak table + if name == "TOTAL": + # Multiple default processes can end up writing their bloat views into a single + # log, particularly on B2G. Eventually, these should be split into multiple + # logs (bug 1068869), but for now, we report the largest leak. + if totalBytesLeaked is not None: + log.warning("leakcheck | %s " + "multiple BloatView byte totals found" + % processString) + else: + totalBytesLeaked = 0 + if bytesLeaked > totalBytesLeaked: + totalBytesLeaked = bytesLeaked + # Throw out the information we had about the previous bloat + # view. + leakedObjectNames = [] + leakedObjectAnalysis = [] + recordLeakedObjects = True + else: + recordLeakedObjects = False + if size < 0 or bytesLeaked < 0 or numLeaked < 0: + log.error("TEST-UNEXPECTED-FAIL | leakcheck | %s negative leaks caught!" + % processString) + continue + if name != "TOTAL" and numLeaked != 0 and recordLeakedObjects: + leakedObjectNames.append(name) + leakedObjectAnalysis.append("TEST-INFO | leakcheck | %s leaked %d %s" + % (processString, numLeaked, name)) + + log.info('\n'.join(leakedObjectAnalysis)) + + if totalBytesLeaked is None: + # We didn't see a line with name 'TOTAL' + if crashedOnPurpose: + log.info("TEST-INFO | leakcheck | %s deliberate crash and thus no leak log" + % processString) + elif ignoreMissingLeaks: + log.info("TEST-INFO | leakcheck | %s ignoring missing output line for total leaks" + % processString) + else: + log.error("TEST-UNEXPECTED-FAIL | leakcheck | %s missing output line for total leaks!" + % processString) + log.info("TEST-INFO | leakcheck | missing output line from log file %s" + % leakLogFileName) + return + + if totalBytesLeaked == 0: + log.info("TEST-PASS | leakcheck | %s no leaks detected!" % + processString) + return + + # Create a comma delimited string of the first N leaked objects found, + # to aid with bug summary matching in TBPL. Note: The order of the objects + # had no significance (they're sorted alphabetically). + maxSummaryObjects = 5 + leakedObjectSummary = ', '.join(leakedObjectNames[:maxSummaryObjects]) + if len(leakedObjectNames) > maxSummaryObjects: + leakedObjectSummary += ', ...' + + message = "leakcheck | %s %d bytes leaked (%s)" % ( + processString, totalBytesLeaked, leakedObjectSummary) + + # totalBytesLeaked will include any expected leaks, so it can be off + # by a few thousand bytes. + if totalBytesLeaked > leakThreshold: + log.error("TEST-UNEXPECTED-FAIL | %s" % message) + else: + log.warning(message) + + +def process_leak_log(leak_log_file, leak_thresholds=None, + ignore_missing_leaks=None, log=None, + stack_fixer=None): + """Process the leak log, including separate leak logs created + by child processes. + + Use this function if you want an additional PASS/FAIL summary. + It must be used with the |XPCOM_MEM_BLOAT_LOG| environment variable. + + The base of leak_log_file for a non-default process needs to end with + _proctype_pid12345.log + "proctype" is a string denoting the type of the process, which should + be the result of calling XRE_ChildProcessTypeToString(). 12345 is + a series of digits that is the pid for the process. The .log is + optional. + + All other file names are treated as being for default processes. + + leak_thresholds should be a dict mapping process types to leak thresholds, + in bytes. If a process type is not present in the dict the threshold + will be 0. + + ignore_missing_leaks should be a list of process types. If a process + creates a leak log without a TOTAL, then we report an error if it isn't + in the list ignore_missing_leaks. + """ + + log = log or _get_default_logger() + + leakLogFile = leak_log_file + if not os.path.exists(leakLogFile): + log.warning( + "leakcheck | refcount logging is off, so leaks can't be detected!") + return + + leakThresholds = leak_thresholds or {} + ignoreMissingLeaks = ignore_missing_leaks or [] + + # This list is based on kGeckoProcessTypeString. ipdlunittest processes likely + # are not going to produce leak logs we will ever see. + knownProcessTypes = ["default", "plugin", "tab", "geckomediaplugin", "gpu"] + + for processType in knownProcessTypes: + log.info("TEST-INFO | leakcheck | %s process: leak threshold set at %d bytes" + % (processType, leakThresholds.get(processType, 0))) + + for processType in leakThresholds: + if processType not in knownProcessTypes: + log.error("TEST-UNEXPECTED-FAIL | leakcheck | " + "Unknown process type %s in leakThresholds" % processType) + + (leakLogFileDir, leakFileBase) = os.path.split(leakLogFile) + if leakFileBase[-4:] == ".log": + leakFileBase = leakFileBase[:-4] + fileNameRegExp = re.compile(r"_([a-z]*)_pid\d*.log$") + else: + fileNameRegExp = re.compile(r"_([a-z]*)_pid\d*$") + + for fileName in os.listdir(leakLogFileDir): + if fileName.find(leakFileBase) != -1: + thisFile = os.path.join(leakLogFileDir, fileName) + m = fileNameRegExp.search(fileName) + if m: + processType = m.group(1) + else: + processType = "default" + if processType not in knownProcessTypes: + log.error("TEST-UNEXPECTED-FAIL | leakcheck | " + "Leak log with unknown process type %s" % processType) + leakThreshold = leakThresholds.get(processType, 0) + process_single_leak_file(thisFile, processType, leakThreshold, + processType in ignoreMissingLeaks, + log=log, stackFixer=stack_fixer) diff --git a/testing/mozbase/mozleak/setup.py b/testing/mozbase/mozleak/setup.py new file mode 100644 index 000000000..76eb64a9f --- /dev/null +++ b/testing/mozbase/mozleak/setup.py @@ -0,0 +1,26 @@ +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this file, +# You can obtain one at http://mozilla.org/MPL/2.0/. + +from setuptools import setup + + +PACKAGE_NAME = 'mozleak' +PACKAGE_VERSION = '0.1' + + +setup( + name=PACKAGE_NAME, + version=PACKAGE_VERSION, + description="Library for extracting memory leaks from leak logs files", + long_description="see http://mozbase.readthedocs.org/", + classifiers=[], # Get strings from http://pypi.python.org/pypi?%3Aaction=list_classifiers + keywords='mozilla', + author='Mozilla Automation and Tools team', + author_email='tools@lists.mozilla.org', + url='https://wiki.mozilla.org/Auto-tools/Projects/Mozbase', + license='MPL', + packages=['mozleak'], + zip_safe=False, + install_requires=[], +) diff --git a/testing/mozbase/mozlog/mozlog/__init__.py b/testing/mozbase/mozlog/mozlog/__init__.py new file mode 100644 index 000000000..1fe4dc738 --- /dev/null +++ b/testing/mozbase/mozlog/mozlog/__init__.py @@ -0,0 +1,30 @@ +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, You can obtain one at http://mozilla.org/MPL/2.0/. + +""" +Mozlog aims to standardize log handling and formatting within Mozilla. + +It implements a JSON-based structured logging protocol with convenience +facilities for recording test results. + +The old unstructured module is deprecated. It simply wraps Python's +logging_ module and adds a few convenience methods for logging test +results and events. +""" + +import sys + +from . import commandline +from . import structuredlog +from . import unstructured +from .structuredlog import get_default_logger, set_default_logger +from .proxy import get_proxy_logger + +# Backwards compatibility shim for consumers that use mozlog.structured +structured = sys.modules[__name__] +sys.modules['{}.structured'.format(__name__)] = structured + +__all__ = ['commandline', 'structuredlog', 'unstructured', + 'get_default_logger', 'set_default_logger', 'get_proxy_logger', + 'structured'] diff --git a/testing/mozbase/mozlog/mozlog/commandline.py b/testing/mozbase/mozlog/mozlog/commandline.py new file mode 100644 index 000000000..107708154 --- /dev/null +++ b/testing/mozbase/mozlog/mozlog/commandline.py @@ -0,0 +1,282 @@ +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, You can obtain one at http://mozilla.org/MPL/2.0/. + +import argparse +import optparse +import os +import sys +from collections import defaultdict + +from . import handlers +from . import formatters +from .structuredlog import StructuredLogger, set_default_logger + +log_formatters = { + 'raw': (formatters.JSONFormatter, "Raw structured log messages"), + 'unittest': (formatters.UnittestFormatter, "Unittest style output"), + 'xunit': (formatters.XUnitFormatter, "xUnit compatible XML"), + 'html': (formatters.HTMLFormatter, "HTML report"), + 'mach': (formatters.MachFormatter, "Human-readable output"), + 'tbpl': (formatters.TbplFormatter, "TBPL style log format"), + 'errorsummary': (formatters.ErrorSummaryFormatter, argparse.SUPPRESS), +} + +TEXT_FORMATTERS = ('raw', 'mach') +"""a subset of formatters for non test harnesses related applications""" + + +def level_filter_wrapper(formatter, level): + return handlers.LogLevelFilter(formatter, level) + + +def verbose_wrapper(formatter, verbose): + formatter.verbose = verbose + return formatter + + +def compact_wrapper(formatter, compact): + formatter.compact = compact + return formatter + + +def buffer_handler_wrapper(handler, buffer_limit): + if buffer_limit == "UNLIMITED": + buffer_limit = None + else: + buffer_limit = int(buffer_limit) + return handlers.BufferHandler(handler, buffer_limit) + + +def valgrind_handler_wrapper(handler): + return handlers.ValgrindHandler(handler) + + +def default_formatter_options(log_type, overrides): + formatter_option_defaults = { + "raw": { + "level": "debug" + } + } + rv = {"verbose": False, + "level": "info"} + rv.update(formatter_option_defaults.get(log_type, {})) + + if overrides is not None: + rv.update(overrides) + + return rv + +fmt_options = { + #