diff options
Diffstat (limited to 'testing/web-platform/harness/docs')
| -rw-r--r-- | testing/web-platform/harness/docs/Makefile | 177 | ||||
| -rw-r--r-- | testing/web-platform/harness/docs/architecture.svg | 1 | ||||
| -rw-r--r-- | testing/web-platform/harness/docs/conf.py | 267 | ||||
| -rw-r--r-- | testing/web-platform/harness/docs/design.rst | 106 | ||||
| -rw-r--r-- | testing/web-platform/harness/docs/expectation.rst | 248 | ||||
| -rw-r--r-- | testing/web-platform/harness/docs/index.rst | 24 | ||||
| -rw-r--r-- | testing/web-platform/harness/docs/make.bat | 242 | ||||
| -rw-r--r-- | testing/web-platform/harness/docs/usage.rst | 238 |
8 files changed, 1303 insertions, 0 deletions
diff --git a/testing/web-platform/harness/docs/Makefile b/testing/web-platform/harness/docs/Makefile new file mode 100644 index 000000000..d02b6c5e7 --- /dev/null +++ b/testing/web-platform/harness/docs/Makefile @@ -0,0 +1,177 @@ +# Makefile for Sphinx documentation +# + +# You can set these variables from the command line. +SPHINXOPTS = +SPHINXBUILD = sphinx-build +PAPER = +BUILDDIR = _build + +# User-friendly check for sphinx-build +ifeq ($(shell which $(SPHINXBUILD) >/dev/null 2>&1; echo $$?), 1) +$(error The '$(SPHINXBUILD)' command was not found. Make sure you have Sphinx installed, then set the SPHINXBUILD environment variable to point to the full path of the '$(SPHINXBUILD)' executable. Alternatively you can add the directory with the executable to your PATH. If you don't have Sphinx installed, grab it from http://sphinx-doc.org/) +endif + +# Internal variables. +PAPEROPT_a4 = -D latex_paper_size=a4 +PAPEROPT_letter = -D latex_paper_size=letter +ALLSPHINXOPTS = -d $(BUILDDIR)/doctrees $(PAPEROPT_$(PAPER)) $(SPHINXOPTS) . +# the i18n builder cannot share the environment and doctrees with the others +I18NSPHINXOPTS = $(PAPEROPT_$(PAPER)) $(SPHINXOPTS) . + +.PHONY: help clean html dirhtml singlehtml pickle json htmlhelp qthelp devhelp epub latex latexpdf text man changes linkcheck doctest gettext + +help: + @echo "Please use \`make <target>' where <target> is one of" + @echo " html to make standalone HTML files" + @echo " dirhtml to make HTML files named index.html in directories" + @echo " singlehtml to make a single large HTML file" + @echo " pickle to make pickle files" + @echo " json to make JSON files" + @echo " htmlhelp to make HTML files and a HTML help project" + @echo " qthelp to make HTML files and a qthelp project" + @echo " devhelp to make HTML files and a Devhelp project" + @echo " epub to make an epub" + @echo " latex to make LaTeX files, you can set PAPER=a4 or PAPER=letter" + @echo " latexpdf to make LaTeX files and run them through pdflatex" + @echo " latexpdfja to make LaTeX files and run them through platex/dvipdfmx" + @echo " text to make text files" + @echo " man to make manual pages" + @echo " texinfo to make Texinfo files" + @echo " info to make Texinfo files and run them through makeinfo" + @echo " gettext to make PO message catalogs" + @echo " changes to make an overview of all changed/added/deprecated items" + @echo " xml to make Docutils-native XML files" + @echo " pseudoxml to make pseudoxml-XML files for display purposes" + @echo " linkcheck to check all external links for integrity" + @echo " doctest to run all doctests embedded in the documentation (if enabled)" + +clean: + rm -rf $(BUILDDIR)/* + +html: + $(SPHINXBUILD) -b html $(ALLSPHINXOPTS) $(BUILDDIR)/html + @echo + @echo "Build finished. The HTML pages are in $(BUILDDIR)/html." + +dirhtml: + $(SPHINXBUILD) -b dirhtml $(ALLSPHINXOPTS) $(BUILDDIR)/dirhtml + @echo + @echo "Build finished. The HTML pages are in $(BUILDDIR)/dirhtml." + +singlehtml: + $(SPHINXBUILD) -b singlehtml $(ALLSPHINXOPTS) $(BUILDDIR)/singlehtml + @echo + @echo "Build finished. The HTML page is in $(BUILDDIR)/singlehtml." + +pickle: + $(SPHINXBUILD) -b pickle $(ALLSPHINXOPTS) $(BUILDDIR)/pickle + @echo + @echo "Build finished; now you can process the pickle files." + +json: + $(SPHINXBUILD) -b json $(ALLSPHINXOPTS) $(BUILDDIR)/json + @echo + @echo "Build finished; now you can process the JSON files." + +htmlhelp: + $(SPHINXBUILD) -b htmlhelp $(ALLSPHINXOPTS) $(BUILDDIR)/htmlhelp + @echo + @echo "Build finished; now you can run HTML Help Workshop with the" \ + ".hhp project file in $(BUILDDIR)/htmlhelp." + +qthelp: + $(SPHINXBUILD) -b qthelp $(ALLSPHINXOPTS) $(BUILDDIR)/qthelp + @echo + @echo "Build finished; now you can run "qcollectiongenerator" with the" \ + ".qhcp project file in $(BUILDDIR)/qthelp, like this:" + @echo "# qcollectiongenerator $(BUILDDIR)/qthelp/wptrunner.qhcp" + @echo "To view the help file:" + @echo "# assistant -collectionFile $(BUILDDIR)/qthelp/wptrunner.qhc" + +devhelp: + $(SPHINXBUILD) -b devhelp $(ALLSPHINXOPTS) $(BUILDDIR)/devhelp + @echo + @echo "Build finished." + @echo "To view the help file:" + @echo "# mkdir -p $$HOME/.local/share/devhelp/wptrunner" + @echo "# ln -s $(BUILDDIR)/devhelp $$HOME/.local/share/devhelp/wptrunner" + @echo "# devhelp" + +epub: + $(SPHINXBUILD) -b epub $(ALLSPHINXOPTS) $(BUILDDIR)/epub + @echo + @echo "Build finished. The epub file is in $(BUILDDIR)/epub." + +latex: + $(SPHINXBUILD) -b latex $(ALLSPHINXOPTS) $(BUILDDIR)/latex + @echo + @echo "Build finished; the LaTeX files are in $(BUILDDIR)/latex." + @echo "Run \`make' in that directory to run these through (pdf)latex" \ + "(use \`make latexpdf' here to do that automatically)." + +latexpdf: + $(SPHINXBUILD) -b latex $(ALLSPHINXOPTS) $(BUILDDIR)/latex + @echo "Running LaTeX files through pdflatex..." + $(MAKE) -C $(BUILDDIR)/latex all-pdf + @echo "pdflatex finished; the PDF files are in $(BUILDDIR)/latex." + +latexpdfja: + $(SPHINXBUILD) -b latex $(ALLSPHINXOPTS) $(BUILDDIR)/latex + @echo "Running LaTeX files through platex and dvipdfmx..." + $(MAKE) -C $(BUILDDIR)/latex all-pdf-ja + @echo "pdflatex finished; the PDF files are in $(BUILDDIR)/latex." + +text: + $(SPHINXBUILD) -b text $(ALLSPHINXOPTS) $(BUILDDIR)/text + @echo + @echo "Build finished. The text files are in $(BUILDDIR)/text." + +man: + $(SPHINXBUILD) -b man $(ALLSPHINXOPTS) $(BUILDDIR)/man + @echo + @echo "Build finished. The manual pages are in $(BUILDDIR)/man." + +texinfo: + $(SPHINXBUILD) -b texinfo $(ALLSPHINXOPTS) $(BUILDDIR)/texinfo + @echo + @echo "Build finished. The Texinfo files are in $(BUILDDIR)/texinfo." + @echo "Run \`make' in that directory to run these through makeinfo" \ + "(use \`make info' here to do that automatically)." + +info: + $(SPHINXBUILD) -b texinfo $(ALLSPHINXOPTS) $(BUILDDIR)/texinfo + @echo "Running Texinfo files through makeinfo..." + make -C $(BUILDDIR)/texinfo info + @echo "makeinfo finished; the Info files are in $(BUILDDIR)/texinfo." + +gettext: + $(SPHINXBUILD) -b gettext $(I18NSPHINXOPTS) $(BUILDDIR)/locale + @echo + @echo "Build finished. The message catalogs are in $(BUILDDIR)/locale." + +changes: + $(SPHINXBUILD) -b changes $(ALLSPHINXOPTS) $(BUILDDIR)/changes + @echo + @echo "The overview file is in $(BUILDDIR)/changes." + +linkcheck: + $(SPHINXBUILD) -b linkcheck $(ALLSPHINXOPTS) $(BUILDDIR)/linkcheck + @echo + @echo "Link check complete; look for any errors in the above output " \ + "or in $(BUILDDIR)/linkcheck/output.txt." + +doctest: + $(SPHINXBUILD) -b doctest $(ALLSPHINXOPTS) $(BUILDDIR)/doctest + @echo "Testing of doctests in the sources finished, look at the " \ + "results in $(BUILDDIR)/doctest/output.txt." + +xml: + $(SPHINXBUILD) -b xml $(ALLSPHINXOPTS) $(BUILDDIR)/xml + @echo + @echo "Build finished. The XML files are in $(BUILDDIR)/xml." + +pseudoxml: + $(SPHINXBUILD) -b pseudoxml $(ALLSPHINXOPTS) $(BUILDDIR)/pseudoxml + @echo + @echo "Build finished. The pseudo-XML files are in $(BUILDDIR)/pseudoxml." diff --git a/testing/web-platform/harness/docs/architecture.svg b/testing/web-platform/harness/docs/architecture.svg new file mode 100644 index 000000000..b8d5aa21c --- /dev/null +++ b/testing/web-platform/harness/docs/architecture.svg @@ -0,0 +1 @@ +<svg xmlns="http://www.w3.org/2000/svg" xmlns:xlink="http://www.w3.org/1999/xlink" width="780px" height="1087px" version="1.1"><defs><linearGradient x1="0%" y1="0%" x2="0%" y2="100%" id="mx-gradient-a9c4eb-1-a9c4eb-1-s-0"><stop offset="0%" style="stop-color:#A9C4EB"/><stop offset="100%" style="stop-color:#A9C4EB"/></linearGradient></defs><g transform="translate(0.5,0.5)"><rect x="498" y="498" width="120" height="60" fill="#e6d0de" stroke="#000000" pointer-events="none"/><g transform="translate(500,521)"><switch><foreignObject pointer-events="all" width="116" height="15" requiredFeatures="http://www.w3.org/TR/SVG11/feature#Extensibility"><div xmlns="http://www.w3.org/1999/xhtml" style="display: inline-block; font-size: 12px; font-family: Helvetica; color: rgb(0, 0, 0); line-height: 1.26; vertical-align: top; width: 116px; white-space: normal; text-align: center;">TestRunner</div></foreignObject><text x="58" y="14" fill="#000000" text-anchor="middle" font-size="12px" font-family="Helvetica">[Not supported by viewer]</text></switch></g><rect x="338" y="778" width="120" height="60" fill="#f19c99" stroke="#000000" pointer-events="none"/><g transform="translate(340,801)"><switch><foreignObject pointer-events="all" width="116" height="15" requiredFeatures="http://www.w3.org/TR/SVG11/feature#Extensibility"><div xmlns="http://www.w3.org/1999/xhtml" style="display: inline-block; font-size: 12px; font-family: Helvetica; color: rgb(0, 0, 0); line-height: 1.26; vertical-align: top; width: 116px; white-space: normal; text-align: center;">Product under test</div></foreignObject><text x="58" y="14" fill="#000000" text-anchor="middle" font-size="12px" font-family="Helvetica">[Not supported by viewer]</text></switch></g><rect x="338" y="388" width="120" height="60" fill="#e6d0de" stroke="#000000" pointer-events="none"/><g transform="translate(340,411)"><switch><foreignObject pointer-events="all" width="116" height="15" requiredFeatures="http://www.w3.org/TR/SVG11/feature#Extensibility"><div xmlns="http://www.w3.org/1999/xhtml" style="display: inline-block; font-size: 12px; font-family: Helvetica; color: rgb(0, 0, 0); line-height: 1.26; vertical-align: top; width: 116px; white-space: normal; text-align: center;">TestRunnerManager</div></foreignObject><text x="58" y="14" fill="#000000" text-anchor="middle" font-size="12px" font-family="Helvetica">[Not supported by viewer]</text></switch></g><rect x="338" y="228" width="120" height="60" fill="#e6d0de" stroke="#000000" pointer-events="none"/><g transform="translate(340,251)"><switch><foreignObject pointer-events="all" width="116" height="15" requiredFeatures="http://www.w3.org/TR/SVG11/feature#Extensibility"><div xmlns="http://www.w3.org/1999/xhtml" style="display: inline-block; font-size: 12px; font-family: Helvetica; color: rgb(0, 0, 0); line-height: 1.26; vertical-align: top; width: 116px; white-space: normal; text-align: center;">ManagerGroup</div></foreignObject><text x="58" y="14" fill="#000000" text-anchor="middle" font-size="12px" font-family="Helvetica">[Not supported by viewer]</text></switch></g><rect x="658" y="608" width="120" height="60" fill="#ffce9f" stroke="#000000" pointer-events="none"/><g transform="translate(660,631)"><switch><foreignObject pointer-events="all" width="116" height="15" requiredFeatures="http://www.w3.org/TR/SVG11/feature#Extensibility"><div xmlns="http://www.w3.org/1999/xhtml" style="display: inline-block; font-size: 12px; font-family: Helvetica; color: rgb(0, 0, 0); line-height: 1.26; vertical-align: top; width: 116px; white-space: normal; text-align: center;">Executor</div></foreignObject><text x="58" y="14" fill="#000000" text-anchor="middle" font-size="12px" font-family="Helvetica">[Not supported by viewer]</text></switch></g><rect x="338" y="498" width="120" height="60" fill="url(#mx-gradient-a9c4eb-1-a9c4eb-1-s-0)" stroke="#000000" pointer-events="none"/><g transform="translate(340,521)"><switch><foreignObject pointer-events="all" width="116" height="15" requiredFeatures="http://www.w3.org/TR/SVG11/feature#Extensibility"><div xmlns="http://www.w3.org/1999/xhtml" style="display: inline-block; font-size: 12px; font-family: Helvetica; color: rgb(0, 0, 0); line-height: 1.26; vertical-align: top; width: 116px; white-space: normal; text-align: center;">Browser</div></foreignObject><text x="58" y="14" fill="#000000" text-anchor="middle" font-size="12px" font-family="Helvetica">[Not supported by viewer]</text></switch></g><path d="M 398 288 L 398 382" fill="none" stroke="#000000" stroke-miterlimit="10" pointer-events="none"/><path d="M 398 387 L 395 380 L 398 382 L 402 380 Z" fill="#000000" stroke="#000000" stroke-miterlimit="10" pointer-events="none"/><path d="M 398 448 L 398 492" fill="none" stroke="#000000" stroke-miterlimit="10" pointer-events="none"/><path d="M 398 497 L 395 490 L 398 492 L 402 490 Z" fill="#000000" stroke="#000000" stroke-miterlimit="10" pointer-events="none"/><path d="M 618 528 L 684 603" fill="none" stroke="#000000" stroke-miterlimit="10" pointer-events="none"/><path d="M 687 607 L 680 604 L 684 603 L 685 600 Z" fill="#000000" stroke="#000000" stroke-miterlimit="10" pointer-events="none"/><rect x="498" y="608" width="120" height="60" fill="#a9c4eb" stroke="#000000" pointer-events="none"/><g transform="translate(500,631)"><switch><foreignObject pointer-events="all" width="116" height="15" requiredFeatures="http://www.w3.org/TR/SVG11/feature#Extensibility"><div xmlns="http://www.w3.org/1999/xhtml" style="display: inline-block; font-size: 12px; font-family: Helvetica; color: rgb(0, 0, 0); line-height: 1.26; vertical-align: top; width: 116px; white-space: normal; text-align: center;">ExecutorBrowser</div></foreignObject><text x="58" y="14" fill="#000000" text-anchor="middle" font-size="12px" font-family="Helvetica">[Not supported by viewer]</text></switch></g><path d="M 624 638 L 658 638" fill="none" stroke="#000000" stroke-miterlimit="10" pointer-events="none"/><path d="M 619 638 L 626 635 L 624 638 L 626 642 Z" fill="#000000" stroke="#000000" stroke-miterlimit="10" pointer-events="none"/><path d="M 428 448 L 552 496" fill="none" stroke="#000000" stroke-miterlimit="10" stroke-dasharray="3 3" pointer-events="none"/><path d="M 557 498 L 549 498 L 552 496 L 552 492 Z" fill="#000000" stroke="#000000" stroke-miterlimit="10" pointer-events="none"/><path d="M 398 558 L 398 772" fill="none" stroke="#000000" stroke-miterlimit="10" stroke-dasharray="3 3" pointer-events="none"/><path d="M 398 777 L 395 770 L 398 772 L 402 770 Z" fill="#000000" stroke="#000000" stroke-miterlimit="10" pointer-events="none"/><rect x="338" y="48" width="120" height="60" fill="#e6d0de" stroke="#000000" pointer-events="none"/><g transform="translate(340,71)"><switch><foreignObject pointer-events="all" width="116" height="15" requiredFeatures="http://www.w3.org/TR/SVG11/feature#Extensibility"><div xmlns="http://www.w3.org/1999/xhtml" style="display: inline-block; font-size: 12px; font-family: Helvetica; color: rgb(0, 0, 0); line-height: 1.26; vertical-align: top; width: 116px; white-space: normal; text-align: center;">run_tests</div></foreignObject><text x="58" y="14" fill="#000000" text-anchor="middle" font-size="12px" font-family="Helvetica">[Not supported by viewer]</text></switch></g><path d="M 458 78 L 652 78" fill="none" stroke="#000000" stroke-miterlimit="10" pointer-events="none"/><path d="M 657 78 L 650 82 L 652 78 L 650 75 Z" fill="#000000" stroke="#000000" stroke-miterlimit="10" pointer-events="none"/><rect x="658" y="48" width="120" height="60" fill="#e6d0de" stroke="#000000" pointer-events="none"/><g transform="translate(660,71)"><switch><foreignObject pointer-events="all" width="116" height="15" requiredFeatures="http://www.w3.org/TR/SVG11/feature#Extensibility"><div xmlns="http://www.w3.org/1999/xhtml" style="display: inline-block; font-size: 12px; font-family: Helvetica; color: rgb(0, 0, 0); line-height: 1.26; vertical-align: top; width: 116px; white-space: normal; text-align: center;">TestLoader</div></foreignObject><text x="58" y="14" fill="#000000" text-anchor="middle" font-size="12px" font-family="Helvetica">[Not supported by viewer]</text></switch></g><rect x="71" y="48" width="120" height="60" fill="#e6d0de" stroke="#000000" pointer-events="none"/><g transform="translate(73,71)"><switch><foreignObject pointer-events="all" width="116" height="15" requiredFeatures="http://www.w3.org/TR/SVG11/feature#Extensibility"><div xmlns="http://www.w3.org/1999/xhtml" style="display: inline-block; font-size: 12px; font-family: Helvetica; color: rgb(0, 0, 0); line-height: 1.26; vertical-align: top; width: 116px; white-space: normal; text-align: center;">TestEnvironment</div></foreignObject><text x="58" y="14" fill="#000000" text-anchor="middle" font-size="12px" font-family="Helvetica">[Not supported by viewer]</text></switch></g><rect x="151" y="618" width="120" height="60" fill="#b9e0a5" stroke="#000000" pointer-events="none"/><g transform="translate(153,641)"><switch><foreignObject pointer-events="all" width="116" height="15" requiredFeatures="http://www.w3.org/TR/SVG11/feature#Extensibility"><div xmlns="http://www.w3.org/1999/xhtml" style="display: inline-block; font-size: 12px; font-family: Helvetica; color: rgb(0, 0, 0); line-height: 1.26; vertical-align: top; width: 116px; white-space: normal; text-align: center;">wptserve</div></foreignObject><text x="58" y="14" fill="#000000" text-anchor="middle" font-size="12px" font-family="Helvetica">[Not supported by viewer]</text></switch></g><rect x="1" y="618" width="120" height="60" fill="#b9e0a5" stroke="#000000" pointer-events="none"/><g transform="translate(3,641)"><switch><foreignObject pointer-events="all" width="116" height="15" requiredFeatures="http://www.w3.org/TR/SVG11/feature#Extensibility"><div xmlns="http://www.w3.org/1999/xhtml" style="display: inline-block; font-size: 12px; font-family: Helvetica; color: rgb(0, 0, 0); line-height: 1.26; vertical-align: top; width: 116px; white-space: normal; text-align: center;">pywebsocket</div></foreignObject><text x="58" y="14" fill="#000000" text-anchor="middle" font-size="12px" font-family="Helvetica">[Not supported by viewer]</text></switch></g><path d="M 338 78 L 197 78" fill="none" stroke="#000000" stroke-miterlimit="10" pointer-events="none"/><path d="M 192 78 L 199 75 L 197 78 L 199 82 Z" fill="#000000" stroke="#000000" stroke-miterlimit="10" pointer-events="none"/><path d="M 101 308 L 62 612" fill="none" stroke="#000000" stroke-miterlimit="10" stroke-dasharray="3 3" pointer-events="none"/><path d="M 61 617 L 59 610 L 62 612 L 66 610 Z" fill="#000000" stroke="#000000" stroke-miterlimit="10" pointer-events="none"/><path d="M 161 308 L 204 612" fill="none" stroke="#000000" stroke-miterlimit="10" stroke-dasharray="3 3" pointer-events="none"/><path d="M 204 617 L 200 610 L 204 612 L 207 609 Z" fill="#000000" stroke="#000000" stroke-miterlimit="10" pointer-events="none"/><path d="M 338 823 L 61 678" fill="none" stroke="#000000" stroke-miterlimit="10" stroke-dasharray="3 3" pointer-events="none"/><path d="M 211 678 L 338 793" fill="none" stroke="#000000" stroke-miterlimit="10" stroke-dasharray="3 3" pointer-events="none"/><path d="M 398 108 L 398 222" fill="none" stroke="#000000" stroke-miterlimit="10" pointer-events="none"/><path d="M 398 227 L 395 220 L 398 222 L 402 220 Z" fill="#000000" stroke="#000000" stroke-miterlimit="10" pointer-events="none"/><path d="M 706 288 L 618 513" fill="none" stroke="#000000" stroke-miterlimit="10" stroke-dasharray="3 3" pointer-events="none"/><rect x="658" y="388" width="70" height="40" fill="none" stroke="none" pointer-events="none"/><g fill="#000000" font-family="Helvetica" text-anchor="middle" font-size="12px"><text x="693" y="412">Queue.get</text></g><path d="M 458 808 L 718 668" fill="none" stroke="#000000" stroke-miterlimit="10" stroke-dasharray="3 3" pointer-events="none"/><rect x="71" y="248" width="120" height="60" fill="#b9e0a5" stroke="#000000" pointer-events="none"/><g transform="translate(73,271)"><switch><foreignObject pointer-events="all" width="116" height="15" requiredFeatures="http://www.w3.org/TR/SVG11/feature#Extensibility"><div xmlns="http://www.w3.org/1999/xhtml" style="display: inline-block; font-size: 12px; font-family: Helvetica; color: rgb(0, 0, 0); line-height: 1.26; vertical-align: top; width: 116px; white-space: normal; text-align: center;">serve.py</div></foreignObject><text x="58" y="14" fill="#000000" text-anchor="middle" font-size="12px" font-family="Helvetica">[Not supported by viewer]</text></switch></g><path d="M 131 108 L 131 242" fill="none" stroke="#000000" stroke-miterlimit="10" stroke-dasharray="3 3" pointer-events="none"/><path d="M 131 247 L 128 240 L 131 242 L 135 240 Z" fill="#000000" stroke="#000000" stroke-miterlimit="10" pointer-events="none"/><path d="M 88 973 L 132 973" fill="none" stroke="#000000" stroke-miterlimit="10" pointer-events="none"/><path d="M 137 973 L 130 977 L 132 973 L 130 970 Z" fill="#000000" stroke="#000000" stroke-miterlimit="10" pointer-events="none"/><rect x="138" y="1018" width="180" height="30" fill="none" stroke="none" pointer-events="none"/><g fill="#000000" font-family="Helvetica" text-anchor="middle" font-size="12px"><text x="228" y="1037">Communication (cross process)</text></g><path d="M 88 1002 L 132 1002" fill="none" stroke="#000000" stroke-miterlimit="10" stroke-dasharray="3 3" pointer-events="none"/><path d="M 137 1002 L 130 1006 L 132 1002 L 130 999 Z" fill="#000000" stroke="#000000" stroke-miterlimit="10" pointer-events="none"/><rect x="138" y="958" width="180" height="30" fill="none" stroke="none" pointer-events="none"/><g fill="#000000" font-family="Helvetica" text-anchor="middle" font-size="12px"><text x="228" y="977">Ownership (same process)</text></g><path d="M 88 1033 L 138 1033" fill="none" stroke="#000000" stroke-miterlimit="10" stroke-dasharray="3 3" pointer-events="none"/><rect x="143" y="988" width="180" height="30" fill="none" stroke="none" pointer-events="none"/><g fill="#000000" font-family="Helvetica" text-anchor="middle" font-size="12px"><text x="233" y="1007">Ownership (cross process)</text></g><rect x="428" y="966" width="50" height="15" fill="#e6d0de" stroke="#000000" pointer-events="none"/><rect x="428" y="990" width="50" height="15" fill="#a9c4eb" stroke="#000000" pointer-events="none"/><rect x="428" y="1015" width="50" height="15" fill="#ffce9f" stroke="#000000" pointer-events="none"/><rect x="428" y="1063" width="50" height="15" fill="#f19c99" stroke="#000000" pointer-events="none"/><rect x="428" y="1038" width="50" height="15" fill="#b9e0a5" stroke="#000000" pointer-events="none"/><rect x="485" y="958" width="90" height="30" fill="none" stroke="none" pointer-events="none"/><g fill="#000000" font-family="Helvetica" text-anchor="middle" font-size="12px"><text x="530" y="977">wptrunner class</text></g><rect x="486" y="983" width="150" height="30" fill="none" stroke="none" pointer-events="none"/><g fill="#000000" font-family="Helvetica" text-anchor="middle" font-size="12px"><text x="561" y="1002">Per-product wptrunner class</text></g><rect x="486" y="1008" width="150" height="30" fill="none" stroke="none" pointer-events="none"/><g fill="#000000" font-family="Helvetica" text-anchor="middle" font-size="12px"><text x="561" y="1027">Per-protocol wptrunner class</text></g><rect x="491" y="1031" width="150" height="30" fill="none" stroke="none" pointer-events="none"/><g fill="#000000" font-family="Helvetica" text-anchor="middle" font-size="12px"><text x="566" y="1050">Web-platform-tests component</text></g><rect x="486" y="1055" width="90" height="30" fill="none" stroke="none" pointer-events="none"/><g fill="#000000" font-family="Helvetica" text-anchor="middle" font-size="12px"><text x="531" y="1074">Browser process</text></g><path d="M 398 8 L 398 42" fill="none" stroke="#000000" stroke-miterlimit="10" pointer-events="none"/><path d="M 398 47 L 395 40 L 398 42 L 402 40 Z" fill="#000000" stroke="#000000" stroke-miterlimit="10" pointer-events="none"/><rect x="478" y="388" width="120" height="60" fill-opacity="0.5" fill="#e6d0de" stroke="#000000" stroke-opacity="0.5" pointer-events="none"/><g transform="translate(480,411)"><switch><foreignObject pointer-events="all" width="116" height="15" requiredFeatures="http://www.w3.org/TR/SVG11/feature#Extensibility"><div xmlns="http://www.w3.org/1999/xhtml" style="display: inline-block; font-size: 12px; font-family: Helvetica; color: rgb(0, 0, 0); line-height: 1.26; vertical-align: top; width: 116px; white-space: normal; text-align: center;">TestRunnerManager</div></foreignObject><text x="58" y="14" fill="#000000" text-anchor="middle" font-size="12px" font-family="Helvetica">[Not supported by viewer]</text></switch></g><path d="M 398 288 L 533 384" fill="none" stroke="#000000" stroke-miterlimit="10" pointer-events="none"/><path d="M 537 387 L 529 386 L 533 384 L 533 380 Z" fill="#000000" stroke="#000000" stroke-miterlimit="10" pointer-events="none"/><rect x="198" y="388" width="120" height="60" fill-opacity="0.5" fill="#e6d0de" stroke="#000000" stroke-opacity="0.5" pointer-events="none"/><g transform="translate(200,411)"><switch><foreignObject pointer-events="all" width="116" height="15" requiredFeatures="http://www.w3.org/TR/SVG11/feature#Extensibility"><div xmlns="http://www.w3.org/1999/xhtml" style="display: inline-block; font-size: 12px; font-family: Helvetica; color: rgb(0, 0, 0); line-height: 1.26; vertical-align: top; width: 116px; white-space: normal; text-align: center;">TestRunnerManager</div></foreignObject><text x="58" y="14" fill="#000000" text-anchor="middle" font-size="12px" font-family="Helvetica">[Not supported by viewer]</text></switch></g><path d="M 398 288 L 263 384" fill="none" stroke="#000000" stroke-miterlimit="10" pointer-events="none"/><path d="M 259 387 L 263 380 L 263 384 L 267 386 Z" fill="#000000" stroke="#000000" stroke-miterlimit="10" pointer-events="none"/><rect x="575" y="748" width="110" height="40" fill="none" stroke="none" pointer-events="none"/><g fill="#000000" font-family="Helvetica" text-anchor="middle" font-size="12px"><text x="630" y="758">Browser control</text><text x="630" y="772">protocol</text><text x="630" y="786">(e.g. WebDriver)</text></g><rect x="258" y="708" width="80" height="40" fill="none" stroke="none" pointer-events="none"/><g fill="#000000" font-family="Helvetica" text-anchor="middle" font-size="12px"><text x="298" y="732">HTTP</text></g><rect x="111" y="728" width="80" height="40" fill="none" stroke="none" pointer-events="none"/><g fill="#000000" font-family="Helvetica" text-anchor="middle" font-size="12px"><text x="151" y="752">websockets</text></g><rect x="658" y="228" width="120" height="60" fill="#e6d0de" stroke="#000000" pointer-events="none"/><g transform="translate(660,251)"><switch><foreignObject pointer-events="all" width="116" height="15" requiredFeatures="http://www.w3.org/TR/SVG11/feature#Extensibility"><div xmlns="http://www.w3.org/1999/xhtml" style="display: inline-block; font-size: 12px; font-family: Helvetica; color: rgb(0, 0, 0); line-height: 1.26; vertical-align: top; width: 116px; white-space: normal; text-align: center;">Tests Queue</div></foreignObject><text x="58" y="14" fill="#000000" text-anchor="middle" font-size="12px" font-family="Helvetica">[Not supported by viewer]</text></switch></g><path d="M 718 108 L 718 222" fill="none" stroke="#000000" stroke-miterlimit="10" pointer-events="none"/><path d="M 718 227 L 715 220 L 718 222 L 722 220 Z" fill="#000000" stroke="#000000" stroke-miterlimit="10" pointer-events="none"/><path d="M 428 970 L 428 970" fill="none" stroke="#000000" stroke-miterlimit="10" pointer-events="none"/></g></svg> diff --git a/testing/web-platform/harness/docs/conf.py b/testing/web-platform/harness/docs/conf.py new file mode 100644 index 000000000..39e5cc4f0 --- /dev/null +++ b/testing/web-platform/harness/docs/conf.py @@ -0,0 +1,267 @@ +# -*- coding: utf-8 -*- +# +# wptrunner documentation build configuration file, created by +# sphinx-quickstart on Mon May 19 18:14:20 2014. +# +# This file is execfile()d with the current directory set to its +# containing dir. +# +# Note that not all possible configuration values are present in this +# autogenerated file. +# +# All configuration values have a default; values that are commented out +# serve to show the default. + +import sys +import os + +# If extensions (or modules to document with autodoc) are in another directory, +# add these directories to sys.path here. If the directory is relative to the +# documentation root, use os.path.abspath to make it absolute, like shown here. +#sys.path.insert(0, os.path.abspath('.')) + +# -- General configuration ------------------------------------------------ + +# If your documentation needs a minimal Sphinx version, state it here. +#needs_sphinx = '1.0' + +# Add any Sphinx extension module names here, as strings. They can be +# extensions coming with Sphinx (named 'sphinx.ext.*') or your custom +# ones. +extensions = [ + 'sphinx.ext.autodoc', + 'sphinx.ext.intersphinx', + 'sphinx.ext.viewcode', +] + +# Add any paths that contain templates here, relative to this directory. +templates_path = ['_templates'] + +# The suffix of source filenames. +source_suffix = '.rst' + +# The encoding of source files. +#source_encoding = 'utf-8-sig' + +# The master toctree document. +master_doc = 'index' + +# General information about the project. +project = u'wptrunner' +copyright = u'' + +# The version info for the project you're documenting, acts as replacement for +# |version| and |release|, also used in various other places throughout the +# built documents. +# +# The short X.Y version. +version = '0.3' +# The full version, including alpha/beta/rc tags. +release = '0.3' + +# The language for content autogenerated by Sphinx. Refer to documentation +# for a list of supported languages. +#language = None + +# There are two options for replacing |today|: either, you set today to some +# non-false value, then it is used: +#today = '' +# Else, today_fmt is used as the format for a strftime call. +#today_fmt = '%B %d, %Y' + +# List of patterns, relative to source directory, that match files and +# directories to ignore when looking for source files. +exclude_patterns = ['_build'] + +# The reST default role (used for this markup: `text`) to use for all +# documents. +#default_role = None + +# If true, '()' will be appended to :func: etc. cross-reference text. +#add_function_parentheses = True + +# If true, the current module name will be prepended to all description +# unit titles (such as .. function::). +#add_module_names = True + +# If true, sectionauthor and moduleauthor directives will be shown in the +# output. They are ignored by default. +#show_authors = False + +# The name of the Pygments (syntax highlighting) style to use. +pygments_style = 'sphinx' + +# A list of ignored prefixes for module index sorting. +#modindex_common_prefix = [] + +# If true, keep warnings as "system message" paragraphs in the built documents. +#keep_warnings = False + + +# -- Options for HTML output ---------------------------------------------- + +# The theme to use for HTML and HTML Help pages. See the documentation for +# a list of builtin themes. +html_theme = 'default' + +# Theme options are theme-specific and customize the look and feel of a theme +# further. For a list of options available for each theme, see the +# documentation. +#html_theme_options = {} + +# Add any paths that contain custom themes here, relative to this directory. +#html_theme_path = [] + +# The name for this set of Sphinx documents. If None, it defaults to +# "<project> v<release> documentation". +#html_title = None + +# A shorter title for the navigation bar. Default is the same as html_title. +#html_short_title = None + +# The name of an image file (relative to this directory) to place at the top +# of the sidebar. +#html_logo = None + +# The name of an image file (within the static path) to use as favicon of the +# docs. This file should be a Windows icon file (.ico) being 16x16 or 32x32 +# pixels large. +#html_favicon = None + +# Add any paths that contain custom static files (such as style sheets) here, +# relative to this directory. They are copied after the builtin static files, +# so a file named "default.css" will overwrite the builtin "default.css". +html_static_path = ['_static'] + +# Add any extra paths that contain custom files (such as robots.txt or +# .htaccess) here, relative to this directory. These files are copied +# directly to the root of the documentation. +#html_extra_path = [] + +# If not '', a 'Last updated on:' timestamp is inserted at every page bottom, +# using the given strftime format. +#html_last_updated_fmt = '%b %d, %Y' + +# If true, SmartyPants will be used to convert quotes and dashes to +# typographically correct entities. +#html_use_smartypants = True + +# Custom sidebar templates, maps document names to template names. +#html_sidebars = {} + +# Additional templates that should be rendered to pages, maps page names to +# template names. +#html_additional_pages = {} + +# If false, no module index is generated. +#html_domain_indices = True + +# If false, no index is generated. +#html_use_index = True + +# If true, the index is split into individual pages for each letter. +#html_split_index = False + +# If true, links to the reST sources are added to the pages. +#html_show_sourcelink = True + +# If true, "Created using Sphinx" is shown in the HTML footer. Default is True. +#html_show_sphinx = True + +# If true, "(C) Copyright ..." is shown in the HTML footer. Default is True. +#html_show_copyright = True + +# If true, an OpenSearch description file will be output, and all pages will +# contain a <link> tag referring to it. The value of this option must be the +# base URL from which the finished HTML is served. +#html_use_opensearch = '' + +# This is the file name suffix for HTML files (e.g. ".xhtml"). +#html_file_suffix = None + +# Output file base name for HTML help builder. +htmlhelp_basename = 'wptrunnerdoc' + + +# -- Options for LaTeX output --------------------------------------------- + +latex_elements = { +# The paper size ('letterpaper' or 'a4paper'). +#'papersize': 'letterpaper', + +# The font size ('10pt', '11pt' or '12pt'). +#'pointsize': '10pt', + +# Additional stuff for the LaTeX preamble. +#'preamble': '', +} + +# Grouping the document tree into LaTeX files. List of tuples +# (source start file, target name, title, +# author, documentclass [howto, manual, or own class]). +latex_documents = [ + ('index', 'wptrunner.tex', u'wptrunner Documentation', + u'James Graham', 'manual'), +] + +# The name of an image file (relative to this directory) to place at the top of +# the title page. +#latex_logo = None + +# For "manual" documents, if this is true, then toplevel headings are parts, +# not chapters. +#latex_use_parts = False + +# If true, show page references after internal links. +#latex_show_pagerefs = False + +# If true, show URL addresses after external links. +#latex_show_urls = False + +# Documents to append as an appendix to all manuals. +#latex_appendices = [] + +# If false, no module index is generated. +#latex_domain_indices = True + + +# -- Options for manual page output --------------------------------------- + +# One entry per manual page. List of tuples +# (source start file, name, description, authors, manual section). +man_pages = [ + ('index', 'wptrunner', u'wptrunner Documentation', + [u'James Graham'], 1) +] + +# If true, show URL addresses after external links. +#man_show_urls = False + + +# -- Options for Texinfo output ------------------------------------------- + +# Grouping the document tree into Texinfo files. List of tuples +# (source start file, target name, title, author, +# dir menu entry, description, category) +texinfo_documents = [ + ('index', 'wptrunner', u'wptrunner Documentation', + u'James Graham', 'wptrunner', 'One line description of project.', + 'Miscellaneous'), +] + +# Documents to append as an appendix to all manuals. +#texinfo_appendices = [] + +# If false, no module index is generated. +#texinfo_domain_indices = True + +# How to display URL addresses: 'footnote', 'no', or 'inline'. +#texinfo_show_urls = 'footnote' + +# If true, do not generate a @detailmenu in the "Top" node's menu. +#texinfo_no_detailmenu = False + + +# Example configuration for intersphinx: refer to the Python standard library. +intersphinx_mapping = {'python': ('http://docs.python.org/', None), + 'mozlog': ('http://mozbase.readthedocs.org/en/latest/', None)} diff --git a/testing/web-platform/harness/docs/design.rst b/testing/web-platform/harness/docs/design.rst new file mode 100644 index 000000000..bf108a008 --- /dev/null +++ b/testing/web-platform/harness/docs/design.rst @@ -0,0 +1,106 @@ +wptrunner Design +================ + +The design of wptrunner is intended to meet the following +requirements: + + * Possible to run tests from W3C web-platform-tests. + + * Tests should be run as fast as possible. In particular it should + not be necessary to restart the browser between tests, or similar. + + * As far as possible, the tests should run in a "normal" browser and + browsing context. In particular many tests assume that they are + running in a top-level browsing context, so we must avoid the use + of an ``iframe`` test container. + + * It must be possible to deal with all kinds of behaviour of the + browser runder test, for example, crashing, hanging, etc. + + * It should be possible to add support for new platforms and browsers + with minimal code changes. + + * It must be possible to run tests in parallel to further improve + performance. + + * Test output must be in a machine readable form. + +Architecture +------------ + +In order to meet the above requirements, wptrunner is designed to +push as much of the test scheduling as possible into the harness. This +allows the harness to monitor the state of the browser and perform +appropriate action if it gets into an unwanted state e.g. kill the +browser if it appears to be hung. + +The harness will typically communicate with the browser via some remote +control protocol such as WebDriver. However for browsers where no such +protocol is supported, other implementation strategies are possible, +typically at the expense of speed. + +The overall architecture of wptrunner is shown in the diagram below: + +.. image:: architecture.svg + +The main entry point to the code is :py:func:`run_tests` in +``wptrunner.py``. This is responsible for setting up the test +environment, loading the list of tests to be executed, and invoking +the remainder of the code to actually execute some tests. + +The test environment is encapsulated in the +:py:class:`TestEnvironment` class. This defers to code in +``web-platform-tests`` which actually starts the required servers to +run the tests. + +The set of tests to run is defined by the +:py:class:`TestLoader`. This is constructed with a +:py:class:`TestFilter` (not shown), which takes any filter arguments +from the command line to restrict the set of tests that will be +run. The :py:class:`TestLoader` reads both the ``web-platform-tests`` +JSON manifest and the expectation data stored in ini files and +produces a :py:class:`multiprocessing.Queue` of tests to run, and +their expected results. + +Actually running the tests happens through the +:py:class:`ManagerGroup` object. This takes the :py:class:`Queue` of +tests to be run and starts a :py:class:`testrunner.TestRunnerManager` for each +instance of the browser under test that will be started. These +:py:class:`TestRunnerManager` instances are each started in their own +thread. + +A :py:class:`TestRunnerManager` coordinates starting the product under +test, and outputting results from the test. In the case that the test +has timed out or the browser has crashed, it has to restart the +browser to ensure the test run can continue. The functionality for +initialising the browser under test, and probing its state +(e.g. whether the process is still alive) is implemented through a +:py:class:`Browser` object. An implementation of this class must be +provided for each product that is supported. + +The functionality for actually running the tests is provided by a +:py:class:`TestRunner` object. :py:class:`TestRunner` instances are +run in their own child process created with the +:py:mod:`multiprocessing` module. This allows them to run concurrently +and to be killed and restarted as required. Communication between the +:py:class:`TestRunnerManager` and the :py:class:`TestRunner` is +provided by a pair of queues, one for sending messages in each +direction. In particular test results are sent from the +:py:class:`TestRunner` to the :py:class:`TestRunnerManager` using one +of these queues. + +The :py:class:`TestRunner` object is generic in that the same +:py:class:`TestRunner` is used regardless of the product under +test. However the details of how to run the test may vary greatly with +the product since different products support different remote control +protocols (or none at all). These protocol-specific parts are placed +in the :py:class:`Executor` object. There is typically a different +:py:class:`Executor` class for each combination of control protocol +and test type. The :py:class:`TestRunner` is responsible for pulling +each test off the :py:class:`Queue` of tests and passing it down to +the :py:class:`Executor`. + +The executor often requires access to details of the particular +browser instance that it is testing so that it knows e.g. which port +to connect to to send commands to the browser. These details are +encapsulated in the :py:class:`ExecutorBrowser` class. diff --git a/testing/web-platform/harness/docs/expectation.rst b/testing/web-platform/harness/docs/expectation.rst new file mode 100644 index 000000000..6a0c77684 --- /dev/null +++ b/testing/web-platform/harness/docs/expectation.rst @@ -0,0 +1,248 @@ +Expectation Data +================ + +Introduction +------------ + +For use in continuous integration systems, and other scenarios where +regression tracking is required, wptrunner supports storing and +loading the expected result of each test in a test run. Typically +these expected results will initially be generated by running the +testsuite in a baseline build. They may then be edited by humans as +new features are added to the product that change the expected +results. The expected results may also vary for a single product +depending on the platform on which it is run. Therefore, the raw +structured log data is not a suitable format for storing these +files. Instead something is required that is: + + * Human readable + + * Human editable + + * Machine readable / writable + + * Capable of storing test id / result pairs + + * Suitable for storing in a version control system (i.e. text-based) + +The need for different results per platform means either having +multiple expectation files for each platform, or having a way to +express conditional values within a certain file. The former would be +rather cumbersome for humans updating the expectation files, so the +latter approach has been adopted, leading to the requirement: + + * Capable of storing result values that are conditional on the platform. + +There are few extant formats that meet these requirements, so +wptrunner uses a bespoke ``expectation manifest`` format, which is +closely based on the standard ``ini`` format. + +Directory Layout +---------------- + +Expectation manifest files must be stored under the ``metadata`` +directory passed to the test runner. The directory layout follows that +of web-platform-tests with each test path having a corresponding +manifest file. Tests that differ only by query string, or reftests +with the same test path but different ref paths share the same +reference file. The file name is taken from the last /-separated part +of the path, suffixed with ``.ini``. + +As an optimisation, files which produce only default results +(i.e. ``PASS`` or ``OK``) don't require a corresponding manifest file. + +For example a test with url:: + + /spec/section/file.html?query=param + +would have an expectation file :: + + metadata/spec/section/file.html.ini + + +.. _wptupdate-label: + +Generating Expectation Files +---------------------------- + +wptrunner provides the tool ``wptupdate`` to generate expectation +files from the results of a set of baseline test runs. The basic +syntax for this is:: + + wptupdate [options] [logfile]... + +Each ``logfile`` is a structured log file from a previous run. These +can be generated from wptrunner using the ``--log-raw`` option +e.g. ``--log-raw=structured.log``. The default behaviour is to update +all the test data for the particular combination of hardware and OS +used in the run corresponding to the log data, whilst leaving any +other expectations untouched. + +wptupdate takes several useful options: + +``--sync`` + Pull the latest version of web-platform-tests from the + upstream specified in the config file. If this is specified in + combination with logfiles, it is assumed that the results in the log + files apply to the post-update tests. + +``--no-check-clean`` + Don't attempt to check if the working directory is clean before + doing the update (assuming that the working directory is a git or + mercurial tree). + +``--patch`` + Create a a git commit, or a mq patch, with the changes made by wptupdate. + +``--ignore-existing`` + Overwrite all the expectation data for any tests that have a result + in the passed log files, not just data for the same platform. + +Examples +~~~~~~~~ + +Update the local copy of web-platform-tests without changing the +expectation data and commit (or create a mq patch for) the result:: + + wptupdate --patch --sync + +Update all the expectations from a set of cross-platform test runs:: + + wptupdate --no-check-clean --patch osx.log linux.log windows.log + +Add expectation data for some new tests that are expected to be +platform-independent:: + + wptupdate --no-check-clean --patch --ignore-existing tests.log + +Manifest Format +--------------- +The format of the manifest files is based on the ini format. Files are +divided into sections, each (apart from the root section) having a +heading enclosed in square braces. Within each section are key-value +pairs. There are several notable differences from standard .ini files, +however: + + * Sections may be hierarchically nested, with significant whitespace + indicating nesting depth. + + * Only ``:`` is valid as a key/value separator + +A simple example of a manifest file is:: + + root_key: root_value + + [section] + section_key: section_value + + [subsection] + subsection_key: subsection_value + + [another_section] + another_key: another_value + +Conditional Values +~~~~~~~~~~~~~~~~~~ + +In order to support values that depend on some external data, the +right hand side of a key/value pair can take a set of conditionals +rather than a plain value. These values are placed on a new line +following the key, with significant indentation. Conditional values +are prefixed with ``if`` and terminated with a colon, for example:: + + key: + if cond1: value1 + if cond2: value2 + value3 + +In this example, the value associated with ``key`` is determined by +first evaluating ``cond1`` against external data. If that is true, +``key`` is assigned the value ``value1``, otherwise ``cond2`` is +evaluated in the same way. If both ``cond1`` and ``cond2`` are false, +the unconditional ``value3`` is used. + +Conditions themselves use a Python-like expression syntax. Operands +can either be variables, corresponding to data passed in, numbers +(integer or floating point; exponential notation is not supported) or +quote-delimited strings. Equality is tested using ``==`` and +inequality by ``!=``. The operators ``and``, ``or`` and ``not`` are +used in the expected way. Parentheses can also be used for +grouping. For example:: + + key: + if (a == 2 or a == 3) and b == "abc": value1 + if a == 1 or b != "abc": value2 + value3 + +Here ``a`` and ``b`` are variables, the value of which will be +supplied when the manifest is used. + +Expectation Manifests +--------------------- + +When used for expectation data, manifests have the following format: + + * A section per test URL described by the manifest, with the section + heading being the part of the test URL following the last ``/`` in + the path (this allows multiple tests in a single manifest file with + the same path part of the URL, but different query parts). + + * A subsection per subtest, with the heading being the title of the + subtest. + + * A key ``type`` indicating the test type. This takes the values + ``testharness`` and ``reftest``. + + * For reftests, keys ``reftype`` indicating the reference type + (``==`` or ``!=``) and ``refurl`` indicating the URL of the + reference. + + * A key ``expected`` giving the expectation value of each (sub)test. + + * A key ``disabled`` which can be set to any value to indicate that + the (sub)test is disabled and should either not be run (for tests) + or that its results should be ignored (subtests). + + * A key ``restart-after`` which can be set to any value to indicate that + the runner should restart the browser after running this test (e.g. to + clear out unwanted state). + + * Variables ``debug``, ``os``, ``version``, ``processor`` and + ``bits`` that describe the configuration of the browser under + test. ``debug`` is a boolean indicating whether a build is a debug + build. ``os`` is a string indicating the operating system, and + ``version`` a string indicating the particular version of that + operating system. ``processor`` is a string indicating the + processor architecture and ``bits`` an integer indicating the + number of bits. This information is typically provided by + :py:mod:`mozinfo`. + + * Top level keys are taken as defaults for the whole file. So, for + example, a top level key with ``expected: FAIL`` would indicate + that all tests and subtests in the file are expected to fail, + unless they have an ``expected`` key of their own. + +An simple example manifest might look like:: + + [test.html?variant=basic] + type: testharness + + [Test something unsupported] + expected: FAIL + + [test.html?variant=broken] + expected: ERROR + + [test.html?variant=unstable] + disabled: http://test.bugs.example.org/bugs/12345 + +A more complex manifest with conditional properties might be:: + + [canvas_test.html] + expected: + if os == "osx": FAIL + if os == "windows" and version == "XP": FAIL + PASS + +Note that ``PASS`` in the above works, but is unnecessary; ``PASS`` +(or ``OK``) is always the default expectation for (sub)tests. diff --git a/testing/web-platform/harness/docs/index.rst b/testing/web-platform/harness/docs/index.rst new file mode 100644 index 000000000..5147d3e31 --- /dev/null +++ b/testing/web-platform/harness/docs/index.rst @@ -0,0 +1,24 @@ +.. wptrunner documentation master file, created by + sphinx-quickstart on Mon May 19 18:14:20 2014. + You can adapt this file completely to your liking, but it should at least + contain the root `toctree` directive. + +Welcome to wptrunner's documentation! +===================================== + +Contents: + +.. toctree:: + :maxdepth: 2 + + usage + expectation + design + +Indices and tables +================== + +* :ref:`genindex` +* :ref:`modindex` +* :ref:`search` + diff --git a/testing/web-platform/harness/docs/make.bat b/testing/web-platform/harness/docs/make.bat new file mode 100644 index 000000000..959c1615a --- /dev/null +++ b/testing/web-platform/harness/docs/make.bat @@ -0,0 +1,242 @@ +@ECHO OFF
+
+REM Command file for Sphinx documentation
+
+if "%SPHINXBUILD%" == "" (
+ set SPHINXBUILD=sphinx-build
+)
+set BUILDDIR=_build
+set ALLSPHINXOPTS=-d %BUILDDIR%/doctrees %SPHINXOPTS% .
+set I18NSPHINXOPTS=%SPHINXOPTS% .
+if NOT "%PAPER%" == "" (
+ set ALLSPHINXOPTS=-D latex_paper_size=%PAPER% %ALLSPHINXOPTS%
+ set I18NSPHINXOPTS=-D latex_paper_size=%PAPER% %I18NSPHINXOPTS%
+)
+
+if "%1" == "" goto help
+
+if "%1" == "help" (
+ :help
+ echo.Please use `make ^<target^>` where ^<target^> is one of
+ echo. html to make standalone HTML files
+ echo. dirhtml to make HTML files named index.html in directories
+ echo. singlehtml to make a single large HTML file
+ echo. pickle to make pickle files
+ echo. json to make JSON files
+ echo. htmlhelp to make HTML files and a HTML help project
+ echo. qthelp to make HTML files and a qthelp project
+ echo. devhelp to make HTML files and a Devhelp project
+ echo. epub to make an epub
+ echo. latex to make LaTeX files, you can set PAPER=a4 or PAPER=letter
+ echo. text to make text files
+ echo. man to make manual pages
+ echo. texinfo to make Texinfo files
+ echo. gettext to make PO message catalogs
+ echo. changes to make an overview over all changed/added/deprecated items
+ echo. xml to make Docutils-native XML files
+ echo. pseudoxml to make pseudoxml-XML files for display purposes
+ echo. linkcheck to check all external links for integrity
+ echo. doctest to run all doctests embedded in the documentation if enabled
+ goto end
+)
+
+if "%1" == "clean" (
+ for /d %%i in (%BUILDDIR%\*) do rmdir /q /s %%i
+ del /q /s %BUILDDIR%\*
+ goto end
+)
+
+
+%SPHINXBUILD% 2> nul
+if errorlevel 9009 (
+ echo.
+ echo.The 'sphinx-build' command was not found. Make sure you have Sphinx
+ echo.installed, then set the SPHINXBUILD environment variable to point
+ echo.to the full path of the 'sphinx-build' executable. Alternatively you
+ echo.may add the Sphinx directory to PATH.
+ echo.
+ echo.If you don't have Sphinx installed, grab it from
+ echo.http://sphinx-doc.org/
+ exit /b 1
+)
+
+if "%1" == "html" (
+ %SPHINXBUILD% -b html %ALLSPHINXOPTS% %BUILDDIR%/html
+ if errorlevel 1 exit /b 1
+ echo.
+ echo.Build finished. The HTML pages are in %BUILDDIR%/html.
+ goto end
+)
+
+if "%1" == "dirhtml" (
+ %SPHINXBUILD% -b dirhtml %ALLSPHINXOPTS% %BUILDDIR%/dirhtml
+ if errorlevel 1 exit /b 1
+ echo.
+ echo.Build finished. The HTML pages are in %BUILDDIR%/dirhtml.
+ goto end
+)
+
+if "%1" == "singlehtml" (
+ %SPHINXBUILD% -b singlehtml %ALLSPHINXOPTS% %BUILDDIR%/singlehtml
+ if errorlevel 1 exit /b 1
+ echo.
+ echo.Build finished. The HTML pages are in %BUILDDIR%/singlehtml.
+ goto end
+)
+
+if "%1" == "pickle" (
+ %SPHINXBUILD% -b pickle %ALLSPHINXOPTS% %BUILDDIR%/pickle
+ if errorlevel 1 exit /b 1
+ echo.
+ echo.Build finished; now you can process the pickle files.
+ goto end
+)
+
+if "%1" == "json" (
+ %SPHINXBUILD% -b json %ALLSPHINXOPTS% %BUILDDIR%/json
+ if errorlevel 1 exit /b 1
+ echo.
+ echo.Build finished; now you can process the JSON files.
+ goto end
+)
+
+if "%1" == "htmlhelp" (
+ %SPHINXBUILD% -b htmlhelp %ALLSPHINXOPTS% %BUILDDIR%/htmlhelp
+ if errorlevel 1 exit /b 1
+ echo.
+ echo.Build finished; now you can run HTML Help Workshop with the ^
+.hhp project file in %BUILDDIR%/htmlhelp.
+ goto end
+)
+
+if "%1" == "qthelp" (
+ %SPHINXBUILD% -b qthelp %ALLSPHINXOPTS% %BUILDDIR%/qthelp
+ if errorlevel 1 exit /b 1
+ echo.
+ echo.Build finished; now you can run "qcollectiongenerator" with the ^
+.qhcp project file in %BUILDDIR%/qthelp, like this:
+ echo.^> qcollectiongenerator %BUILDDIR%\qthelp\wptrunner.qhcp
+ echo.To view the help file:
+ echo.^> assistant -collectionFile %BUILDDIR%\qthelp\wptrunner.ghc
+ goto end
+)
+
+if "%1" == "devhelp" (
+ %SPHINXBUILD% -b devhelp %ALLSPHINXOPTS% %BUILDDIR%/devhelp
+ if errorlevel 1 exit /b 1
+ echo.
+ echo.Build finished.
+ goto end
+)
+
+if "%1" == "epub" (
+ %SPHINXBUILD% -b epub %ALLSPHINXOPTS% %BUILDDIR%/epub
+ if errorlevel 1 exit /b 1
+ echo.
+ echo.Build finished. The epub file is in %BUILDDIR%/epub.
+ goto end
+)
+
+if "%1" == "latex" (
+ %SPHINXBUILD% -b latex %ALLSPHINXOPTS% %BUILDDIR%/latex
+ if errorlevel 1 exit /b 1
+ echo.
+ echo.Build finished; the LaTeX files are in %BUILDDIR%/latex.
+ goto end
+)
+
+if "%1" == "latexpdf" (
+ %SPHINXBUILD% -b latex %ALLSPHINXOPTS% %BUILDDIR%/latex
+ cd %BUILDDIR%/latex
+ make all-pdf
+ cd %BUILDDIR%/..
+ echo.
+ echo.Build finished; the PDF files are in %BUILDDIR%/latex.
+ goto end
+)
+
+if "%1" == "latexpdfja" (
+ %SPHINXBUILD% -b latex %ALLSPHINXOPTS% %BUILDDIR%/latex
+ cd %BUILDDIR%/latex
+ make all-pdf-ja
+ cd %BUILDDIR%/..
+ echo.
+ echo.Build finished; the PDF files are in %BUILDDIR%/latex.
+ goto end
+)
+
+if "%1" == "text" (
+ %SPHINXBUILD% -b text %ALLSPHINXOPTS% %BUILDDIR%/text
+ if errorlevel 1 exit /b 1
+ echo.
+ echo.Build finished. The text files are in %BUILDDIR%/text.
+ goto end
+)
+
+if "%1" == "man" (
+ %SPHINXBUILD% -b man %ALLSPHINXOPTS% %BUILDDIR%/man
+ if errorlevel 1 exit /b 1
+ echo.
+ echo.Build finished. The manual pages are in %BUILDDIR%/man.
+ goto end
+)
+
+if "%1" == "texinfo" (
+ %SPHINXBUILD% -b texinfo %ALLSPHINXOPTS% %BUILDDIR%/texinfo
+ if errorlevel 1 exit /b 1
+ echo.
+ echo.Build finished. The Texinfo files are in %BUILDDIR%/texinfo.
+ goto end
+)
+
+if "%1" == "gettext" (
+ %SPHINXBUILD% -b gettext %I18NSPHINXOPTS% %BUILDDIR%/locale
+ if errorlevel 1 exit /b 1
+ echo.
+ echo.Build finished. The message catalogs are in %BUILDDIR%/locale.
+ goto end
+)
+
+if "%1" == "changes" (
+ %SPHINXBUILD% -b changes %ALLSPHINXOPTS% %BUILDDIR%/changes
+ if errorlevel 1 exit /b 1
+ echo.
+ echo.The overview file is in %BUILDDIR%/changes.
+ goto end
+)
+
+if "%1" == "linkcheck" (
+ %SPHINXBUILD% -b linkcheck %ALLSPHINXOPTS% %BUILDDIR%/linkcheck
+ if errorlevel 1 exit /b 1
+ echo.
+ echo.Link check complete; look for any errors in the above output ^
+or in %BUILDDIR%/linkcheck/output.txt.
+ goto end
+)
+
+if "%1" == "doctest" (
+ %SPHINXBUILD% -b doctest %ALLSPHINXOPTS% %BUILDDIR%/doctest
+ if errorlevel 1 exit /b 1
+ echo.
+ echo.Testing of doctests in the sources finished, look at the ^
+results in %BUILDDIR%/doctest/output.txt.
+ goto end
+)
+
+if "%1" == "xml" (
+ %SPHINXBUILD% -b xml %ALLSPHINXOPTS% %BUILDDIR%/xml
+ if errorlevel 1 exit /b 1
+ echo.
+ echo.Build finished. The XML files are in %BUILDDIR%/xml.
+ goto end
+)
+
+if "%1" == "pseudoxml" (
+ %SPHINXBUILD% -b pseudoxml %ALLSPHINXOPTS% %BUILDDIR%/pseudoxml
+ if errorlevel 1 exit /b 1
+ echo.
+ echo.Build finished. The pseudo-XML files are in %BUILDDIR%/pseudoxml.
+ goto end
+)
+
+:end
diff --git a/testing/web-platform/harness/docs/usage.rst b/testing/web-platform/harness/docs/usage.rst new file mode 100644 index 000000000..8e74a4320 --- /dev/null +++ b/testing/web-platform/harness/docs/usage.rst @@ -0,0 +1,238 @@ +Getting Started +=============== + +Installing wptrunner +-------------------- + +The easiest way to install wptrunner is into a virtualenv, using pip:: + + virtualenv wptrunner + cd wptrunner + source bin/activate + pip install wptrunner + +This will install the base dependencies for wptrunner, but not any +extra dependencies required to test against specific browsers. In +order to do this you must use use the extra requirements files in +``$VIRTUAL_ENV/requirements/requirements_browser.txt``. For example, +in order to test against Firefox you would have to run:: + + pip install -r requirements/requirements_firefox.txt + +If you intend to work on the code, the ``-e`` option to pip should be +used in combination with a source checkout i.e. inside a virtual +environment created as above:: + + git clone https://github.com/w3c/wptrunner.git + cd wptrunner + pip install -e ./ + +In addition to the dependencies installed by pip, wptrunner requires +a copy of the web-platform-tests repository. This can be located +anywhere on the filesystem, but the easiest option is to put it +under the same parent directory as the wptrunner checkout:: + + git clone https://github.com/w3c/web-platform-tests.git + +It is also necessary to generate a web-platform-tests ``MANIFEST.json`` +file. It's recommended to also put that under the same parent directory as +the wptrunner checkout, in a directory named ``meta``:: + + mkdir meta + cd web-platform-tests + python manifest --path ../meta/MANIFEST.json + +The ``MANIFEST.json`` file needs to be regenerated each time the +web-platform-tests checkout is updated. To aid with the update process +there is a tool called ``wptupdate``, which is described in +:ref:`wptupdate-label`. + +Running the Tests +----------------- + +A test run is started using the ``wptrunner`` command. The command +takes multiple options, of which the following are most significant: + +``--product`` (defaults to `firefox`) + The product to test against: `b2g`, `chrome`, `firefox`, or `servo`. + +``--binary`` (required if product is `firefox` or `servo`) + The path to a binary file for the product (browser) to test against. + +``--webdriver-binary`` (required if product is `chrome`) + The path to a `*driver` binary; e.g., a `chromedriver` binary. + +``--certutil-binary`` (required if product is `firefox` [#]_) + The path to a `certutil` binary (for tests that must be run over https). + +``--metadata`` (required only when not `using default paths`_) + The path to a directory containing test metadata. [#]_ + +``--tests`` (required only when not `using default paths`_) + The path to a directory containing a web-platform-tests checkout. + +``--prefs-root`` (required only when testing a Firefox binary) + The path to a directory containing Firefox test-harness preferences. [#]_ + +``--config`` (should default to `wptrunner.default.ini`) + The path to the config (ini) file. + +.. [#] The ``--certutil-binary`` option is required when the product is + ``firefox`` unless ``--ssl-type=none`` is specified. + +.. [#] The ``--metadata`` path is to a directory that contains: + + * a ``MANIFEST.json`` file (the web-platform-tests documentation has + instructions on generating this file) + * (optionally) any expectation files (see :ref:`wptupdate-label`) + +.. [#] Example ``--prefs-root`` value: ``~/mozilla-central/testing/profiles``. + +There are also a variety of other command-line options available; use +``--help`` to list them. + +The following examples show how to start wptrunner with various options. + +------------------ +Starting wptrunner +------------------ + +The examples below assume the following directory layout, +though no specific folder structure is required:: + + ~/testtwf/wptrunner # wptrunner checkout + ~/testtwf/web-platform-tests # web-platform-tests checkout + ~/testtwf/meta # metadata + +To test a Firefox Nightly build in an OS X environment, you might start +wptrunner using something similar to the following example:: + + wptrunner --metadata=~/testtwf/meta/ --tests=~/testtwf/web-platform-tests/ \ + --binary=~/mozilla-central/obj-x86_64-apple-darwin14.3.0/dist/Nightly.app/Contents/MacOS/firefox \ + --certutil-binary=~/mozilla-central/obj-x86_64-apple-darwin14.3.0/security/nss/cmd/certutil/certutil \ + --prefs-root=~/mozilla-central/testing/profiles + + +And to test a Chromium build in an OS X environment, you might start +wptrunner using something similar to the following example:: + + wptrunner --metadata=~/testtwf/meta/ --tests=~/testtwf/web-platform-tests/ \ + --binary=~/chromium/src/out/Release/Chromium.app/Contents/MacOS/Chromium \ + --webdriver-binary=/usr/local/bin/chromedriver --product=chrome + +-------------------- +Running test subsets +-------------------- + +To restrict a test run just to tests in a particular web-platform-tests +subdirectory, specify the directory name in the positional arguments after +the options; for example, run just the tests in the `dom` subdirectory:: + + wptrunner --metadata=~/testtwf/meta --tests=~/testtwf/web-platform-tests/ \ + --binary=/path/to/firefox --certutil-binary=/path/to/certutil \ + --prefs-root=/path/to/testing/profiles \ + dom + +------------------- +Running in parallel +------------------- + +To speed up the testing process, use the ``--processes`` option to have +wptrunner run multiple browser instances in parallel. For example, to +have wptrunner attempt to run tests against with six browser instances +in parallel, specify ``--processes=6``. But note that behaviour in this +mode is necessarily less deterministic than with ``--processes=1`` (the +default), so there may be more noise in the test results. + +------------------- +Using default paths +------------------- + +The (otherwise-required) ``--tests`` and ``--metadata`` command-line +options/flags be omitted if any configuration file is found that +contains a section specifying the ``tests`` and ``metadata`` keys. + +See the `Configuration File`_ section for more information about +configuration files, including information about their expected +locations. + +The content of the ``wptrunner.default.ini`` default configuration file +makes wptrunner look for tests (that is, a web-platform-tests checkout) +as a subdirectory of the current directory named ``tests``, and for +metadata files in a subdirectory of the current directory named ``meta``. + +Output +------ + +wptrunner uses the :py:mod:`mozlog` package for output. This +structures events such as test results or log messages as JSON objects +that can then be fed to other tools for interpretation. More details +about the message format are given in the +:py:mod:`mozlog` documentation. + +By default the raw JSON messages are dumped to stdout. This is +convenient for piping into other tools, but not ideal for humans +reading the output. :py:mod:`mozlog` comes with several other +formatters, which are accessible through command line options. The +general format of these options is ``--log-name=dest``, where ``name`` +is the name of the format and ``dest`` is a path to a destination +file, or ``-`` for stdout. The raw JSON data is written by the ``raw`` +formatter so, the default setup corresponds to ``--log-raw=-``. + +A reasonable output format for humans is provided as ``mach``. So in +order to output the full raw log to a file and a human-readable +summary to stdout, one might pass the options:: + + --log-raw=output.log --log-mach=- + +Configuration File +------------------ + +wptrunner uses a ``.ini`` file to control some configuration +sections. The file has three sections; ``[products]``, +``[manifest:default]`` and ``[web-platform-tests]``. + +``[products]`` is used to +define the set of available products. By default this section is empty +which means that all the products distributed with wptrunner are +enabled (although their dependencies may not be installed). The set +of enabled products can be set by using the product name as the +key. For built in products the value is empty. It is also possible to +provide the path to a script implementing the browser functionality +e.g.:: + + [products] + chrome = + netscape4 = path/to/netscape.py + +``[manifest:default]`` specifies the default paths for the tests and metadata, +relative to the config file. For example:: + + [manifest:default] + tests = ~/testtwf/web-platform-tests + metadata = ~/testtwf/meta + + +``[web-platform-tests]`` is used to set the properties of the upstream +repository when updating the paths. ``remote_url`` specifies the git +url to pull from; ``branch`` the branch to sync against and +``sync_path`` the local path, relative to the configuration file, to +use when checking out the tests e.g.:: + + [web-platform-tests] + remote_url = https://github.com/w3c/web-platform-tests.git + branch = master + sync_path = sync + +A configuration file must contain all the above fields; falling back +to the default values for unspecified fields is not yet supported. + +The ``wptrunner`` and ``wptupdate`` commands will use configuration +files in the following order: + + * Any path supplied with a ``--config`` flag to the command. + + * A file called ``wptrunner.ini`` in the current directory + + * The default configuration file (``wptrunner.default.ini`` in the + source directory) |
