diff options
Diffstat (limited to 'testing/web-platform/tests/tools/wptserve')
58 files changed, 4925 insertions, 0 deletions
diff --git a/testing/web-platform/tests/tools/wptserve/.gitignore b/testing/web-platform/tests/tools/wptserve/.gitignore new file mode 100644 index 000000000..8e87d3884 --- /dev/null +++ b/testing/web-platform/tests/tools/wptserve/.gitignore @@ -0,0 +1,40 @@ +*.py[cod] +*~ +\#* + +docs/_build/ + +# C extensions +*.so + +# Packages +*.egg +*.egg-info +dist +build +eggs +parts +bin +var +sdist +develop-eggs +.installed.cfg +lib +lib64 + +# Installer logs +pip-log.txt + +# Unit test / coverage reports +.coverage +.tox +nosetests.xml +tests/functional/html/* + +# Translations +*.mo + +# Mr Developer +.mr.developer.cfg +.project +.pydevproject diff --git a/testing/web-platform/tests/tools/wptserve/.travis.yml b/testing/web-platform/tests/tools/wptserve/.travis.yml new file mode 100644 index 000000000..00183731b --- /dev/null +++ b/testing/web-platform/tests/tools/wptserve/.travis.yml @@ -0,0 +1,24 @@ +language: python + +sudo: false + +cache: + directories: + - $HOME/.cache/pip + +matrix: + include: + - python: 2.7 + env: TOXENV=py27 + - python: pypy + env: TOXENV=pypy + +install: + - pip install -U tox codecov + +script: + - tox + +after_success: + - coverage combine + - codecov diff --git a/testing/web-platform/tests/tools/wptserve/LICENSE b/testing/web-platform/tests/tools/wptserve/LICENSE new file mode 100644 index 000000000..45896e6be --- /dev/null +++ b/testing/web-platform/tests/tools/wptserve/LICENSE @@ -0,0 +1,30 @@ +W3C 3-clause BSD License + +http://www.w3.org/Consortium/Legal/2008/03-bsd-license.html + +Redistribution and use in source and binary forms, with or without +modification, are permitted provided that the following conditions are +met: + +* Redistributions of works must retain the original copyright notice, + this list of conditions and the following disclaimer. + +* Redistributions in binary form must reproduce the original copyright + notice, this list of conditions and the following disclaimer in the + documentation and/or other materials provided with the distribution. + +* Neither the name of the W3C nor the names of its contributors may be + used to endorse or promote products derived from this work without + specific prior written permission. + +THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +"AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. diff --git a/testing/web-platform/tests/tools/wptserve/MANIFEST.in b/testing/web-platform/tests/tools/wptserve/MANIFEST.in new file mode 100644 index 000000000..4bf448352 --- /dev/null +++ b/testing/web-platform/tests/tools/wptserve/MANIFEST.in @@ -0,0 +1 @@ +include README.md
\ No newline at end of file diff --git a/testing/web-platform/tests/tools/wptserve/README.md b/testing/web-platform/tests/tools/wptserve/README.md new file mode 100644 index 000000000..c0c88e2c3 --- /dev/null +++ b/testing/web-platform/tests/tools/wptserve/README.md @@ -0,0 +1,4 @@ +wptserve +======== + +Web server designed for use with web-platform-tests diff --git a/testing/web-platform/tests/tools/wptserve/docs/Makefile b/testing/web-platform/tests/tools/wptserve/docs/Makefile new file mode 100644 index 000000000..250b6c864 --- /dev/null +++ b/testing/web-platform/tests/tools/wptserve/docs/Makefile @@ -0,0 +1,153 @@ +# Makefile for Sphinx documentation +# + +# You can set these variables from the command line. +SPHINXOPTS = +SPHINXBUILD = sphinx-build +PAPER = +BUILDDIR = _build + +# Internal variables. +PAPEROPT_a4 = -D latex_paper_size=a4 +PAPEROPT_letter = -D latex_paper_size=letter +ALLSPHINXOPTS = -d $(BUILDDIR)/doctrees $(PAPEROPT_$(PAPER)) $(SPHINXOPTS) . +# the i18n builder cannot share the environment and doctrees with the others +I18NSPHINXOPTS = $(PAPEROPT_$(PAPER)) $(SPHINXOPTS) . + +.PHONY: help clean html dirhtml singlehtml pickle json htmlhelp qthelp devhelp epub latex latexpdf text man changes linkcheck doctest gettext + +help: + @echo "Please use \`make <target>' where <target> is one of" + @echo " html to make standalone HTML files" + @echo " dirhtml to make HTML files named index.html in directories" + @echo " singlehtml to make a single large HTML file" + @echo " pickle to make pickle files" + @echo " json to make JSON files" + @echo " htmlhelp to make HTML files and a HTML help project" + @echo " qthelp to make HTML files and a qthelp project" + @echo " devhelp to make HTML files and a Devhelp project" + @echo " epub to make an epub" + @echo " latex to make LaTeX files, you can set PAPER=a4 or PAPER=letter" + @echo " latexpdf to make LaTeX files and run them through pdflatex" + @echo " text to make text files" + @echo " man to make manual pages" + @echo " texinfo to make Texinfo files" + @echo " info to make Texinfo files and run them through makeinfo" + @echo " gettext to make PO message catalogs" + @echo " changes to make an overview of all changed/added/deprecated items" + @echo " linkcheck to check all external links for integrity" + @echo " doctest to run all doctests embedded in the documentation (if enabled)" + +clean: + -rm -rf $(BUILDDIR)/* + +html: + $(SPHINXBUILD) -b html $(ALLSPHINXOPTS) $(BUILDDIR)/html + @echo + @echo "Build finished. The HTML pages are in $(BUILDDIR)/html." + +dirhtml: + $(SPHINXBUILD) -b dirhtml $(ALLSPHINXOPTS) $(BUILDDIR)/dirhtml + @echo + @echo "Build finished. The HTML pages are in $(BUILDDIR)/dirhtml." + +singlehtml: + $(SPHINXBUILD) -b singlehtml $(ALLSPHINXOPTS) $(BUILDDIR)/singlehtml + @echo + @echo "Build finished. The HTML page is in $(BUILDDIR)/singlehtml." + +pickle: + $(SPHINXBUILD) -b pickle $(ALLSPHINXOPTS) $(BUILDDIR)/pickle + @echo + @echo "Build finished; now you can process the pickle files." + +json: + $(SPHINXBUILD) -b json $(ALLSPHINXOPTS) $(BUILDDIR)/json + @echo + @echo "Build finished; now you can process the JSON files." + +htmlhelp: + $(SPHINXBUILD) -b htmlhelp $(ALLSPHINXOPTS) $(BUILDDIR)/htmlhelp + @echo + @echo "Build finished; now you can run HTML Help Workshop with the" \ + ".hhp project file in $(BUILDDIR)/htmlhelp." + +qthelp: + $(SPHINXBUILD) -b qthelp $(ALLSPHINXOPTS) $(BUILDDIR)/qthelp + @echo + @echo "Build finished; now you can run "qcollectiongenerator" with the" \ + ".qhcp project file in $(BUILDDIR)/qthelp, like this:" + @echo "# qcollectiongenerator $(BUILDDIR)/qthelp/wptserve.qhcp" + @echo "To view the help file:" + @echo "# assistant -collectionFile $(BUILDDIR)/qthelp/wptserve.qhc" + +devhelp: + $(SPHINXBUILD) -b devhelp $(ALLSPHINXOPTS) $(BUILDDIR)/devhelp + @echo + @echo "Build finished." + @echo "To view the help file:" + @echo "# mkdir -p $$HOME/.local/share/devhelp/wptserve" + @echo "# ln -s $(BUILDDIR)/devhelp $$HOME/.local/share/devhelp/wptserve" + @echo "# devhelp" + +epub: + $(SPHINXBUILD) -b epub $(ALLSPHINXOPTS) $(BUILDDIR)/epub + @echo + @echo "Build finished. The epub file is in $(BUILDDIR)/epub." + +latex: + $(SPHINXBUILD) -b latex $(ALLSPHINXOPTS) $(BUILDDIR)/latex + @echo + @echo "Build finished; the LaTeX files are in $(BUILDDIR)/latex." + @echo "Run \`make' in that directory to run these through (pdf)latex" \ + "(use \`make latexpdf' here to do that automatically)." + +latexpdf: + $(SPHINXBUILD) -b latex $(ALLSPHINXOPTS) $(BUILDDIR)/latex + @echo "Running LaTeX files through pdflatex..." + $(MAKE) -C $(BUILDDIR)/latex all-pdf + @echo "pdflatex finished; the PDF files are in $(BUILDDIR)/latex." + +text: + $(SPHINXBUILD) -b text $(ALLSPHINXOPTS) $(BUILDDIR)/text + @echo + @echo "Build finished. The text files are in $(BUILDDIR)/text." + +man: + $(SPHINXBUILD) -b man $(ALLSPHINXOPTS) $(BUILDDIR)/man + @echo + @echo "Build finished. The manual pages are in $(BUILDDIR)/man." + +texinfo: + $(SPHINXBUILD) -b texinfo $(ALLSPHINXOPTS) $(BUILDDIR)/texinfo + @echo + @echo "Build finished. The Texinfo files are in $(BUILDDIR)/texinfo." + @echo "Run \`make' in that directory to run these through makeinfo" \ + "(use \`make info' here to do that automatically)." + +info: + $(SPHINXBUILD) -b texinfo $(ALLSPHINXOPTS) $(BUILDDIR)/texinfo + @echo "Running Texinfo files through makeinfo..." + make -C $(BUILDDIR)/texinfo info + @echo "makeinfo finished; the Info files are in $(BUILDDIR)/texinfo." + +gettext: + $(SPHINXBUILD) -b gettext $(I18NSPHINXOPTS) $(BUILDDIR)/locale + @echo + @echo "Build finished. The message catalogs are in $(BUILDDIR)/locale." + +changes: + $(SPHINXBUILD) -b changes $(ALLSPHINXOPTS) $(BUILDDIR)/changes + @echo + @echo "The overview file is in $(BUILDDIR)/changes." + +linkcheck: + $(SPHINXBUILD) -b linkcheck $(ALLSPHINXOPTS) $(BUILDDIR)/linkcheck + @echo + @echo "Link check complete; look for any errors in the above output " \ + "or in $(BUILDDIR)/linkcheck/output.txt." + +doctest: + $(SPHINXBUILD) -b doctest $(ALLSPHINXOPTS) $(BUILDDIR)/doctest + @echo "Testing of doctests in the sources finished, look at the " \ + "results in $(BUILDDIR)/doctest/output.txt." diff --git a/testing/web-platform/tests/tools/wptserve/docs/conf.py b/testing/web-platform/tests/tools/wptserve/docs/conf.py new file mode 100644 index 000000000..eae1c20cc --- /dev/null +++ b/testing/web-platform/tests/tools/wptserve/docs/conf.py @@ -0,0 +1,243 @@ +# -*- coding: utf-8 -*- +# +# wptserve documentation build configuration file, created by +# sphinx-quickstart on Wed Aug 14 17:23:24 2013. +# +# This file is execfile()d with the current directory set to its containing dir. +# +# Note that not all possible configuration values are present in this +# autogenerated file. +# +# All configuration values have a default; values that are commented out +# serve to show the default. + +import sys, os +sys.path.insert(0, os.path.abspath("..")) + +# If extensions (or modules to document with autodoc) are in another directory, +# add these directories to sys.path here. If the directory is relative to the +# documentation root, use os.path.abspath to make it absolute, like shown here. +#sys.path.insert(0, os.path.abspath('.')) + +# -- General configuration ----------------------------------------------------- + +# If your documentation needs a minimal Sphinx version, state it here. +#needs_sphinx = '1.0' + +# Add any Sphinx extension module names here, as strings. They can be extensions +# coming with Sphinx (named 'sphinx.ext.*') or your custom ones. +extensions = ['sphinx.ext.autodoc', 'sphinx.ext.viewcode'] + +# Add any paths that contain templates here, relative to this directory. +templates_path = ['_templates'] + +# The suffix of source filenames. +source_suffix = '.rst' + +# The encoding of source files. +#source_encoding = 'utf-8-sig' + +# The master toctree document. +master_doc = 'index' + +# General information about the project. +project = u'wptserve' +copyright = u'2013, Mozilla Foundation and other wptserve contributers' + +# The version info for the project you're documenting, acts as replacement for +# |version| and |release|, also used in various other places throughout the +# built documents. +# +# The short X.Y version. +version = '0.1' +# The full version, including alpha/beta/rc tags. +release = '0.1' + +# The language for content autogenerated by Sphinx. Refer to documentation +# for a list of supported languages. +#language = None + +# There are two options for replacing |today|: either, you set today to some +# non-false value, then it is used: +#today = '' +# Else, today_fmt is used as the format for a strftime call. +#today_fmt = '%B %d, %Y' + +# List of patterns, relative to source directory, that match files and +# directories to ignore when looking for source files. +exclude_patterns = ['_build'] + +# The reST default role (used for this markup: `text`) to use for all documents. +#default_role = None + +# If true, '()' will be appended to :func: etc. cross-reference text. +#add_function_parentheses = True + +# If true, the current module name will be prepended to all description +# unit titles (such as .. function::). +#add_module_names = True + +# If true, sectionauthor and moduleauthor directives will be shown in the +# output. They are ignored by default. +#show_authors = False + +# The name of the Pygments (syntax highlighting) style to use. +pygments_style = 'sphinx' + +# A list of ignored prefixes for module index sorting. +#modindex_common_prefix = [] + + +# -- Options for HTML output --------------------------------------------------- + +# The theme to use for HTML and HTML Help pages. See the documentation for +# a list of builtin themes. +html_theme = 'default' + +# Theme options are theme-specific and customize the look and feel of a theme +# further. For a list of options available for each theme, see the +# documentation. +#html_theme_options = {} + +# Add any paths that contain custom themes here, relative to this directory. +#html_theme_path = [] + +# The name for this set of Sphinx documents. If None, it defaults to +# "<project> v<release> documentation". +#html_title = None + +# A shorter title for the navigation bar. Default is the same as html_title. +#html_short_title = None + +# The name of an image file (relative to this directory) to place at the top +# of the sidebar. +#html_logo = None + +# The name of an image file (within the static path) to use as favicon of the +# docs. This file should be a Windows icon file (.ico) being 16x16 or 32x32 +# pixels large. +#html_favicon = None + +# Add any paths that contain custom static files (such as style sheets) here, +# relative to this directory. They are copied after the builtin static files, +# so a file named "default.css" will overwrite the builtin "default.css". +html_static_path = ['_static'] + +# If not '', a 'Last updated on:' timestamp is inserted at every page bottom, +# using the given strftime format. +#html_last_updated_fmt = '%b %d, %Y' + +# If true, SmartyPants will be used to convert quotes and dashes to +# typographically correct entities. +#html_use_smartypants = True + +# Custom sidebar templates, maps document names to template names. +#html_sidebars = {} + +# Additional templates that should be rendered to pages, maps page names to +# template names. +#html_additional_pages = {} + +# If false, no module index is generated. +#html_domain_indices = True + +# If false, no index is generated. +#html_use_index = True + +# If true, the index is split into individual pages for each letter. +#html_split_index = False + +# If true, links to the reST sources are added to the pages. +#html_show_sourcelink = True + +# If true, "Created using Sphinx" is shown in the HTML footer. Default is True. +#html_show_sphinx = True + +# If true, "(C) Copyright ..." is shown in the HTML footer. Default is True. +#html_show_copyright = True + +# If true, an OpenSearch description file will be output, and all pages will +# contain a <link> tag referring to it. The value of this option must be the +# base URL from which the finished HTML is served. +#html_use_opensearch = '' + +# This is the file name suffix for HTML files (e.g. ".xhtml"). +#html_file_suffix = None + +# Output file base name for HTML help builder. +htmlhelp_basename = 'wptservedoc' + + +# -- Options for LaTeX output -------------------------------------------------- + +latex_elements = { +# The paper size ('letterpaper' or 'a4paper'). +#'papersize': 'letterpaper', + +# The font size ('10pt', '11pt' or '12pt'). +#'pointsize': '10pt', + +# Additional stuff for the LaTeX preamble. +#'preamble': '', +} + +# Grouping the document tree into LaTeX files. List of tuples +# (source start file, target name, title, author, documentclass [howto/manual]). +latex_documents = [ + ('index', 'wptserve.tex', u'wptserve Documentation', + u'James Graham', 'manual'), +] + +# The name of an image file (relative to this directory) to place at the top of +# the title page. +#latex_logo = None + +# For "manual" documents, if this is true, then toplevel headings are parts, +# not chapters. +#latex_use_parts = False + +# If true, show page references after internal links. +#latex_show_pagerefs = False + +# If true, show URL addresses after external links. +#latex_show_urls = False + +# Documents to append as an appendix to all manuals. +#latex_appendices = [] + +# If false, no module index is generated. +#latex_domain_indices = True + + +# -- Options for manual page output -------------------------------------------- + +# One entry per manual page. List of tuples +# (source start file, name, description, authors, manual section). +man_pages = [ + ('index', 'wptserve', u'wptserve Documentation', + [u'James Graham'], 1) +] + +# If true, show URL addresses after external links. +#man_show_urls = False + + +# -- Options for Texinfo output ------------------------------------------------ + +# Grouping the document tree into Texinfo files. List of tuples +# (source start file, target name, title, author, +# dir menu entry, description, category) +texinfo_documents = [ + ('index', 'wptserve', u'wptserve Documentation', + u'James Graham', 'wptserve', 'One line description of project.', + 'Miscellaneous'), +] + +# Documents to append as an appendix to all manuals. +#texinfo_appendices = [] + +# If false, no module index is generated. +#texinfo_domain_indices = True + +# How to display URL addresses: 'footnote', 'no', or 'inline'. +#texinfo_show_urls = 'footnote' diff --git a/testing/web-platform/tests/tools/wptserve/docs/handlers.rst b/testing/web-platform/tests/tools/wptserve/docs/handlers.rst new file mode 100644 index 000000000..c15aab635 --- /dev/null +++ b/testing/web-platform/tests/tools/wptserve/docs/handlers.rst @@ -0,0 +1,111 @@ +Handlers +======== + +Handlers are functions that have the general signature:: + + handler(request, response) + +It is expected that the handler will use information from +the request (e.g. the path) either to populate the response +object with the data to send, or to directly write to the +output stream via the ResponseWriter instance associated with +the request. If a handler writes to the output stream then the +server will not attempt additional writes, i.e. the choice to write +directly in the handler or not is all-or-nothing. + +A number of general-purpose handler functions are provided by default: + +.. _handlers.Python: + +Python Handlers +--------------- + +Python handlers are functions which provide a higher-level API over +manually updating the response object, by causing the return value of +the function to provide (part of) the response. There are three +possible sets of values that may be returned:: + + + (status, headers, content) + (headers, content) + content + +Here `status` is either a tuple (status code, message) or simply a +integer status code, `headers` is a list of (field name, value) pairs, +and `content` is a string or an iterable returning strings. Such a +function may also update the response manually. For example one may +use `response.headers.set` to set a response header, and only return +the content. One may even use this kind of handler, but manipulate +the output socket directly, in which case the return value of the +function, and the properties of the response object, will be ignored. + +The most common way to make a user function into a python handler is +to use the provided `wptserve.handlers.handler` decorator:: + + from wptserve.handlers import handler + + @handler + def test(request, response): + return [("X-Test": "PASS"), ("Content-Type", "text/plain")], "test" + + #Later, assuming we have a Router object called 'router' + + router.register("GET", "/test", test) + +JSON Handlers +------------- + +This is a specialisation of the python handler type specifically +designed to facilitate providing JSON responses. The API is largely +the same as for a normal python handler, but the `content` part of the +return value is JSON encoded, and a default Content-Type header of +`application/json` is added. Again this handler is usually used as a +decorator:: + + from wptserve.handlers import json_handler + + @json_handler + def test(request, response): + return {"test": "PASS"} + +Python File Handlers +-------------------- + +Python file handlers are designed to provide a vaguely PHP-like interface +where each resource corresponds to a particular python file on the +filesystem. Typically this is hooked up to a route like ``("*", +"*.py", python_file_handler)``, meaning that any .py file will be +treated as a handler file (note that this makes python files unsafe in +much the same way that .php files are when using PHP). + +Unlike PHP, the python files don't work by outputting text to stdout +from the global scope. Instead they must define a single function +`main` with the signature:: + + main(request, response) + +This function then behaves just like those described in +:ref:`handlers.Python` above. + +asis Handlers +------------- + +These are used to serve files as literal byte streams including the +HTTP status line, headers and body. In the default configuration this +handler is invoked for all files with a .asis extension. + +File Handlers +------------- + +File handlers are used to serve static files. By default the content +type of these files is set by examining the file extension. However +this can be overridden, or additional headers supplied, by providing a +file with the same name as the file being served but an additional +.headers suffix, i.e. test.html has its headers set from +test.html.headers. The format of the .headers file is plaintext, with +each line containing:: + + Header-Name: header_value + +In addition headers can be set for a whole directory of files (but not +subdirectories), using a file called `__dir__.headers`. diff --git a/testing/web-platform/tests/tools/wptserve/docs/index.rst b/testing/web-platform/tests/tools/wptserve/docs/index.rst new file mode 100644 index 000000000..a9f630c76 --- /dev/null +++ b/testing/web-platform/tests/tools/wptserve/docs/index.rst @@ -0,0 +1,35 @@ +.. wptserve documentation master file, created by + sphinx-quickstart on Wed Aug 14 17:23:24 2013. + You can adapt this file completely to your liking, but it should at least + contain the root `toctree` directive. + +Web Platform Test Server +======================== + +A python-based HTTP server specifically targeted at being used for +testing the web platform. This means that extreme flexibility — +including the possibility of HTTP non-conformance — in the response is +supported. + +Contents: + +.. toctree:: + :maxdepth: 2 + + introduction + server + router + request + response + stash + handlers + pipes + + +Indices and tables +================== + +* :ref:`genindex` +* :ref:`modindex` +* :ref:`search` + diff --git a/testing/web-platform/tests/tools/wptserve/docs/introduction.rst b/testing/web-platform/tests/tools/wptserve/docs/introduction.rst new file mode 100644 index 000000000..b585a983a --- /dev/null +++ b/testing/web-platform/tests/tools/wptserve/docs/introduction.rst @@ -0,0 +1,51 @@ +Introduction +============ + +wptserve has been designed with the specific goal of making a server +that is suitable for writing tests for the web platform. This means +that it cannot use common abstractions over HTTP such as WSGI, since +these assume that the goal is to generate a well-formed HTTP +response. Testcases, however, often require precise control of the +exact bytes sent over the wire and their timing. The full list of +design goals for the server are: + +* Suitable to run on individual test machines and over the public internet. + +* Support plain TCP and SSL servers. + +* Serve static files with the minimum of configuration. + +* Allow headers to be overwritten on a per-file and per-directory + basis. + +* Full customisation of headers sent (e.g. altering or omitting + "mandatory" headers). + +* Simple per-client state. + +* Complex logic in tests, up to precise control over the individual + bytes sent and the timing of sending them. + +Request Handling +---------------- + +At the high level, the design of the server is based around similar +concepts to those found in common web frameworks like Django, Pyramid +or Flask. In particular the lifecycle of a typical request will be +familiar to users of these systems. Incoming requests are parsed and a +:doc:`Request <request>` object is constructed. This object is passed +to a :ref:`Router <router.Interface>` instance, which is +responsible for mapping the request method and path to a handler +function. This handler is passed two arguments; the request object and +a :doc:`Response <response>` object. In cases where only simple +responses are required, the handler function may fill in the +properties of the response object and the server will take care of +constructing the response. However each Response also contains a +:ref:`ResponseWriter <response.Interface>` which can be +used to directly control the TCP socket. + +By default there are several built-in handler functions that provide a +higher level API than direct manipulation of the Response +object. These are documented in :doc:`handlers`. + + diff --git a/testing/web-platform/tests/tools/wptserve/docs/make.bat b/testing/web-platform/tests/tools/wptserve/docs/make.bat new file mode 100644 index 000000000..40c71ff5d --- /dev/null +++ b/testing/web-platform/tests/tools/wptserve/docs/make.bat @@ -0,0 +1,190 @@ +@ECHO OFF + +REM Command file for Sphinx documentation + +if "%SPHINXBUILD%" == "" ( + set SPHINXBUILD=sphinx-build +) +set BUILDDIR=_build +set ALLSPHINXOPTS=-d %BUILDDIR%/doctrees %SPHINXOPTS% . +set I18NSPHINXOPTS=%SPHINXOPTS% . +if NOT "%PAPER%" == "" ( + set ALLSPHINXOPTS=-D latex_paper_size=%PAPER% %ALLSPHINXOPTS% + set I18NSPHINXOPTS=-D latex_paper_size=%PAPER% %I18NSPHINXOPTS% +) + +if "%1" == "" goto help + +if "%1" == "help" ( + :help + echo.Please use `make ^<target^>` where ^<target^> is one of + echo. html to make standalone HTML files + echo. dirhtml to make HTML files named index.html in directories + echo. singlehtml to make a single large HTML file + echo. pickle to make pickle files + echo. json to make JSON files + echo. htmlhelp to make HTML files and a HTML help project + echo. qthelp to make HTML files and a qthelp project + echo. devhelp to make HTML files and a Devhelp project + echo. epub to make an epub + echo. latex to make LaTeX files, you can set PAPER=a4 or PAPER=letter + echo. text to make text files + echo. man to make manual pages + echo. texinfo to make Texinfo files + echo. gettext to make PO message catalogs + echo. changes to make an overview over all changed/added/deprecated items + echo. linkcheck to check all external links for integrity + echo. doctest to run all doctests embedded in the documentation if enabled + goto end +) + +if "%1" == "clean" ( + for /d %%i in (%BUILDDIR%\*) do rmdir /q /s %%i + del /q /s %BUILDDIR%\* + goto end +) + +if "%1" == "html" ( + %SPHINXBUILD% -b html %ALLSPHINXOPTS% %BUILDDIR%/html + if errorlevel 1 exit /b 1 + echo. + echo.Build finished. The HTML pages are in %BUILDDIR%/html. + goto end +) + +if "%1" == "dirhtml" ( + %SPHINXBUILD% -b dirhtml %ALLSPHINXOPTS% %BUILDDIR%/dirhtml + if errorlevel 1 exit /b 1 + echo. + echo.Build finished. The HTML pages are in %BUILDDIR%/dirhtml. + goto end +) + +if "%1" == "singlehtml" ( + %SPHINXBUILD% -b singlehtml %ALLSPHINXOPTS% %BUILDDIR%/singlehtml + if errorlevel 1 exit /b 1 + echo. + echo.Build finished. The HTML pages are in %BUILDDIR%/singlehtml. + goto end +) + +if "%1" == "pickle" ( + %SPHINXBUILD% -b pickle %ALLSPHINXOPTS% %BUILDDIR%/pickle + if errorlevel 1 exit /b 1 + echo. + echo.Build finished; now you can process the pickle files. + goto end +) + +if "%1" == "json" ( + %SPHINXBUILD% -b json %ALLSPHINXOPTS% %BUILDDIR%/json + if errorlevel 1 exit /b 1 + echo. + echo.Build finished; now you can process the JSON files. + goto end +) + +if "%1" == "htmlhelp" ( + %SPHINXBUILD% -b htmlhelp %ALLSPHINXOPTS% %BUILDDIR%/htmlhelp + if errorlevel 1 exit /b 1 + echo. + echo.Build finished; now you can run HTML Help Workshop with the ^ +.hhp project file in %BUILDDIR%/htmlhelp. + goto end +) + +if "%1" == "qthelp" ( + %SPHINXBUILD% -b qthelp %ALLSPHINXOPTS% %BUILDDIR%/qthelp + if errorlevel 1 exit /b 1 + echo. + echo.Build finished; now you can run "qcollectiongenerator" with the ^ +.qhcp project file in %BUILDDIR%/qthelp, like this: + echo.^> qcollectiongenerator %BUILDDIR%\qthelp\wptserve.qhcp + echo.To view the help file: + echo.^> assistant -collectionFile %BUILDDIR%\qthelp\wptserve.ghc + goto end +) + +if "%1" == "devhelp" ( + %SPHINXBUILD% -b devhelp %ALLSPHINXOPTS% %BUILDDIR%/devhelp + if errorlevel 1 exit /b 1 + echo. + echo.Build finished. + goto end +) + +if "%1" == "epub" ( + %SPHINXBUILD% -b epub %ALLSPHINXOPTS% %BUILDDIR%/epub + if errorlevel 1 exit /b 1 + echo. + echo.Build finished. The epub file is in %BUILDDIR%/epub. + goto end +) + +if "%1" == "latex" ( + %SPHINXBUILD% -b latex %ALLSPHINXOPTS% %BUILDDIR%/latex + if errorlevel 1 exit /b 1 + echo. + echo.Build finished; the LaTeX files are in %BUILDDIR%/latex. + goto end +) + +if "%1" == "text" ( + %SPHINXBUILD% -b text %ALLSPHINXOPTS% %BUILDDIR%/text + if errorlevel 1 exit /b 1 + echo. + echo.Build finished. The text files are in %BUILDDIR%/text. + goto end +) + +if "%1" == "man" ( + %SPHINXBUILD% -b man %ALLSPHINXOPTS% %BUILDDIR%/man + if errorlevel 1 exit /b 1 + echo. + echo.Build finished. The manual pages are in %BUILDDIR%/man. + goto end +) + +if "%1" == "texinfo" ( + %SPHINXBUILD% -b texinfo %ALLSPHINXOPTS% %BUILDDIR%/texinfo + if errorlevel 1 exit /b 1 + echo. + echo.Build finished. The Texinfo files are in %BUILDDIR%/texinfo. + goto end +) + +if "%1" == "gettext" ( + %SPHINXBUILD% -b gettext %I18NSPHINXOPTS% %BUILDDIR%/locale + if errorlevel 1 exit /b 1 + echo. + echo.Build finished. The message catalogs are in %BUILDDIR%/locale. + goto end +) + +if "%1" == "changes" ( + %SPHINXBUILD% -b changes %ALLSPHINXOPTS% %BUILDDIR%/changes + if errorlevel 1 exit /b 1 + echo. + echo.The overview file is in %BUILDDIR%/changes. + goto end +) + +if "%1" == "linkcheck" ( + %SPHINXBUILD% -b linkcheck %ALLSPHINXOPTS% %BUILDDIR%/linkcheck + if errorlevel 1 exit /b 1 + echo. + echo.Link check complete; look for any errors in the above output ^ +or in %BUILDDIR%/linkcheck/output.txt. + goto end +) + +if "%1" == "doctest" ( + %SPHINXBUILD% -b doctest %ALLSPHINXOPTS% %BUILDDIR%/doctest + if errorlevel 1 exit /b 1 + echo. + echo.Testing of doctests in the sources finished, look at the ^ +results in %BUILDDIR%/doctest/output.txt. + goto end +) + +:end diff --git a/testing/web-platform/tests/tools/wptserve/docs/pipes.rst b/testing/web-platform/tests/tools/wptserve/docs/pipes.rst new file mode 100644 index 000000000..c606140d4 --- /dev/null +++ b/testing/web-platform/tests/tools/wptserve/docs/pipes.rst @@ -0,0 +1,157 @@ +Pipes +====== + +Pipes are functions that may be used when serving files to alter parts +of the response. These are invoked by adding a pipe= query parameter +taking a | separated list of pipe functions and parameters. The pipe +functions are applied to the response from left to right. For example:: + + GET /sample.txt?pipe=slice(1,200)|status(404). + +This would serve bytes 1 to 199, inclusive, of foo.txt with the HTTP status +code 404. + +There are several built-in pipe functions, and it is possible to add +more using the `@pipe` decorator on a function, if required. + +.. note:: + Because of the way pipes compose, using some pipe functions prevents the + content-length of the response from being known in advance. In these cases + the server will close the connection to indicate the end of the response, + preventing the use of HTTP 1.1 keepalive. + +Built-In Pipes +-------------- + +sub +~~~ + +Used to substitute variables from the server environment, or from the +request into the response. + +Substitutions are marked in a file using a block delimited by `{{` +and `}}`. Inside the block the following variables are available: + + `{{host}}` + The host name of the server excluding any subdomain part. + + `{{domains[]}}` + The domain name of a particular subdomain + e.g. `{{domains[www]}}` for the `www` subdomain. + + `{{ports[][]}}` + The port number of servers, by protocol + e.g. `{{ports[http][0]}}` for the first (and, depending on setup, + possibly only) http server + + `{{headers[]}}` + The HTTP headers in the request + e.g. `{{headers[X-Test]}}` for a hypothetical `X-Test` header. + + `{{GET[]}}` + The query parameters for the request + e.g. `{{GET[id]}}` for an id parameter sent with the request. + +So, for example, to write a javascript file called `xhr.js` that +depends on the host name of the server, without hardcoding, one might +write:: + + var server_url = http://{{host}}:{{ports[http][0]}}/path/to/resource; + //Create the actual XHR and so on + +The file would then be included as: + + <script src="xhr.js?pipe=sub"></script> + +This pipe can also be enabled by using a filename `*.sub.ext`, e.g. the file above could be called `xhr.sub.js`. + +status +~~~~~~ + +Used to set the HTTP status of the response, for example:: + + example.js?pipe=status(410) + +headers +~~~~~~~ + +Used to add or replace http headers in the response. Takes two or +three arguments; the header name, the header value and whether to +append the header rather than replace an existing header (default: +False). So, for example, a request for:: + + example.html?pipe=header(Content-Type,text/plain) + +causes example.html to be returned with a text/plain content type +whereas:: + + example.html?pipe=header(Content-Type,text/plain,True) + +Will cause example.html to be returned with both text/html and +text/plain content-type headers. + +slice +~~~~~ + +Used to send only part of a response body. Takes the start and, +optionally, end bytes as arguments, although either can be null to +indicate the start or end of the file, respectively. So for example:: + + example.txt?pipe=slice(10,20) + +Would result in a response with a body containing 10 bytes of +example.txt including byte 10 but excluding byte 20. + +:: + + example.txt?pipe=slice(10) + +Would cause all bytes from byte 10 of example.txt to be sent, but:: + + example.txt?pipe=slice(null,20) + +Would send the first 20 bytes of example.txt. + +trickle +~~~~~~~ + +.. note:: + Using this function will force a connection close. + +Used to send the body of a response in chunks with delays. Takes a +single argument that is a microsyntax consisting of colon-separated +commands. There are three types of commands: + +* Bare numbers represent a number of bytes to send + +* Numbers prefixed `d` indicate a delay in seconds + +* Numbers prefixed `r` must only appear at the end of the command, and + indicate that the preceding N items must be repeated until there is + no more content to send. The number of items to repeat must be even. + +In the absence of a repetition command, the entire remainder of the content is +sent at once when the command list is exhausted. So for example:: + + example.txt?pipe=trickle(d1) + +causes a 1s delay before sending the entirety of example.txt. + +:: + + example.txt?pipe=trickle(100:d1) + +causes 100 bytes of example.txt to be sent, followed by a 1s delay, +and then the remainder of the file to be sent. On the other hand:: + + example.txt?pipe=trickle(100:d1:r2) + +Will cause the file to be sent in 100 byte chunks separated by a 1s +delay until the whole content has been sent. + + +:mod:`Interface <pipes>` +------------------------ + +.. automodule:: wptserve.pipes + :members: diff --git a/testing/web-platform/tests/tools/wptserve/docs/request.rst b/testing/web-platform/tests/tools/wptserve/docs/request.rst new file mode 100644 index 000000000..790e4f0bb --- /dev/null +++ b/testing/web-platform/tests/tools/wptserve/docs/request.rst @@ -0,0 +1,10 @@ +Request +======= + +Request object. + +:mod:`Interface <request>` +-------------------------- + +.. automodule:: wptserve.request + :members: diff --git a/testing/web-platform/tests/tools/wptserve/docs/response.rst b/testing/web-platform/tests/tools/wptserve/docs/response.rst new file mode 100644 index 000000000..23075dfb3 --- /dev/null +++ b/testing/web-platform/tests/tools/wptserve/docs/response.rst @@ -0,0 +1,41 @@ +Response +======== + +Response object. This object is used to control the response that will +be sent to the HTTP client. A handler function will take the response +object and fill in various parts of the response. For example, a plain +text response with the body 'Some example content' could be produced as:: + + def handler(request, response): + response.headers.set("Content-Type", "text/plain") + response.content = "Some example content" + +The response object also gives access to a ResponseWriter, which +allows direct access to the response socket. For example, one could +write a similar response but with more explicit control as follows:: + + import time + + def handler(request, response): + response.add_required_headers = False # Don't implicitly add HTTP headers + response.writer.write_status(200) + response.writer.write_header("Content-Type", "text/plain") + response.writer.write_header("Content-Length", len("Some example content")) + response.writer.end_headers() + response.writer.write("Some ") + time.sleep(1) + response.writer.write("example content") + +Note that when writing the response directly like this it is always +necessary to either set the Content-Length header or set +`response.close_connection = True`. Without one of these, the client +will not be able to determine where the response body ends and will +continue to load indefinitely. + +.. _response.Interface: + +:mod:`Interface <response>` +--------------------------- + +.. automodule:: wptserve.response + :members: diff --git a/testing/web-platform/tests/tools/wptserve/docs/router.rst b/testing/web-platform/tests/tools/wptserve/docs/router.rst new file mode 100644 index 000000000..21d67d222 --- /dev/null +++ b/testing/web-platform/tests/tools/wptserve/docs/router.rst @@ -0,0 +1,78 @@ +Router +====== + +The router is used to match incoming requests to request handler +functions. Typically users don't interact with the router directly, +but instead send a list of routes to register when starting the +server. However it is also possible to add routes after starting the +server by calling the `register` method on the server's `router` +property. + +Routes are represented by a three item tuple:: + + (methods, path_match, handler) + +`methods` is either a string or a list of strings indicating the HTTP +methods to match. In cases where all methods should match there is a +special sentinel value `any_method` provided as a property of the +`router` module that can be used. + +`path_match` is an expression that will be evaluated against the +request path to decide if the handler should match. These expressions +follow a custom syntax intended to make matching URLs straightforward +and, in particular, to be easier to use than raw regexp for URL +matching. There are three possible components of a match expression: + +* Literals. These match any character. The special characters \*, \{ + and \} must be escaped by prefixing them with a \\. + +* Match groups. These match any character other than / and save the + result as a named group. They are delimited by curly braces; for + example:: + + {abc} + + would create a match group with the name `abc`. + +* Stars. These are denoted with a `*` and match any character + including /. There can be at most one star + per pattern and it must follow any match groups. + +Path expressions always match the entire request path and a leading / +in the expression is implied even if it is not explicitly +provided. This means that `/foo` and `foo` are equivalent. + +For example, the following pattern matches all requests for resources with the +extension `.py`:: + + *.py + +The following expression matches anything directly under `/resources` +with a `.html` extension, and places the "filename" in the `name` +group:: + + /resources/{name}.html + +The groups, including anything that matches a `*` are available in the +request object through the `route_match` property. This is a +dictionary mapping the group names, and any match for `*` to the +matching part of the route. For example, given a route:: + + /api/{sub_api}/* + +and the request path `/api/test/html/test.html`, `route_match` would +be:: + + {"sub_api": "html", "*": "html/test.html"} + +`handler` is a function taking a request and a response object that is +responsible for constructing the response to the HTTP request. See +:doc:`handlers` for more details on handler functions. + +.. _router.Interface: + +:mod:`Interface <wptserve>` +--------------------------- + +.. automodule:: wptserve.router + :members: diff --git a/testing/web-platform/tests/tools/wptserve/docs/server.rst b/testing/web-platform/tests/tools/wptserve/docs/server.rst new file mode 100644 index 000000000..732f9fdc7 --- /dev/null +++ b/testing/web-platform/tests/tools/wptserve/docs/server.rst @@ -0,0 +1,20 @@ +Server +====== + +Basic server classes and router. + +The following example creates a server that serves static files from +the `files` subdirectory of the current directory and causes it to +run on port 8080 until it is killed:: + + from wptserve import server, handlers + + httpd = server.WebTestHttpd(port=8080, doc_root="./files/", + routes=[("GET", "*", handlers.file_handler)]) + httpd.start(block=True) + +:mod:`Interface <wptserve>` +--------------------------- + +.. automodule:: wptserve.server + :members: diff --git a/testing/web-platform/tests/tools/wptserve/docs/stash.rst b/testing/web-platform/tests/tools/wptserve/docs/stash.rst new file mode 100644 index 000000000..821c2d344 --- /dev/null +++ b/testing/web-platform/tests/tools/wptserve/docs/stash.rst @@ -0,0 +1,31 @@ +Stash +===== + +Object for storing cross-request state. This is unusual in that keys +must be UUIDs, in order to prevent different clients setting the same +key, and values are write-once, read-once to minimise the chances of +state persisting indefinitely. The stash defines two operations; +`put`, to add state and `take` to remove state. Furthermore, the view +of the stash is path-specific; by default a request will only see the +part of the stash corresponding to its own path. + +A typical example of using a stash to store state might be:: + + @handler + def handler(request, response): + # We assume this is a string representing a UUID + key = request.GET.first("id") + + if request.method == "POST": + request.server.stash.put(key, "Some sample value") + return "Added value to stash" + else: + value = request.server.stash.take(key) + assert request.server.stash.take(key) is None + return key + +:mod:`Interface <stash>` +------------------------ + +.. automodule:: wptserve.stash + :members: diff --git a/testing/web-platform/tests/tools/wptserve/setup.py b/testing/web-platform/tests/tools/wptserve/setup.py new file mode 100644 index 000000000..956e20922 --- /dev/null +++ b/testing/web-platform/tests/tools/wptserve/setup.py @@ -0,0 +1,23 @@ +from setuptools import setup + +PACKAGE_VERSION = '1.4.0' +deps = [] + +setup(name='wptserve', + version=PACKAGE_VERSION, + description="Python webserver intended for in web browser testing", + long_description=open("README.md").read(), + # Get strings from http://pypi.python.org/pypi?%3Aaction=list_classifiers + classifiers=["Development Status :: 5 - Production/Stable", + "License :: OSI Approved :: BSD License", + "Topic :: Internet :: WWW/HTTP :: HTTP Servers"], + keywords='', + author='James Graham', + author_email='james@hoppipolla.co.uk', + url='http://wptserve.readthedocs.org/', + license='BSD', + packages=['wptserve'], + include_package_data=True, + zip_safe=False, + install_requires=deps + ) diff --git a/testing/web-platform/tests/tools/wptserve/tests/functional/__init__.py b/testing/web-platform/tests/tools/wptserve/tests/functional/__init__.py new file mode 100644 index 000000000..e69de29bb --- /dev/null +++ b/testing/web-platform/tests/tools/wptserve/tests/functional/__init__.py diff --git a/testing/web-platform/tests/tools/wptserve/tests/functional/base.py b/testing/web-platform/tests/tools/wptserve/tests/functional/base.py new file mode 100644 index 000000000..eae7e87d9 --- /dev/null +++ b/testing/web-platform/tests/tools/wptserve/tests/functional/base.py @@ -0,0 +1,65 @@ +from __future__ import print_function + +import base64 +import logging +import os +import unittest +import urllib +import urllib2 +import urlparse + +import wptserve + +logging.basicConfig() + +wptserve.logger.set_logger(logging.getLogger()) + +here = os.path.split(__file__)[0] +doc_root = os.path.join(here, "docroot") + +class Request(urllib2.Request): + def __init__(self, *args, **kwargs): + urllib2.Request.__init__(self, *args, **kwargs) + self.method = "GET" + + def get_method(self): + return self.method + + def add_data(self, data): + if hasattr(data, "iteritems"): + data = urllib.urlencode(data) + print(data) + self.add_header("Content-Length", str(len(data))) + urllib2.Request.add_data(self, data) + +class TestUsingServer(unittest.TestCase): + def setUp(self): + self.server = wptserve.server.WebTestHttpd(host="localhost", + port=0, + use_ssl=False, + certificate=None, + doc_root=doc_root) + self.server.start(False) + + def tearDown(self): + self.server.stop() + + def abs_url(self, path, query=None): + return urlparse.urlunsplit(("http", "%s:%i" % (self.server.host, self.server.port), path, query, None)) + + def request(self, path, query=None, method="GET", headers=None, body=None, auth=None): + req = Request(self.abs_url(path, query)) + req.method = method + if headers is None: + headers = {} + + for name, value in headers.iteritems(): + req.add_header(name, value) + + if body is not None: + req.add_data(body) + + if auth is not None: + req.add_header("Authorization", "Basic %s" % base64.b64encode('%s:%s' % auth)) + + return urllib2.urlopen(req) diff --git a/testing/web-platform/tests/tools/wptserve/tests/functional/docroot/document.txt b/testing/web-platform/tests/tools/wptserve/tests/functional/docroot/document.txt new file mode 100644 index 000000000..611dccd84 --- /dev/null +++ b/testing/web-platform/tests/tools/wptserve/tests/functional/docroot/document.txt @@ -0,0 +1 @@ +This is a test document diff --git a/testing/web-platform/tests/tools/wptserve/tests/functional/docroot/invalid.py b/testing/web-platform/tests/tools/wptserve/tests/functional/docroot/invalid.py new file mode 100644 index 000000000..017d4d9d6 --- /dev/null +++ b/testing/web-platform/tests/tools/wptserve/tests/functional/docroot/invalid.py @@ -0,0 +1,3 @@ +# Oops... +def main(request, response + return "FAIL" diff --git a/testing/web-platform/tests/tools/wptserve/tests/functional/docroot/no_main.py b/testing/web-platform/tests/tools/wptserve/tests/functional/docroot/no_main.py new file mode 100644 index 000000000..cee379fe1 --- /dev/null +++ b/testing/web-platform/tests/tools/wptserve/tests/functional/docroot/no_main.py @@ -0,0 +1,3 @@ +# Oops... +def mian(request, response): + return "FAIL" diff --git a/testing/web-platform/tests/tools/wptserve/tests/functional/docroot/sub.sub.txt b/testing/web-platform/tests/tools/wptserve/tests/functional/docroot/sub.sub.txt new file mode 100644 index 000000000..4302db16a --- /dev/null +++ b/testing/web-platform/tests/tools/wptserve/tests/functional/docroot/sub.sub.txt @@ -0,0 +1 @@ +{{host}} {{domains[]}} {{ports[http][0]}} diff --git a/testing/web-platform/tests/tools/wptserve/tests/functional/docroot/sub.txt b/testing/web-platform/tests/tools/wptserve/tests/functional/docroot/sub.txt new file mode 100644 index 000000000..4302db16a --- /dev/null +++ b/testing/web-platform/tests/tools/wptserve/tests/functional/docroot/sub.txt @@ -0,0 +1 @@ +{{host}} {{domains[]}} {{ports[http][0]}} diff --git a/testing/web-platform/tests/tools/wptserve/tests/functional/docroot/sub_headers.sub.txt b/testing/web-platform/tests/tools/wptserve/tests/functional/docroot/sub_headers.sub.txt new file mode 100644 index 000000000..ee021eb86 --- /dev/null +++ b/testing/web-platform/tests/tools/wptserve/tests/functional/docroot/sub_headers.sub.txt @@ -0,0 +1 @@ +{{headers[X-Test]}} diff --git a/testing/web-platform/tests/tools/wptserve/tests/functional/docroot/sub_headers.txt b/testing/web-platform/tests/tools/wptserve/tests/functional/docroot/sub_headers.txt new file mode 100644 index 000000000..ee021eb86 --- /dev/null +++ b/testing/web-platform/tests/tools/wptserve/tests/functional/docroot/sub_headers.txt @@ -0,0 +1 @@ +{{headers[X-Test]}} diff --git a/testing/web-platform/tests/tools/wptserve/tests/functional/docroot/sub_params.sub.txt b/testing/web-platform/tests/tools/wptserve/tests/functional/docroot/sub_params.sub.txt new file mode 100644 index 000000000..8323878d6 --- /dev/null +++ b/testing/web-platform/tests/tools/wptserve/tests/functional/docroot/sub_params.sub.txt @@ -0,0 +1 @@ +{{GET[test]}} diff --git a/testing/web-platform/tests/tools/wptserve/tests/functional/docroot/sub_params.txt b/testing/web-platform/tests/tools/wptserve/tests/functional/docroot/sub_params.txt new file mode 100644 index 000000000..8323878d6 --- /dev/null +++ b/testing/web-platform/tests/tools/wptserve/tests/functional/docroot/sub_params.txt @@ -0,0 +1 @@ +{{GET[test]}} diff --git a/testing/web-platform/tests/tools/wptserve/tests/functional/docroot/subdir/file.txt b/testing/web-platform/tests/tools/wptserve/tests/functional/docroot/subdir/file.txt new file mode 100644 index 000000000..06d84d30d --- /dev/null +++ b/testing/web-platform/tests/tools/wptserve/tests/functional/docroot/subdir/file.txt @@ -0,0 +1 @@ +I am here to ensure that my containing directory exists. diff --git a/testing/web-platform/tests/tools/wptserve/tests/functional/docroot/test.asis b/testing/web-platform/tests/tools/wptserve/tests/functional/docroot/test.asis new file mode 100644 index 000000000..b05ba7da8 --- /dev/null +++ b/testing/web-platform/tests/tools/wptserve/tests/functional/docroot/test.asis @@ -0,0 +1,5 @@ +HTTP/1.1 202 Giraffe
+X-TEST: PASS
+Content-Length: 7
+
+Content
\ No newline at end of file diff --git a/testing/web-platform/tests/tools/wptserve/tests/functional/docroot/test_string.py b/testing/web-platform/tests/tools/wptserve/tests/functional/docroot/test_string.py new file mode 100644 index 000000000..8fa605bb1 --- /dev/null +++ b/testing/web-platform/tests/tools/wptserve/tests/functional/docroot/test_string.py @@ -0,0 +1,3 @@ +def main(request, response): + response.headers.set("Content-Type", "text/plain") + return "PASS" diff --git a/testing/web-platform/tests/tools/wptserve/tests/functional/docroot/test_tuple_2.py b/testing/web-platform/tests/tools/wptserve/tests/functional/docroot/test_tuple_2.py new file mode 100644 index 000000000..fa791fbdd --- /dev/null +++ b/testing/web-platform/tests/tools/wptserve/tests/functional/docroot/test_tuple_2.py @@ -0,0 +1,2 @@ +def main(request, response): + return [("Content-Type", "text/html"), ("X-Test", "PASS")], "PASS" diff --git a/testing/web-platform/tests/tools/wptserve/tests/functional/docroot/test_tuple_3.py b/testing/web-platform/tests/tools/wptserve/tests/functional/docroot/test_tuple_3.py new file mode 100644 index 000000000..2c2656d04 --- /dev/null +++ b/testing/web-platform/tests/tools/wptserve/tests/functional/docroot/test_tuple_3.py @@ -0,0 +1,2 @@ +def main(request, response): + return (202, "Giraffe"), [("Content-Type", "text/html"), ("X-Test", "PASS")], "PASS" diff --git a/testing/web-platform/tests/tools/wptserve/tests/functional/docroot/with_headers.txt b/testing/web-platform/tests/tools/wptserve/tests/functional/docroot/with_headers.txt new file mode 100644 index 000000000..45ce1a079 --- /dev/null +++ b/testing/web-platform/tests/tools/wptserve/tests/functional/docroot/with_headers.txt @@ -0,0 +1 @@ +Test document with custom headers diff --git a/testing/web-platform/tests/tools/wptserve/tests/functional/docroot/with_headers.txt.sub.headers b/testing/web-platform/tests/tools/wptserve/tests/functional/docroot/with_headers.txt.sub.headers new file mode 100644 index 000000000..71494fccf --- /dev/null +++ b/testing/web-platform/tests/tools/wptserve/tests/functional/docroot/with_headers.txt.sub.headers @@ -0,0 +1,6 @@ +Custom-Header: PASS +Another-Header: {{$id:uuid()}} +Same-Value-Header: {{$id}} +Double-Header: PA +Double-Header: SS +Content-Type: text/html diff --git a/testing/web-platform/tests/tools/wptserve/tests/functional/test_cookies.py b/testing/web-platform/tests/tools/wptserve/tests/functional/test_cookies.py new file mode 100644 index 000000000..d1080b4bf --- /dev/null +++ b/testing/web-platform/tests/tools/wptserve/tests/functional/test_cookies.py @@ -0,0 +1,61 @@ +import unittest + +import wptserve +from .base import TestUsingServer + +class TestResponseSetCookie(TestUsingServer): + def test_name_value(self): + @wptserve.handlers.handler + def handler(request, response): + response.set_cookie("name", "value") + return "Test" + + route = ("GET", "/test/name_value", handler) + self.server.router.register(*route) + resp = self.request(route[1]) + + self.assertEqual(resp.info()["Set-Cookie"], "name=value; Path=/") + + def test_unset(self): + @wptserve.handlers.handler + def handler(request, response): + response.set_cookie("name", "value") + response.unset_cookie("name") + return "Test" + + route = ("GET", "/test/unset", handler) + self.server.router.register(*route) + resp = self.request(route[1]) + + self.assertTrue("Set-Cookie" not in resp.info()) + + def test_delete(self): + @wptserve.handlers.handler + def handler(request, response): + response.delete_cookie("name") + return "Test" + + route = ("GET", "/test/delete", handler) + self.server.router.register(*route) + resp = self.request(route[1]) + + parts = dict(item.split("=") for + item in resp.info()["Set-Cookie"].split("; ") if item) + + self.assertEqual(parts["name"], "") + self.assertEqual(parts["Path"], "/") + #Should also check that expires is in the past + +class TestRequestCookies(TestUsingServer): + def test_set_cookie(self): + @wptserve.handlers.handler + def handler(request, response): + return request.cookies["name"].value + + route = ("GET", "/test/set_cookie", handler) + self.server.router.register(*route) + resp = self.request(route[1], headers={"Cookie": "name=value"}) + self.assertEqual(resp.read(), b"value") + +if __name__ == '__main__': + unittest.main() diff --git a/testing/web-platform/tests/tools/wptserve/tests/functional/test_handlers.py b/testing/web-platform/tests/tools/wptserve/tests/functional/test_handlers.py new file mode 100644 index 000000000..9189725cb --- /dev/null +++ b/testing/web-platform/tests/tools/wptserve/tests/functional/test_handlers.py @@ -0,0 +1,299 @@ +import json +import os +import pytest +import unittest +import urllib2 +import uuid + +import wptserve +from .base import TestUsingServer, doc_root + +class TestFileHandler(TestUsingServer): + def test_GET(self): + resp = self.request("/document.txt") + self.assertEqual(200, resp.getcode()) + self.assertEqual("text/plain", resp.info()["Content-Type"]) + self.assertEqual(open(os.path.join(doc_root, "document.txt"), 'rb').read(), resp.read()) + + def test_headers(self): + resp = self.request("/with_headers.txt") + self.assertEqual(200, resp.getcode()) + self.assertEqual("text/html", resp.info()["Content-Type"]) + self.assertEqual("PASS", resp.info()["Custom-Header"]) + # This will fail if it isn't a valid uuid + uuid.UUID(resp.info()["Another-Header"]) + self.assertEqual(resp.info()["Same-Value-Header"], resp.info()["Another-Header"]) + self.assertEqual(resp.info()["Double-Header"], "PA, SS") + + + def test_range(self): + resp = self.request("/document.txt", headers={"Range":"bytes=10-19"}) + self.assertEqual(206, resp.getcode()) + data = resp.read() + expected = open(os.path.join(doc_root, "document.txt"), 'rb').read() + self.assertEqual(10, len(data)) + self.assertEqual("bytes 10-19/%i" % len(expected), resp.info()['Content-Range']) + self.assertEqual("10", resp.info()['Content-Length']) + self.assertEqual(expected[10:20], data) + + def test_range_no_end(self): + resp = self.request("/document.txt", headers={"Range":"bytes=10-"}) + self.assertEqual(206, resp.getcode()) + data = resp.read() + expected = open(os.path.join(doc_root, "document.txt"), 'rb').read() + self.assertEqual(len(expected) - 10, len(data)) + self.assertEqual("bytes 10-%i/%i" % (len(expected) - 1, len(expected)), resp.info()['Content-Range']) + self.assertEqual(expected[10:], data) + + def test_range_no_start(self): + resp = self.request("/document.txt", headers={"Range":"bytes=-10"}) + self.assertEqual(206, resp.getcode()) + data = resp.read() + expected = open(os.path.join(doc_root, "document.txt"), 'rb').read() + self.assertEqual(10, len(data)) + self.assertEqual("bytes %i-%i/%i" % (len(expected) - 10, len(expected) - 1, len(expected)), + resp.info()['Content-Range']) + self.assertEqual(expected[-10:], data) + + def test_multiple_ranges(self): + resp = self.request("/document.txt", headers={"Range":"bytes=1-2,5-7,6-10"}) + self.assertEqual(206, resp.getcode()) + data = resp.read() + expected = open(os.path.join(doc_root, "document.txt"), 'rb').read() + self.assertTrue(resp.info()["Content-Type"].startswith("multipart/byteranges; boundary=")) + boundary = resp.info()["Content-Type"].split("boundary=")[1] + parts = data.split("--" + boundary) + self.assertEqual("\r\n", parts[0]) + self.assertEqual("--", parts[-1]) + expected_parts = [("1-2", expected[1:3]), ("5-10", expected[5:11])] + for expected_part, part in zip(expected_parts, parts[1:-1]): + header_string, body = part.split("\r\n\r\n") + headers = dict(item.split(": ", 1) for item in header_string.split("\r\n") if item.strip()) + self.assertEqual(headers["Content-Type"], "text/plain") + self.assertEqual(headers["Content-Range"], "bytes %s/%i" % (expected_part[0], len(expected))) + self.assertEqual(expected_part[1] + "\r\n", body) + + def test_range_invalid(self): + with self.assertRaises(urllib2.HTTPError) as cm: + self.request("/document.txt", headers={"Range":"bytes=11-10"}) + self.assertEqual(cm.exception.code, 416) + + expected = open(os.path.join(doc_root, "document.txt"), 'rb').read() + with self.assertRaises(urllib2.HTTPError) as cm: + self.request("/document.txt", headers={"Range":"bytes=%i-%i" % (len(expected), len(expected) + 10)}) + self.assertEqual(cm.exception.code, 416) + + def test_sub_config(self): + resp = self.request("/sub.sub.txt") + expected = b"localhost localhost %i" % self.server.port + assert resp.read().rstrip() == expected + + def test_sub_headers(self): + resp = self.request("/sub_headers.sub.txt", headers={"X-Test": "PASS"}) + expected = b"PASS" + assert resp.read().rstrip() == expected + + def test_sub_params(self): + resp = self.request("/sub_params.sub.txt", query="test=PASS") + expected = b"PASS" + assert resp.read().rstrip() == expected + + +class TestFunctionHandler(TestUsingServer): + def test_string_rv(self): + @wptserve.handlers.handler + def handler(request, response): + return "test data" + + route = ("GET", "/test/test_string_rv", handler) + self.server.router.register(*route) + resp = self.request(route[1]) + self.assertEqual(200, resp.getcode()) + self.assertEqual("9", resp.info()["Content-Length"]) + self.assertEqual("test data", resp.read()) + + def test_tuple_1_rv(self): + @wptserve.handlers.handler + def handler(request, response): + return () + + route = ("GET", "/test/test_tuple_1_rv", handler) + self.server.router.register(*route) + + with pytest.raises(urllib2.HTTPError) as cm: + self.request(route[1]) + + assert cm.value.code == 500 + + def test_tuple_2_rv(self): + @wptserve.handlers.handler + def handler(request, response): + return [("Content-Length", 4), ("test-header", "test-value")], "test data" + + route = ("GET", "/test/test_tuple_2_rv", handler) + self.server.router.register(*route) + resp = self.request(route[1]) + self.assertEqual(200, resp.getcode()) + self.assertEqual("4", resp.info()["Content-Length"]) + self.assertEqual("test-value", resp.info()["test-header"]) + self.assertEqual("test", resp.read()) + + def test_tuple_3_rv(self): + @wptserve.handlers.handler + def handler(request, response): + return 202, [("test-header", "test-value")], "test data" + + route = ("GET", "/test/test_tuple_3_rv", handler) + self.server.router.register(*route) + resp = self.request(route[1]) + self.assertEqual(202, resp.getcode()) + self.assertEqual("test-value", resp.info()["test-header"]) + self.assertEqual("test data", resp.read()) + + def test_tuple_3_rv_1(self): + @wptserve.handlers.handler + def handler(request, response): + return (202, "Some Status"), [("test-header", "test-value")], "test data" + + route = ("GET", "/test/test_tuple_3_rv_1", handler) + self.server.router.register(*route) + resp = self.request(route[1]) + self.assertEqual(202, resp.getcode()) + self.assertEqual("Some Status", resp.msg) + self.assertEqual("test-value", resp.info()["test-header"]) + self.assertEqual("test data", resp.read()) + + def test_tuple_4_rv(self): + @wptserve.handlers.handler + def handler(request, response): + return 202, [("test-header", "test-value")], "test data", "garbage" + + route = ("GET", "/test/test_tuple_1_rv", handler) + self.server.router.register(*route) + + with pytest.raises(urllib2.HTTPError) as cm: + self.request(route[1]) + + assert cm.value.code == 500 + + def test_none_rv(self): + @wptserve.handlers.handler + def handler(request, response): + return None + + route = ("GET", "/test/test_none_rv", handler) + self.server.router.register(*route) + resp = self.request(route[1]) + assert resp.getcode() == 200 + assert "Content-Length" not in resp.info() + assert resp.read() == b"" + + +class TestJSONHandler(TestUsingServer): + def test_json_0(self): + @wptserve.handlers.json_handler + def handler(request, response): + return {"data": "test data"} + + route = ("GET", "/test/test_json_0", handler) + self.server.router.register(*route) + resp = self.request(route[1]) + self.assertEqual(200, resp.getcode()) + self.assertEqual({"data": "test data"}, json.load(resp)) + + def test_json_tuple_2(self): + @wptserve.handlers.json_handler + def handler(request, response): + return [("Test-Header", "test-value")], {"data": "test data"} + + route = ("GET", "/test/test_json_tuple_2", handler) + self.server.router.register(*route) + resp = self.request(route[1]) + self.assertEqual(200, resp.getcode()) + self.assertEqual("test-value", resp.info()["test-header"]) + self.assertEqual({"data": "test data"}, json.load(resp)) + + def test_json_tuple_3(self): + @wptserve.handlers.json_handler + def handler(request, response): + return (202, "Giraffe"), [("Test-Header", "test-value")], {"data": "test data"} + + route = ("GET", "/test/test_json_tuple_2", handler) + self.server.router.register(*route) + resp = self.request(route[1]) + self.assertEqual(202, resp.getcode()) + self.assertEqual("Giraffe", resp.msg) + self.assertEqual("test-value", resp.info()["test-header"]) + self.assertEqual({"data": "test data"}, json.load(resp)) + +class TestPythonHandler(TestUsingServer): + def test_string(self): + resp = self.request("/test_string.py") + self.assertEqual(200, resp.getcode()) + self.assertEqual("text/plain", resp.info()["Content-Type"]) + self.assertEqual("PASS", resp.read()) + + def test_tuple_2(self): + resp = self.request("/test_tuple_2.py") + self.assertEqual(200, resp.getcode()) + self.assertEqual("text/html", resp.info()["Content-Type"]) + self.assertEqual("PASS", resp.info()["X-Test"]) + self.assertEqual("PASS", resp.read()) + + def test_tuple_3(self): + resp = self.request("/test_tuple_3.py") + self.assertEqual(202, resp.getcode()) + self.assertEqual("Giraffe", resp.msg) + self.assertEqual("text/html", resp.info()["Content-Type"]) + self.assertEqual("PASS", resp.info()["X-Test"]) + self.assertEqual("PASS", resp.read()) + + def test_no_main(self): + with pytest.raises(urllib2.HTTPError) as cm: + self.request("/no_main.py") + + assert cm.value.code == 500 + + def test_invalid(self): + with pytest.raises(urllib2.HTTPError) as cm: + self.request("/invalid.py") + + assert cm.value.code == 500 + + def test_missing(self): + with pytest.raises(urllib2.HTTPError) as cm: + self.request("/missing.py") + + assert cm.value.code == 404 + + +class TestDirectoryHandler(TestUsingServer): + def test_directory(self): + resp = self.request("/") + self.assertEqual(200, resp.getcode()) + self.assertEqual("text/html", resp.info()["Content-Type"]) + #Add a check that the response is actually sane + + def test_subdirectory_trailing_slash(self): + resp = self.request("/subdir/") + assert resp.getcode() == 200 + assert resp.info()["Content-Type"] == "text/html" + + def test_subdirectory_no_trailing_slash(self): + with pytest.raises(urllib2.HTTPError) as cm: + self.request("/subdir") + + assert cm.value.code == 404 + + +class TestAsIsHandler(TestUsingServer): + def test_as_is(self): + resp = self.request("/test.asis") + self.assertEqual(202, resp.getcode()) + self.assertEqual("Giraffe", resp.msg) + self.assertEqual("PASS", resp.info()["X-Test"]) + self.assertEqual("Content", resp.read()) + #Add a check that the response is actually sane + +if __name__ == '__main__': + unittest.main() diff --git a/testing/web-platform/tests/tools/wptserve/tests/functional/test_pipes.py b/testing/web-platform/tests/tools/wptserve/tests/functional/test_pipes.py new file mode 100644 index 000000000..af5068108 --- /dev/null +++ b/testing/web-platform/tests/tools/wptserve/tests/functional/test_pipes.py @@ -0,0 +1,77 @@ +import os +import unittest +import time + +from .base import TestUsingServer, doc_root + +class TestStatus(TestUsingServer): + def test_status(self): + resp = self.request("/document.txt", query="pipe=status(202)") + self.assertEqual(resp.getcode(), 202) + +class TestHeader(TestUsingServer): + def test_not_set(self): + resp = self.request("/document.txt", query="pipe=header(X-TEST,PASS)") + self.assertEqual(resp.info()["X-TEST"], "PASS") + + def test_set(self): + resp = self.request("/document.txt", query="pipe=header(Content-Type,text/html)") + self.assertEqual(resp.info()["Content-Type"], "text/html") + + def test_multiple(self): + resp = self.request("/document.txt", query="pipe=header(X-Test,PASS)|header(Content-Type,text/html)") + self.assertEqual(resp.info()["X-TEST"], "PASS") + self.assertEqual(resp.info()["Content-Type"], "text/html") + + def test_multiple_same(self): + resp = self.request("/document.txt", query="pipe=header(Content-Type,FAIL)|header(Content-Type,text/html)") + self.assertEqual(resp.info()["Content-Type"], "text/html") + + def test_multiple_append(self): + resp = self.request("/document.txt", query="pipe=header(X-Test,1)|header(X-Test,2,True)") + self.assertEqual(resp.info()["X-Test"], "1, 2") + +class TestSlice(TestUsingServer): + def test_both_bounds(self): + resp = self.request("/document.txt", query="pipe=slice(1,10)") + expected = open(os.path.join(doc_root, "document.txt"), 'rb').read() + self.assertEqual(resp.read(), expected[1:10]) + + def test_no_upper(self): + resp = self.request("/document.txt", query="pipe=slice(1)") + expected = open(os.path.join(doc_root, "document.txt"), 'rb').read() + self.assertEqual(resp.read(), expected[1:]) + + def test_no_lower(self): + resp = self.request("/document.txt", query="pipe=slice(null,10)") + expected = open(os.path.join(doc_root, "document.txt"), 'rb').read() + self.assertEqual(resp.read(), expected[:10]) + +class TestSub(TestUsingServer): + def test_sub_config(self): + resp = self.request("/sub.txt", query="pipe=sub") + expected = "localhost localhost %i" % self.server.port + self.assertEqual(resp.read().rstrip(), expected) + + def test_sub_headers(self): + resp = self.request("/sub_headers.txt", query="pipe=sub", headers={"X-Test": "PASS"}) + expected = "PASS" + self.assertEqual(resp.read().rstrip(), expected) + + def test_sub_params(self): + resp = self.request("/sub_params.txt", query="test=PASS&pipe=sub") + expected = "PASS" + self.assertEqual(resp.read().rstrip(), expected) + +class TestTrickle(TestUsingServer): + def test_trickle(self): + #Actually testing that the response trickles in is not that easy + t0 = time.time() + resp = self.request("/document.txt", query="pipe=trickle(1:d2:5:d1:r2)") + t1 = time.time() + expected = open(os.path.join(doc_root, "document.txt"), 'rb').read() + self.assertEqual(resp.read(), expected) + self.assertGreater(6, t1-t0) + +if __name__ == '__main__': + unittest.main() diff --git a/testing/web-platform/tests/tools/wptserve/tests/functional/test_request.py b/testing/web-platform/tests/tools/wptserve/tests/functional/test_request.py new file mode 100644 index 000000000..40dfe7703 --- /dev/null +++ b/testing/web-platform/tests/tools/wptserve/tests/functional/test_request.py @@ -0,0 +1,82 @@ +import unittest + +import wptserve +from .base import TestUsingServer + +class TestInputFile(TestUsingServer): + def test_seek(self): + @wptserve.handlers.handler + def handler(request, response): + rv = [] + f = request.raw_input + f.seek(5) + rv.append(f.read(2)) + rv.append(f.tell()) + f.seek(0) + rv.append(f.readline()) + rv.append(f.tell()) + rv.append(f.read(-1)) + rv.append(f.tell()) + f.seek(0) + rv.append(f.read()) + f.seek(0) + rv.extend(f.readlines()) + + return " ".join(str(item) for item in rv) + + route = ("POST", "/test/test_seek", handler) + self.server.router.register(*route) + resp = self.request(route[1], method="POST", body="12345ab\ncdef") + self.assertEqual(200, resp.getcode()) + self.assertEqual(["ab", "7", "12345ab\n", "8", "cdef", "12", + "12345ab\ncdef", "12345ab\n", "cdef"], + resp.read().split(" ")) + + def test_iter(self): + @wptserve.handlers.handler + def handler(request, response): + f = request.raw_input + return " ".join(line for line in f) + + route = ("POST", "/test/test_iter", handler) + self.server.router.register(*route) + resp = self.request(route[1], method="POST", body="12345\nabcdef\r\nzyxwv") + self.assertEqual(200, resp.getcode()) + self.assertEqual(["12345\n", "abcdef\r\n", "zyxwv"], resp.read().split(" ")) + +class TestRequest(TestUsingServer): + def test_body(self): + @wptserve.handlers.handler + def handler(request, response): + request.raw_input.seek(5) + return request.body + + route = ("POST", "/test/test_body", handler) + self.server.router.register(*route) + resp = self.request(route[1], method="POST", body="12345ab\ncdef") + self.assertEqual("12345ab\ncdef", resp.read()) + + def test_route_match(self): + @wptserve.handlers.handler + def handler(request, response): + return request.route_match["match"] + " " + request.route_match["*"] + + route = ("GET", "/test/{match}_*", handler) + self.server.router.register(*route) + resp = self.request("/test/some_route") + self.assertEqual("some route", resp.read()) + +class TestAuth(TestUsingServer): + def test_auth(self): + @wptserve.handlers.handler + def handler(request, response): + return " ".join((request.auth.username, request.auth.password)) + + route = ("GET", "/test/test_auth", handler) + self.server.router.register(*route) + resp = self.request(route[1], auth=("test", "PASS")) + self.assertEqual(200, resp.getcode()) + self.assertEqual(["test", "PASS"], resp.read().split(" ")) + +if __name__ == '__main__': + unittest.main() diff --git a/testing/web-platform/tests/tools/wptserve/tests/functional/test_response.py b/testing/web-platform/tests/tools/wptserve/tests/functional/test_response.py new file mode 100644 index 000000000..e9808b54e --- /dev/null +++ b/testing/web-platform/tests/tools/wptserve/tests/functional/test_response.py @@ -0,0 +1,47 @@ +import unittest +from types import MethodType + +import wptserve +from .base import TestUsingServer + +def send_body_as_header(self): + if self._response.add_required_headers: + self.write_default_headers() + + self.write("X-Body: ") + self._headers_complete = True + +class TestResponse(TestUsingServer): + def test_head_without_body(self): + @wptserve.handlers.handler + def handler(request, response): + response.writer.end_headers = MethodType(send_body_as_header, + response.writer, + wptserve.response.ResponseWriter) + return [("X-Test", "TEST")], "body\r\n" + + route = ("GET", "/test/test_head_without_body", handler) + self.server.router.register(*route) + resp = self.request(route[1], method="HEAD") + self.assertEqual("6", resp.info()['Content-Length']) + self.assertEqual("TEST", resp.info()['x-Test']) + self.assertEqual("", resp.info()['x-body']) + + def test_head_with_body(self): + @wptserve.handlers.handler + def handler(request, response): + response.send_body_for_head_request = True + response.writer.end_headers = MethodType(send_body_as_header, + response.writer, + wptserve.response.ResponseWriter) + return [("X-Test", "TEST")], "body\r\n" + + route = ("GET", "/test/test_head_with_body", handler) + self.server.router.register(*route) + resp = self.request(route[1], method="HEAD") + self.assertEqual("6", resp.info()['Content-Length']) + self.assertEqual("TEST", resp.info()['x-Test']) + self.assertEqual("body", resp.info()['X-Body']) + +if __name__ == '__main__': + unittest.main() diff --git a/testing/web-platform/tests/tools/wptserve/tests/functional/test_server.py b/testing/web-platform/tests/tools/wptserve/tests/functional/test_server.py new file mode 100644 index 000000000..7681f4412 --- /dev/null +++ b/testing/web-platform/tests/tools/wptserve/tests/functional/test_server.py @@ -0,0 +1,41 @@ +import unittest +import urllib2 + +import wptserve +from .base import TestUsingServer + +class TestFileHandler(TestUsingServer): + def test_not_handled(self): + with self.assertRaises(urllib2.HTTPError) as cm: + resp = self.request("/not_existing") + + self.assertEqual(cm.exception.code, 404) + +class TestRewriter(TestUsingServer): + def test_rewrite(self): + @wptserve.handlers.handler + def handler(request, response): + return request.request_path + + route = ("GET", "/test/rewritten", handler) + self.server.rewriter.register("GET", "/test/original", route[1]) + self.server.router.register(*route) + resp = self.request("/test/original") + self.assertEqual(200, resp.getcode()) + self.assertEqual("/test/rewritten", resp.read()) + +class TestRequestHandler(TestUsingServer): + def test_exception(self): + @wptserve.handlers.handler + def handler(request, response): + raise Exception + + route = ("GET", "/test/raises", handler) + self.server.router.register(*route) + with self.assertRaises(urllib2.HTTPError) as cm: + resp = self.request("/test/raises") + + self.assertEqual(cm.exception.code, 500) + +if __name__ == "__main__": + unittest.main() diff --git a/testing/web-platform/tests/tools/wptserve/tests/functional/test_stash.py b/testing/web-platform/tests/tools/wptserve/tests/functional/test_stash.py new file mode 100644 index 000000000..134293d34 --- /dev/null +++ b/testing/web-platform/tests/tools/wptserve/tests/functional/test_stash.py @@ -0,0 +1,41 @@ +import unittest +import uuid + +import wptserve +from wptserve.router import any_method +from wptserve.stash import StashServer +from .base import TestUsingServer + +class TestResponseSetCookie(TestUsingServer): + def run(self, result=None): + with StashServer(None, authkey=str(uuid.uuid4())): + super(TestResponseSetCookie, self).run(result) + + def test_put_take(self): + @wptserve.handlers.handler + def handler(request, response): + if request.method == "POST": + request.server.stash.put(request.POST.first("id"), request.POST.first("data")) + data = "OK" + elif request.method == "GET": + data = request.server.stash.take(request.GET.first("id")) + if data is None: + return "NOT FOUND" + return data + + id = str(uuid.uuid4()) + route = (any_method, "/test/put_take", handler) + self.server.router.register(*route) + + resp = self.request(route[1], method="POST", body={"id": id, "data": "Sample data"}) + self.assertEqual(resp.read(), "OK") + + resp = self.request(route[1], query="id=" + id) + self.assertEqual(resp.read(), "Sample data") + + resp = self.request(route[1], query="id=" + id) + self.assertEqual(resp.read(), "NOT FOUND") + + +if __name__ == '__main__': + unittest.main() diff --git a/testing/web-platform/tests/tools/wptserve/tox.ini b/testing/web-platform/tests/tools/wptserve/tox.ini new file mode 100644 index 000000000..9532ca4c2 --- /dev/null +++ b/testing/web-platform/tests/tools/wptserve/tox.ini @@ -0,0 +1,17 @@ +[tox] +envlist = py27,pypy + +[testenv] +deps = + coverage + flake8 + pytest + +commands = + coverage run -m pytest tests/functional + flake8 + +[flake8] +ignore = E128,E129,E221,E226,E231,E251,E265,E302,E303,E402,E901,F821,F841 +max-line-length = 141 +exclude=docs,.git,__pycache__,.tox,.eggs,*.egg,tests/functional/docroot/ diff --git a/testing/web-platform/tests/tools/wptserve/wptserve/__init__.py b/testing/web-platform/tests/tools/wptserve/wptserve/__init__.py new file mode 100644 index 000000000..a286bfe0b --- /dev/null +++ b/testing/web-platform/tests/tools/wptserve/wptserve/__init__.py @@ -0,0 +1,3 @@ +from .server import WebTestHttpd, WebTestServer, Router # noqa: F401 +from .request import Request # noqa: F401 +from .response import Response # noqa: F401 diff --git a/testing/web-platform/tests/tools/wptserve/wptserve/constants.py b/testing/web-platform/tests/tools/wptserve/wptserve/constants.py new file mode 100644 index 000000000..bd36344a4 --- /dev/null +++ b/testing/web-platform/tests/tools/wptserve/wptserve/constants.py @@ -0,0 +1,92 @@ +from . import utils + +content_types = utils.invert_dict({"text/html": ["htm", "html"], + "application/json": ["json"], + "application/xhtml+xml": ["xht", "xhtm", "xhtml"], + "application/xml": ["xml"], + "application/x-xpinstall": ["xpi"], + "text/javascript": ["js"], + "text/css": ["css"], + "text/plain": ["txt", "md"], + "image/svg+xml": ["svg"], + "image/gif": ["gif"], + "image/jpeg": ["jpg", "jpeg"], + "image/png": ["png"], + "image/bmp": ["bmp"], + "text/event-stream": ["event_stream"], + "text/cache-manifest": ["manifest"], + "video/mp4": ["mp4", "m4v"], + "audio/mp4": ["m4a"], + "audio/mpeg": ["mp3"], + "video/webm": ["webm"], + "audio/webm": ["weba"], + "video/ogg": ["ogg", "ogv"], + "audio/ogg": ["oga"], + "audio/x-wav": ["wav"], + "text/vtt": ["vtt"],}) + +response_codes = { + 100: ('Continue', 'Request received, please continue'), + 101: ('Switching Protocols', + 'Switching to new protocol; obey Upgrade header'), + + 200: ('OK', 'Request fulfilled, document follows'), + 201: ('Created', 'Document created, URL follows'), + 202: ('Accepted', + 'Request accepted, processing continues off-line'), + 203: ('Non-Authoritative Information', 'Request fulfilled from cache'), + 204: ('No Content', 'Request fulfilled, nothing follows'), + 205: ('Reset Content', 'Clear input form for further input.'), + 206: ('Partial Content', 'Partial content follows.'), + + 300: ('Multiple Choices', + 'Object has several resources -- see URI list'), + 301: ('Moved Permanently', 'Object moved permanently -- see URI list'), + 302: ('Found', 'Object moved temporarily -- see URI list'), + 303: ('See Other', 'Object moved -- see Method and URL list'), + 304: ('Not Modified', + 'Document has not changed since given time'), + 305: ('Use Proxy', + 'You must use proxy specified in Location to access this ' + 'resource.'), + 307: ('Temporary Redirect', + 'Object moved temporarily -- see URI list'), + + 400: ('Bad Request', + 'Bad request syntax or unsupported method'), + 401: ('Unauthorized', + 'No permission -- see authorization schemes'), + 402: ('Payment Required', + 'No payment -- see charging schemes'), + 403: ('Forbidden', + 'Request forbidden -- authorization will not help'), + 404: ('Not Found', 'Nothing matches the given URI'), + 405: ('Method Not Allowed', + 'Specified method is invalid for this resource.'), + 406: ('Not Acceptable', 'URI not available in preferred format.'), + 407: ('Proxy Authentication Required', 'You must authenticate with ' + 'this proxy before proceeding.'), + 408: ('Request Timeout', 'Request timed out; try again later.'), + 409: ('Conflict', 'Request conflict.'), + 410: ('Gone', + 'URI no longer exists and has been permanently removed.'), + 411: ('Length Required', 'Client must specify Content-Length.'), + 412: ('Precondition Failed', 'Precondition in headers is false.'), + 413: ('Request Entity Too Large', 'Entity is too large.'), + 414: ('Request-URI Too Long', 'URI is too long.'), + 415: ('Unsupported Media Type', 'Entity body in unsupported format.'), + 416: ('Requested Range Not Satisfiable', + 'Cannot satisfy request range.'), + 417: ('Expectation Failed', + 'Expect condition could not be satisfied.'), + + 500: ('Internal Server Error', 'Server got itself in trouble'), + 501: ('Not Implemented', + 'Server does not support this operation'), + 502: ('Bad Gateway', 'Invalid responses from another server/proxy.'), + 503: ('Service Unavailable', + 'The server cannot process the request due to a high load'), + 504: ('Gateway Timeout', + 'The gateway server did not receive a timely response'), + 505: ('HTTP Version Not Supported', 'Cannot fulfill request.'), +} diff --git a/testing/web-platform/tests/tools/wptserve/wptserve/handlers.py b/testing/web-platform/tests/tools/wptserve/wptserve/handlers.py new file mode 100644 index 000000000..c40321dfe --- /dev/null +++ b/testing/web-platform/tests/tools/wptserve/wptserve/handlers.py @@ -0,0 +1,370 @@ +import cgi +import json +import os +import traceback +import urllib +import urlparse + +from .constants import content_types +from .pipes import Pipeline, template +from .ranges import RangeParser +from .request import Authentication +from .response import MultipartContent +from .utils import HTTPException + +__all__ = ["file_handler", "python_script_handler", + "FunctionHandler", "handler", "json_handler", + "as_is_handler", "ErrorHandler", "BasicAuthHandler"] + + +def guess_content_type(path): + ext = os.path.splitext(path)[1].lstrip(".") + if ext in content_types: + return content_types[ext] + + return "application/octet-stream" + + + +def filesystem_path(base_path, request, url_base="/"): + if base_path is None: + base_path = request.doc_root + + path = urllib.unquote(request.url_parts.path) + + if path.startswith(url_base): + path = path[len(url_base):] + + if ".." in path: + raise HTTPException(404) + + new_path = os.path.join(base_path, path) + + # Otherwise setting path to / allows access outside the root directory + if not new_path.startswith(base_path): + raise HTTPException(404) + + return new_path + +class DirectoryHandler(object): + def __init__(self, base_path=None, url_base="/"): + self.base_path = base_path + self.url_base = url_base + + def __repr__(self): + return "<%s base_path:%s url_base:%s>" % (self.__class__.__name__, self.base_path, self.url_base) + + def __call__(self, request, response): + url_path = request.url_parts.path + + if not url_path.endswith("/"): + raise HTTPException(404) + + path = filesystem_path(self.base_path, request, self.url_base) + + assert os.path.isdir(path) + + response.headers = [("Content-Type", "text/html")] + response.content = """<!doctype html> +<meta name="viewport" content="width=device-width"> +<title>Directory listing for %(path)s</title> +<h1>Directory listing for %(path)s</h1> +<ul> +%(items)s +</ul> +""" % {"path": cgi.escape(url_path), + "items": "\n".join(self.list_items(url_path, path))} # flake8: noqa + + def list_items(self, base_path, path): + assert base_path.endswith("/") + + # TODO: this won't actually list all routes, only the + # ones that correspond to a real filesystem path. It's + # not possible to list every route that will match + # something, but it should be possible to at least list the + # statically defined ones + + if base_path != "/": + link = urlparse.urljoin(base_path, "..") + yield ("""<li class="dir"><a href="%(link)s">%(name)s</a></li>""" % + {"link": link, "name": ".."}) + for item in sorted(os.listdir(path)): + link = cgi.escape(urllib.quote(item)) + if os.path.isdir(os.path.join(path, item)): + link += "/" + class_ = "dir" + else: + class_ = "file" + yield ("""<li class="%(class)s"><a href="%(link)s">%(name)s</a></li>""" % + {"link": link, "name": cgi.escape(item), "class": class_}) + + +class FileHandler(object): + def __init__(self, base_path=None, url_base="/"): + self.base_path = base_path + self.url_base = url_base + self.directory_handler = DirectoryHandler(self.base_path, self.url_base) + + def __repr__(self): + return "<%s base_path:%s url_base:%s>" % (self.__class__.__name__, self.base_path, self.url_base) + + def __call__(self, request, response): + path = filesystem_path(self.base_path, request, self.url_base) + + if os.path.isdir(path): + return self.directory_handler(request, response) + try: + #This is probably racy with some other process trying to change the file + file_size = os.stat(path).st_size + response.headers.update(self.get_headers(request, path)) + if "Range" in request.headers: + try: + byte_ranges = RangeParser()(request.headers['Range'], file_size) + except HTTPException as e: + if e.code == 416: + response.headers.set("Content-Range", "bytes */%i" % file_size) + raise + else: + byte_ranges = None + data = self.get_data(response, path, byte_ranges) + response.content = data + query = urlparse.parse_qs(request.url_parts.query) + + pipeline = None + if "pipe" in query: + pipeline = Pipeline(query["pipe"][-1]) + elif os.path.splitext(path)[0].endswith(".sub"): + ml_extensions = {".html", ".htm", ".xht", ".xhtml", ".xml", ".svg"} + escape_type = "html" if os.path.splitext(path)[1] in ml_extensions else "none" + pipeline = Pipeline("sub(%s)" % escape_type) + + if pipeline is not None: + response = pipeline(request, response) + + return response + + except (OSError, IOError): + raise HTTPException(404) + + def get_headers(self, request, path): + rv = (self.load_headers(request, os.path.join(os.path.split(path)[0], "__dir__")) + + self.load_headers(request, path)) + + if not any(key.lower() == "content-type" for (key, _) in rv): + rv.insert(0, ("Content-Type", guess_content_type(path))) + + return rv + + def load_headers(self, request, path): + headers_path = path + ".sub.headers" + if os.path.exists(headers_path): + use_sub = True + else: + headers_path = path + ".headers" + use_sub = False + + try: + with open(headers_path) as headers_file: + data = headers_file.read() + except IOError: + return [] + else: + if use_sub: + data = template(request, data, escape_type="none") + return [tuple(item.strip() for item in line.split(":", 1)) + for line in data.splitlines() if line] + + def get_data(self, response, path, byte_ranges): + """Return either the handle to a file, or a string containing + the content of a chunk of the file, if we have a range request.""" + if byte_ranges is None: + return open(path, 'rb') + else: + with open(path, 'rb') as f: + response.status = 206 + if len(byte_ranges) > 1: + parts_content_type, content = self.set_response_multipart(response, + byte_ranges, + f) + for byte_range in byte_ranges: + content.append_part(self.get_range_data(f, byte_range), + parts_content_type, + [("Content-Range", byte_range.header_value())]) + return content + else: + response.headers.set("Content-Range", byte_ranges[0].header_value()) + return self.get_range_data(f, byte_ranges[0]) + + def set_response_multipart(self, response, ranges, f): + parts_content_type = response.headers.get("Content-Type") + if parts_content_type: + parts_content_type = parts_content_type[-1] + else: + parts_content_type = None + content = MultipartContent() + response.headers.set("Content-Type", "multipart/byteranges; boundary=%s" % content.boundary) + return parts_content_type, content + + def get_range_data(self, f, byte_range): + f.seek(byte_range.lower) + return f.read(byte_range.upper - byte_range.lower) + + +file_handler = FileHandler() + + +class PythonScriptHandler(object): + def __init__(self, base_path=None, url_base="/"): + self.base_path = base_path + self.url_base = url_base + + def __repr__(self): + return "<%s base_path:%s url_base:%s>" % (self.__class__.__name__, self.base_path, self.url_base) + + def __call__(self, request, response): + path = filesystem_path(self.base_path, request, self.url_base) + + try: + environ = {"__file__": path} + execfile(path, environ, environ) + if "main" in environ: + handler = FunctionHandler(environ["main"]) + handler(request, response) + else: + raise HTTPException(500, "No main function in script %s" % path) + except IOError: + raise HTTPException(404) + +python_script_handler = PythonScriptHandler() + +class FunctionHandler(object): + def __init__(self, func): + self.func = func + + def __call__(self, request, response): + try: + rv = self.func(request, response) + except Exception: + msg = traceback.format_exc() + raise HTTPException(500, message=msg) + if rv is not None: + if isinstance(rv, tuple): + if len(rv) == 3: + status, headers, content = rv + response.status = status + elif len(rv) == 2: + headers, content = rv + else: + raise HTTPException(500) + response.headers.update(headers) + else: + content = rv + response.content = content + + +#The generic name here is so that this can be used as a decorator +def handler(func): + return FunctionHandler(func) + + +class JsonHandler(object): + def __init__(self, func): + self.func = func + + def __call__(self, request, response): + return FunctionHandler(self.handle_request)(request, response) + + def handle_request(self, request, response): + rv = self.func(request, response) + response.headers.set("Content-Type", "application/json") + enc = json.dumps + if isinstance(rv, tuple): + rv = list(rv) + value = tuple(rv[:-1] + [enc(rv[-1])]) + length = len(value[-1]) + else: + value = enc(rv) + length = len(value) + response.headers.set("Content-Length", length) + return value + +def json_handler(func): + return JsonHandler(func) + +class AsIsHandler(object): + def __init__(self, base_path=None, url_base="/"): + self.base_path = base_path + self.url_base = url_base + + def __call__(self, request, response): + path = filesystem_path(self.base_path, request, self.url_base) + + try: + with open(path) as f: + response.writer.write_content(f.read()) + response.close_connection = True + except IOError: + raise HTTPException(404) + +as_is_handler = AsIsHandler() + +class BasicAuthHandler(object): + def __init__(self, handler, user, password): + """ + A Basic Auth handler + + :Args: + - handler: a secondary handler for the request after authentication is successful (example file_handler) + - user: string of the valid user name or None if any / all credentials are allowed + - password: string of the password required + """ + self.user = user + self.password = password + self.handler = handler + + def __call__(self, request, response): + if "authorization" not in request.headers: + response.status = 401 + response.headers.set("WWW-Authenticate", "Basic") + return response + else: + auth = Authentication(request.headers) + if self.user is not None and (self.user != auth.username or self.password != auth.password): + response.set_error(403, "Invalid username or password") + return response + return self.handler(request, response) + +basic_auth_handler = BasicAuthHandler(file_handler, None, None) + +class ErrorHandler(object): + def __init__(self, status): + self.status = status + + def __call__(self, request, response): + response.set_error(self.status) + + +class StaticHandler(object): + def __init__(self, path, format_args, content_type, **headers): + """Hander that reads a file from a path and substitutes some fixed data + + :param path: Path to the template file to use + :param format_args: Dictionary of values to substitute into the template file + :param content_type: Content type header to server the response with + :param headers: List of headers to send with responses""" + + with open(path) as f: + self.data = f.read() % format_args + + self.resp_headers = [("Content-Type", content_type)] + for k, v in headers.iteritems(): + resp_headers.append((k.replace("_", "-"), v)) + + self.handler = handler(self.handle_request) + + def handle_request(self, request, response): + return self.resp_headers, self.data + + def __call__(self, request, response): + rv = self.handler(request, response) + return rv diff --git a/testing/web-platform/tests/tools/wptserve/wptserve/logger.py b/testing/web-platform/tests/tools/wptserve/wptserve/logger.py new file mode 100644 index 000000000..6c91492c7 --- /dev/null +++ b/testing/web-platform/tests/tools/wptserve/wptserve/logger.py @@ -0,0 +1,29 @@ +class NoOpLogger(object): + def critical(self, msg): + pass + + def error(self, msg): + pass + + def info(self, msg): + pass + + def warning(self, msg): + pass + + def debug(self, msg): + pass + +logger = NoOpLogger() +_set_logger = False + +def set_logger(new_logger): + global _set_logger + if _set_logger: + raise Exception("Logger must be set at most once") + global logger + logger = new_logger + _set_logger = True + +def get_logger(): + return logger diff --git a/testing/web-platform/tests/tools/wptserve/wptserve/pipes.py b/testing/web-platform/tests/tools/wptserve/wptserve/pipes.py new file mode 100644 index 000000000..41f7dd33e --- /dev/null +++ b/testing/web-platform/tests/tools/wptserve/wptserve/pipes.py @@ -0,0 +1,449 @@ +from cgi import escape +import gzip as gzip_module +import re +import time +import types +import uuid +from cStringIO import StringIO + + +def resolve_content(response): + rv = "".join(item for item in response.iter_content(read_file=True)) + if type(rv) == unicode: + rv = rv.encode(response.encoding) + return rv + + +class Pipeline(object): + pipes = {} + + def __init__(self, pipe_string): + self.pipe_functions = self.parse(pipe_string) + + def parse(self, pipe_string): + functions = [] + for item in PipeTokenizer().tokenize(pipe_string): + if not item: + break + if item[0] == "function": + functions.append((self.pipes[item[1]], [])) + elif item[0] == "argument": + functions[-1][1].append(item[1]) + return functions + + def __call__(self, request, response): + for func, args in self.pipe_functions: + response = func(request, response, *args) + return response + + +class PipeTokenizer(object): + def __init__(self): + #This whole class can likely be replaced by some regexps + self.state = None + + def tokenize(self, string): + self.string = string + self.state = self.func_name_state + self._index = 0 + while self.state: + yield self.state() + yield None + + def get_char(self): + if self._index >= len(self.string): + return None + rv = self.string[self._index] + self._index += 1 + return rv + + def func_name_state(self): + rv = "" + while True: + char = self.get_char() + if char is None: + self.state = None + if rv: + return ("function", rv) + else: + return None + elif char == "(": + self.state = self.argument_state + return ("function", rv) + elif char == "|": + if rv: + return ("function", rv) + else: + rv += char + + def argument_state(self): + rv = "" + while True: + char = self.get_char() + if char is None: + self.state = None + return ("argument", rv) + elif char == "\\": + rv += self.get_escape() + if rv is None: + #This should perhaps be an error instead + return ("argument", rv) + elif char == ",": + return ("argument", rv) + elif char == ")": + self.state = self.func_name_state + return ("argument", rv) + else: + rv += char + + def get_escape(self): + char = self.get_char() + escapes = {"n": "\n", + "r": "\r", + "t": "\t"} + return escapes.get(char, char) + + +class pipe(object): + def __init__(self, *arg_converters): + self.arg_converters = arg_converters + self.max_args = len(self.arg_converters) + self.min_args = 0 + opt_seen = False + for item in self.arg_converters: + if not opt_seen: + if isinstance(item, opt): + opt_seen = True + else: + self.min_args += 1 + else: + if not isinstance(item, opt): + raise ValueError("Non-optional argument cannot follow optional argument") + + def __call__(self, f): + def inner(request, response, *args): + if not (self.min_args <= len(args) <= self.max_args): + raise ValueError("Expected between %d and %d args, got %d" % + (self.min_args, self.max_args, len(args))) + arg_values = tuple(f(x) for f, x in zip(self.arg_converters, args)) + return f(request, response, *arg_values) + Pipeline.pipes[f.__name__] = inner + #We actually want the undecorated function in the main namespace + return f + + +class opt(object): + def __init__(self, f): + self.f = f + + def __call__(self, arg): + return self.f(arg) + + +def nullable(func): + def inner(arg): + if arg.lower() == "null": + return None + else: + return func(arg) + return inner + + +def boolean(arg): + if arg.lower() in ("true", "1"): + return True + elif arg.lower() in ("false", "0"): + return False + raise ValueError + + +@pipe(int) +def status(request, response, code): + """Alter the status code. + + :param code: Status code to use for the response.""" + response.status = code + return response + + +@pipe(str, str, opt(boolean)) +def header(request, response, name, value, append=False): + """Set a HTTP header. + + Replaces any existing HTTP header of the same name unless + append is set, in which case the header is appended without + replacement. + + :param name: Name of the header to set. + :param value: Value to use for the header. + :param append: True if existing headers should not be replaced + """ + if not append: + response.headers.set(name, value) + else: + response.headers.append(name, value) + return response + + +@pipe(str) +def trickle(request, response, delays): + """Send the response in parts, with time delays. + + :param delays: A string of delays and amounts, in bytes, of the + response to send. Each component is separated by + a colon. Amounts in bytes are plain integers, whilst + delays are floats prefixed with a single d e.g. + d1:100:d2 + Would cause a 1 second delay, would then send 100 bytes + of the file, and then cause a 2 second delay, before sending + the remainder of the file. + + If the last token is of the form rN, instead of sending the + remainder of the file, the previous N instructions will be + repeated until the whole file has been sent e.g. + d1:100:d2:r2 + Causes a delay of 1s, then 100 bytes to be sent, then a 2s delay + and then a further 100 bytes followed by a two second delay + until the response has been fully sent. + """ + def parse_delays(): + parts = delays.split(":") + rv = [] + for item in parts: + if item.startswith("d"): + item_type = "delay" + item = item[1:] + value = float(item) + elif item.startswith("r"): + item_type = "repeat" + value = int(item[1:]) + if not value % 2 == 0: + raise ValueError + else: + item_type = "bytes" + value = int(item) + if len(rv) and rv[-1][0] == item_type: + rv[-1][1] += value + else: + rv.append((item_type, value)) + return rv + + delays = parse_delays() + if not delays: + return response + content = resolve_content(response) + offset = [0] + + def add_content(delays, repeat=False): + for i, (item_type, value) in enumerate(delays): + if item_type == "bytes": + yield content[offset[0]:offset[0] + value] + offset[0] += value + elif item_type == "delay": + time.sleep(value) + elif item_type == "repeat": + if i != len(delays) - 1: + continue + while offset[0] < len(content): + for item in add_content(delays[-(value + 1):-1], True): + yield item + + if not repeat and offset[0] < len(content): + yield content[offset[0]:] + + response.content = add_content(delays) + return response + + +@pipe(nullable(int), opt(nullable(int))) +def slice(request, response, start, end=None): + """Send a byte range of the response body + + :param start: The starting offset. Follows python semantics including + negative numbers. + + :param end: The ending offset, again with python semantics and None + (spelled "null" in a query string) to indicate the end of + the file. + """ + content = resolve_content(response) + response.content = content[start:end] + return response + + +class ReplacementTokenizer(object): + def ident(scanner, token): + return ("ident", token) + + def index(scanner, token): + token = token[1:-1] + try: + token = int(token) + except ValueError: + token = unicode(token, "utf8") + return ("index", token) + + def var(scanner, token): + token = token[:-1] + return ("var", token) + + def tokenize(self, string): + return self.scanner.scan(string)[0] + + scanner = re.Scanner([(r"\$\w+:", var), + (r"\$?\w+(?:\(\))?", ident), + (r"\[[^\]]*\]", index)]) + + +class FirstWrapper(object): + def __init__(self, params): + self.params = params + + def __getitem__(self, key): + try: + return self.params.first(key) + except KeyError: + return "" + + +@pipe(opt(nullable(str))) +def sub(request, response, escape_type="html"): + """Substitute environment information about the server and request into the script. + + :param escape_type: String detailing the type of escaping to use. Known values are + "html" and "none", with "html" the default for historic reasons. + + The format is a very limited template language. Substitutions are + enclosed by {{ and }}. There are several avaliable substitutions: + + host + A simple string value and represents the primary host from which the + tests are being run. + domains + A dictionary of available domains indexed by subdomain name. + ports + A dictionary of lists of ports indexed by protocol. + location + A dictionary of parts of the request URL. Valid keys are + 'server, 'scheme', 'host', 'hostname', 'port', 'path' and 'query'. + 'server' is scheme://host:port, 'host' is hostname:port, and query + includes the leading '?', but other delimiters are omitted. + headers + A dictionary of HTTP headers in the request. + GET + A dictionary of query parameters supplied with the request. + uuid() + A pesudo-random UUID suitable for usage with stash + + So for example in a setup running on localhost with a www + subdomain and a http server on ports 80 and 81:: + + {{host}} => localhost + {{domains[www]}} => www.localhost + {{ports[http][1]}} => 81 + + + It is also possible to assign a value to a variable name, which must start with + the $ character, using the ":" syntax e.g. + + {{$id:uuid()} + + Later substitutions in the same file may then refer to the variable + by name e.g. + + {{$id}} + """ + content = resolve_content(response) + + new_content = template(request, content, escape_type=escape_type) + + response.content = new_content + return response + +def template(request, content, escape_type="html"): + #TODO: There basically isn't any error handling here + tokenizer = ReplacementTokenizer() + + variables = {} + + def config_replacement(match): + content, = match.groups() + + tokens = tokenizer.tokenize(content) + + if tokens[0][0] == "var": + variable = tokens[0][1] + tokens = tokens[1:] + else: + variable = None + + assert tokens[0][0] == "ident" and all(item[0] == "index" for item in tokens[1:]), tokens + + field = tokens[0][1] + + if field in variables: + value = variables[field] + elif field == "headers": + value = request.headers + elif field == "GET": + value = FirstWrapper(request.GET) + elif field in request.server.config: + value = request.server.config[tokens[0][1]] + elif field == "location": + value = {"server": "%s://%s:%s" % (request.url_parts.scheme, + request.url_parts.hostname, + request.url_parts.port), + "scheme": request.url_parts.scheme, + "host": "%s:%s" % (request.url_parts.hostname, + request.url_parts.port), + "hostname": request.url_parts.hostname, + "port": request.url_parts.port, + "path": request.url_parts.path, + "pathname": request.url_parts.path, + "query": "?%s" % request.url_parts.query} + elif field == "uuid()": + value = str(uuid.uuid4()) + elif field == "url_base": + value = request.url_base + else: + raise Exception("Undefined template variable %s" % field) + + for item in tokens[1:]: + value = value[item[1]] + + assert isinstance(value, (int,) + types.StringTypes), tokens + + if variable is not None: + variables[variable] = value + + escape_func = {"html": lambda x:escape(x, quote=True), + "none": lambda x:x}[escape_type] + + #Should possibly support escaping for other contexts e.g. script + #TODO: read the encoding of the response + return escape_func(unicode(value)).encode("utf-8") + + template_regexp = re.compile(r"{{([^}]*)}}") + new_content = template_regexp.sub(config_replacement, content) + + return new_content + +@pipe() +def gzip(request, response): + """This pipe gzip-encodes response data. + + It sets (or overwrites) these HTTP headers: + Content-Encoding is set to gzip + Content-Length is set to the length of the compressed content + """ + content = resolve_content(response) + response.headers.set("Content-Encoding", "gzip") + + out = StringIO() + with gzip_module.GzipFile(fileobj=out, mode="w") as f: + f.write(content) + response.content = out.getvalue() + + response.headers.set("Content-Length", len(response.content)) + + return response diff --git a/testing/web-platform/tests/tools/wptserve/wptserve/ranges.py b/testing/web-platform/tests/tools/wptserve/wptserve/ranges.py new file mode 100644 index 000000000..976cb1781 --- /dev/null +++ b/testing/web-platform/tests/tools/wptserve/wptserve/ranges.py @@ -0,0 +1,90 @@ +from .utils import HTTPException + + +class RangeParser(object): + def __call__(self, header, file_size): + prefix = "bytes=" + if not header.startswith(prefix): + raise HTTPException(416, message="Unrecognised range type %s" % (header,)) + + parts = header[len(prefix):].split(",") + ranges = [] + for item in parts: + components = item.split("-") + if len(components) != 2: + raise HTTPException(416, "Bad range specifier %s" % (item)) + data = [] + for component in components: + if component == "": + data.append(None) + else: + try: + data.append(int(component)) + except ValueError: + raise HTTPException(416, "Bad range specifier %s" % (item)) + try: + ranges.append(Range(data[0], data[1], file_size)) + except ValueError: + raise HTTPException(416, "Bad range specifier %s" % (item)) + + return self.coalesce_ranges(ranges, file_size) + + def coalesce_ranges(self, ranges, file_size): + rv = [] + target = None + for current in reversed(sorted(ranges)): + if target is None: + target = current + else: + new = target.coalesce(current) + target = new[0] + if len(new) > 1: + rv.append(new[1]) + rv.append(target) + + return rv[::-1] + + +class Range(object): + def __init__(self, lower, upper, file_size): + self.file_size = file_size + self.lower, self.upper = self._abs(lower, upper) + if self.lower >= self.upper or self.lower >= self.file_size: + raise ValueError + + def __repr__(self): + return "<Range %s-%s>" % (self.lower, self.upper) + + def __lt__(self, other): + return self.lower < other.lower + + def __gt__(self, other): + return self.lower > other.lower + + def __eq__(self, other): + return self.lower == other.lower and self.upper == other.upper + + def _abs(self, lower, upper): + if lower is None and upper is None: + lower, upper = 0, self.file_size + elif lower is None: + lower, upper = max(0, self.file_size - upper), self.file_size + elif upper is None: + lower, upper = lower, self.file_size + else: + lower, upper = lower, min(self.file_size, upper + 1) + + return lower, upper + + def coalesce(self, other): + assert self.file_size == other.file_size + + if (self.upper < other.lower or self.lower > other.upper): + return sorted([self, other]) + else: + return [Range(min(self.lower, other.lower), + max(self.upper, other.upper) - 1, + self.file_size)] + + def header_value(self): + return "bytes %i-%i/%i" % (self.lower, self.upper - 1, self.file_size) diff --git a/testing/web-platform/tests/tools/wptserve/wptserve/request.py b/testing/web-platform/tests/tools/wptserve/wptserve/request.py new file mode 100644 index 000000000..6b8a7cef8 --- /dev/null +++ b/testing/web-platform/tests/tools/wptserve/wptserve/request.py @@ -0,0 +1,589 @@ +import base64 +import cgi +import Cookie +import StringIO +import tempfile +import urlparse + +from . import stash +from .utils import HTTPException + +missing = object() + + +class Server(object): + """Data about the server environment + + .. attribute:: config + + Environment configuration information with information about the + various servers running, their hostnames and ports. + + .. attribute:: stash + + Stash object holding state stored on the server between requests. + + """ + config = None + + def __init__(self, request): + self._stash = None + self._request = request + + @property + def stash(self): + if self._stash is None: + address, authkey = stash.load_env_config() + self._stash = stash.Stash(self._request.url_parts.path, address, authkey) + return self._stash + + +class InputFile(object): + max_buffer_size = 1024*1024 + + def __init__(self, rfile, length): + """File-like object used to provide a seekable view of request body data""" + self._file = rfile + self.length = length + + self._file_position = 0 + + if length > self.max_buffer_size: + self._buf = tempfile.TemporaryFile(mode="rw+b") + else: + self._buf = StringIO.StringIO() + + @property + def _buf_position(self): + rv = self._buf.tell() + assert rv <= self._file_position + return rv + + def read(self, bytes=-1): + assert self._buf_position <= self._file_position + + if bytes < 0: + bytes = self.length - self._buf_position + bytes_remaining = min(bytes, self.length - self._buf_position) + + if bytes_remaining == 0: + return "" + + if self._buf_position != self._file_position: + buf_bytes = min(bytes_remaining, self._file_position - self._buf_position) + old_data = self._buf.read(buf_bytes) + bytes_remaining -= buf_bytes + else: + old_data = "" + + assert self._buf_position == self._file_position, ( + "Before reading buffer position (%i) didn't match file position (%i)" % + (self._buf_position, self._file_position)) + new_data = self._file.read(bytes_remaining) + self._buf.write(new_data) + self._file_position += bytes_remaining + assert self._buf_position == self._file_position, ( + "After reading buffer position (%i) didn't match file position (%i)" % + (self._buf_position, self._file_position)) + + return old_data + new_data + + def tell(self): + return self._buf_position + + def seek(self, offset): + if offset > self.length or offset < 0: + raise ValueError + if offset <= self._file_position: + self._buf.seek(offset) + else: + self.read(offset - self._file_position) + + def readline(self, max_bytes=None): + if max_bytes is None: + max_bytes = self.length - self._buf_position + + if self._buf_position < self._file_position: + data = self._buf.readline(max_bytes) + if data.endswith("\n") or len(data) == max_bytes: + return data + else: + data = "" + + assert self._buf_position == self._file_position + + initial_position = self._file_position + found = False + buf = [] + max_bytes -= len(data) + while not found: + readahead = self.read(min(2, max_bytes)) + max_bytes -= len(readahead) + for i, c in enumerate(readahead): + if c == "\n": + buf.append(readahead[:i+1]) + found = True + break + if not found: + buf.append(readahead) + if not readahead or not max_bytes: + break + new_data = "".join(buf) + data += new_data + self.seek(initial_position + len(new_data)) + return data + + def readlines(self): + rv = [] + while True: + data = self.readline() + if data: + rv.append(data) + else: + break + return rv + + def next(self): + data = self.readline() + if data: + return data + else: + raise StopIteration + + def __iter__(self): + return self + + +class Request(object): + """Object representing a HTTP request. + + .. attribute:: doc_root + + The local directory to use as a base when resolving paths + + .. attribute:: route_match + + Regexp match object from matching the request path to the route + selected for the request. + + .. attribute:: protocol_version + + HTTP version specified in the request. + + .. attribute:: method + + HTTP method in the request. + + .. attribute:: request_path + + Request path as it appears in the HTTP request. + + .. attribute:: url_base + + The prefix part of the path; typically / unless the handler has a url_base set + + .. attribute:: url + + Absolute URL for the request. + + .. attribute:: headers + + List of request headers. + + .. attribute:: raw_input + + File-like object representing the body of the request. + + .. attribute:: url_parts + + Parts of the requested URL as obtained by urlparse.urlsplit(path) + + .. attribute:: request_line + + Raw request line + + .. attribute:: headers + + RequestHeaders object providing a dictionary-like representation of + the request headers. + + .. attribute:: body + + Request body as a string + + .. attribute:: GET + + MultiDict representing the parameters supplied with the request. + Note that these may be present on non-GET requests; the name is + chosen to be familiar to users of other systems such as PHP. + + .. attribute:: POST + + MultiDict representing the request body parameters. Most parameters + are present as string values, but file uploads have file-like + values. + + .. attribute:: cookies + + Cookies object representing cookies sent with the request with a + dictionary-like interface. + + .. attribute:: auth + + Object with username and password properties representing any + credentials supplied using HTTP authentication. + + .. attribute:: server + + Server object containing information about the server environment. + """ + + def __init__(self, request_handler): + self.doc_root = request_handler.server.router.doc_root + self.route_match = None # Set by the router + + self.protocol_version = request_handler.protocol_version + self.method = request_handler.command + + scheme = request_handler.server.scheme + host = request_handler.headers.get("Host") + port = request_handler.server.server_address[1] + + if host is None: + host = request_handler.server.server_address[0] + else: + if ":" in host: + host, port = host.split(":", 1) + + self.request_path = request_handler.path + self.url_base = "/" + + if self.request_path.startswith(scheme + "://"): + self.url = request_handler.path + else: + self.url = "%s://%s:%s%s" % (scheme, + host, + port, + self.request_path) + self.url_parts = urlparse.urlsplit(self.url) + + self._raw_headers = request_handler.headers + + self.request_line = request_handler.raw_requestline + + self._headers = None + + self.raw_input = InputFile(request_handler.rfile, + int(self.headers.get("Content-Length", 0))) + self._body = None + + self._GET = None + self._POST = None + self._cookies = None + self._auth = None + + self.server = Server(self) + + def __repr__(self): + return "<Request %s %s>" % (self.method, self.url) + + @property + def GET(self): + if self._GET is None: + params = urlparse.parse_qsl(self.url_parts.query, keep_blank_values=True) + self._GET = MultiDict() + for key, value in params: + self._GET.add(key, value) + return self._GET + + @property + def POST(self): + if self._POST is None: + #Work out the post parameters + pos = self.raw_input.tell() + self.raw_input.seek(0) + fs = cgi.FieldStorage(fp=self.raw_input, + environ={"REQUEST_METHOD": self.method}, + headers=self.headers, + keep_blank_values=True) + self._POST = MultiDict.from_field_storage(fs) + self.raw_input.seek(pos) + return self._POST + + @property + def cookies(self): + if self._cookies is None: + parser = Cookie.BaseCookie() + cookie_headers = self.headers.get("cookie", "") + parser.load(cookie_headers) + cookies = Cookies() + for key, value in parser.iteritems(): + cookies[key] = CookieValue(value) + self._cookies = cookies + return self._cookies + + @property + def headers(self): + if self._headers is None: + self._headers = RequestHeaders(self._raw_headers) + return self._headers + + @property + def body(self): + if self._body is None: + pos = self.raw_input.tell() + self.raw_input.seek(0) + self._body = self.raw_input.read() + self.raw_input.seek(pos) + return self._body + + @property + def auth(self): + if self._auth is None: + self._auth = Authentication(self.headers) + return self._auth + + +class RequestHeaders(dict): + """Dictionary-like API for accessing request headers.""" + def __init__(self, items): + for key, value in zip(items.keys(), items.values()): + key = key.lower() + if key in self: + self[key].append(value) + else: + dict.__setitem__(self, key, [value]) + + def __getitem__(self, key): + """Get all headers of a certain (case-insensitive) name. If there is + more than one, the values are returned comma separated""" + values = dict.__getitem__(self, key.lower()) + if len(values) == 1: + return values[0] + else: + return ", ".join(values) + + def __setitem__(self, name, value): + raise Exception + + def get(self, key, default=None): + """Get a string representing all headers with a particular value, + with multiple headers separated by a comma. If no header is found + return a default value + + :param key: The header name to look up (case-insensitive) + :param default: The value to return in the case of no match + """ + try: + return self[key] + except KeyError: + return default + + def get_list(self, key, default=missing): + """Get all the header values for a particular field name as + a list""" + try: + return dict.__getitem__(self, key.lower()) + except KeyError: + if default is not missing: + return default + else: + raise + + def __contains__(self, key): + return dict.__contains__(self, key.lower()) + + def iteritems(self): + for item in self: + yield item, self[item] + + def itervalues(self): + for item in self: + yield self[item] + +class CookieValue(object): + """Representation of cookies. + + Note that cookies are considered read-only and the string value + of the cookie will not change if you update the field values. + However this is not enforced. + + .. attribute:: key + + The name of the cookie. + + .. attribute:: value + + The value of the cookie + + .. attribute:: expires + + The expiry date of the cookie + + .. attribute:: path + + The path of the cookie + + .. attribute:: comment + + The comment of the cookie. + + .. attribute:: domain + + The domain with which the cookie is associated + + .. attribute:: max_age + + The max-age value of the cookie. + + .. attribute:: secure + + Whether the cookie is marked as secure + + .. attribute:: httponly + + Whether the cookie is marked as httponly + + """ + def __init__(self, morsel): + self.key = morsel.key + self.value = morsel.value + + for attr in ["expires", "path", + "comment", "domain", "max-age", + "secure", "version", "httponly"]: + setattr(self, attr.replace("-", "_"), morsel[attr]) + + self._str = morsel.OutputString() + + def __str__(self): + return self._str + + def __repr__(self): + return self._str + + def __eq__(self, other): + """Equality comparison for cookies. Compares to other cookies + based on value alone and on non-cookies based on the equality + of self.value with the other object so that a cookie with value + "ham" compares equal to the string "ham" + """ + if hasattr(other, "value"): + return self.value == other.value + return self.value == other + + +class MultiDict(dict): + """Dictionary type that holds multiple values for each + key""" + #TODO: this should perhaps also order the keys + def __init__(self): + pass + + def __setitem__(self, name, value): + dict.__setitem__(self, name, [value]) + + def add(self, name, value): + if name in self: + dict.__getitem__(self, name).append(value) + else: + dict.__setitem__(self, name, [value]) + + def __getitem__(self, key): + """Get the first value with a given key""" + #TODO: should this instead be the last value? + return self.first(key) + + def first(self, key, default=missing): + """Get the first value with a given key + + :param key: The key to lookup + :param default: The default to return if key is + not found (throws if nothing is + specified) + """ + if key in self and dict.__getitem__(self, key): + return dict.__getitem__(self, key)[0] + elif default is not missing: + return default + raise KeyError + + def last(self, key, default=missing): + """Get the last value with a given key + + :param key: The key to lookup + :param default: The default to return if key is + not found (throws if nothing is + specified) + """ + if key in self and dict.__getitem__(self, key): + return dict.__getitem__(self, key)[-1] + elif default is not missing: + return default + raise KeyError + + def get_list(self, key): + """Get all values with a given key as a list + + :param key: The key to lookup + """ + return dict.__getitem__(self, key) + + @classmethod + def from_field_storage(cls, fs): + self = cls() + if fs.list is None: + return self + for key in fs: + values = fs[key] + if not isinstance(values, list): + values = [values] + + for value in values: + if value.filename: + value = value + else: + value = value.value + self.add(key, value) + return self + + +class Cookies(MultiDict): + """MultiDict specialised for Cookie values""" + def __init__(self): + pass + + def __getitem__(self, key): + return self.last(key) + + +class Authentication(object): + """Object for dealing with HTTP Authentication + + .. attribute:: username + + The username supplied in the HTTP Authorization + header, or None + + .. attribute:: password + + The password supplied in the HTTP Authorization + header, or None + """ + def __init__(self, headers): + self.username = None + self.password = None + + auth_schemes = {"Basic": self.decode_basic} + + if "authorization" in headers: + header = headers.get("authorization") + auth_type, data = header.split(" ", 1) + if auth_type in auth_schemes: + self.username, self.password = auth_schemes[auth_type](data) + else: + raise HTTPException(400, "Unsupported authentication scheme %s" % auth_type) + + def decode_basic(self, data): + decoded_data = base64.decodestring(data) + return decoded_data.split(":", 1) diff --git a/testing/web-platform/tests/tools/wptserve/wptserve/response.py b/testing/web-platform/tests/tools/wptserve/wptserve/response.py new file mode 100644 index 000000000..6c073feea --- /dev/null +++ b/testing/web-platform/tests/tools/wptserve/wptserve/response.py @@ -0,0 +1,473 @@ +from collections import OrderedDict +from datetime import datetime, timedelta +import Cookie +import json +import types +import uuid +import socket + +from .constants import response_codes +from .logger import get_logger + +missing = object() + +class Response(object): + """Object representing the response to a HTTP request + + :param handler: RequestHandler being used for this response + :param request: Request that this is the response for + + .. attribute:: request + + Request associated with this Response. + + .. attribute:: encoding + + The encoding to use when converting unicode to strings for output. + + .. attribute:: add_required_headers + + Boolean indicating whether mandatory headers should be added to the + response. + + .. attribute:: send_body_for_head_request + + Boolean, default False, indicating whether the body content should be + sent when the request method is HEAD. + + .. attribute:: explicit_flush + + Boolean indicating whether output should be flushed automatically or only + when requested. + + .. attribute:: writer + + The ResponseWriter for this response + + .. attribute:: status + + Status tuple (code, message). Can be set to an integer, in which case the + message part is filled in automatically, or a tuple. + + .. attribute:: headers + + List of HTTP headers to send with the response. Each item in the list is a + tuple of (name, value). + + .. attribute:: content + + The body of the response. This can either be a string or a iterable of response + parts. If it is an iterable, any item may be a string or a function of zero + parameters which, when called, returns a string.""" + + def __init__(self, handler, request): + self.request = request + self.encoding = "utf8" + + self.add_required_headers = True + self.send_body_for_head_request = False + self.explicit_flush = False + self.close_connection = False + + self.writer = ResponseWriter(handler, self) + + self._status = (200, None) + self.headers = ResponseHeaders() + self.content = [] + + self.logger = get_logger() + + @property + def status(self): + return self._status + + @status.setter + def status(self, value): + if hasattr(value, "__len__"): + if len(value) != 2: + raise ValueError + else: + self._status = (int(value[0]), str(value[1])) + else: + self._status = (int(value), None) + + def set_cookie(self, name, value, path="/", domain=None, max_age=None, + expires=None, secure=False, httponly=False, comment=None): + """Set a cookie to be sent with a Set-Cookie header in the + response + + :param name: String name of the cookie + :param value: String value of the cookie + :param max_age: datetime.timedelta int representing the time (in seconds) + until the cookie expires + :param path: String path to which the cookie applies + :param domain: String domain to which the cookie applies + :param secure: Boolean indicating whether the cookie is marked as secure + :param httponly: Boolean indicating whether the cookie is marked as + HTTP Only + :param comment: String comment + :param expires: datetime.datetime or datetime.timedelta indicating a + time or interval from now when the cookie expires + + """ + days = dict((i+1, name) for i, name in enumerate(["jan", "feb", "mar", + "apr", "may", "jun", + "jul", "aug", "sep", + "oct", "nov", "dec"])) + if value is None: + value = '' + max_age = 0 + expires = timedelta(days=-1) + + if isinstance(expires, timedelta): + expires = datetime.utcnow() + expires + + if expires is not None: + expires_str = expires.strftime("%d %%s %Y %H:%M:%S GMT") + expires_str = expires_str % days[expires.month] + expires = expires_str + + if max_age is not None: + if hasattr(max_age, "total_seconds"): + max_age = int(max_age.total_seconds()) + max_age = "%.0d" % max_age + + m = Cookie.Morsel() + + def maybe_set(key, value): + if value is not None and value is not False: + m[key] = value + + m.set(name, value, value) + maybe_set("path", path) + maybe_set("domain", domain) + maybe_set("comment", comment) + maybe_set("expires", expires) + maybe_set("max-age", max_age) + maybe_set("secure", secure) + maybe_set("httponly", httponly) + + self.headers.append("Set-Cookie", m.OutputString()) + + def unset_cookie(self, name): + """Remove a cookie from those that are being sent with the response""" + cookies = self.headers.get("Set-Cookie") + parser = Cookie.BaseCookie() + for cookie in cookies: + parser.load(cookie) + + if name in parser.keys(): + del self.headers["Set-Cookie"] + for m in parser.values(): + if m.key != name: + self.headers.append(("Set-Cookie", m.OutputString())) + + def delete_cookie(self, name, path="/", domain=None): + """Delete a cookie on the client by setting it to the empty string + and to expire in the past""" + self.set_cookie(name, None, path=path, domain=domain, max_age=0, + expires=timedelta(days=-1)) + + def iter_content(self, read_file=False): + """Iterator returning chunks of response body content. + + If any part of the content is a function, this will be called + and the resulting value (if any) returned. + + :param read_file: - boolean controlling the behaviour when content + is a file handle. When set to False the handle will be returned directly + allowing the file to be passed to the output in small chunks. When set to + True, the entire content of the file will be returned as a string facilitating + non-streaming operations like template substitution. + """ + if isinstance(self.content, types.StringTypes): + yield self.content + elif hasattr(self.content, "read"): + if read_file: + yield self.content.read() + else: + yield self.content + else: + for item in self.content: + if hasattr(item, "__call__"): + value = item() + else: + value = item + if value: + yield value + + def write_status_headers(self): + """Write out the status line and headers for the response""" + self.writer.write_status(*self.status) + for item in self.headers: + self.writer.write_header(*item) + self.writer.end_headers() + + def write_content(self): + """Write out the response content""" + if self.request.method != "HEAD" or self.send_body_for_head_request: + for item in self.iter_content(): + self.writer.write_content(item) + + def write(self): + """Write the whole response""" + self.write_status_headers() + self.write_content() + + def set_error(self, code, message=""): + """Set the response status headers and body to indicate an + error""" + err = {"code": code, + "message": message} + data = json.dumps({"error": err}) + self.status = code + self.headers = [("Content-Type", "application/json"), + ("Content-Length", len(data))] + self.content = data + if code == 500: + self.logger.error(message) + + +class MultipartContent(object): + def __init__(self, boundary=None, default_content_type=None): + self.items = [] + if boundary is None: + boundary = str(uuid.uuid4()) + self.boundary = boundary + self.default_content_type = default_content_type + + def __call__(self): + boundary = "--" + self.boundary + rv = ["", boundary] + for item in self.items: + rv.append(str(item)) + rv.append(boundary) + rv[-1] += "--" + return "\r\n".join(rv) + + def append_part(self, data, content_type=None, headers=None): + if content_type is None: + content_type = self.default_content_type + self.items.append(MultipartPart(data, content_type, headers)) + + def __iter__(self): + #This is hackish; when writing the response we need an iterable + #or a string. For a multipart/byterange response we want an + #iterable that contains a single callable; the MultipartContent + #object itself + yield self + + +class MultipartPart(object): + def __init__(self, data, content_type=None, headers=None): + self.headers = ResponseHeaders() + + if content_type is not None: + self.headers.set("Content-Type", content_type) + + if headers is not None: + for name, value in headers: + if name.lower() == "content-type": + func = self.headers.set + else: + func = self.headers.append + func(name, value) + + self.data = data + + def __str__(self): + rv = [] + for item in self.headers: + rv.append("%s: %s" % item) + rv.append("") + rv.append(self.data) + return "\r\n".join(rv) + + +class ResponseHeaders(object): + """Dictionary-like object holding the headers for the response""" + def __init__(self): + self.data = OrderedDict() + + def set(self, key, value): + """Set a header to a specific value, overwriting any previous header + with the same name + + :param key: Name of the header to set + :param value: Value to set the header to + """ + self.data[key.lower()] = (key, [value]) + + def append(self, key, value): + """Add a new header with a given name, not overwriting any existing + headers with the same name + + :param key: Name of the header to add + :param value: Value to set for the header + """ + if key.lower() in self.data: + self.data[key.lower()][1].append(value) + else: + self.set(key, value) + + def get(self, key, default=missing): + """Get the set values for a particular header.""" + try: + return self[key] + except KeyError: + if default is missing: + return [] + return default + + def __getitem__(self, key): + """Get a list of values for a particular header + + """ + return self.data[key.lower()][1] + + def __delitem__(self, key): + del self.data[key.lower()] + + def __contains__(self, key): + return key.lower() in self.data + + def __setitem__(self, key, value): + self.set(key, value) + + def __iter__(self): + for key, values in self.data.itervalues(): + for value in values: + yield key, value + + def items(self): + return list(self) + + def update(self, items_iter): + for name, value in items_iter: + self.append(name, value) + + def __repr__(self): + return repr(self.data) + + +class ResponseWriter(object): + """Object providing an API to write out a HTTP response. + + :param handler: The RequestHandler being used. + :param response: The Response associated with this writer. + + After each part of the response is written, the output is + flushed unless response.explicit_flush is False, in which case + the user must call .flush() explicitly.""" + def __init__(self, handler, response): + self._wfile = handler.wfile + self._response = response + self._handler = handler + self._headers_seen = set() + self._headers_complete = False + self.content_written = False + self.request = response.request + self.file_chunk_size = 32 * 1024 + + def write_status(self, code, message=None): + """Write out the status line of a response. + + :param code: The integer status code of the response. + :param message: The message of the response. Defaults to the message commonly used + with the status code.""" + if message is None: + if code in response_codes: + message = response_codes[code][0] + else: + message = '' + self.write("%s %d %s\r\n" % + (self._response.request.protocol_version, code, message)) + + def write_header(self, name, value): + """Write out a single header for the response. + + :param name: Name of the header field + :param value: Value of the header field + """ + self._headers_seen.add(name.lower()) + self.write("%s: %s\r\n" % (name, value)) + if not self._response.explicit_flush: + self.flush() + + def write_default_headers(self): + for name, f in [("Server", self._handler.version_string), + ("Date", self._handler.date_time_string)]: + if name.lower() not in self._headers_seen: + self.write_header(name, f()) + + if (type(self._response.content) in (str, unicode) and + "content-length" not in self._headers_seen): + #Would be nice to avoid double-encoding here + self.write_header("Content-Length", len(self.encode(self._response.content))) + + def end_headers(self): + """Finish writing headers and write the separator. + + Unless add_required_headers on the response is False, + this will also add HTTP-mandated headers that have not yet been supplied + to the response headers""" + + if self._response.add_required_headers: + self.write_default_headers() + + self.write("\r\n") + if "content-length" not in self._headers_seen: + self._response.close_connection = True + if not self._response.explicit_flush: + self.flush() + self._headers_complete = True + + def write_content(self, data): + """Write the body of the response.""" + if isinstance(data, types.StringTypes): + self.write(data) + else: + self.write_content_file(data) + if not self._response.explicit_flush: + self.flush() + + def write(self, data): + """Write directly to the response, converting unicode to bytes + according to response.encoding. Does not flush.""" + self.content_written = True + try: + self._wfile.write(self.encode(data)) + except socket.error: + # This can happen if the socket got closed by the remote end + pass + + def write_content_file(self, data): + """Write a file-like object directly to the response in chunks. + Does not flush.""" + self.content_written = True + while True: + buf = data.read(self.file_chunk_size) + if not buf: + break + try: + self._wfile.write(buf) + except socket.error: + break + data.close() + + def encode(self, data): + """Convert unicode to bytes according to response.encoding.""" + if isinstance(data, str): + return data + elif isinstance(data, unicode): + return data.encode(self._response.encoding) + else: + raise ValueError + + def flush(self): + """Flush the output.""" + try: + self._wfile.flush() + except socket.error: + # This can happen if the socket got closed by the remote end + pass diff --git a/testing/web-platform/tests/tools/wptserve/wptserve/router.py b/testing/web-platform/tests/tools/wptserve/wptserve/router.py new file mode 100644 index 000000000..a35e098e6 --- /dev/null +++ b/testing/web-platform/tests/tools/wptserve/wptserve/router.py @@ -0,0 +1,168 @@ +import itertools +import re +import types + +from .logger import get_logger + +any_method = object() + +class RouteTokenizer(object): + def literal(self, scanner, token): + return ("literal", token) + + def slash(self, scanner, token): + return ("slash", None) + + def group(self, scanner, token): + return ("group", token[1:-1]) + + def star(self, scanner, token): + return ("star", token[1:-3]) + + def scan(self, input_str): + scanner = re.Scanner([(r"/", self.slash), + (r"{\w*}", self.group), + (r"\*", self.star), + (r"(?:\\.|[^{\*/])*", self.literal),]) + return scanner.scan(input_str) + +class RouteCompiler(object): + def __init__(self): + self.reset() + + def reset(self): + self.star_seen = False + + def compile(self, tokens): + self.reset() + + func_map = {"slash":self.process_slash, + "literal":self.process_literal, + "group":self.process_group, + "star":self.process_star} + + re_parts = ["^"] + + if not tokens or tokens[0][0] != "slash": + tokens = itertools.chain([("slash", None)], tokens) + + for token in tokens: + re_parts.append(func_map[token[0]](token)) + + if self.star_seen: + re_parts.append(")") + re_parts.append("$") + + return re.compile("".join(re_parts)) + + def process_literal(self, token): + return re.escape(token[1]) + + def process_slash(self, token): + return "/" + + def process_group(self, token): + if self.star_seen: + raise ValueError("Group seen after star in regexp") + return "(?P<%s>[^/]+)" % token[1] + + def process_star(self, token): + if self.star_seen: + raise ValueError("Star seen after star in regexp") + self.star_seen = True + return "(.*" + +def compile_path_match(route_pattern): + """tokens: / or literal or match or *""" + + tokenizer = RouteTokenizer() + tokens, unmatched = tokenizer.scan(route_pattern) + + assert unmatched == "", unmatched + + compiler = RouteCompiler() + + return compiler.compile(tokens) + +class Router(object): + """Object for matching handler functions to requests. + + :param doc_root: Absolute path of the filesystem location from + which to serve tests + :param routes: Initial routes to add; a list of three item tuples + (method, path_pattern, handler_function), defined + as for register() + """ + + def __init__(self, doc_root, routes): + self.doc_root = doc_root + self.routes = [] + self.logger = get_logger() + for route in reversed(routes): + self.register(*route) + + def register(self, methods, path, handler): + """Register a handler for a set of paths. + + :param methods: Set of methods this should match. "*" is a + special value indicating that all methods should + be matched. + + :param path_pattern: Match pattern that will be used to determine if + a request path matches this route. Match patterns + consist of either literal text, match groups, + denoted {name}, which match any character except /, + and, at most one \*, which matches and character and + creates a match group to the end of the string. + If there is no leading "/" on the pattern, this is + automatically implied. For example:: + + api/{resource}/*.json + + Would match `/api/test/data.json` or + `/api/test/test2/data.json`, but not `/api/test/data.py`. + + The match groups are made available in the request object + as a dictionary through the route_match property. For + example, given the route pattern above and the path + `/api/test/data.json`, the route_match property would + contain:: + + {"resource": "test", "*": "data.json"} + + :param handler: Function that will be called to process matching + requests. This must take two parameters, the request + object and the response object. + + """ + if type(methods) in types.StringTypes or methods in (any_method, "*"): + methods = [methods] + for method in methods: + self.routes.append((method, compile_path_match(path), handler)) + self.logger.debug("Route pattern: %s" % self.routes[-1][1].pattern) + + def get_handler(self, request): + """Get a handler for a request or None if there is no handler. + + :param request: Request to get a handler for. + :rtype: Callable or None + """ + for method, regexp, handler in reversed(self.routes): + if (request.method == method or + method in (any_method, "*") or + (request.method == "HEAD" and method == "GET")): + m = regexp.match(request.url_parts.path) + if m: + if not hasattr(handler, "__class__"): + name = handler.__name__ + else: + name = handler.__class__.__name__ + self.logger.debug("Found handler %s" % name) + + match_parts = m.groupdict().copy() + if len(match_parts) < len(m.groups()): + match_parts["*"] = m.groups()[-1] + request.route_match = match_parts + + return handler + return None diff --git a/testing/web-platform/tests/tools/wptserve/wptserve/routes.py b/testing/web-platform/tests/tools/wptserve/wptserve/routes.py new file mode 100644 index 000000000..b6e380001 --- /dev/null +++ b/testing/web-platform/tests/tools/wptserve/wptserve/routes.py @@ -0,0 +1,6 @@ +from . import handlers +from .router import any_method +routes = [(any_method, "*.py", handlers.python_script_handler), + ("GET", "*.asis", handlers.as_is_handler), + ("GET", "*", handlers.file_handler), + ] diff --git a/testing/web-platform/tests/tools/wptserve/wptserve/server.py b/testing/web-platform/tests/tools/wptserve/wptserve/server.py new file mode 100644 index 000000000..31929efd6 --- /dev/null +++ b/testing/web-platform/tests/tools/wptserve/wptserve/server.py @@ -0,0 +1,461 @@ +import BaseHTTPServer +import errno +import os +import socket +from SocketServer import ThreadingMixIn +import ssl +import sys +import threading +import time +import traceback +import types +import urlparse + +from . import routes as default_routes +from .logger import get_logger +from .request import Server, Request +from .response import Response +from .router import Router +from .utils import HTTPException + + +"""HTTP server designed for testing purposes. + +The server is designed to provide flexibility in the way that +requests are handled, and to provide control both of exactly +what bytes are put on the wire for the response, and in the +timing of sending those bytes. + +The server is based on the stdlib HTTPServer, but with some +notable differences in the way that requests are processed. +Overall processing is handled by a WebTestRequestHandler, +which is a subclass of BaseHTTPRequestHandler. This is responsible +for parsing the incoming request. A RequestRewriter is then +applied and may change the request data if it matches a +supplied rule. + +Once the request data had been finalised, Request and Reponse +objects are constructed. These are used by the other parts of the +system to read information about the request and manipulate the +response. + +Each request is handled by a particular handler function. The +mapping between Request and the appropriate handler is determined +by a Router. By default handlers are installed to interpret files +under the document root with .py extensions as executable python +files (see handlers.py for the api for such files), .asis files as +bytestreams to be sent literally and all other files to be served +statically. + +The handler functions are responsible for either populating the +fields of the response object, which will then be written when the +handler returns, or for directly writing to the output stream. +""" + + +class RequestRewriter(object): + def __init__(self, rules): + """Object for rewriting the request path. + + :param rules: Initial rules to add; a list of three item tuples + (method, input_path, output_path), defined as for + register() + """ + self.rules = {} + for rule in reversed(rules): + self.register(*rule) + self.logger = get_logger() + + def register(self, methods, input_path, output_path): + """Register a rewrite rule. + + :param methods: Set of methods this should match. "*" is a + special value indicating that all methods should + be matched. + + :param input_path: Path to match for the initial request. + + :param output_path: Path to replace the input path with in + the request. + """ + if type(methods) in types.StringTypes: + methods = [methods] + self.rules[input_path] = (methods, output_path) + + def rewrite(self, request_handler): + """Rewrite the path in a BaseHTTPRequestHandler instance, if + it matches a rule. + + :param request_handler: BaseHTTPRequestHandler for which to + rewrite the request. + """ + split_url = urlparse.urlsplit(request_handler.path) + if split_url.path in self.rules: + methods, destination = self.rules[split_url.path] + if "*" in methods or request_handler.command in methods: + self.logger.debug("Rewriting request path %s to %s" % + (request_handler.path, destination)) + new_url = list(split_url) + new_url[2] = destination + new_url = urlparse.urlunsplit(new_url) + request_handler.path = new_url + + +class WebTestServer(ThreadingMixIn, BaseHTTPServer.HTTPServer): + allow_reuse_address = True + acceptable_errors = (errno.EPIPE, errno.ECONNABORTED) + request_queue_size = 2000 + + # Ensure that we don't hang on shutdown waiting for requests + daemon_threads = True + + def __init__(self, server_address, RequestHandlerClass, router, rewriter, bind_hostname, + config=None, use_ssl=False, key_file=None, certificate=None, + encrypt_after_connect=False, latency=None, **kwargs): + """Server for HTTP(s) Requests + + :param server_address: tuple of (server_name, port) + + :param RequestHandlerClass: BaseHTTPRequestHandler-like class to use for + handling requests. + + :param router: Router instance to use for matching requests to handler + functions + + :param rewriter: RequestRewriter-like instance to use for preprocessing + requests before they are routed + + :param config: Dictionary holding environment configuration settings for + handlers to read, or None to use the default values. + + :param use_ssl: Boolean indicating whether the server should use SSL + + :param key_file: Path to key file to use if SSL is enabled. + + :param certificate: Path to certificate to use if SSL is enabled. + + :param encrypt_after_connect: For each connection, don't start encryption + until a CONNECT message has been received. + This enables the server to act as a + self-proxy. + + :param bind_hostname True to bind the server to both the hostname and + port specified in the server_address parameter. + False to bind the server only to the port in the + server_address parameter, but not to the hostname. + :param latency: Delay in ms to wait before seving each response, or + callable that returns a delay in ms + """ + self.router = router + self.rewriter = rewriter + + self.scheme = "https" if use_ssl else "http" + self.logger = get_logger() + + self.latency = latency + + if bind_hostname: + hostname_port = server_address + else: + hostname_port = ("",server_address[1]) + + #super doesn't work here because BaseHTTPServer.HTTPServer is old-style + BaseHTTPServer.HTTPServer.__init__(self, hostname_port, RequestHandlerClass, **kwargs) + + if config is not None: + Server.config = config + else: + self.logger.debug("Using default configuration") + Server.config = {"host": server_address[0], + "domains": {"": server_address[0]}, + "ports": {"http": [self.server_address[1]]}} + + + self.key_file = key_file + self.certificate = certificate + self.encrypt_after_connect = use_ssl and encrypt_after_connect + + if use_ssl and not encrypt_after_connect: + self.socket = ssl.wrap_socket(self.socket, + keyfile=self.key_file, + certfile=self.certificate, + server_side=True) + + def handle_error(self, request, client_address): + error = sys.exc_info()[1] + + if ((isinstance(error, socket.error) and + isinstance(error.args, tuple) and + error.args[0] in self.acceptable_errors) or + (isinstance(error, IOError) and + error.errno in self.acceptable_errors)): + pass # remote hang up before the result is sent + else: + self.logger.error(traceback.format_exc()) + + +class WebTestRequestHandler(BaseHTTPServer.BaseHTTPRequestHandler): + """RequestHandler for WebTestHttpd""" + + protocol_version = "HTTP/1.1" + + def handle_one_request(self): + response = None + self.logger = get_logger() + try: + self.close_connection = False + request_line_is_valid = self.get_request_line() + + if self.close_connection: + return + + request_is_valid = self.parse_request() + if not request_is_valid: + #parse_request() actually sends its own error responses + return + + self.server.rewriter.rewrite(self) + + request = Request(self) + response = Response(self, request) + + if request.method == "CONNECT": + self.handle_connect(response) + return + + if not request_line_is_valid: + response.set_error(414) + response.write() + return + + self.logger.debug("%s %s" % (request.method, request.request_path)) + handler = self.server.router.get_handler(request) + + # If the handler we used for the request had a non-default base path + # set update the doc_root of the request to reflect this + if hasattr(handler, "base_path") and handler.base_path: + request.doc_root = handler.base_path + if hasattr(handler, "url_base") and handler.url_base != "/": + request.url_base = handler.url_base + + if self.server.latency is not None: + if callable(self.server.latency): + latency = self.server.latency() + else: + latency = self.server.latency + self.logger.warning("Latency enabled. Sleeping %i ms" % latency) + time.sleep(latency / 1000.) + + if handler is None: + response.set_error(404) + else: + try: + handler(request, response) + except HTTPException as e: + response.set_error(e.code, e.message) + except Exception as e: + if e.message: + err = [e.message] + else: + err = [] + err.append(traceback.format_exc()) + response.set_error(500, "\n".join(err)) + self.logger.debug("%i %s %s (%s) %i" % (response.status[0], + request.method, + request.request_path, + request.headers.get('Referer'), + request.raw_input.length)) + + if not response.writer.content_written: + response.write() + + # If we want to remove this in the future, a solution is needed for + # scripts that produce a non-string iterable of content, since these + # can't set a Content-Length header. A notable example of this kind of + # problem is with the trickle pipe i.e. foo.js?pipe=trickle(d1) + if response.close_connection: + self.close_connection = True + + if not self.close_connection: + # Ensure that the whole request has been read from the socket + request.raw_input.read() + + except socket.timeout as e: + self.log_error("Request timed out: %r", e) + self.close_connection = True + return + + except Exception as e: + err = traceback.format_exc() + if response: + response.set_error(500, err) + response.write() + self.logger.error(err) + + def get_request_line(self): + try: + self.raw_requestline = self.rfile.readline(65537) + except socket.error: + self.close_connection = True + return False + if len(self.raw_requestline) > 65536: + self.requestline = '' + self.request_version = '' + self.command = '' + return False + if not self.raw_requestline: + self.close_connection = True + return True + + def handle_connect(self, response): + self.logger.debug("Got CONNECT") + response.status = 200 + response.write() + if self.server.encrypt_after_connect: + self.logger.debug("Enabling SSL for connection") + self.request = ssl.wrap_socket(self.connection, + keyfile=self.server.key_file, + certfile=self.server.certificate, + server_side=True) + self.setup() + return + + +class WebTestHttpd(object): + """ + :param host: Host from which to serve (default: 127.0.0.1) + :param port: Port from which to serve (default: 8000) + :param server_cls: Class to use for the server (default depends on ssl vs non-ssl) + :param handler_cls: Class to use for the RequestHandler + :param use_ssl: Use a SSL server if no explicit server_cls is supplied + :param key_file: Path to key file to use if ssl is enabled + :param certificate: Path to certificate file to use if ssl is enabled + :param encrypt_after_connect: For each connection, don't start encryption + until a CONNECT message has been received. + This enables the server to act as a + self-proxy. + :param router_cls: Router class to use when matching URLs to handlers + :param doc_root: Document root for serving files + :param routes: List of routes with which to initialize the router + :param rewriter_cls: Class to use for request rewriter + :param rewrites: List of rewrites with which to initialize the rewriter_cls + :param config: Dictionary holding environment configuration settings for + handlers to read, or None to use the default values. + :param bind_hostname: Boolean indicating whether to bind server to hostname. + :param latency: Delay in ms to wait before seving each response, or + callable that returns a delay in ms + + HTTP server designed for testing scenarios. + + Takes a router class which provides one method get_handler which takes a Request + and returns a handler function. + + .. attribute:: host + + The host name or ip address of the server + + .. attribute:: port + + The port on which the server is running + + .. attribute:: router + + The Router object used to associate requests with resources for this server + + .. attribute:: rewriter + + The Rewriter object used for URL rewriting + + .. attribute:: use_ssl + + Boolean indicating whether the server is using ssl + + .. attribute:: started + + Boolean indictaing whether the server is running + + """ + def __init__(self, host="127.0.0.1", port=8000, + server_cls=None, handler_cls=WebTestRequestHandler, + use_ssl=False, key_file=None, certificate=None, encrypt_after_connect=False, + router_cls=Router, doc_root=os.curdir, routes=None, + rewriter_cls=RequestRewriter, bind_hostname=True, rewrites=None, + latency=None, config=None): + + if routes is None: + routes = default_routes.routes + + self.host = host + + self.router = router_cls(doc_root, routes) + self.rewriter = rewriter_cls(rewrites if rewrites is not None else []) + + self.use_ssl = use_ssl + self.logger = get_logger() + + if server_cls is None: + server_cls = WebTestServer + + if use_ssl: + if key_file is not None: + assert os.path.exists(key_file) + assert certificate is not None and os.path.exists(certificate) + + try: + self.httpd = server_cls((host, port), + handler_cls, + self.router, + self.rewriter, + config=config, + bind_hostname=bind_hostname, + use_ssl=use_ssl, + key_file=key_file, + certificate=certificate, + encrypt_after_connect=encrypt_after_connect, + latency=latency) + self.started = False + + _host, self.port = self.httpd.socket.getsockname() + except Exception: + self.logger.error('Init failed! You may need to modify your hosts file. Refer to README.md.') + raise + + def start(self, block=False): + """Start the server. + + :param block: True to run the server on the current thread, blocking, + False to run on a separate thread.""" + self.logger.info("Starting http server on %s:%s" % (self.host, self.port)) + self.started = True + if block: + self.httpd.serve_forever() + else: + self.server_thread = threading.Thread(target=self.httpd.serve_forever) + self.server_thread.setDaemon(True) # don't hang on exit + self.server_thread.start() + + def stop(self): + """ + Stops the server. + + If the server is not running, this method has no effect. + """ + if self.started: + try: + self.httpd.shutdown() + self.httpd.server_close() + self.server_thread.join() + self.server_thread = None + self.logger.info("Stopped http server on %s:%s" % (self.host, self.port)) + except AttributeError: + pass + self.started = False + self.httpd = None + + def get_url(self, path="/", query=None, fragment=None): + if not self.started: + return None + + return urlparse.urlunsplit(("http" if not self.use_ssl else "https", + "%s:%s" % (self.host, self.port), + path, query, fragment)) diff --git a/testing/web-platform/tests/tools/wptserve/wptserve/stash.py b/testing/web-platform/tests/tools/wptserve/wptserve/stash.py new file mode 100644 index 000000000..b6bd6eed4 --- /dev/null +++ b/testing/web-platform/tests/tools/wptserve/wptserve/stash.py @@ -0,0 +1,143 @@ +import base64 +import json +import os +import uuid +from multiprocessing.managers import BaseManager, DictProxy + +class ServerDictManager(BaseManager): + shared_data = {} + +def _get_shared(): + return ServerDictManager.shared_data + +ServerDictManager.register("get_dict", + callable=_get_shared, + proxytype=DictProxy) + +class ClientDictManager(BaseManager): + pass + +ClientDictManager.register("get_dict") + +class StashServer(object): + def __init__(self, address=None, authkey=None): + self.address = address + self.authkey = authkey + self.manager = None + + def __enter__(self): + self.manager, self.address, self.authkey = start_server(self.address, self.authkey) + store_env_config(self.address, self.authkey) + + def __exit__(self, *args, **kwargs): + if self.manager is not None: + self.manager.shutdown() + +def load_env_config(): + address, authkey = json.loads(os.environ["WPT_STASH_CONFIG"]) + if isinstance(address, list): + address = tuple(address) + else: + address = str(address) + authkey = base64.decodestring(authkey) + return address, authkey + +def store_env_config(address, authkey): + authkey = base64.encodestring(authkey) + os.environ["WPT_STASH_CONFIG"] = json.dumps((address, authkey)) + +def start_server(address=None, authkey=None): + manager = ServerDictManager(address, authkey) + manager.start() + + return (manager, manager._address, manager._authkey) + + +#TODO: Consider expiring values after some fixed time for long-running +#servers + +class Stash(object): + """Key-value store for persisting data across HTTP/S and WS/S requests. + + This data store is specifically designed for persisting data across server + requests. The synchronization is achieved by using the BaseManager from + the multiprocessing module so different processes can acccess the same data. + + Stash can be used interchangeably between HTTP, HTTPS, WS and WSS servers. + A thing to note about WS/S servers is that they require additional steps in + the handlers for accessing the same underlying shared data in the Stash. + This can usually be achieved by using load_env_config(). When using Stash + interchangeably between HTTP/S and WS/S request, the path part of the key + should be expliclitly specified if accessing the same key/value subset. + + The store has several unusual properties. Keys are of the form (path, + uuid), where path is, by default, the path in the HTTP request and + uuid is a unique id. In addition, the store is write-once, read-once, + i.e. the value associated with a particular key cannot be changed once + written and the read operation (called "take") is destructive. Taken together, + these properties make it difficult for data to accidentally leak + between different resources or different requests for the same + resource. + """ + + _proxy = None + + def __init__(self, default_path, address=None, authkey=None): + self.default_path = default_path + self.data = self._get_proxy(address, authkey) + + def _get_proxy(self, address=None, authkey=None): + if address is None and authkey is None: + Stash._proxy = {} + + if Stash._proxy is None: + manager = ClientDictManager(address, authkey) + manager.connect() + Stash._proxy = manager.get_dict() + + return Stash._proxy + + def _wrap_key(self, key, path): + if path is None: + path = self.default_path + # This key format is required to support using the path. Since the data + # passed into the stash can be a DictProxy which wouldn't detect changes + # when writing to a subdict. + return (str(path), str(uuid.UUID(key))) + + def put(self, key, value, path=None): + """Place a value in the shared stash. + + :param key: A UUID to use as the data's key. + :param value: The data to store. This can be any python object. + :param path: The path that has access to read the data (by default + the current request path)""" + if value is None: + raise ValueError("SharedStash value may not be set to None") + internal_key = self._wrap_key(key, path) + if internal_key in self.data: + raise StashError("Tried to overwrite existing shared stash value " + "for key %s (old value was %s, new value is %s)" % + (internal_key, self.data[str(internal_key)], value)) + else: + self.data[internal_key] = value + + def take(self, key, path=None): + """Remove a value from the shared stash and return it. + + :param key: A UUID to use as the data's key. + :param path: The path that has access to read the data (by default + the current request path)""" + internal_key = self._wrap_key(key, path) + value = self.data.get(internal_key, None) + if value is not None: + try: + self.data.pop(internal_key) + except KeyError: + # Silently continue when pop error occurs. + pass + + return value + +class StashError(Exception): + pass diff --git a/testing/web-platform/tests/tools/wptserve/wptserve/utils.py b/testing/web-platform/tests/tools/wptserve/wptserve/utils.py new file mode 100644 index 000000000..e57ff196a --- /dev/null +++ b/testing/web-platform/tests/tools/wptserve/wptserve/utils.py @@ -0,0 +1,14 @@ +def invert_dict(dict): + rv = {} + for key, values in dict.iteritems(): + for value in values: + if value in rv: + raise ValueError + rv[value] = key + return rv + + +class HTTPException(Exception): + def __init__(self, code, message=""): + self.code = code + self.message = message diff --git a/testing/web-platform/tests/tools/wptserve/wptserve/wptserve.py b/testing/web-platform/tests/tools/wptserve/wptserve/wptserve.py new file mode 100755 index 000000000..816c8a5a6 --- /dev/null +++ b/testing/web-platform/tests/tools/wptserve/wptserve/wptserve.py @@ -0,0 +1,33 @@ +#!/usr/bin/env python +import argparse +import os + +import server + +def abs_path(path): + return os.path.abspath(path) + + +def parse_args(): + parser = argparse.ArgumentParser(description="HTTP server designed for extreme flexibility " + "required in testing situations.") + parser.add_argument("document_root", action="store", type=abs_path, + help="Root directory to serve files from") + parser.add_argument("--port", "-p", dest="port", action="store", + type=int, default=8000, + help="Port number to run server on") + parser.add_argument("--host", "-H", dest="host", action="store", + type=str, default="127.0.0.1", + help="Host to run server on") + return parser.parse_args() + + +def main(): + args = parse_args() + httpd = server.WebTestHttpd(host=args.host, port=args.port, + use_ssl=False, certificate=None, + doc_root=args.document_root) + httpd.start() + +if __name__ == "__main__": + main() |