Add m-esr52 at 52.6.0

1 files changed, 446 insertions, 0 deletions

diff --git a/xpcom/ds/Tokenizer.h b/xpcom/ds/Tokenizer.h
new file mode 100644
index 000000000..b4aad9ed9
--- /dev/null
+++ b/xpcom/ds/Tokenizer.h
@@ -0,0 +1,446 @@
+/* -*- Mode: C++; tab-width: 8; indent-tabs-mode: nil; c-basic-offset: 2 -*- */
+/* vim: set ts=8 sts=2 et sw=2 tw=80: */
+/* This Source Code Form is subject to the terms of the Mozilla Public
+ * License, v. 2.0. If a copy of the MPL was not distributed with this
+ * file, You can obtain one at http://mozilla.org/MPL/2.0/. */
+
+#ifndef Tokenizer_h__
+#define Tokenizer_h__
+
+#include "nsString.h"
+#include "mozilla/CheckedInt.h"
+#include "mozilla/UniquePtr.h"
+#include "nsTArray.h"
+
+namespace mozilla {
+
+class TokenizerBase
+{
+public:
+  /**
+   * The analyzer works with elements in the input cut to a sequence of token
+   * where each token has an elementary type
+   */
+  enum TokenType : uint32_t
+  {
+    TOKEN_UNKNOWN,
+    TOKEN_RAW,
+    TOKEN_ERROR,
+    TOKEN_INTEGER,
+    TOKEN_WORD,
+    TOKEN_CHAR,
+    TOKEN_WS,
+    TOKEN_EOL,
+    TOKEN_EOF,
+    TOKEN_CUSTOM0 = 1000
+  };
+
+  enum ECaseSensitivity
+  {
+    CASE_SENSITIVE,
+    CASE_INSENSITIVE
+  };
+
+  /**
+   * Class holding the type and the value of a token.  It can be manually created
+   * to allow checks against it via methods of Tokenizer or are results of some of
+   * the Tokenizer's methods.
+   */
+  class Token
+  {
+    TokenType mType;
+    nsDependentCSubstring mWord;
+    nsCString mCustom;
+    char mChar;
+    uint64_t mInteger;
+    ECaseSensitivity mCustomCaseInsensitivity;
+    bool mCustomEnabled;
+
+    // If this token is a result of the parsing process, this member is referencing
+    // a sub-string in the input buffer.  If this is externally created Token this
+    // member is left an empty string.
+    nsDependentCSubstring mFragment;
+
+    friend class TokenizerBase;
+    void AssignFragment(nsACString::const_char_iterator begin,
+                        nsACString::const_char_iterator end);
+
+    static Token Raw();
+
+  public:
+    Token();
+    Token(const Token& aOther);
+    Token& operator=(const Token& aOther);
+
+    // Static constructors of tokens by type and value
+    static Token Word(const nsACString& aWord);
+    static Token Char(const char aChar);
+    static Token Number(const uint64_t aNumber);
+    static Token Whitespace();
+    static Token NewLine();
+    static Token EndOfFile();
+    static Token Error();
+
+    // Compares the two tokens, type must be identical and value
+    // of one of the tokens must be 'any' or equal.
+    bool Equals(const Token& aOther) const;
+
+    TokenType Type() const { return mType; }
+    char AsChar() const;
+    nsDependentCSubstring AsString() const;
+    uint64_t AsInteger() const;
+
+    nsDependentCSubstring Fragment() const { return mFragment; }
+  };
+
+  /**
+   * Consumers may register a custom string that, when found in the input, is considered
+   * a token and returned by Next*() and accepted by Check*() methods.
+   * AddCustomToken() returns a reference to a token that can then be comapred using
+   * Token::Equals() againts the output from Next*() or be passed to Check*().
+   */
+  Token AddCustomToken(const nsACString& aValue, ECaseSensitivity aCaseInsensitivity, bool aEnabled = true);
+  template <uint32_t N>
+  Token AddCustomToken(const char(&aValue)[N], ECaseSensitivity aCaseInsensitivity, bool aEnabled = true)
+  {
+    return AddCustomToken(nsDependentCSubstring(aValue, N - 1), aCaseInsensitivity, aEnabled);
+  }
+  void RemoveCustomToken(Token& aToken);
+  /**
+   * Only applies to a custom type of a Token (see AddCustomToken above.)
+   * This turns on and off token recognition.  When a custom token is disabled,
+   * it's ignored as never added as a custom token.
+   */
+  void EnableCustomToken(Token const& aToken, bool aEnable);
+
+  /**
+   * Mode of tokenization.
+   * FULL tokenization, the default, recognizes built-in tokens and any custom tokens,
+   * if added.
+   * CUSTOM_ONLY will only recognize custom tokens, the rest is seen as 'raw'.
+   * This mode can be understood as a 'binary' mode.
+   */
+  enum class Mode
+  {
+    FULL,
+    CUSTOM_ONLY
+  };
+  void SetTokenizingMode(Mode aMode);
+
+  /**
+   * Return false iff the last Check*() call has returned false or when we've read past
+   * the end of the input string.
+   */
+  MOZ_MUST_USE bool HasFailed() const;
+
+protected:
+  explicit TokenizerBase(const char* aWhitespaces = nullptr,
+                         const char* aAdditionalWordChars = nullptr);
+
+  // false if we have already read the EOF token.
+  bool HasInput() const;
+  // Main parsing function, it doesn't shift the read cursor, just returns the next
+  // token position.
+  nsACString::const_char_iterator Parse(Token& aToken) const;
+  // Is read cursor at the end?
+  bool IsEnd(const nsACString::const_char_iterator& caret) const;
+  // True, when we are at the end of the input data, but it has not been marked
+  // as complete yet.  In that case we cannot proceed with providing a multi-char token.
+  bool IsPending(const nsACString::const_char_iterator & caret) const;
+  // Is read cursor on a character that is a word start?
+  bool IsWordFirst(const char aInput) const;
+  // Is read cursor on a character that is an in-word letter?
+  bool IsWord(const char aInput) const;
+  // Is read cursor on a character that is a valid number?
+  // TODO - support multiple radix
+  bool IsNumber(const char aInput) const;
+  // Is equal to the given custom token?
+  bool IsCustom(const nsACString::const_char_iterator& caret,
+                const Token& aCustomToken, uint32_t* aLongest = nullptr) const;
+
+  // Friendly helper to assign a fragment on a Token
+  static void AssignFragment(Token& aToken,
+                             nsACString::const_char_iterator begin,
+                             nsACString::const_char_iterator end);
+
+  // true iff we have already read the EOF token
+  bool mPastEof;
+  // true iff the last Check*() call has returned false, reverts to true on Rollback() call
+  bool mHasFailed;
+  // true if the input string is final (finished), false when we expect more data
+  // yet to be fed to the tokenizer (see IncrementalTokenizer derived class).
+  bool mInputFinished;
+  // custom only vs full tokenizing mode, see the Parse() method
+  Mode mMode;
+  // minimal raw data chunked delivery during incremental feed
+  uint32_t mMinRawDelivery;
+
+  // Customizable list of whitespaces
+  const char* mWhitespaces;
+  // Additinal custom word characters
+  const char* mAdditionalWordChars;
+
+  // All these point to the original buffer passed to the constructor or to the incremental
+  // buffer after FeedInput.
+  nsACString::const_char_iterator mCursor; // Position of the current (actually next to read) token start
+  nsACString::const_char_iterator mEnd; // End of the input position
+
+  // This is the list of tokens user has registered with AddCustomToken()
+  nsTArray<UniquePtr<Token>> mCustomTokens;
+  uint32_t mNextCustomTokenID;
+
+private:
+  TokenizerBase() = delete;
+  TokenizerBase(const TokenizerBase&) = delete;
+  TokenizerBase(TokenizerBase&&) = delete;
+  TokenizerBase(const TokenizerBase&&) = delete;
+  TokenizerBase &operator=(const TokenizerBase&) = delete;
+};
+
+/**
+ * This is a simple implementation of a lexical analyzer or maybe better
+ * called a tokenizer.  It doesn't allow any user dictionaries or
+ * user define token types.
+ *
+ * It is limited only to ASCII input for now. UTF-8 or any other input
+ * encoding must yet be implemented.
+ */
+class Tokenizer : public TokenizerBase
+{
+public:
+  /**
+   * @param aSource
+   *    The string to parse.
+   *    IMPORTANT NOTE: Tokenizer doesn't ensure the input string buffer lifetime.
+   *    It's up to the consumer to make sure the string's buffer outlives the Tokenizer!
+   * @param aWhitespaces
+   *    If non-null Tokenizer will use this custom set of whitespaces for CheckWhite()
+   *    and SkipWhites() calls.
+   *    By default the list consists of space and tab.
+   * @param aAdditionalWordChars
+   *    If non-null it will be added to the list of characters that consist a word.
+   *    This is useful when you want to accept e.g. '-' in HTTP headers.
+   *    By default a word character is consider any character for which upper case
+   *    is different from lower case.
+   *
+   * If there is an overlap between aWhitespaces and aAdditionalWordChars, the check for
+   * word characters is made first.
+   */
+  explicit Tokenizer(const nsACString& aSource,
+                     const char* aWhitespaces = nullptr,
+                     const char* aAdditionalWordChars = nullptr);
+  explicit Tokenizer(const char* aSource,
+                     const char* aWhitespaces = nullptr,
+                     const char* aAdditionalWordChars = nullptr);
+
+  /**
+   * When there is still anything to read from the input, tokenize it, store the token type
+   * and value to aToken result and shift the cursor past this just parsed token.  Each call
+   * to Next() reads another token from the input and shifts the cursor.
+   * Returns false if we have passed the end of the input.
+   */
+  MOZ_MUST_USE
+  bool Next(Token& aToken);
+
+  /**
+   * Parse the token on the input read cursor position, check its type is equal to aTokenType
+   * and if so, put it into aResult, shift the cursor and return true.  Otherwise, leave
+   * the input read cursor position intact and return false.
+   */
+  MOZ_MUST_USE
+  bool Check(const TokenType aTokenType, Token& aResult);
+  /**
+   * Same as above method, just compares both token type and token value passed in aToken.
+   * When both the type and the value equals, shift the cursor and return true.  Otherwise
+   * return false.
+   */
+  MOZ_MUST_USE
+  bool Check(const Token& aToken);
+
+  /**
+   * SkipWhites method (below) may also skip new line characters automatically.
+   */
+  enum WhiteSkipping {
+    /**
+     * SkipWhites will only skip what is defined as a white space (default).
+     */
+    DONT_INCLUDE_NEW_LINE = 0,
+    /**
+     * SkipWhites will skip definited white spaces as well as new lines
+     * automatically.
+     */
+    INCLUDE_NEW_LINE = 1
+  };
+
+  /**
+   * Skips any occurence of whitespaces specified in mWhitespaces member,
+   * optionally skip also new lines.
+   */
+  void SkipWhites(WhiteSkipping aIncludeNewLines = DONT_INCLUDE_NEW_LINE);
+
+  /**
+   * Skips all tokens until the given one is found or EOF is hit.  The token
+   * or EOF are next to read.
+   */
+  void SkipUntil(Token const& aToken);
+
+  // These are mostly shortcuts for the Check() methods above.
+
+  /**
+   * Check whitespace character is present.
+   */
+  MOZ_MUST_USE
+  bool CheckWhite() { return Check(Token::Whitespace()); }
+  /**
+   * Check there is a single character on the read cursor position.  If so, shift the read
+   * cursor position and return true.  Otherwise false.
+   */
+  MOZ_MUST_USE
+  bool CheckChar(const char aChar) { return Check(Token::Char(aChar)); }
+  /**
+   * This is a customizable version of CheckChar.  aClassifier is a function called with
+   * value of the character on the current input read position.  If this user function
+   * returns true, read cursor is shifted and true returned.  Otherwise false.
+   * The user classifiction function is not called when we are at or past the end and
+   * false is immediately returned.
+   */
+  MOZ_MUST_USE
+  bool CheckChar(bool (*aClassifier)(const char aChar));
+  /**
+   * Check for a whole expected word.
+   */
+  MOZ_MUST_USE
+  bool CheckWord(const nsACString& aWord) { return Check(Token::Word(aWord)); }
+  /**
+   * Shortcut for literal const word check with compile time length calculation.
+   */
+  template <uint32_t N>
+  MOZ_MUST_USE
+  bool CheckWord(const char (&aWord)[N]) { return Check(Token::Word(nsDependentCString(aWord, N - 1))); }
+  /**
+   * Checks \r, \n or \r\n.
+   */
+  MOZ_MUST_USE
+  bool CheckEOL() { return Check(Token::NewLine()); }
+  /**
+   * Checks we are at the end of the input string reading.  If so, shift past the end
+   * and returns true.  Otherwise does nothing and returns false.
+   */
+  MOZ_MUST_USE
+  bool CheckEOF() { return Check(Token::EndOfFile()); }
+
+  /**
+   * These are shortcuts to obtain the value immediately when the token type matches.
+   */
+  MOZ_MUST_USE bool ReadChar(char* aValue);
+  MOZ_MUST_USE bool ReadChar(bool (*aClassifier)(const char aChar),
+                             char* aValue);
+  MOZ_MUST_USE bool ReadWord(nsACString& aValue);
+  MOZ_MUST_USE bool ReadWord(nsDependentCSubstring& aValue);
+
+  /**
+   * This is an integer read helper.  It returns false and doesn't move the read
+   * cursor when any of the following happens:
+   *  - the token at the read cursor is not an integer
+   *  - the final number doesn't fit the T type
+   * Otherwise true is returned, aValue is filled with the integral number
+   * and the cursor is moved forward.
+   */
+  template <typename T>
+  MOZ_MUST_USE bool ReadInteger(T* aValue)
+  {
+    MOZ_RELEASE_ASSERT(aValue);
+
+    nsACString::const_char_iterator rollback = mRollback;
+    nsACString::const_char_iterator cursor = mCursor;
+    Token t;
+    if (!Check(TOKEN_INTEGER, t)) {
+      return false;
+    }
+
+    mozilla::CheckedInt<T> checked(t.AsInteger());
+    if (!checked.isValid()) {
+      // Move to a state as if Check() call has failed
+      mRollback = rollback;
+      mCursor = cursor;
+      mHasFailed = true;
+      return false;
+    }
+
+    *aValue = checked.value();
+    return true;
+  }
+
+  /**
+   * Returns the read cursor position back as it was before the last call of any parsing
+   * method of Tokenizer (Next, Check*, Skip*, Read*) so that the last operation
+   * can be repeated.
+   * Rollback cannot be used multiple times, it only reverts the last successfull parse
+   * operation.  It also cannot be used before any parsing operation has been called
+   * on the Tokenizer.
+   */
+  void Rollback();
+
+  /**
+   * Record() and Claim() are collecting the input as it is being parsed to obtain
+   * a substring between particular syntax bounderies defined by any recursive
+   * descent parser or simple parser the Tokenizer is used to read the input for.
+   * Inlucsion of a token that has just been parsed can be controlled using an arguemnt.
+   */
+  enum ClaimInclusion {
+    /**
+     * Include resulting (or passed) token of the last lexical analyzer operation in the result.
+     */
+    INCLUDE_LAST,
+    /**
+     * Do not include it.
+     */
+    EXCLUDE_LAST
+  };
+
+  /**
+   * Start the process of recording.  Based on aInclude value the begining of the recorded
+   * sub-string is at the current position (EXCLUDE_LAST) or at the position before the last
+   * parsed token (INCLUDE_LAST).
+   */
+  void Record(ClaimInclusion aInclude = EXCLUDE_LAST);
+  /**
+   * Claim result of the record started with Record() call before.  Depending on aInclude
+   * the ending of the sub-string result includes or excludes the last parsed or checked
+   * token.
+   */
+  void Claim(nsACString& aResult, ClaimInclusion aInclude = EXCLUDE_LAST);
+  void Claim(nsDependentCSubstring& aResult, ClaimInclusion aInclude = EXCLUDE_LAST);
+
+  /**
+   * If aToken is found, aResult is set to the substring between the current
+   * position and the position of aToken, potentially including aToken depending
+   * on aInclude.
+   * If aToken isn't found aResult is set to the substring between the current
+   * position and the end of the string.
+   * If aToken is found, the method returns true. Otherwise it returns false.
+   *
+   * Calling Rollback() after ReadUntil() will return the read cursor to the
+   * position it had before ReadUntil was called.
+   */
+  MOZ_MUST_USE bool ReadUntil(Token const& aToken, nsDependentCSubstring& aResult,
+                              ClaimInclusion aInclude = EXCLUDE_LAST);
+  MOZ_MUST_USE bool ReadUntil(Token const& aToken, nsACString& aResult,
+                              ClaimInclusion aInclude = EXCLUDE_LAST);
+
+protected:
+  // All these point to the original buffer passed to the Tokenizer's constructor
+  nsACString::const_char_iterator mRecord; // Position where the recorded sub-string for Claim() is
+  nsACString::const_char_iterator mRollback; // Position of the previous token start
+
+private:
+  Tokenizer() = delete;
+  Tokenizer(const Tokenizer&) = delete;
+  Tokenizer(Tokenizer&&) = delete;
+  Tokenizer(const Tokenizer&&) = delete;
+  Tokenizer &operator=(const Tokenizer&) = delete;
+};
+
+} // mozilla
+
+#endif // Tokenizer_h__