1317375 - Implement "Template Literals Revision / Lifting Template Literal Restriction" ECMAScript proposal

11 files changed, 202 insertions, 68 deletions

diff --git a/js/src/frontend/BytecodeEmitter.cpp b/js/src/frontend/BytecodeEmitter.cpp
index 7f9fa8a5d..44787dc52 100644
--- a/js/src/frontend/BytecodeEmitter.cpp
+++ b/js/src/frontend/BytecodeEmitter.cpp
@@ -3069,6 +3069,7 @@ BytecodeEmitter::checkSideEffects(ParseNode* pn, bool* answer)
       case PNK_TRUE:
       case PNK_FALSE:
       case PNK_NULL:
+      case PNK_RAW_UNDEFINED:
       case PNK_ELISION:
       case PNK_GENERATOR:
       case PNK_NUMBER:
@@ -6242,6 +6243,9 @@ ParseNode::getConstantValue(ExclusiveContext* cx, AllowConstantObjects allowObje
       case PNK_NULL:
         vp.setNull();
         return true;
+      case PNK_RAW_UNDEFINED:
+        vp.setUndefined();
+        return true;
       case PNK_CALLSITEOBJ:
       case PNK_ARRAY: {
         unsigned count;
@@ -10636,6 +10640,7 @@ BytecodeEmitter::emitTree(ParseNode* pn, EmitLineNumberNote emitLineNote)
       case PNK_TRUE:
       case PNK_FALSE:
       case PNK_NULL:
+      case PNK_RAW_UNDEFINED:
         if (!emit1(pn->getOp()))
             return false;
         break;
diff --git a/js/src/frontend/FoldConstants.cpp b/js/src/frontend/FoldConstants.cpp
index 6f62ffac6..441a72a45 100644
--- a/js/src/frontend/FoldConstants.cpp
+++ b/js/src/frontend/FoldConstants.cpp
@@ -378,6 +378,7 @@ ContainsHoistedDeclaration(ExclusiveContext* cx, ParseNode* node, bool* result)
       case PNK_TRUE:
       case PNK_FALSE:
       case PNK_NULL:
+      case PNK_RAW_UNDEFINED:
       case PNK_THIS:
       case PNK_ELISION:
       case PNK_NUMBER:
@@ -468,6 +469,7 @@ IsEffectless(ParseNode* node)
            node->isKind(PNK_TEMPLATE_STRING) ||
            node->isKind(PNK_NUMBER) ||
            node->isKind(PNK_NULL) ||
+           node->isKind(PNK_RAW_UNDEFINED) ||
            node->isKind(PNK_FUNCTION) ||
            node->isKind(PNK_GENEXP);
 }
@@ -492,6 +494,7 @@ Boolish(ParseNode* pn)
 
       case PNK_FALSE:
       case PNK_NULL:
+      case PNK_RAW_UNDEFINED:
         return Falsy;
 
       case PNK_VOID: {
@@ -1643,6 +1646,7 @@ Fold(ExclusiveContext* cx, ParseNode** pnp, Parser<FullParseHandler>& parser, bo
       case PNK_TRUE:
       case PNK_FALSE:
       case PNK_NULL:
+      case PNK_RAW_UNDEFINED:
       case PNK_ELISION:
       case PNK_NUMBER:
       case PNK_DEBUGGER:
diff --git a/js/src/frontend/FullParseHandler.h b/js/src/frontend/FullParseHandler.h
index 3938d9743..f0fc7700d 100644
--- a/js/src/frontend/FullParseHandler.h
+++ b/js/src/frontend/FullParseHandler.h
@@ -183,6 +183,10 @@ class FullParseHandler
         return new_<NullLiteral>(pos);
     }
 
+    ParseNode* newRawUndefinedLiteral(const TokenPos& pos) {
+        return new_<RawUndefinedLiteral>(pos);
+    }
+
     // The Boxer object here is any object that can allocate ObjectBoxes.
     // Specifically, a Boxer has a .newObjectBox(T) method that accepts a
     // Rooted<RegExpObject*> argument and returns an ObjectBox*.
diff --git a/js/src/frontend/NameFunctions.cpp b/js/src/frontend/NameFunctions.cpp
index ce1318f0b..dc54d0a88 100644
--- a/js/src/frontend/NameFunctions.cpp
+++ b/js/src/frontend/NameFunctions.cpp
@@ -316,7 +316,8 @@ class NameResolver
             return false;
 
         // Next is the callsite object node.  This node only contains
-        // internal strings and an array -- no user-controlled expressions.
+        // internal strings or undefined and an array -- no user-controlled
+        // expressions.
         element = element->pn_next;
 #ifdef DEBUG
         {
@@ -326,7 +327,7 @@ class NameResolver
             for (ParseNode* kid = array->pn_head; kid; kid = kid->pn_next)
                 MOZ_ASSERT(kid->isKind(PNK_TEMPLATE_STRING));
             for (ParseNode* next = array->pn_next; next; next = next->pn_next)
-                MOZ_ASSERT(next->isKind(PNK_TEMPLATE_STRING));
+                MOZ_ASSERT(next->isKind(PNK_TEMPLATE_STRING) || next->isKind(PNK_RAW_UNDEFINED));
         }
 #endif
 
@@ -382,6 +383,7 @@ class NameResolver
           case PNK_TRUE:
           case PNK_FALSE:
           case PNK_NULL:
+          case PNK_RAW_UNDEFINED:
           case PNK_ELISION:
           case PNK_GENERATOR:
           case PNK_NUMBER:
diff --git a/js/src/frontend/ParseNode.cpp b/js/src/frontend/ParseNode.cpp
index 91f17625c..5fe64e3d3 100644
--- a/js/src/frontend/ParseNode.cpp
+++ b/js/src/frontend/ParseNode.cpp
@@ -190,6 +190,7 @@ PushNodeChildren(ParseNode* pn, NodeStack* stack)
       case PNK_TRUE:
       case PNK_FALSE:
       case PNK_NULL:
+      case PNK_RAW_UNDEFINED:
       case PNK_ELISION:
       case PNK_GENERATOR:
       case PNK_NUMBER:
@@ -685,6 +686,7 @@ NullaryNode::dump()
       case PNK_TRUE:  fprintf(stderr, "#true");  break;
       case PNK_FALSE: fprintf(stderr, "#false"); break;
       case PNK_NULL:  fprintf(stderr, "#null");  break;
+      case PNK_RAW_UNDEFINED: fprintf(stderr, "#undefined"); break;
 
       case PNK_NUMBER: {
         ToCStringBuf cbuf;
diff --git a/js/src/frontend/ParseNode.h b/js/src/frontend/ParseNode.h
index c3523afe4..c0669d85e 100644
--- a/js/src/frontend/ParseNode.h
+++ b/js/src/frontend/ParseNode.h
@@ -54,6 +54,7 @@ class ObjectBox;
     F(TRUE) \
     F(FALSE) \
     F(NULL) \
+    F(RAW_UNDEFINED) \
     F(THIS) \
     F(FUNCTION) \
     F(MODULE) \
@@ -406,7 +407,8 @@ IsTypeofKind(ParseNodeKind kind)
  * PNK_NUMBER   dval        pn_dval: double value of numeric literal
  * PNK_TRUE,    nullary     pn_op: JSOp bytecode
  * PNK_FALSE,
- * PNK_NULL
+ * PNK_NULL,
+ * PNK_RAW_UNDEFINED
  *
  * PNK_THIS,        unary   pn_kid: '.this' Name if function `this`, else nullptr
  * PNK_SUPERBASE    unary   pn_kid: '.this' Name
@@ -686,7 +688,8 @@ class ParseNode
                isKind(PNK_STRING) ||
                isKind(PNK_TRUE) ||
                isKind(PNK_FALSE) ||
-               isKind(PNK_NULL);
+               isKind(PNK_NULL) ||
+               isKind(PNK_RAW_UNDEFINED);
     }
 
     /* Return true if this node appears in a Directive Prologue. */
@@ -1141,6 +1144,16 @@ class NullLiteral : public ParseNode
     explicit NullLiteral(const TokenPos& pos) : ParseNode(PNK_NULL, JSOP_NULL, PN_NULLARY, pos) { }
 };
 
+// This is only used internally, currently just for tagged templates.
+// It represents the value 'undefined' (aka `void 0`), like NullLiteral
+// represents the value 'null'.
+class RawUndefinedLiteral : public ParseNode
+{
+  public:
+    explicit RawUndefinedLiteral(const TokenPos& pos)
+      : ParseNode(PNK_RAW_UNDEFINED, JSOP_UNDEFINED, PN_NULLARY, pos) { }
+};
+
 class BooleanLiteral : public ParseNode
 {
   public:
@@ -1361,6 +1374,7 @@ ParseNode::isConstant()
       case PNK_STRING:
       case PNK_TEMPLATE_STRING:
       case PNK_NULL:
+      case PNK_RAW_UNDEFINED:
       case PNK_FALSE:
       case PNK_TRUE:
         return true;
diff --git a/js/src/frontend/Parser.cpp b/js/src/frontend/Parser.cpp
index 66ada79f2..0ff10f73f 100644
--- a/js/src/frontend/Parser.cpp
+++ b/js/src/frontend/Parser.cpp
@@ -3123,7 +3123,7 @@ template <typename ParseHandler>
 typename ParseHandler::Node
 Parser<ParseHandler>::templateLiteral(YieldHandling yieldHandling)
 {
-    Node pn = noSubstitutionTemplate();
+    Node pn = noSubstitutionUntaggedTemplate();
     if (!pn)
         return null();
 
@@ -3136,7 +3136,7 @@ Parser<ParseHandler>::templateLiteral(YieldHandling yieldHandling)
         if (!addExprAndGetNextTemplStrToken(yieldHandling, nodeList, &tt))
             return null();
 
-        pn = noSubstitutionTemplate();
+        pn = noSubstitutionUntaggedTemplate();
         if (!pn)
             return null();
 
@@ -3365,7 +3365,7 @@ template <typename ParseHandler>
 bool
 Parser<ParseHandler>::appendToCallSiteObj(Node callSiteObj)
 {
-    Node cookedNode = noSubstitutionTemplate();
+    Node cookedNode = noSubstitutionTaggedTemplate();
     if (!cookedNode)
         return false;
 
@@ -8753,8 +8753,23 @@ Parser<ParseHandler>::stringLiteral()
 
 template <typename ParseHandler>
 typename ParseHandler::Node
-Parser<ParseHandler>::noSubstitutionTemplate()
+Parser<ParseHandler>::noSubstitutionTaggedTemplate()
 {
+    if (tokenStream.hasInvalidTemplateEscape()) {
+        tokenStream.clearInvalidTemplateEscape();
+        return handler.newRawUndefinedLiteral(pos());
+    }
+
+    return handler.newTemplateStringLiteral(stopStringCompression(), pos());
+}
+
+template <typename ParseHandler>
+typename ParseHandler::Node
+Parser<ParseHandler>::noSubstitutionUntaggedTemplate()
+{
+    if (!tokenStream.checkForInvalidTemplateEscapeError())
+        return null();
+
     return handler.newTemplateStringLiteral(stopStringCompression(), pos());
 }
 
@@ -9485,7 +9500,7 @@ Parser<ParseHandler>::primaryExpr(YieldHandling yieldHandling, TripledotHandling
         return templateLiteral(yieldHandling);
 
       case TOK_NO_SUBS_TEMPLATE:
-        return noSubstitutionTemplate();
+        return noSubstitutionUntaggedTemplate();
 
       case TOK_STRING:
         return stringLiteral();
diff --git a/js/src/frontend/Parser.h b/js/src/frontend/Parser.h
index e36a6e8b2..b850dc00b 100644
--- a/js/src/frontend/Parser.h
+++ b/js/src/frontend/Parser.h
@@ -1052,7 +1052,8 @@ class Parser final : public ParserBase, private JS::AutoGCRooter
     JSAtom* stopStringCompression();
 
     Node stringLiteral();
-    Node noSubstitutionTemplate();
+    Node noSubstitutionTaggedTemplate();
+    Node noSubstitutionUntaggedTemplate();
     Node templateLiteral(YieldHandling yieldHandling);
     bool taggedTemplate(YieldHandling yieldHandling, Node nodeList, TokenKind tt);
     bool appendToCallSiteObj(Node callSiteObj);
diff --git a/js/src/frontend/SyntaxParseHandler.h b/js/src/frontend/SyntaxParseHandler.h
index 9dc3c1072..aa85a631e 100644
--- a/js/src/frontend/SyntaxParseHandler.h
+++ b/js/src/frontend/SyntaxParseHandler.h
@@ -224,6 +224,7 @@ class SyntaxParseHandler
 
     Node newThisLiteral(const TokenPos& pos, Node thisName) { return NodeGeneric; }
     Node newNullLiteral(const TokenPos& pos) { return NodeGeneric; }
+    Node newRawUndefinedLiteral(const TokenPos& pos) { return NodeGeneric; }
 
     template <class Boxer>
     Node newRegExp(RegExpObject* reobj, const TokenPos& pos, Boxer& boxer) { return NodeGeneric; }
diff --git a/js/src/frontend/TokenStream.cpp b/js/src/frontend/TokenStream.cpp
index 186b7a8ab..6a7934e8f 100644
--- a/js/src/frontend/TokenStream.cpp
+++ b/js/src/frontend/TokenStream.cpp
@@ -1918,55 +1918,6 @@ TokenStream::getTokenInternal(TokenKind* ttp, Modifier modifier)
 }
 
 bool
-TokenStream::matchBracedUnicode(bool* matched, uint32_t* cp)
-{
-    int32_t c;
-    if (!peekChar(&c))
-        return false;
-    if (c != '{') {
-        *matched = false;
-        return true;
-    }
-
-    consumeKnownChar('{');
-
-    uint32_t start = userbuf.offset();
-
-    bool first = true;
-    uint32_t code = 0;
-    do {
-        int32_t c = getCharIgnoreEOL();
-        if (c == EOF) {
-            error(JSMSG_MALFORMED_ESCAPE, "Unicode");
-            return false;
-        }
-        if (c == '}') {
-            if (first) {
-                error(JSMSG_MALFORMED_ESCAPE, "Unicode");
-                return false;
-            }
-            break;
-        }
-
-        if (!JS7_ISHEX(c)) {
-            error(JSMSG_MALFORMED_ESCAPE, "Unicode");
-            return false;
-        }
-
-        code = (code << 4) | JS7_UNHEX(c);
-        if (code > unicode::NonBMPMax) {
-            errorAt(start, JSMSG_UNICODE_OVERFLOW, "escape sequence");
-            return false;
-        }
-        first = false;
-    } while (true);
-
-    *matched = true;
-    *cp = code;
-    return true;
-}
-
-bool
 TokenStream::getStringOrTemplateToken(int untilChar, Token** tp)
 {
     int c;
@@ -1988,6 +1939,10 @@ TokenStream::getStringOrTemplateToken(int untilChar, Token** tp)
         }
 
         if (c == '\\') {
+            // When parsing templates, we don't immediately report errors for
+            // invalid escapes; these are handled by the parser.
+            // In those cases we don't append to tokenbuf, since it won't be
+            // read.
             switch (c = getChar()) {
               case 'b': c = '\b'; break;
               case 'f': c = '\f'; break;
@@ -2003,11 +1958,73 @@ TokenStream::getStringOrTemplateToken(int untilChar, Token** tp)
 
               // Unicode character specification.
               case 'u': {
-                bool matched;
-                uint32_t code;
-                if (!matchBracedUnicode(&matched, &code))
+                uint32_t code = 0;
+
+                int32_t c2;
+                if (!peekChar(&c2))
                     return false;
-                if (matched) {
+
+                uint32_t start = userbuf.offset() - 2;
+
+                if (c2 == '{') {
+                    consumeKnownChar('{');
+
+                    bool first = true;
+                    bool valid = true;
+                    do {
+                        int32_t c = getCharIgnoreEOL();
+                        if (c == EOF) {
+                            if (parsingTemplate) {
+                                setInvalidTemplateEscape(start, InvalidEscapeType::Unicode);
+                                valid = false;
+                                break;
+                            }
+                            reportInvalidEscapeError(start, InvalidEscapeType::Unicode);
+                            return false;
+                        }
+                        if (c == '}') {
+                            if (first) {
+                                if (parsingTemplate) {
+                                    setInvalidTemplateEscape(start, InvalidEscapeType::Unicode);
+                                    valid = false;
+                                    break;
+                                }
+                                reportInvalidEscapeError(start, InvalidEscapeType::Unicode);
+                                return false;
+                            }
+                            break;
+                        }
+
+                        if (!JS7_ISHEX(c)) {
+                            if (parsingTemplate) {
+                                // We put the character back so that we read
+                                // it on the next pass, which matters if it
+                                // was '`' or '\'.
+                                ungetCharIgnoreEOL(c);
+                                setInvalidTemplateEscape(start, InvalidEscapeType::Unicode);
+                                valid = false;
+                                break;
+                            }
+                            reportInvalidEscapeError(start, InvalidEscapeType::Unicode);
+                            return false;
+                        }
+
+                        code = (code << 4) | JS7_UNHEX(c);
+                        if (code > unicode::NonBMPMax) {
+                            if (parsingTemplate) {
+                                setInvalidTemplateEscape(start + 3, InvalidEscapeType::UnicodeOverflow);
+                                valid = false;
+                                break;
+                            }
+                            reportInvalidEscapeError(start + 3, InvalidEscapeType::UnicodeOverflow);
+                            return false;
+                        }
+
+                        first = false;
+                    } while (true);
+
+                    if (!valid)
+                        continue;
 
                     MOZ_ASSERT(code <= unicode::NonBMPMax);
                     if (code < unicode::NonBMPMin) {
@@ -2030,7 +2047,11 @@ TokenStream::getStringOrTemplateToken(int untilChar, Token** tp)
                     c = (c << 4) + JS7_UNHEX(cp[3]);
                     skipChars(4);
                 } else {
-                    error(JSMSG_MALFORMED_ESCAPE, "Unicode");
+                    if (parsingTemplate) {
+                        setInvalidTemplateEscape(start, InvalidEscapeType::Unicode);
+                        continue;
+                    }
+                    reportInvalidEscapeError(start, InvalidEscapeType::Unicode);
                     return false;
                 }
                 break;
@@ -2043,7 +2064,12 @@ TokenStream::getStringOrTemplateToken(int untilChar, Token** tp)
                     c = (JS7_UNHEX(cp[0]) << 4) + JS7_UNHEX(cp[1]);
                     skipChars(2);
                 } else {
-                    error(JSMSG_MALFORMED_ESCAPE, "hexadecimal");
+                    uint32_t start = userbuf.offset() - 2;
+                    if (parsingTemplate) {
+                        setInvalidTemplateEscape(start, InvalidEscapeType::Hexadecimal);
+                        continue;
+                    }
+                    reportInvalidEscapeError(start, InvalidEscapeType::Hexadecimal);
                     return false;
                 }
                 break;
@@ -2060,8 +2086,8 @@ TokenStream::getStringOrTemplateToken(int untilChar, Token** tp)
                     // Strict mode code allows only \0, then a non-digit.
                     if (val != 0 || JS7_ISDEC(c)) {
                         if (parsingTemplate) {
-                            error(JSMSG_DEPRECATED_OCTAL);
-                            return false;
+                            setInvalidTemplateEscape(userbuf.offset() - 2, InvalidEscapeType::Octal);
+                            continue;
                         }
                         if (!reportStrictModeError(JSMSG_DEPRECATED_OCTAL))
                             return false;
diff --git a/js/src/frontend/TokenStream.h b/js/src/frontend/TokenStream.h
index 4efc31446..fbfefbfe1 100644
--- a/js/src/frontend/TokenStream.h
+++ b/js/src/frontend/TokenStream.h
@@ -80,6 +80,20 @@ struct TokenPos {
 
 enum DecimalPoint { NoDecimal = false, HasDecimal = true };
 
+enum class InvalidEscapeType {
+    // No invalid character escapes.
+    None,
+    // A malformed \x escape.
+    Hexadecimal,
+    // A malformed \u escape.
+    Unicode,
+    // An otherwise well-formed \u escape which represents a
+    // codepoint > 10FFFF.
+    UnicodeOverflow,
+    // An octal escape in a template token.
+    Octal
+};
+
 class TokenStream;
 
 struct Token
@@ -361,6 +375,23 @@ class MOZ_STACK_CLASS TokenStream
     bool hadError() const { return flags.hadError; }
     void clearSawOctalEscape() { flags.sawOctalEscape = false; }
 
+    bool hasInvalidTemplateEscape() const {
+        return invalidTemplateEscapeType != InvalidEscapeType::None;
+    }
+    void clearInvalidTemplateEscape() {
+        invalidTemplateEscapeType = InvalidEscapeType::None;
+    }
+
+    // If there is an invalid escape in a template, report it and return false,
+    // otherwise return true.
+    bool checkForInvalidTemplateEscapeError() {
+        if (invalidTemplateEscapeType == InvalidEscapeType::None)
+            return true;
+
+        reportInvalidEscapeError(invalidTemplateEscapeOffset, invalidTemplateEscapeType);
+        return false;
+    }
+
     // TokenStream-specific error reporters.
     bool reportError(unsigned errorNumber, ...);
     bool reportErrorNoOffset(unsigned errorNumber, ...);
@@ -422,6 +453,33 @@ class MOZ_STACK_CLASS TokenStream
     bool reportStrictModeError(unsigned errorNumber, ...);
     bool strictMode() const { return strictModeGetter && strictModeGetter->strictMode(); }
 
+    void setInvalidTemplateEscape(uint32_t offset, InvalidEscapeType type) {
+        MOZ_ASSERT(type != InvalidEscapeType::None);
+        if (invalidTemplateEscapeType != InvalidEscapeType::None)
+            return;
+        invalidTemplateEscapeOffset = offset;
+        invalidTemplateEscapeType = type;
+    }
+    void reportInvalidEscapeError(uint32_t offset, InvalidEscapeType type) {
+        switch (type) {
+            case InvalidEscapeType::None:
+                MOZ_ASSERT_UNREACHABLE("unexpected InvalidEscapeType");
+                return;
+            case InvalidEscapeType::Hexadecimal:
+                errorAt(offset, JSMSG_MALFORMED_ESCAPE, "hexadecimal");
+                return;
+            case InvalidEscapeType::Unicode:
+                errorAt(offset, JSMSG_MALFORMED_ESCAPE, "Unicode");
+                return;
+            case InvalidEscapeType::UnicodeOverflow:
+                errorAt(offset, JSMSG_UNICODE_OVERFLOW, "escape sequence");
+                return;
+            case InvalidEscapeType::Octal:
+                errorAt(offset, JSMSG_DEPRECATED_OCTAL);
+                return;
+        }
+    }
+
     static JSAtom* atomize(ExclusiveContext* cx, CharBuffer& cb);
     MOZ_MUST_USE bool putIdentInTokenbuf(const char16_t* identStart);
 
@@ -442,6 +500,9 @@ class MOZ_STACK_CLASS TokenStream
     bool awaitIsKeyword = false;
     friend class AutoAwaitIsKeyword;
 
+    uint32_t invalidTemplateEscapeOffset = 0;
+    InvalidEscapeType invalidTemplateEscapeType = InvalidEscapeType::None;
+
   public:
     typedef Token::Modifier Modifier;
     static constexpr Modifier None = Token::None;
@@ -955,7 +1016,6 @@ class MOZ_STACK_CLASS TokenStream
 
     MOZ_MUST_USE bool getTokenInternal(TokenKind* ttp, Modifier modifier);
 
-    MOZ_MUST_USE bool matchBracedUnicode(bool* matched, uint32_t* code);
     MOZ_MUST_USE bool getStringOrTemplateToken(int untilChar, Token** tp);
 
     int32_t getChar();