diff options
Diffstat (limited to 'gfx/angle/src/compiler/preprocessor/Tokenizer.l')
| -rwxr-xr-x | gfx/angle/src/compiler/preprocessor/Tokenizer.l | 357 |
1 files changed, 357 insertions, 0 deletions
diff --git a/gfx/angle/src/compiler/preprocessor/Tokenizer.l b/gfx/angle/src/compiler/preprocessor/Tokenizer.l new file mode 100755 index 000000000..62eb4caa6 --- /dev/null +++ b/gfx/angle/src/compiler/preprocessor/Tokenizer.l @@ -0,0 +1,357 @@ +/* +// +// Copyright (c) 2002-2014 The ANGLE Project Authors. All rights reserved. +// Use of this source code is governed by a BSD-style license that can be +// found in the LICENSE file. +// + +This file contains the Lex specification for GLSL ES preprocessor. +Based on Microsoft Visual Studio 2010 Preprocessor Grammar: +http://msdn.microsoft.com/en-us/library/2scxys89.aspx + +IF YOU MODIFY THIS FILE YOU ALSO NEED TO RUN generate_parser.sh. +*/ + +%top{ +// +// Copyright (c) 2011-2014 The ANGLE Project Authors. All rights reserved. +// Use of this source code is governed by a BSD-style license that can be +// found in the LICENSE file. +// + +// This file is auto-generated by generate_parser.sh. DO NOT EDIT! +} + +%{ +#if defined(_MSC_VER) +#pragma warning(disable: 4005) +#endif + +#include "compiler/preprocessor/Tokenizer.h" + +#include "compiler/preprocessor/DiagnosticsBase.h" +#include "compiler/preprocessor/Token.h" + +#if defined(__GNUC__) +// Triggered by the auto-generated yy_fatal_error function. +#pragma GCC diagnostic ignored "-Wmissing-noreturn" +#elif defined(_MSC_VER) +#pragma warning(disable: 4244) +#endif + +// Workaround for flex using the register keyword, deprecated in C++11. +#ifdef __cplusplus +#if __cplusplus > 199711L +#define register +#endif +#endif + +typedef std::string YYSTYPE; +typedef pp::SourceLocation YYLTYPE; + +// Use the unused yycolumn variable to track file (string) number. +#define yyfileno yycolumn + +#define YY_USER_INIT \ + do { \ + yyfileno = 0; \ + yylineno = 1; \ + yyextra->leadingSpace = false; \ + yyextra->lineStart = true; \ + } while(0); + +#define YY_USER_ACTION \ + do \ + { \ + pp::Input* input = &yyextra->input; \ + pp::Input::Location* scanLoc = &yyextra->scanLoc; \ + while ((scanLoc->sIndex < input->count()) && \ + (scanLoc->cIndex >= input->length(scanLoc->sIndex))) \ + { \ + scanLoc->cIndex -= input->length(scanLoc->sIndex++); \ + ++yyfileno; yylineno = 1; \ + } \ + yylloc->file = yyfileno; \ + yylloc->line = yylineno; \ + scanLoc->cIndex += yyleng; \ + } while(0); + +#define YY_INPUT(buf, result, maxSize) \ + result = yyextra->input.read(buf, maxSize, &yylineno); + +%} + +%option noyywrap nounput never-interactive +%option reentrant bison-bridge bison-locations +%option prefix="pp" +%option extra-type="pp::Tokenizer::Context*" +%x COMMENT + +NEWLINE \n|\r|\r\n +IDENTIFIER [_a-zA-Z][_a-zA-Z0-9]* +PUNCTUATOR [][<>(){}.+-/*%^|&~=!:;,?] + +DECIMAL_CONSTANT [1-9][0-9]*[uU]? +OCTAL_CONSTANT 0[0-7]*[uU]? +HEXADECIMAL_CONSTANT 0[xX][0-9a-fA-F]+[uU]? + +DIGIT [0-9] +EXPONENT_PART [eE][+-]?{DIGIT}+ +FRACTIONAL_CONSTANT ({DIGIT}*"."{DIGIT}+)|({DIGIT}+".") + +%% + + /* Line comment */ +"//"[^\r\n]* + + /* Block comment */ + /* Line breaks are just counted - not returned. */ + /* The comment is replaced by a single space. */ +"/*" { BEGIN(COMMENT); } +<COMMENT>[^*\r\n]+ +<COMMENT>"*" +<COMMENT>{NEWLINE} { ++yylineno; } +<COMMENT>"*/" { + yyextra->leadingSpace = true; + BEGIN(INITIAL); +} + +# { + // # is only valid at start of line for preprocessor directives. + yylval->assign(1, yytext[0]); + return yyextra->lineStart ? pp::Token::PP_HASH : pp::Token::PP_OTHER; +} + +{IDENTIFIER} { + yylval->assign(yytext, yyleng); + return pp::Token::IDENTIFIER; +} + +({DECIMAL_CONSTANT}[uU]?)|({OCTAL_CONSTANT}[uU]?)|({HEXADECIMAL_CONSTANT}[uU]?) { + yylval->assign(yytext, yyleng); + return pp::Token::CONST_INT; +} + +({DIGIT}+{EXPONENT_PART}[fF]?)|({FRACTIONAL_CONSTANT}{EXPONENT_PART}?[fF]?) { + yylval->assign(yytext, yyleng); + return pp::Token::CONST_FLOAT; +} + + /* Anything that starts with a {DIGIT} or .{DIGIT} must be a number. */ + /* Rule to catch all invalid integers and floats. */ +({DIGIT}+[_a-zA-Z0-9.]*)|("."{DIGIT}+[_a-zA-Z0-9.]*) { + yylval->assign(yytext, yyleng); + return pp::Token::PP_NUMBER; +} + +"++" { + yylval->assign(yytext, yyleng); + return pp::Token::OP_INC; +} +"--" { + yylval->assign(yytext, yyleng); + return pp::Token::OP_DEC; +} +"<<" { + yylval->assign(yytext, yyleng); + return pp::Token::OP_LEFT; +} +">>" { + yylval->assign(yytext, yyleng); + return pp::Token::OP_RIGHT; +} +"<=" { + yylval->assign(yytext, yyleng); + return pp::Token::OP_LE; +} +">=" { + yylval->assign(yytext, yyleng); + return pp::Token::OP_GE; +} +"==" { + yylval->assign(yytext, yyleng); + return pp::Token::OP_EQ; +} +"!=" { + yylval->assign(yytext, yyleng); + return pp::Token::OP_NE; +} +"&&" { + yylval->assign(yytext, yyleng); + return pp::Token::OP_AND; +} +"^^" { + yylval->assign(yytext, yyleng); + return pp::Token::OP_XOR; +} +"||" { + yylval->assign(yytext, yyleng); + return pp::Token::OP_OR; +} +"+=" { + yylval->assign(yytext, yyleng); + return pp::Token::OP_ADD_ASSIGN; +} +"-=" { + yylval->assign(yytext, yyleng); + return pp::Token::OP_SUB_ASSIGN; +} +"*=" { + yylval->assign(yytext, yyleng); + return pp::Token::OP_MUL_ASSIGN; +} +"/=" { + yylval->assign(yytext, yyleng); + return pp::Token::OP_DIV_ASSIGN; +} +"%=" { + yylval->assign(yytext, yyleng); + return pp::Token::OP_MOD_ASSIGN; +} +"<<=" { + yylval->assign(yytext, yyleng); + return pp::Token::OP_LEFT_ASSIGN; +} +">>=" { + yylval->assign(yytext, yyleng); + return pp::Token::OP_RIGHT_ASSIGN; +} +"&=" { + yylval->assign(yytext, yyleng); + return pp::Token::OP_AND_ASSIGN; +} +"^=" { + yylval->assign(yytext, yyleng); + return pp::Token::OP_XOR_ASSIGN; +} +"|=" { + yylval->assign(yytext, yyleng); + return pp::Token::OP_OR_ASSIGN; +} + +{PUNCTUATOR} { + yylval->assign(1, yytext[0]); + return yytext[0]; +} + +[ \t\v\f]+ { yyextra->leadingSpace = true; } + +{NEWLINE} { + ++yylineno; + yylval->assign(1, '\n'); + return '\n'; +} + +\\{NEWLINE} { ++yylineno; } + +. { + yylval->assign(1, yytext[0]); + return pp::Token::PP_OTHER; +} + +<*><<EOF>> { + // YY_USER_ACTION is not invoked for handling EOF. + // Set the location for EOF token manually. + pp::Input* input = &yyextra->input; + pp::Input::Location* scanLoc = &yyextra->scanLoc; + yy_size_t sIndexMax = input->count() ? input->count() - 1 : 0; + if (scanLoc->sIndex != sIndexMax) + { + // We can only reach here if there are empty strings at the + // end of the input. + scanLoc->sIndex = sIndexMax; scanLoc->cIndex = 0; + // FIXME: this is not 64-bit clean. + yyfileno = static_cast<int>(sIndexMax); yylineno = 1; + } + yylloc->file = yyfileno; + yylloc->line = yylineno; + yylval->clear(); + + if (YY_START == COMMENT) + { + yyextra->diagnostics->report(pp::Diagnostics::PP_EOF_IN_COMMENT, + pp::SourceLocation(yyfileno, yylineno), + ""); + } + yyterminate(); +} + +%% + +namespace pp { + +Tokenizer::Tokenizer(Diagnostics *diagnostics) : mHandle(nullptr), mMaxTokenSize(256) +{ + mContext.diagnostics = diagnostics; +} + +Tokenizer::~Tokenizer() +{ + destroyScanner(); +} + +bool Tokenizer::init(size_t count, const char * const string[], const int length[]) +{ + if ((count > 0) && (string == 0)) + return false; + + mContext.input = Input(count, string, length); + return initScanner(); +} + +void Tokenizer::setFileNumber(int file) +{ + // We use column number as file number. + // See macro yyfileno. + yyset_column(file, mHandle); +} + +void Tokenizer::setLineNumber(int line) +{ + yyset_lineno(line, mHandle); +} + +void Tokenizer::setMaxTokenSize(size_t maxTokenSize) +{ + mMaxTokenSize = maxTokenSize; +} + +void Tokenizer::lex(Token *token) +{ + token->type = yylex(&token->text, &token->location, mHandle); + if (token->text.size() > mMaxTokenSize) + { + mContext.diagnostics->report(Diagnostics::PP_TOKEN_TOO_LONG, + token->location, token->text); + token->text.erase(mMaxTokenSize); + } + + token->flags = 0; + + token->setAtStartOfLine(mContext.lineStart); + mContext.lineStart = token->type == '\n'; + + token->setHasLeadingSpace(mContext.leadingSpace); + mContext.leadingSpace = false; +} + +bool Tokenizer::initScanner() +{ + if ((mHandle == nullptr) && yylex_init_extra(&mContext, &mHandle)) + return false; + + yyrestart(0, mHandle); + return true; +} + +void Tokenizer::destroyScanner() +{ + if (mHandle == nullptr) + return; + + yylex_destroy(mHandle); + mHandle = nullptr; +} + +} // namespace pp + |