From d3ceb2186dc79130508093b3d0c944771a53534f Mon Sep 17 00:00:00 2001 From: Mikko Rasa Date: Sun, 14 Feb 2021 17:42:04 +0200 Subject: [PATCH] Split tokenizer and preprocessor out of the GLSL parser --- source/glsl/parser.cpp | 556 +++++++++-------------------------- source/glsl/parser.h | 55 +--- source/glsl/preprocessor.cpp | 85 ++++++ source/glsl/preprocessor.h | 37 +++ source/glsl/syntax.cpp | 50 ++++ source/glsl/syntax.h | 24 ++ source/glsl/tokenizer.cpp | 206 +++++++++++++ source/glsl/tokenizer.h | 54 ++++ 8 files changed, 605 insertions(+), 462 deletions(-) create mode 100644 source/glsl/preprocessor.cpp create mode 100644 source/glsl/preprocessor.h create mode 100644 source/glsl/tokenizer.cpp create mode 100644 source/glsl/tokenizer.h diff --git a/source/glsl/parser.cpp b/source/glsl/parser.cpp index 3126fae6..eaf5706f 100644 --- a/source/glsl/parser.cpp +++ b/source/glsl/parser.cpp @@ -1,4 +1,3 @@ -#include #include #include #include "glsl_error.h" @@ -12,58 +11,14 @@ namespace Msp { namespace GL { namespace SL { -Parser::Operator Parser::operators[] = -{ - { "[", 2, BINARY, LEFT_TO_RIGHT }, - { "(", 2, BINARY, LEFT_TO_RIGHT }, - { ".", 2, BINARY, LEFT_TO_RIGHT }, - { "++", 2, POSTFIX, LEFT_TO_RIGHT }, - { "--", 2, POSTFIX, LEFT_TO_RIGHT }, - { "++", 3, PREFIX, RIGHT_TO_LEFT }, - { "--", 3, PREFIX, RIGHT_TO_LEFT }, - { "+", 3, PREFIX, RIGHT_TO_LEFT }, - { "-", 3, PREFIX, RIGHT_TO_LEFT }, - { "~", 3, PREFIX, RIGHT_TO_LEFT }, - { "!", 3, PREFIX, RIGHT_TO_LEFT }, - { "*", 4, BINARY, LEFT_TO_RIGHT }, - { "/", 4, BINARY, LEFT_TO_RIGHT }, - { "%", 4, BINARY, LEFT_TO_RIGHT }, - { "+", 5, BINARY, LEFT_TO_RIGHT }, - { "-", 5, BINARY, LEFT_TO_RIGHT }, - { "<<", 6, BINARY, LEFT_TO_RIGHT }, - { ">>", 6, BINARY, LEFT_TO_RIGHT }, - { "<", 7, BINARY, LEFT_TO_RIGHT }, - { ">", 7, BINARY, LEFT_TO_RIGHT }, - { "<=", 7, BINARY, LEFT_TO_RIGHT }, - { ">=", 7, BINARY, LEFT_TO_RIGHT }, - { "==", 8, BINARY, LEFT_TO_RIGHT }, - { "!=", 8, BINARY, LEFT_TO_RIGHT }, - { "&", 9, BINARY, LEFT_TO_RIGHT }, - { "^", 10, BINARY, LEFT_TO_RIGHT }, - { "|", 11, BINARY, LEFT_TO_RIGHT }, - { "&&", 12, BINARY, LEFT_TO_RIGHT }, - { "^^", 13, BINARY, LEFT_TO_RIGHT }, - { "||", 14, BINARY, LEFT_TO_RIGHT }, - { "?", 15, BINARY, RIGHT_TO_LEFT }, - { ":", 15, BINARY, RIGHT_TO_LEFT }, - { "=", 16, BINARY, RIGHT_TO_LEFT }, - { "+=", 16, BINARY, RIGHT_TO_LEFT }, - { "-=", 16, BINARY, RIGHT_TO_LEFT }, - { "*=", 16, BINARY, RIGHT_TO_LEFT }, - { "/=", 16, BINARY, RIGHT_TO_LEFT }, - { "%=", 16, BINARY, RIGHT_TO_LEFT }, - { "<<=", 16, BINARY, RIGHT_TO_LEFT }, - { ">>=", 16, BINARY, RIGHT_TO_LEFT }, - { "&=", 16, BINARY, RIGHT_TO_LEFT }, - { "^=", 16, BINARY, RIGHT_TO_LEFT }, - { "|=", 16, BINARY, RIGHT_TO_LEFT }, - { ",", 17, BINARY, LEFT_TO_RIGHT }, - { { 0 }, 18, NO_OPERATOR, LEFT_TO_RIGHT } -}; - Parser::Parser(): + preprocessor(tokenizer), module(0) -{ } +{ + tokenizer.signal_preprocess.connect(sigc::mem_fun(&preprocessor, &Preprocessor::preprocess)); + preprocessor.signal_version.connect(sigc::mem_fun(this, &Parser::set_required_version)); + preprocessor.signal_stage_change.connect(sigc::mem_fun(this, &Parser::stage_change)); +} Parser::~Parser() { @@ -73,16 +28,14 @@ Parser::~Parser() Module &Parser::parse(const string &s, const string &n, unsigned i) { source = s; - location.name = n; source_index = i; - parse_source(); + parse_source(n); return *module; } Module &Parser::parse(IO::Base &io, const string &n, unsigned i) { source = string(); - location.name = n; source_index = i; while(!io.eof()) { @@ -90,11 +43,11 @@ Module &Parser::parse(IO::Base &io, const string &n, unsigned i) unsigned len = io.read(buffer, sizeof(buffer)); source.append(buffer, len); } - parse_source(); + parse_source(n); return *module; } -void Parser::parse_source() +void Parser::parse_source(const string &name) { while(1) { @@ -110,194 +63,51 @@ void Parser::parse_source() delete module; module = new Module; cur_stage = &module->shared; - iter = source.begin(); - source_end = source.end(); - location.line = 1; - allow_preprocess = true; + tokenizer.begin(name, source); while(RefPtr statement = parse_global_declaration()) cur_stage->content.body.push_back(statement); } -const string &Parser::peek_token(unsigned index) -{ - while(next_tokens.size()<=index) - next_tokens.push_back(parse_token_()); - return (last_token = next_tokens[index]); -} - -const string &Parser::parse_token() -{ - if(!next_tokens.empty()) - { - last_token = next_tokens.front(); - next_tokens.pop_front(); - return last_token; - } - - return (last_token = parse_token_()); -} - -string Parser::parse_token_() -{ - while(1) - { - skip_comment_and_whitespace(); - if(iter==source_end) - return string(); - else if(allow_preprocess && *iter=='#') - { - allow_preprocess = false; - SetForScope > clear_tokens(next_tokens, deque()); - preprocess(); - } - else if(isalpha(*iter) || *iter=='_') - return parse_identifier(); - else if(isdigit(*iter)) - return parse_number(); - else - return parse_other(); - } -} - -string Parser::parse_identifier() -{ - string ident; - while(iter!=source_end) - { - if(isalnum(*iter) || *iter=='_') - ident += *iter++; - else - break; - } - - return ident; -} - -string Parser::parse_number() -{ - bool accept_sign = false; - string number; - while(iter!=source_end) - { - if(isdigit(*iter) || *iter=='.') - number += *iter++; - else if(*iter=='e' || *iter=='E') - { - number += *iter++; - accept_sign = true; - } - else if(accept_sign && (*iter=='+' || *iter=='-')) - number += *iter++; - else - break; - } - - return number; -} - -string Parser::parse_other() +void Parser::set_required_version(const Version &ver) { - if(iter==source_end) - return string(); - - string token(1, *iter++); - for(unsigned i=1; (i<3 && iter!=source_end); ++i) - { - bool matched = false; - for(const Operator *j=operators; (!matched && j->type); ++j) - { - matched = (j->token[i]==*iter); - for(unsigned k=0; (matched && ktoken[k]); ++k) - matched = (j->token[k]==token[k]); - } - - if(!matched) - break; - - token += *iter++; - } - - return token; + cur_stage->required_version = ver; } -void Parser::skip_comment_and_whitespace() +void Parser::stage_change(StageType stage) { - unsigned comment = 0; - while(iter!=source_end) - { - if(comment==0) - { - if(*iter=='/') - comment = 1; - else if(!isspace(*iter)) - break; - } - else if(comment==1) - { - if(*iter=='/') - comment = 2; - else if(*iter=='*') - comment = 3; - else - { - comment = 0; - --iter; - break; - } - } - else if(comment==2) - { - if(*iter=='\n') - comment = 0; - } - else if(comment==3 && *iter=='*') - comment = 4; - else if(comment==4) - { - if(*iter=='/') - comment = 0; - else if(*iter!='*') - comment = 3; - } - - if(*iter=='\n') - { - ++location.line; - allow_preprocess = (comment<3); - } + if(!allow_stage_change) + throw invalid_shader_source(tokenizer.get_location(), "Changing stage not allowed here"); + else if(stage<=cur_stage->type) + throw invalid_shader_source(tokenizer.get_location(), "Stage '%s' not allowed here", stage); - ++iter; - } -} + module->stages.push_back(stage); -void Parser::expect(const string &token) -{ - string parsed = parse_token(); - if(parsed!=token) - throw parse_error(location, parsed, format("'%s'", token)); + if(cur_stage->type!=SHARED) + module->stages.back().previous = cur_stage; + cur_stage = &module->stages.back(); } string Parser::expect_type() { - string token = parse_token(); + string token = tokenizer.parse_token(); if(!is_type(token)) - throw parse_error(location, token, "a type"); + throw parse_error(tokenizer.get_location(), token, "a type"); return token; } string Parser::expect_identifier() { - string token = parse_token(); + string token = tokenizer.parse_token(); if(!is_identifier(token)) - throw parse_error(location, token, "an identifier"); + throw parse_error(tokenizer.get_location(), token, "an identifier"); return token; } bool Parser::check(const string &token) { - bool result = (peek_token()==token); + bool result = (tokenizer.peek_token()==token); if(result) - parse_token(); + tokenizer.parse_token(); return result; } @@ -347,94 +157,10 @@ bool Parser::is_identifier(const string &token) return re.match(token); } -void Parser::preprocess() -{ - expect("#"); - - string::const_iterator line_end = iter; - for(; (line_end!=source_end && *line_end!='\n'); ++line_end) ; - SetForScope stop_at_line_end(source_end, line_end); - - string token = peek_token(); - if(token=="pragma") - preprocess_pragma(); - else if(token=="version") - preprocess_version(); - else if(token=="define" || token=="undef" || token=="if" || token=="ifdef" || token=="ifndef" || token=="else" || - token=="elif" || token=="endif" || token=="error" || token=="extension" || token=="line") - throw invalid_shader_source(location, "Unsupported preprocessor directive '%s'", token); - else if(!token.empty()) - throw parse_error(location, token, "a preprocessor directive"); - - iter = line_end; -} - -void Parser::preprocess_version() -{ - expect("version"); - string token = parse_token(); - unsigned version = lexical_cast(token); - cur_stage->required_version = Version(version/100, version%100); - - token = parse_token(); - if(!token.empty()) - throw parse_error(location, token, "end of line"); -} - -void Parser::preprocess_pragma() -{ - expect("pragma"); - string token = parse_token(); - if(token=="MSP") - preprocess_pragma_msp(); -} - -void Parser::preprocess_pragma_msp() -{ - string token = peek_token(); - if(token=="stage") - preprocess_stage(); - else - throw invalid_shader_source(location, "Unrecognized MSP pragma '%s'", token); - - token = parse_token(); - if(!token.empty()) - throw parse_error(location, token, "end of line"); -} - -void Parser::preprocess_stage() -{ - if(!allow_stage_change) - throw invalid_shader_source(location, "Changing stage not allowed here"); - - expect("stage"); - expect("("); - string token = expect_identifier(); - StageType stage = SHARED; - if(token=="vertex") - stage = VERTEX; - else if(token=="geometry") - stage = GEOMETRY; - else if(token=="fragment") - stage = FRAGMENT; - else - throw parse_error(location, token, "stage identifier"); - expect(")"); - - if(stage<=cur_stage->type) - throw invalid_shader_source(location, "Stage '%s' not allowed here", token); - - module->stages.push_back(stage); - - if(cur_stage->type!=SHARED) - module->stages.back().previous = cur_stage; - cur_stage = &module->stages.back(); -} - RefPtr Parser::parse_global_declaration() { allow_stage_change = true; - string token = peek_token(); + string token = tokenizer.peek_token(); allow_stage_change = false; if(token=="import") @@ -444,15 +170,15 @@ RefPtr Parser::parse_global_declaration() else if(token=="layout") { RefPtr layout = parse_layout(); - token = peek_token(); - if(is_interface_qualifier(token) && peek_token(1)==";") + token = tokenizer.peek_token(); + if(is_interface_qualifier(token) && tokenizer.peek_token(1)==";") { RefPtr iface_lo = new InterfaceLayout; iface_lo->source = source_index; - iface_lo->line = location.line; + iface_lo->line = tokenizer.get_location().line; iface_lo->layout.qualifiers = layout->qualifiers; - iface_lo->interface = parse_token(); - expect(";"); + iface_lo->interface = tokenizer.parse_token(); + tokenizer.expect(";"); return iface_lo; } else @@ -466,7 +192,7 @@ RefPtr Parser::parse_global_declaration() return parse_struct_declaration(); else if(is_interface_qualifier(token)) { - string next = peek_token(1); + string next = tokenizer.peek_token(1); if(is_type(next) || is_qualifier(next)) return parse_variable_declaration(); else @@ -476,7 +202,7 @@ RefPtr Parser::parse_global_declaration() return parse_variable_declaration(); else if(is_type(token)) { - if(peek_token(2)=="(") + if(tokenizer.peek_token(2)=="(") return parse_function_declaration(); else return parse_variable_declaration(); @@ -484,12 +210,12 @@ RefPtr Parser::parse_global_declaration() else if(token.empty()) return 0; else - throw parse_error(location, token, "a global declaration"); + throw parse_error(tokenizer.get_location(), token, "a global declaration"); } RefPtr Parser::parse_statement() { - string token = peek_token(); + string token = tokenizer.peek_token(); if(token=="if") return parse_conditional(); else if(token=="for") @@ -504,9 +230,9 @@ RefPtr Parser::parse_statement() { RefPtr jump = new Jump; jump->source = source_index; - jump->line = location.line; - jump->keyword = parse_token(); - expect(";"); + jump->line = tokenizer.get_location().line; + jump->keyword = tokenizer.parse_token(); + tokenizer.expect(";"); return jump; } @@ -516,88 +242,88 @@ RefPtr Parser::parse_statement() { RefPtr expr = new ExpressionStatement; expr->source = source_index; - expr->line = location.line; + expr->line = tokenizer.get_location().line; expr->expression = parse_expression(); - expect(";"); + tokenizer.expect(";"); return expr; } else - throw parse_error(location, token, "a statement"); + throw parse_error(tokenizer.get_location(), token, "a statement"); } RefPtr Parser::parse_import() { if(cur_stage->type!=SHARED) - throw invalid_shader_source(location, "Imports are only allowed in the shared section"); + throw invalid_shader_source(tokenizer.get_location(), "Imports are only allowed in the shared section"); - expect("import"); + tokenizer.expect("import"); RefPtr import = new Import; import->source = source_index; - import->line = location.line; + import->line = tokenizer.get_location().line; import->module = expect_identifier(); - expect(";"); + tokenizer.expect(";"); return import; } RefPtr Parser::parse_precision() { - expect("precision"); + tokenizer.expect("precision"); RefPtr precision = new Precision; precision->source = source_index; - precision->line = location.line; + precision->line = tokenizer.get_location().line; - precision->precision = parse_token(); + precision->precision = tokenizer.parse_token(); if(!is_precision_qualifier(precision->precision)) - throw parse_error(location, precision->precision, "a precision qualifier"); + throw parse_error(tokenizer.get_location(), precision->precision, "a precision qualifier"); - precision->type = parse_token(); + precision->type = tokenizer.parse_token(); // Not entirely accurate; only float, int and sampler types are allowed if(!is_builtin_type(precision->type)) - throw parse_error(location, precision->type, "a builtin type"); + throw parse_error(tokenizer.get_location(), precision->type, "a builtin type"); - expect(";"); + tokenizer.expect(";"); return precision; } RefPtr Parser::parse_layout() { - expect("layout"); - expect("("); + tokenizer.expect("layout"); + tokenizer.expect("("); RefPtr layout = new Layout; while(1) { - string token = parse_token(); + string token = tokenizer.parse_token(); if(token==")") - throw parse_error(location, token, "a layout qualifier name"); + throw parse_error(tokenizer.get_location(), token, "a layout qualifier name"); layout->qualifiers.push_back(Layout::Qualifier()); Layout::Qualifier &qual = layout->qualifiers.back(); qual.identifier = token; if(check("=")) - qual.value = parse_token(); + qual.value = tokenizer.parse_token(); - if(peek_token()==")") + if(tokenizer.peek_token()==")") break; - expect(","); + tokenizer.expect(","); } - expect(")"); + tokenizer.expect(")"); return layout; } void Parser::parse_block(Block &block, bool require_braces) { - bool have_braces = (require_braces || peek_token()=="{"); + bool have_braces = (require_braces || tokenizer.peek_token()=="{"); if(have_braces) - expect("{"); + tokenizer.expect("{"); if(have_braces) { - while(peek_token()!="}") + while(tokenizer.peek_token()!="}") block.body.push_back(parse_statement()); } else @@ -606,7 +332,7 @@ void Parser::parse_block(Block &block, bool require_braces) block.use_braces = (require_braces || block.body.size()!=1); if(have_braces) - expect("}"); + tokenizer.expect("}"); } RefPtr Parser::parse_expression(unsigned precedence) @@ -615,11 +341,11 @@ RefPtr Parser::parse_expression(unsigned precedence) VariableReference *left_var = 0; while(1) { - string token = peek_token(); + string token = tokenizer.peek_token(); const Operator *oper = 0; - for(Operator *i=operators; (!oper && i->type); ++i) - if(token==i->token && (!left || i->type!=PREFIX) && (left || i->type!=POSTFIX)) + for(const Operator *i=Operator::operators; (!oper && i->type); ++i) + if(token==i->token && (!left || i->type!=Operator::PREFIX) && (left || i->type!=Operator::POSTFIX)) oper = i; if(token==";" || token==")" || token=="]" || token=="," || (oper && precedence && oper->precedence>=precedence)) @@ -627,52 +353,52 @@ RefPtr Parser::parse_expression(unsigned precedence) if(left) return left; else - throw parse_error(location, token, "an expression"); + throw parse_error(tokenizer.get_location(), token, "an expression"); } else if(left) { if(token=="(") { if(!left_var) - throw invalid_shader_source(location, "Syntax error before '(': function name must be an identifier"); + throw invalid_shader_source(tokenizer.get_location(), "Syntax error before '(': function name must be an identifier"); left = parse_function_call(*left_var); } else if(token==".") { RefPtr memacc = new MemberAccess; memacc->left = left; - parse_token(); + tokenizer.parse_token(); memacc->member = expect_identifier(); left = memacc; } - else if(oper && oper->type==POSTFIX) + else if(oper && oper->type==Operator::POSTFIX) { RefPtr unary = new UnaryExpression; - unary->oper = parse_token(); + unary->oper = tokenizer.parse_token(); unary->prefix = false; unary->expression = left; left = unary; } - else if(oper && oper->type==BINARY) + else if(oper && oper->type==Operator::BINARY) left = parse_binary(left, oper); else - throw parse_error(location, token, "an operator"); + throw parse_error(tokenizer.get_location(), token, "an operator"); left_var = 0; } else { if(token=="(") { - parse_token(); + tokenizer.parse_token(); RefPtr parexpr = new ParenthesizedExpression; parexpr->expression = parse_expression(); - expect(")"); + tokenizer.expect(")"); left = parexpr; } else if(isdigit(token[0]) || token=="true" || token=="false") { RefPtr literal = new Literal; - literal->token = parse_token(); + literal->token = tokenizer.parse_token(); left = literal; } else if(is_identifier(token)) @@ -682,16 +408,16 @@ RefPtr Parser::parse_expression(unsigned precedence) left = var; left_var = var.get(); } - else if(oper && oper->type==PREFIX) + else if(oper && oper->type==Operator::PREFIX) { RefPtr unary = new UnaryExpression; - unary->oper = parse_token(); + unary->oper = tokenizer.parse_token(); unary->prefix = true; unary->expression = parse_expression(oper->precedence); left = unary; } else - throw parse_error(location, token, "an expression"); + throw parse_error(tokenizer.get_location(), token, "an expression"); } } } @@ -700,15 +426,15 @@ RefPtr Parser::parse_binary(const RefPtr &left, co { RefPtr binary = (oper->precedence==16 ? new Assignment : new BinaryExpression); binary->left = left; - binary->oper = parse_token(); + binary->oper = tokenizer.parse_token(); if(binary->oper=="[") { binary->right = parse_expression(); - expect("]"); + tokenizer.expect("]"); binary->after = "]"; } else - binary->right = parse_expression(oper->precedence+(oper->assoc==RIGHT_TO_LEFT)); + binary->right = parse_expression(oper->precedence+(oper->assoc==Operator::RIGHT_TO_LEFT)); return binary; } @@ -717,27 +443,27 @@ RefPtr Parser::parse_function_call(const VariableReference &var) RefPtr call = new FunctionCall; call->name = var.name; call->constructor = is_type(call->name); - expect("("); - while(peek_token()!=")") + tokenizer.expect("("); + while(tokenizer.peek_token()!=")") { if(!call->arguments.empty()) - expect(","); + tokenizer.expect(","); call->arguments.push_back(parse_expression()); } - expect(")"); + tokenizer.expect(")"); return call; } RefPtr Parser::parse_struct_declaration() { - expect("struct"); + tokenizer.expect("struct"); RefPtr strct = new StructDeclaration; strct->source = source_index; - strct->line = location.line; + strct->line = tokenizer.get_location().line; strct->name = expect_identifier(); parse_block(strct->members, true); - expect(";"); + tokenizer.expect(";"); declared_types.insert(strct->name); return strct; @@ -747,12 +473,12 @@ RefPtr Parser::parse_variable_declaration() { RefPtr var = new VariableDeclaration; var->source = source_index; - var->line = location.line; + var->line = tokenizer.get_location().line; - string token = peek_token(); + string token = tokenizer.peek_token(); while(is_qualifier(token)) { - parse_token(); + tokenizer.parse_token(); if(is_interface_qualifier(token)) var->interface = token; else if(is_sampling_qualifier(token)) @@ -763,7 +489,7 @@ RefPtr Parser::parse_variable_declaration() var->precision = token; else if(token=="const") var->constant = true; - token = peek_token(); + token = tokenizer.peek_token(); } var->type = expect_type(); @@ -775,14 +501,14 @@ RefPtr Parser::parse_variable_declaration() if(!check("]")) { var->array_size = parse_expression(); - expect("]"); + tokenizer.expect("]"); } } if(check("=")) var->init_expression = parse_expression(); - expect(";"); + tokenizer.expect(";"); return var; } @@ -790,36 +516,36 @@ RefPtr Parser::parse_function_declaration() { RefPtr func = new FunctionDeclaration; func->source = source_index; - func->line = location.line; + func->line = tokenizer.get_location().line; func->return_type = expect_type(); func->name = expect_identifier(); - expect("("); - while(peek_token()!=")") + tokenizer.expect("("); + while(tokenizer.peek_token()!=")") { if(!func->parameters.empty()) - expect(","); + tokenizer.expect(","); RefPtr var = new VariableDeclaration; - string token = peek_token(); + string token = tokenizer.peek_token(); if(token=="in" || token=="out" || token=="inout") - var->interface = parse_token(); + var->interface = tokenizer.parse_token(); var->type = expect_type(); var->name = expect_identifier(); func->parameters.push_back(var); } - expect(")"); + tokenizer.expect(")"); - string token = peek_token(); + string token = tokenizer.peek_token(); if(token=="{") { func->definition = func.get(); parse_block(func->body, true); } else if(token==";") - parse_token(); + tokenizer.parse_token(); else - throw parse_error(location, token, "'{' or ';'"); + throw parse_error(tokenizer.get_location(), token, "'{' or ';'"); return func; } @@ -828,11 +554,11 @@ RefPtr Parser::parse_interface_block() { RefPtr iface = new InterfaceBlock; iface->source = source_index; - iface->line = location.line; + iface->line = tokenizer.get_location().line; - iface->interface = parse_token(); + iface->interface = tokenizer.parse_token(); if(!is_interface_qualifier(iface->interface)) - throw parse_error(location, iface->interface, "an interface qualifier"); + throw parse_error(tokenizer.get_location(), iface->interface, "an interface qualifier"); iface->name = expect_identifier(); parse_block(iface->members, true); @@ -842,9 +568,9 @@ RefPtr Parser::parse_interface_block() if(check("[")) { iface->array = true; - expect("]"); + tokenizer.expect("]"); } - expect(";"); + tokenizer.expect(";"); } return iface; @@ -852,20 +578,20 @@ RefPtr Parser::parse_interface_block() RefPtr Parser::parse_conditional() { - expect("if"); + tokenizer.expect("if"); RefPtr cond = new Conditional; cond->source = source_index; - cond->line = location.line; - expect("("); + cond->line = tokenizer.get_location().line; + tokenizer.expect("("); cond->condition = parse_expression(); - expect(")"); + tokenizer.expect(")"); parse_block(cond->body, false); - string token = peek_token(); + string token = tokenizer.peek_token(); if(token=="else") { - parse_token(); + tokenizer.parse_token(); parse_block(cond->else_body, false); } @@ -874,12 +600,12 @@ RefPtr Parser::parse_conditional() RefPtr Parser::parse_for() { - expect("for"); + tokenizer.expect("for"); RefPtr loop = new Iteration; loop->source = source_index; - loop->line = location.line; - expect("("); - string token = peek_token(); + loop->line = tokenizer.get_location().line; + tokenizer.expect("("); + string token = tokenizer.peek_token(); if(is_type(token)) loop->init_statement = parse_statement(); else @@ -890,14 +616,14 @@ RefPtr Parser::parse_for() expr->expression = parse_expression(); loop->init_statement = expr; } - expect(";"); + tokenizer.expect(";"); } - if(peek_token()!=";") + if(tokenizer.peek_token()!=";") loop->condition = parse_expression(); - expect(";"); - if(peek_token()!=")") + tokenizer.expect(";"); + if(tokenizer.peek_token()!=")") loop->loop_expression = parse_expression(); - expect(")"); + tokenizer.expect(")"); parse_block(loop->body, false); @@ -906,13 +632,13 @@ RefPtr Parser::parse_for() RefPtr Parser::parse_while() { - expect("while"); + tokenizer.expect("while"); RefPtr loop = new Iteration; loop->source = source_index; - loop->line = location.line; - expect("("); + loop->line = tokenizer.get_location().line; + tokenizer.expect("("); loop->condition = parse_expression(); - expect(")"); + tokenizer.expect(")"); parse_block(loop->body, false); @@ -921,29 +647,29 @@ RefPtr Parser::parse_while() RefPtr Parser::parse_passthrough() { - expect("passthrough"); + tokenizer.expect("passthrough"); RefPtr pass = new Passthrough; pass->source = source_index; - pass->line = location.line; + pass->line = tokenizer.get_location().line; if(cur_stage->type==GEOMETRY) { - expect("["); + tokenizer.expect("["); pass->subscript = parse_expression(); - expect("]"); + tokenizer.expect("]"); } - expect(";"); + tokenizer.expect(";"); return pass; } RefPtr Parser::parse_return() { - expect("return"); + tokenizer.expect("return"); RefPtr ret = new Return; ret->source = source_index; - ret->line = location.line; - if(peek_token()!=";") + ret->line = tokenizer.get_location().line; + if(tokenizer.peek_token()!=";") ret->expression = parse_expression(); - expect(";"); + tokenizer.expect(";"); return ret; } diff --git a/source/glsl/parser.h b/source/glsl/parser.h index 743149d8..6e4bb38b 100644 --- a/source/glsl/parser.h +++ b/source/glsl/parser.h @@ -1,63 +1,30 @@ #ifndef MSP_GL_SL_PARSER_H_ #define MSP_GL_SL_PARSER_H_ -#include -#include #include #include #include +#include "preprocessor.h" #include "syntax.h" +#include "tokenizer.h" namespace Msp { namespace GL { namespace SL { -struct Location -{ - std::string name; - unsigned line; -}; - class Parser { private: - enum OperatorType - { - NO_OPERATOR, - BINARY, - PREFIX, - POSTFIX - }; - - enum Associativity - { - LEFT_TO_RIGHT, - RIGHT_TO_LEFT - }; - - struct Operator - { - const char token[4]; - unsigned precedence; - OperatorType type; - Associativity assoc; - }; - std::string source; + std::string source_name; unsigned source_index; - Location location; - std::string::const_iterator iter; - std::string::const_iterator source_end; - bool allow_preprocess; + Tokenizer tokenizer; + Preprocessor preprocessor; bool allow_stage_change; - std::string last_token; - std::deque next_tokens; Module *module; Stage *cur_stage; std::set declared_types; - static Operator operators[]; - public: Parser(); ~Parser(); @@ -66,16 +33,10 @@ public: Module &parse(IO::Base &, const std::string &, unsigned = 0); private: - void parse_source(); + void parse_source(const std::string &); + void set_required_version(const Version &); + void stage_change(StageType); - const std::string &peek_token(unsigned = 0); - const std::string &parse_token(); - std::string parse_token_(); - std::string parse_identifier(); - std::string parse_number(); - std::string parse_other(); - void skip_comment_and_whitespace(); - void expect(const std::string &); std::string expect_type(); std::string expect_identifier(); bool check(const std::string &); diff --git a/source/glsl/preprocessor.cpp b/source/glsl/preprocessor.cpp new file mode 100644 index 00000000..3c190221 --- /dev/null +++ b/source/glsl/preprocessor.cpp @@ -0,0 +1,85 @@ +#include "glsl_error.h" +#include "preprocessor.h" +#include "tokenizer.h" + +using namespace std; + +namespace Msp { +namespace GL { +namespace SL { + +Preprocessor::Preprocessor(Tokenizer &t): + tokenizer(t) +{ } + +void Preprocessor::preprocess() +{ + tokenizer.expect("#"); + + string token = tokenizer.peek_token(); + if(token=="pragma") + preprocess_pragma(); + else if(token=="version") + preprocess_version(); + else if(token=="define" || token=="undef" || token=="if" || token=="ifdef" || token=="ifndef" || token=="else" || + token=="elif" || token=="endif" || token=="error" || token=="extension" || token=="line") + throw invalid_shader_source(tokenizer.get_location(), "Unsupported preprocessor directive '%s'", token); + else if(!token.empty()) + throw parse_error(tokenizer.get_location(), token, "a preprocessor directive"); +} + +void Preprocessor::preprocess_version() +{ + tokenizer.expect("version"); + string token = tokenizer.parse_token(); + unsigned version = lexical_cast(token); + signal_version.emit(Version(version/100, version%100)); + + token = tokenizer.parse_token(); + if(!token.empty()) + throw parse_error(tokenizer.get_location(), token, "end of line"); +} + +void Preprocessor::preprocess_pragma() +{ + tokenizer.expect("pragma"); + string token = tokenizer.parse_token(); + if(token=="MSP") + preprocess_pragma_msp(); +} + +void Preprocessor::preprocess_pragma_msp() +{ + string token = tokenizer.peek_token(); + if(token=="stage") + preprocess_stage(); + else + throw invalid_shader_source(tokenizer.get_location(), "Unrecognized MSP pragma '%s'", token); + + token = tokenizer.parse_token(); + if(!token.empty()) + throw parse_error(tokenizer.get_location(), token, "end of line"); +} + +void Preprocessor::preprocess_stage() +{ + tokenizer.expect("stage"); + tokenizer.expect("("); + string token = tokenizer.parse_token(); + StageType stage = SHARED; + if(token=="vertex") + stage = VERTEX; + else if(token=="geometry") + stage = GEOMETRY; + else if(token=="fragment") + stage = FRAGMENT; + else + throw parse_error(tokenizer.get_location(), token, "stage identifier"); + tokenizer.expect(")"); + + signal_stage_change.emit(stage); +} + +} // namespace SL +} // namespace GL +} // namespace Msp diff --git a/source/glsl/preprocessor.h b/source/glsl/preprocessor.h new file mode 100644 index 00000000..96970301 --- /dev/null +++ b/source/glsl/preprocessor.h @@ -0,0 +1,37 @@ +#ifndef MSP_GL_SL_PREPROCESSOR_H_ +#define MSP_GL_SL_PREPROCESSOR_H_ + +#include +#include "syntax.h" + +namespace Msp { +namespace GL { +namespace SL { + +class Tokenizer; + +class Preprocessor +{ +public: + sigc::signal signal_version; + sigc::signal signal_stage_change; + +private: + Tokenizer &tokenizer; + +public: + Preprocessor(Tokenizer &); + + void preprocess(); +private: + void preprocess_version(); + void preprocess_pragma(); + void preprocess_pragma_msp(); + void preprocess_stage(); +}; + +} // namespace SL +} // namespace GL +} // namespace Msp + +#endif diff --git a/source/glsl/syntax.cpp b/source/glsl/syntax.cpp index 9705c989..e7a5c155 100644 --- a/source/glsl/syntax.cpp +++ b/source/glsl/syntax.cpp @@ -6,6 +6,56 @@ namespace Msp { namespace GL { namespace SL { +const Operator Operator::operators[] = +{ + { "[", 2, BINARY, LEFT_TO_RIGHT }, + { "(", 2, BINARY, LEFT_TO_RIGHT }, + { ".", 2, BINARY, LEFT_TO_RIGHT }, + { "++", 2, POSTFIX, LEFT_TO_RIGHT }, + { "--", 2, POSTFIX, LEFT_TO_RIGHT }, + { "++", 3, PREFIX, RIGHT_TO_LEFT }, + { "--", 3, PREFIX, RIGHT_TO_LEFT }, + { "+", 3, PREFIX, RIGHT_TO_LEFT }, + { "-", 3, PREFIX, RIGHT_TO_LEFT }, + { "~", 3, PREFIX, RIGHT_TO_LEFT }, + { "!", 3, PREFIX, RIGHT_TO_LEFT }, + { "*", 4, BINARY, LEFT_TO_RIGHT }, + { "/", 4, BINARY, LEFT_TO_RIGHT }, + { "%", 4, BINARY, LEFT_TO_RIGHT }, + { "+", 5, BINARY, LEFT_TO_RIGHT }, + { "-", 5, BINARY, LEFT_TO_RIGHT }, + { "<<", 6, BINARY, LEFT_TO_RIGHT }, + { ">>", 6, BINARY, LEFT_TO_RIGHT }, + { "<", 7, BINARY, LEFT_TO_RIGHT }, + { ">", 7, BINARY, LEFT_TO_RIGHT }, + { "<=", 7, BINARY, LEFT_TO_RIGHT }, + { ">=", 7, BINARY, LEFT_TO_RIGHT }, + { "==", 8, BINARY, LEFT_TO_RIGHT }, + { "!=", 8, BINARY, LEFT_TO_RIGHT }, + { "&", 9, BINARY, LEFT_TO_RIGHT }, + { "^", 10, BINARY, LEFT_TO_RIGHT }, + { "|", 11, BINARY, LEFT_TO_RIGHT }, + { "&&", 12, BINARY, LEFT_TO_RIGHT }, + { "^^", 13, BINARY, LEFT_TO_RIGHT }, + { "||", 14, BINARY, LEFT_TO_RIGHT }, + { "?", 15, BINARY, RIGHT_TO_LEFT }, + { ":", 15, BINARY, RIGHT_TO_LEFT }, + { "=", 16, BINARY, RIGHT_TO_LEFT }, + { "+=", 16, BINARY, RIGHT_TO_LEFT }, + { "-=", 16, BINARY, RIGHT_TO_LEFT }, + { "*=", 16, BINARY, RIGHT_TO_LEFT }, + { "/=", 16, BINARY, RIGHT_TO_LEFT }, + { "%=", 16, BINARY, RIGHT_TO_LEFT }, + { "<<=", 16, BINARY, RIGHT_TO_LEFT }, + { ">>=", 16, BINARY, RIGHT_TO_LEFT }, + { "&=", 16, BINARY, RIGHT_TO_LEFT }, + { "^=", 16, BINARY, RIGHT_TO_LEFT }, + { "|=", 16, BINARY, RIGHT_TO_LEFT }, + { ",", 17, BINARY, LEFT_TO_RIGHT }, + { { 0 }, 18, NO_OPERATOR, LEFT_TO_RIGHT } +}; + + template NodeContainer::NodeContainer(const NodeContainer &c): C(c) diff --git a/source/glsl/syntax.h b/source/glsl/syntax.h index a2630ef0..deb2c780 100644 --- a/source/glsl/syntax.h +++ b/source/glsl/syntax.h @@ -16,6 +16,30 @@ namespace Msp { namespace GL { namespace SL { +struct Operator +{ + enum Type + { + NO_OPERATOR, + BINARY, + PREFIX, + POSTFIX + }; + + enum Associativity + { + LEFT_TO_RIGHT, + RIGHT_TO_LEFT + }; + + char token[4]; + unsigned precedence; + Type type; + Associativity assoc; + + static const Operator operators[]; +}; + struct NodeVisitor; struct Node diff --git a/source/glsl/tokenizer.cpp b/source/glsl/tokenizer.cpp new file mode 100644 index 00000000..be432143 --- /dev/null +++ b/source/glsl/tokenizer.cpp @@ -0,0 +1,206 @@ +#include +#include "glsl_error.h" +#include "preprocessor.h" +#include "syntax.h" +#include "tokenizer.h" + +using namespace std; + +namespace Msp { +namespace GL { +namespace SL { + +Tokenizer::Tokenizer(): + allow_preprocess(true) +{ + static string empty; + iter = empty.begin(); + source_end = empty.end(); + location.line = 0; +} + +void Tokenizer::begin(const string &name, const string &src) +{ + iter = src.begin(); + source_end = src.end(); + location.name = name; + location.line = 1; + allow_preprocess = true; + last_token.clear(); + next_tokens.clear(); +} + +const string &Tokenizer::peek_token(unsigned index) +{ + while(next_tokens.size()<=index) + next_tokens.push_back(parse_token_()); + return (last_token = next_tokens[index]); +} + +const string &Tokenizer::parse_token() +{ + if(!next_tokens.empty()) + { + last_token = next_tokens.front(); + next_tokens.pop_front(); + return last_token; + } + + return (last_token = parse_token_()); +} + +void Tokenizer::expect(const string &token) +{ + string parsed = parse_token(); + if(parsed!=token) + throw parse_error(location, parsed, format("'%s'", token)); +} + +string Tokenizer::parse_token_() +{ + while(1) + { + skip_comment_and_whitespace(); + if(iter==source_end) + return string(); + else if(allow_preprocess && *iter=='#') + { + allow_preprocess = false; + preprocess(); + } + else if(isalpha(*iter) || *iter=='_') + return parse_identifier(); + else if(isdigit(*iter)) + return parse_number(); + else + return parse_other(); + } +} + +void Tokenizer::preprocess() +{ + SetForScope > clear_tokens(next_tokens, deque()); + + string::const_iterator line_end = iter; + for(; (line_end!=source_end && *line_end!='\n'); ++line_end) ; + SetForScope stop_at_line_end(source_end, line_end); + + signal_preprocess.emit(); + + iter = line_end; +} + +string Tokenizer::parse_identifier() +{ + string ident; + while(iter!=source_end) + { + if(isalnum(*iter) || *iter=='_') + ident += *iter++; + else + break; + } + + return ident; +} + +string Tokenizer::parse_number() +{ + bool accept_sign = false; + string number; + while(iter!=source_end) + { + if(isdigit(*iter) || *iter=='.') + number += *iter++; + else if(*iter=='e' || *iter=='E') + { + number += *iter++; + accept_sign = true; + } + else if(accept_sign && (*iter=='+' || *iter=='-')) + number += *iter++; + else + break; + } + + return number; +} + +string Tokenizer::parse_other() +{ + if(iter==source_end) + return string(); + + string token(1, *iter++); + for(unsigned i=1; (i<3 && iter!=source_end); ++i) + { + bool matched = false; + for(const Operator *j=Operator::operators; (!matched && j->type); ++j) + { + matched = (j->token[i]==*iter); + for(unsigned k=0; (matched && ktoken[k]); ++k) + matched = (j->token[k]==token[k]); + } + + if(!matched) + break; + + token += *iter++; + } + + return token; +} + +void Tokenizer::skip_comment_and_whitespace() +{ + unsigned comment = 0; + while(iter!=source_end) + { + if(comment==0) + { + if(*iter=='/') + comment = 1; + else if(!isspace(*iter)) + break; + } + else if(comment==1) + { + if(*iter=='/') + comment = 2; + else if(*iter=='*') + comment = 3; + else + { + comment = 0; + --iter; + break; + } + } + else if(comment==2) + { + if(*iter=='\n') + comment = 0; + } + else if(comment==3 && *iter=='*') + comment = 4; + else if(comment==4) + { + if(*iter=='/') + comment = 0; + else if(*iter!='*') + comment = 3; + } + + if(*iter=='\n') + { + ++location.line; + allow_preprocess = (comment<3); + } + + ++iter; + } +} + +} // namespace SL +} // namespace GL +} // namespace Msp diff --git a/source/glsl/tokenizer.h b/source/glsl/tokenizer.h new file mode 100644 index 00000000..a7329182 --- /dev/null +++ b/source/glsl/tokenizer.h @@ -0,0 +1,54 @@ +#ifndef MSP_GL_SL_TOKENIZER_H_ +#define MSP_GL_SL_TOKENIZER_H_ + +#include +#include +#include + +namespace Msp { +namespace GL { +namespace SL { + +class Preprocessor; + +struct Location +{ + std::string name; + unsigned line; +}; + +class Tokenizer +{ +public: + sigc::signal signal_preprocess; + +private: + std::string::const_iterator iter; + std::string::const_iterator source_end; + Location location; + bool allow_preprocess; + std::string last_token; + std::deque next_tokens; + +public: + Tokenizer(); + + void begin(const std::string &, const std::string &); + const std::string &peek_token(unsigned = 0); + const std::string &parse_token(); + void expect(const std::string &); + const Location &get_location() const { return location; } +private: + std::string parse_token_(); + void preprocess(); + std::string parse_identifier(); + std::string parse_number(); + std::string parse_other(); + void skip_comment_and_whitespace(); +}; + +} // namespace SL +} // namespace GL +} // namespace Msp + +#endif -- 2.43.0