1 #include <msp/core/algorithm.h>
2 #include <msp/core/raii.h>
3 #include <msp/strings/format.h>
4 #include <msp/strings/regex.h>
5 #include <msp/strings/utils.h>
7 #include "glsl_error.h"
8 #include "modulecache.h"
19 Parser::Parser(ModuleCache *s):
21 preprocessor(tokenizer),
24 tokenizer.signal_preprocess.connect(sigc::mem_fun(&preprocessor, &Preprocessor::preprocess));
25 preprocessor.signal_version.connect(sigc::mem_fun(this, &Parser::set_required_version));
26 preprocessor.signal_source.connect(sigc::mem_fun(this, &Parser::source_reference));
27 preprocessor.signal_stage_change.connect(sigc::mem_fun(this, &Parser::stage_change));
28 preprocessor.signal_line.connect(sigc::mem_fun(this, &Parser::line_change));
31 void Parser::parse(Module &m, const string &s, const string &n, int i)
33 SetForScope<Module *> set_module(module, &m);
38 void Parser::parse(Module &m, IO::Base &io, const string &n, int i)
40 SetForScope<Module *> set_module(module, &m);
45 unsigned len = io.read(buffer, sizeof(buffer));
46 source.append(buffer, len);
51 void Parser::parse_source(const string &name, int index)
53 cur_stage = &module->shared;
57 source_reference(1, name);
59 if(const Stage *builtin = get_builtins(Stage::SHARED))
61 for(const auto &kvp: builtin->types)
62 global_types.insert(kvp.first);
65 tokenizer.begin(source, name);
66 allow_stage_change = true;
67 while(!tokenizer.peek_token().empty())
68 if(RefPtr<Statement> statement = parse_with_recovery(&Parser::parse_global_declaration))
69 cur_stage->content.body.push_back(statement);
72 throw invalid_shader_source(join(errors.begin(), errors.end(), "\n"));
75 void Parser::set_required_version(const Version &ver)
77 cur_stage->required_features.glsl_version = ver;
80 void Parser::source_reference(unsigned index, const string &name)
82 if(index<1 || base_index<0)
83 throw invalid_shader_source(tokenizer.get_location(), "Invalid source reference");
85 module->source_map.set_name(base_index+index-1, name);
88 void Parser::stage_change(Stage::Type stage)
90 if(!allow_stage_change)
91 throw invalid_shader_source(tokenizer.get_location(), "Changing stage not allowed here");
92 else if(stage<=cur_stage->type)
93 throw invalid_shader_source(tokenizer.get_location(), "Stage '%s' not allowed here", Stage::get_stage_name(stage));
95 module->stages.push_back(stage);
97 if(cur_stage->type!=Stage::SHARED)
98 module->stages.back().previous = cur_stage;
99 cur_stage = &module->stages.back();
102 for(const Module *m: imported_modules)
104 auto j = find_member(m->stages, stage, &Stage::type);
105 if(j!=m->stages.end())
107 for(const auto &kvp: j->types)
108 stage_types.insert(kvp.first);
113 void Parser::line_change(int index, unsigned line)
116 source_index = base_index+index-1;
120 index = source_index;
122 string name = module->source_map.get_name(index);
124 name = format("<%d>", index);
125 tokenizer.set_location(Location(name, line));
128 string Parser::expect_type()
130 string token = tokenizer.parse_token();
132 throw parse_error(tokenizer.get_location(), token, "a type");
136 string Parser::expect_identifier()
138 string token = tokenizer.parse_token();
139 if(!is_identifier(token))
140 throw parse_error(tokenizer.get_location(), token, "an identifier");
144 int Parser::expect_integer()
146 string token = tokenizer.parse_token();
148 throw parse_error(tokenizer.get_location(), token, "an integer literal");
149 return lexical_cast<int>(token);
152 bool Parser::check(const string &token)
154 bool result = (tokenizer.peek_token()==token);
156 tokenizer.parse_token();
160 bool Parser::is_interface_qualifier(const string &token)
162 return (token=="uniform" || token=="in" || token=="out");
165 bool Parser::is_sampling_qualifier(const string &token)
167 return (token=="centroid" || token=="sample");
170 bool Parser::is_interpolation_qualifier(const string &token)
172 return (token=="smooth" || token=="flat" || token=="noperspective");
175 bool Parser::is_precision_qualifier(const string &token)
177 return (token=="highp" || token=="mediump" || token=="lowp");
180 bool Parser::is_qualifier(const string &token)
182 return (token=="const" ||
183 is_interface_qualifier(token) ||
184 is_sampling_qualifier(token) ||
185 is_interpolation_qualifier(token) ||
186 is_precision_qualifier(token));
189 bool Parser::is_type(const string &token)
191 return global_types.count(token) || stage_types.count(token);
194 bool Parser::is_identifier(const string &token)
196 static Regex re("^[a-zA-Z_][a-zA-Z0-9_]*$");
197 return re.match(token);
201 RefPtr<T> Parser::create_node()
203 RefPtr<T> node = new T;
204 node->source = source_index;
205 node->line = tokenizer.get_location().line;
210 RefPtr<T> Parser::parse_with_recovery(RefPtr<T> (Parser::*parse_func)())
212 tokenizer.clear_progress_mark();
215 return (this->*parse_func)();
217 catch(const invalid_shader_source &exc)
219 errors.push_back(exc.what());
222 if(tokenizer.get_last_token()!=";" || !tokenizer.get_progress_mark())
224 unsigned scope_level = 0;
227 if(tokenizer.peek_token()=="}" && scope_level==0)
229 if(!tokenizer.get_progress_mark())
230 tokenizer.parse_token();
234 string token = tokenizer.parse_token();
243 else if(token==";" && scope_level==0)
245 else if(token.empty())
253 RefPtr<Statement> Parser::parse_global_declaration()
255 string token = tokenizer.peek_token();
256 SetFlag disallow(allow_stage_change, false);
259 return parse_import();
260 else if(token=="precision")
261 return parse_precision();
262 else if(token=="layout")
264 RefPtr<Layout> layout = parse_layout();
265 token = tokenizer.peek_token();
266 if(is_interface_qualifier(token) && tokenizer.peek_token(1)==";")
268 RefPtr<InterfaceLayout> iface_lo = create_node<InterfaceLayout>();
269 iface_lo->layout.source = layout->source;
270 iface_lo->layout.line = layout->line;
271 iface_lo->layout.qualifiers = layout->qualifiers;
272 iface_lo->interface = tokenizer.parse_token();
273 tokenizer.expect(";");
276 else if(is_interface_qualifier(token) && tokenizer.peek_token(2)=="{")
278 RefPtr<InterfaceBlock> iface = parse_interface_block();
279 iface->layout = layout;
284 RefPtr<VariableDeclaration> var = parse_variable_declaration();
285 var->layout = layout;
289 else if(token=="typedef")
290 return parse_type_declaration();
291 else if(token=="struct")
292 return parse_struct_declaration();
293 else if(is_interface_qualifier(token))
295 string next = tokenizer.peek_token(1);
296 if(is_type(next) || is_qualifier(next))
297 return parse_variable_declaration();
299 return parse_interface_block();
301 else if(is_qualifier(token))
302 return parse_variable_declaration();
303 else if(token=="virtual")
304 return parse_function_declaration();
305 else if(is_type(token))
307 if(tokenizer.peek_token(2)=="(")
308 return parse_function_declaration();
310 return parse_variable_declaration();
312 else if(token.empty())
315 throw parse_error(tokenizer.get_location(), token, "a global declaration");
318 RefPtr<Statement> Parser::parse_statement()
320 string token = tokenizer.peek_token();
322 return parse_conditional();
323 else if(token=="for")
325 else if(token=="while")
326 return parse_while();
327 else if(token=="passthrough")
328 return parse_passthrough();
329 else if(token=="return")
330 return parse_return();
331 else if(token=="break" || token=="continue" || token=="discard")
333 RefPtr<Jump> jump = create_node<Jump>();
334 jump->keyword = tokenizer.parse_token();
335 tokenizer.expect(";");
339 else if(is_qualifier(token) || is_type(token))
340 return parse_variable_declaration();
343 tokenizer.parse_token();
344 throw invalid_shader_source(tokenizer.get_location(), "Empty statement not allowed");
346 else if(!token.empty())
348 RefPtr<ExpressionStatement> expr = create_node<ExpressionStatement>();
349 expr->expression = parse_expression();
350 tokenizer.expect(";");
355 throw parse_error(tokenizer.get_location(), token, "a statement");
358 RefPtr<Import> Parser::parse_import()
360 if(cur_stage->type!=Stage::SHARED)
361 throw invalid_shader_source(tokenizer.get_location(), "Imports are only allowed in the shared section");
363 tokenizer.expect("import");
364 RefPtr<Import> import = create_node<Import>();
365 import->module = expect_identifier();
366 tokenizer.expect(";");
370 const Module &imported_mod = mod_cache->get_module(import->module);
371 imported_modules.push_back(&imported_mod);
372 for(const auto &kvp: imported_mod.shared.types)
373 global_types.insert(kvp.first);
379 RefPtr<Precision> Parser::parse_precision()
381 tokenizer.expect("precision");
382 RefPtr<Precision> precision = create_node<Precision>();
384 precision->precision = tokenizer.parse_token();
385 if(!is_precision_qualifier(precision->precision))
386 throw parse_error(tokenizer.get_location(), precision->precision, "a precision qualifier");
388 // TODO Add validation for this
389 precision->type = expect_type();
391 tokenizer.expect(";");
396 RefPtr<Layout> Parser::parse_layout()
398 tokenizer.expect("layout");
399 tokenizer.expect("(");
400 RefPtr<Layout> layout = create_node<Layout>();
403 string token = tokenizer.parse_token();
405 throw parse_error(tokenizer.get_location(), token, "a layout qualifier name");
407 layout->qualifiers.push_back(token);
408 Layout::Qualifier &qual = layout->qualifiers.back();
410 if((qual.has_value = check("=")))
412 if(qual.name=="constant_id" && tokenizer.peek_token()=="auto")
415 tokenizer.parse_token();
418 qual.value = expect_integer();
421 if(tokenizer.peek_token()==")")
424 tokenizer.expect(",");
426 tokenizer.expect(")");
432 void Parser::parse_block(Block &block, bool require_braces, RefPtr<T> (Parser::*parse_content)())
434 bool have_braces = (require_braces || tokenizer.peek_token()=="{");
436 tokenizer.expect("{");
440 while(tokenizer.peek_token()!="}")
441 if(RefPtr<Statement> node = parse_with_recovery(parse_content))
442 block.body.push_back(node);
445 block.body.push_back((this->*parse_content)());
447 block.use_braces = (require_braces || block.body.size()!=1);
450 tokenizer.expect("}");
453 RefPtr<Expression> Parser::parse_expression(const Operator *outer_oper)
455 unsigned outer_precedence = (outer_oper ? outer_oper->precedence+(outer_oper->assoc==Operator::RIGHT_TO_LEFT) : 20);
456 RefPtr<Expression> left;
457 VariableReference *left_var = 0;
460 string token = tokenizer.peek_token();
462 const Operator *oper = 0;
463 for(const Operator *i=Operator::operators; (!oper && i->type); ++i)
464 if(token==i->token && (!left || i->type!=Operator::PREFIX) && (left || i->type!=Operator::POSTFIX))
467 bool lower_precedence = (oper && oper->type!=Operator::PREFIX && oper->precedence>=outer_precedence);
468 if(token==";" || token==")" || token=="]" || token=="," || token==":" || lower_precedence)
473 throw parse_error(tokenizer.get_location(), token, "an expression");
480 throw invalid_shader_source(tokenizer.get_location(), "Syntax error before '(': function name must be an identifier");
481 left = parse_function_call(*left_var);
485 RefPtr<MemberAccess> memacc = create_node<MemberAccess>();
488 tokenizer.parse_token();
489 memacc->member = expect_identifier();
492 else if(oper && oper->type==Operator::POSTFIX)
494 RefPtr<UnaryExpression> unary = create_node<UnaryExpression>();
496 tokenizer.parse_token();
497 unary->expression = left;
500 else if(oper && oper->type==Operator::BINARY)
501 left = parse_binary(left, *oper);
502 else if(oper && oper->type==Operator::TERNARY)
503 left = parse_ternary(left, *oper);
505 throw parse_error(tokenizer.get_location(), token, "an operator");
512 tokenizer.parse_token();
513 left = parse_expression();
514 tokenizer.expect(")");
516 else if(isdigit(token[0]) || token=="true" || token=="false")
517 left = parse_literal();
518 else if(is_identifier(token))
520 RefPtr<VariableReference> var = create_node<VariableReference>();
521 var->name = expect_identifier();
523 left_var = var.get();
525 else if(oper && oper->type==Operator::PREFIX)
527 RefPtr<UnaryExpression> unary = create_node<UnaryExpression>();
529 tokenizer.parse_token();
530 unary->expression = parse_expression(oper);
534 throw parse_error(tokenizer.get_location(), token, "an expression");
539 RefPtr<Literal> Parser::parse_literal()
541 RefPtr<Literal> literal = create_node<Literal>();
542 literal->token = tokenizer.parse_token();
543 if(isdigit(literal->token[0]))
545 // TODO have the tokenizer return the type of the token
546 if(literal->token.back()=='u')
547 literal->value = lexical_cast<unsigned>(literal->token.substr(0, literal->token.size()-1));
548 else if(literal->token.back()=='f')
549 literal->value = lexical_cast<float>(literal->token.substr(0, literal->token.size()-1));
550 else if(literal->token.find('.')!=string::npos)
551 literal->value = lexical_cast<float>(literal->token);
553 literal->value = lexical_cast<int>(literal->token);
555 else if(literal->token=="true" || literal->token=="false")
556 literal->value = (literal->token=="true");
558 throw parse_error(tokenizer.get_location(), literal->token, "a literal");
563 RefPtr<BinaryExpression> Parser::parse_binary(const RefPtr<Expression> &left, const Operator &oper)
565 RefPtr<BinaryExpression> binary = (oper.precedence==16 ?
566 static_cast<RefPtr<BinaryExpression> >(create_node<Assignment>()) : create_node<BinaryExpression>());
568 binary->oper = &oper;
569 tokenizer.expect(oper.token);
572 binary->right = parse_expression();
573 tokenizer.expect(oper.token2);
576 binary->right = parse_expression(&oper);
580 RefPtr<TernaryExpression> Parser::parse_ternary(const RefPtr<Expression> &cond, const Operator &oper)
582 RefPtr<TernaryExpression> ternary = create_node<TernaryExpression>();
583 ternary->condition = cond;
584 ternary->oper = &oper;
585 tokenizer.expect("?");
586 ternary->true_expr = parse_expression(&oper);
587 tokenizer.expect(":");
588 ternary->false_expr = parse_expression(&oper);
592 RefPtr<FunctionCall> Parser::parse_function_call(const VariableReference &var)
594 RefPtr<FunctionCall> call = create_node<FunctionCall>();
595 call->name = var.name;
596 call->oper = &Operator::get_operator("(", Operator::POSTFIX);
597 tokenizer.expect("(");
598 while(tokenizer.peek_token()!=")")
600 if(!call->arguments.empty())
601 tokenizer.expect(",");
602 call->arguments.push_back(parse_expression());
604 tokenizer.expect(")");
608 RefPtr<TypeDeclaration> Parser::parse_type_declaration()
610 tokenizer.expect("typedef");
612 RefPtr<TypeDeclaration> type;
613 if(tokenizer.peek_token()=="image")
614 type = parse_image_type_declaration();
616 type = parse_basic_type_declaration();
618 tokenizer.expect(";");
619 cur_stage->types[type->name] = type.get();
620 if(cur_stage->type==Stage::SHARED)
621 global_types.insert(type->name);
623 stage_types.insert(type->name);
627 RefPtr<BasicTypeDeclaration> Parser::parse_basic_type_declaration()
629 RefPtr<BasicTypeDeclaration> type = create_node<BasicTypeDeclaration>();
631 if(tokenizer.peek_token()=="vector")
633 type->kind = BasicTypeDeclaration::VECTOR;
635 tokenizer.parse_token();
636 tokenizer.expect("(");
637 type->size = expect_integer();
638 tokenizer.expect(")");
641 type->base = expect_type();
642 type->name = expect_identifier();
644 if(type->kind==BasicTypeDeclaration::ALIAS && check("["))
646 type->kind = BasicTypeDeclaration::ARRAY;
647 tokenizer.expect("]");
653 RefPtr<ImageTypeDeclaration> Parser::parse_image_type_declaration()
655 tokenizer.expect("image");
656 tokenizer.expect("(");
658 RefPtr<ImageTypeDeclaration> type = create_node<ImageTypeDeclaration>();
661 string token = tokenizer.parse_token();
662 if(token=="dimensions")
664 tokenizer.expect("=");
665 token = tokenizer.parse_token();
667 type->dimensions = ImageTypeDeclaration::ONE;
669 type->dimensions = ImageTypeDeclaration::TWO;
671 type->dimensions = ImageTypeDeclaration::THREE;
672 else if(token=="cube")
673 type->dimensions = ImageTypeDeclaration::CUBE;
675 throw parse_error(tokenizer.get_location(), token, "dimensions");
680 tokenizer.expect("]");
683 else if(token=="sampled")
684 type->sampled = true;
685 else if(token=="shadow")
688 throw parse_error(tokenizer.get_location(), token, "image type attribute");
690 token = tokenizer.peek_token();
694 tokenizer.expect(",");
696 tokenizer.expect(")");
698 type->base = expect_type();
699 type->name = expect_identifier();
704 RefPtr<StructDeclaration> Parser::parse_struct_declaration()
706 tokenizer.expect("struct");
707 RefPtr<StructDeclaration> strct = create_node<StructDeclaration>();
709 strct->name = expect_identifier();
710 parse_block(strct->members, true, &Parser::parse_variable_declaration);
711 tokenizer.expect(";");
713 cur_stage->types[strct->name] = strct.get();
714 if(cur_stage->type==Stage::SHARED)
715 global_types.insert(strct->name);
717 stage_types.insert(strct->name);
721 RefPtr<VariableDeclaration> Parser::parse_variable_declaration()
723 RefPtr<VariableDeclaration> var = create_node<VariableDeclaration>();
725 string token = tokenizer.peek_token();
726 while(is_qualifier(token))
728 tokenizer.parse_token();
729 if(is_interface_qualifier(token))
730 var->interface = token;
731 else if(is_sampling_qualifier(token))
732 var->sampling = token;
733 else if(is_interpolation_qualifier(token))
734 var->interpolation = token;
735 else if(is_precision_qualifier(token))
736 var->precision = token;
737 else if(token=="const")
738 var->constant = true;
739 token = tokenizer.peek_token();
742 var->type = expect_type();
743 var->name = expect_identifier();
750 var->array_size = parse_expression();
751 tokenizer.expect("]");
756 var->init_expression = parse_expression();
758 tokenizer.expect(";");
762 RefPtr<VariableDeclaration> Parser::parse_variable_declaration_with_layout()
764 RefPtr<Layout> layout;
765 if(tokenizer.peek_token()=="layout")
766 layout = parse_layout();
768 RefPtr<VariableDeclaration> var = parse_variable_declaration();
769 var->layout = layout;
774 RefPtr<FunctionDeclaration> Parser::parse_function_declaration()
776 RefPtr<FunctionDeclaration> func = create_node<FunctionDeclaration>();
778 func->virtua = check("virtual");
779 func->return_type = expect_type();
780 func->name = expect_identifier();
781 tokenizer.expect("(");
782 while(tokenizer.peek_token()!=")")
784 if(!func->parameters.empty())
785 tokenizer.expect(",");
787 RefPtr<VariableDeclaration> var = create_node<VariableDeclaration>();
788 string token = tokenizer.peek_token();
789 if(token=="in" || token=="out" || token=="inout")
790 var->interface = tokenizer.parse_token();
791 var->type = expect_type();
792 var->name = expect_identifier();
793 func->parameters.push_back(var);
795 tokenizer.expect(")");
797 func->overrd = check("override");
799 string token = tokenizer.peek_token();
802 func->definition = func.get();
803 parse_block(func->body, true, &Parser::parse_statement);
806 tokenizer.parse_token();
808 throw parse_error(tokenizer.get_location(), token, "'{' or ';'");
813 RefPtr<InterfaceBlock> Parser::parse_interface_block()
815 RefPtr<InterfaceBlock> iface = create_node<InterfaceBlock>();
817 iface->interface = tokenizer.parse_token();
818 if(!is_interface_qualifier(iface->interface))
819 throw parse_error(tokenizer.get_location(), iface->interface, "an interface qualifier");
821 iface->block_name = expect_identifier();
822 iface->members = new Block;
823 parse_block(*iface->members, true, &Parser::parse_variable_declaration_with_layout);
826 iface->instance_name = expect_identifier();
830 tokenizer.expect("]");
832 tokenizer.expect(";");
838 RefPtr<Conditional> Parser::parse_conditional()
840 tokenizer.expect("if");
841 RefPtr<Conditional> cond = create_node<Conditional>();
842 tokenizer.expect("(");
843 cond->condition = parse_expression();
844 tokenizer.expect(")");
846 parse_block(cond->body, false, &Parser::parse_statement);
848 string token = tokenizer.peek_token();
851 tokenizer.parse_token();
852 parse_block(cond->else_body, false, &Parser::parse_statement);
858 RefPtr<Iteration> Parser::parse_for()
860 tokenizer.expect("for");
861 RefPtr<Iteration> loop = create_node<Iteration>();
862 tokenizer.expect("(");
863 string token = tokenizer.peek_token();
865 loop->init_statement = parse_statement();
870 RefPtr<ExpressionStatement> expr = create_node<ExpressionStatement>();
871 expr->expression = parse_expression();
872 loop->init_statement = expr;
874 tokenizer.expect(";");
876 if(tokenizer.peek_token()!=";")
877 loop->condition = parse_expression();
878 tokenizer.expect(";");
879 if(tokenizer.peek_token()!=")")
880 loop->loop_expression = parse_expression();
881 tokenizer.expect(")");
883 parse_block(loop->body, false, &Parser::parse_statement);
888 RefPtr<Iteration> Parser::parse_while()
890 tokenizer.expect("while");
891 RefPtr<Iteration> loop = create_node<Iteration>();
892 tokenizer.expect("(");
893 loop->condition = parse_expression();
894 tokenizer.expect(")");
896 parse_block(loop->body, false, &Parser::parse_statement);
901 RefPtr<Passthrough> Parser::parse_passthrough()
903 tokenizer.expect("passthrough");
904 RefPtr<Passthrough> pass = create_node<Passthrough>();
905 if(cur_stage->type==Stage::GEOMETRY)
907 tokenizer.expect("[");
908 pass->subscript = parse_expression();
909 tokenizer.expect("]");
911 tokenizer.expect(";");
915 RefPtr<Return> Parser::parse_return()
917 tokenizer.expect("return");
918 RefPtr<Return> ret = create_node<Return>();
919 if(tokenizer.peek_token()!=";")
920 ret->expression = parse_expression();
921 tokenizer.expect(";");