1 #include <msp/core/algorithm.h>
2 #include <msp/core/raii.h>
3 #include <msp/strings/format.h>
4 #include <msp/strings/regex.h>
5 #include <msp/strings/utils.h>
7 #include "glsl_error.h"
8 #include "modulecache.h"
19 Parser::Parser(ModuleCache *s):
21 preprocessor(tokenizer),
24 tokenizer.signal_preprocess.connect(sigc::mem_fun(&preprocessor, &Preprocessor::preprocess));
25 preprocessor.signal_version.connect(sigc::mem_fun(this, &Parser::set_required_version));
26 preprocessor.signal_source.connect(sigc::mem_fun(this, &Parser::source_reference));
27 preprocessor.signal_stage_change.connect(sigc::mem_fun(this, &Parser::stage_change));
28 preprocessor.signal_line.connect(sigc::mem_fun(this, &Parser::line_change));
31 void Parser::parse(Module &m, const string &s, const string &n, int i)
33 SetForScope<Module *> set_module(module, &m);
38 void Parser::parse(Module &m, IO::Base &io, const string &n, int i)
40 SetForScope<Module *> set_module(module, &m);
45 unsigned len = io.read(buffer, sizeof(buffer));
46 source.append(buffer, len);
51 void Parser::parse_source(const string &name, int index)
53 cur_stage = &module->shared;
57 source_reference(1, name);
59 if(const Stage *builtin = get_builtins(Stage::SHARED))
61 for(map<string, TypeDeclaration *>::const_iterator i=builtin->types.begin(); i!=builtin->types.end(); ++i)
62 global_types.insert(i->first);
66 global_types.insert("void");
67 global_types.insert("bool");
68 global_types.insert("int");
69 global_types.insert("uint");
70 global_types.insert("float");
73 tokenizer.begin(source, name);
74 allow_stage_change = true;
75 while(!tokenizer.peek_token().empty())
76 if(RefPtr<Statement> statement = parse_with_recovery(&Parser::parse_global_declaration))
77 cur_stage->content.body.push_back(statement);
80 throw invalid_shader_source(join(errors.begin(), errors.end(), "\n"));
83 void Parser::set_required_version(const Version &ver)
85 cur_stage->required_features.glsl_version = ver;
88 void Parser::source_reference(unsigned index, const string &name)
90 if(index<1 || base_index<0)
91 throw invalid_shader_source(tokenizer.get_location(), "Invalid source reference");
93 module->source_map.set_name(base_index+index-1, name);
96 void Parser::stage_change(Stage::Type stage)
98 if(!allow_stage_change)
99 throw invalid_shader_source(tokenizer.get_location(), "Changing stage not allowed here");
100 else if(stage<=cur_stage->type)
101 throw invalid_shader_source(tokenizer.get_location(), "Stage '%s' not allowed here", Stage::get_stage_name(stage));
103 module->stages.push_back(stage);
105 if(cur_stage->type!=Stage::SHARED)
106 module->stages.back().previous = cur_stage;
107 cur_stage = &module->stages.back();
110 for(vector<const Module *>::const_iterator i=imported_modules.begin(); i!=imported_modules.end(); ++i)
112 list<Stage>::const_iterator j = find_member((*i)->stages, stage, &Stage::type);
113 if(j!=(*i)->stages.end())
115 for(map<string, TypeDeclaration *>::const_iterator k=j->types.begin(); k!=j->types.end(); ++k)
116 stage_types.insert(k->first);
121 void Parser::line_change(int index, unsigned line)
124 source_index = base_index+index-1;
128 index = source_index;
130 string name = module->source_map.get_name(index);
132 name = format("<%d>", index);
133 tokenizer.set_location(Location(name, line));
136 string Parser::expect_type()
138 string token = tokenizer.parse_token();
140 throw parse_error(tokenizer.get_location(), token, "a type");
144 string Parser::expect_identifier()
146 string token = tokenizer.parse_token();
147 if(!is_identifier(token))
148 throw parse_error(tokenizer.get_location(), token, "an identifier");
152 int Parser::expect_integer()
154 string token = tokenizer.parse_token();
156 throw parse_error(tokenizer.get_location(), token, "an integer literal");
157 return lexical_cast<int>(token);
160 bool Parser::check(const string &token)
162 bool result = (tokenizer.peek_token()==token);
164 tokenizer.parse_token();
168 bool Parser::is_interface_qualifier(const string &token)
170 return (token=="uniform" || token=="in" || token=="out");
173 bool Parser::is_sampling_qualifier(const string &token)
175 return (token=="centroid" || token=="sample");
178 bool Parser::is_interpolation_qualifier(const string &token)
180 return (token=="smooth" || token=="flat" || token=="noperspective");
183 bool Parser::is_precision_qualifier(const string &token)
185 return (token=="highp" || token=="mediump" || token=="lowp");
188 bool Parser::is_qualifier(const string &token)
190 return (token=="const" ||
191 is_interface_qualifier(token) ||
192 is_sampling_qualifier(token) ||
193 is_interpolation_qualifier(token) ||
194 is_precision_qualifier(token));
197 bool Parser::is_type(const string &token)
199 return global_types.count(token) || stage_types.count(token);
202 bool Parser::is_identifier(const string &token)
204 static Regex re("^[a-zA-Z_][a-zA-Z0-9_]*$");
205 return re.match(token);
209 RefPtr<T> Parser::create_node()
211 RefPtr<T> node = new T;
212 node->source = source_index;
213 node->line = tokenizer.get_location().line;
218 RefPtr<T> Parser::parse_with_recovery(RefPtr<T> (Parser::*parse_func)())
220 tokenizer.clear_progress_mark();
223 return (this->*parse_func)();
225 catch(const invalid_shader_source &exc)
227 errors.push_back(exc.what());
230 if(tokenizer.get_last_token()!=";" || !tokenizer.get_progress_mark())
232 unsigned scope_level = 0;
235 if(tokenizer.peek_token()=="}" && scope_level==0)
237 if(!tokenizer.get_progress_mark())
238 tokenizer.parse_token();
242 string token = tokenizer.parse_token();
251 else if(token==";" && scope_level==0)
253 else if(token.empty())
261 RefPtr<Statement> Parser::parse_global_declaration()
263 string token = tokenizer.peek_token();
264 SetFlag disallow(allow_stage_change, false);
267 return parse_import();
268 else if(token=="precision")
269 return parse_precision();
270 else if(token=="layout")
272 RefPtr<Layout> layout = parse_layout();
273 token = tokenizer.peek_token();
274 if(is_interface_qualifier(token) && tokenizer.peek_token(1)==";")
276 RefPtr<InterfaceLayout> iface_lo = create_node<InterfaceLayout>();
277 iface_lo->layout.source = layout->source;
278 iface_lo->layout.line = layout->line;
279 iface_lo->layout.qualifiers = layout->qualifiers;
280 iface_lo->interface = tokenizer.parse_token();
281 tokenizer.expect(";");
284 else if(is_interface_qualifier(token) && tokenizer.peek_token(2)=="{")
286 RefPtr<InterfaceBlock> iface = parse_interface_block();
287 iface->layout = layout;
292 RefPtr<VariableDeclaration> var = parse_variable_declaration();
293 var->layout = layout;
297 else if(token=="typedef")
298 return parse_type_declaration();
299 else if(token=="struct")
300 return parse_struct_declaration();
301 else if(is_interface_qualifier(token))
303 string next = tokenizer.peek_token(1);
304 if(is_type(next) || is_qualifier(next))
305 return parse_variable_declaration();
307 return parse_interface_block();
309 else if(is_qualifier(token))
310 return parse_variable_declaration();
311 else if(token=="virtual")
312 return parse_function_declaration();
313 else if(is_type(token))
315 if(tokenizer.peek_token(2)=="(")
316 return parse_function_declaration();
318 return parse_variable_declaration();
320 else if(token.empty())
323 throw parse_error(tokenizer.get_location(), token, "a global declaration");
326 RefPtr<Statement> Parser::parse_statement()
328 string token = tokenizer.peek_token();
330 return parse_conditional();
331 else if(token=="for")
333 else if(token=="while")
334 return parse_while();
335 else if(token=="passthrough")
336 return parse_passthrough();
337 else if(token=="return")
338 return parse_return();
339 else if(token=="break" || token=="continue" || token=="discard")
341 RefPtr<Jump> jump = create_node<Jump>();
342 jump->keyword = tokenizer.parse_token();
343 tokenizer.expect(";");
347 else if(is_qualifier(token) || is_type(token))
348 return parse_variable_declaration();
351 tokenizer.parse_token();
352 throw invalid_shader_source(tokenizer.get_location(), "Empty statement not allowed");
354 else if(!token.empty())
356 RefPtr<ExpressionStatement> expr = create_node<ExpressionStatement>();
357 expr->expression = parse_expression();
358 tokenizer.expect(";");
363 throw parse_error(tokenizer.get_location(), token, "a statement");
366 RefPtr<Import> Parser::parse_import()
368 if(cur_stage->type!=Stage::SHARED)
369 throw invalid_shader_source(tokenizer.get_location(), "Imports are only allowed in the shared section");
371 tokenizer.expect("import");
372 RefPtr<Import> import = create_node<Import>();
373 import->module = expect_identifier();
374 tokenizer.expect(";");
378 const Module &imported_mod = mod_cache->get_module(import->module);
379 imported_modules.push_back(&imported_mod);
380 for(map<string, TypeDeclaration *>::const_iterator i=imported_mod.shared.types.begin(); i!=imported_mod.shared.types.end(); ++i)
381 global_types.insert(i->first);
387 RefPtr<Precision> Parser::parse_precision()
389 tokenizer.expect("precision");
390 RefPtr<Precision> precision = create_node<Precision>();
392 precision->precision = tokenizer.parse_token();
393 if(!is_precision_qualifier(precision->precision))
394 throw parse_error(tokenizer.get_location(), precision->precision, "a precision qualifier");
396 // TODO Add validation for this
397 precision->type = expect_type();
399 tokenizer.expect(";");
404 RefPtr<Layout> Parser::parse_layout()
406 tokenizer.expect("layout");
407 tokenizer.expect("(");
408 RefPtr<Layout> layout = create_node<Layout>();
411 string token = tokenizer.parse_token();
413 throw parse_error(tokenizer.get_location(), token, "a layout qualifier name");
415 layout->qualifiers.push_back(token);
416 Layout::Qualifier &qual = layout->qualifiers.back();
418 if((qual.has_value = check("=")))
420 if(qual.name=="constant_id" && tokenizer.peek_token()=="auto")
423 tokenizer.parse_token();
426 qual.value = expect_integer();
429 if(tokenizer.peek_token()==")")
432 tokenizer.expect(",");
434 tokenizer.expect(")");
440 void Parser::parse_block(Block &block, bool require_braces, RefPtr<T> (Parser::*parse_content)())
442 bool have_braces = (require_braces || tokenizer.peek_token()=="{");
444 tokenizer.expect("{");
448 while(tokenizer.peek_token()!="}")
449 if(RefPtr<Statement> node = parse_with_recovery(parse_content))
450 block.body.push_back(node);
453 block.body.push_back((this->*parse_content)());
455 block.use_braces = (require_braces || block.body.size()!=1);
458 tokenizer.expect("}");
461 RefPtr<Expression> Parser::parse_expression(const Operator *outer_oper)
463 unsigned outer_precedence = (outer_oper ? outer_oper->precedence+(outer_oper->assoc==Operator::RIGHT_TO_LEFT) : 20);
464 RefPtr<Expression> left;
465 VariableReference *left_var = 0;
468 string token = tokenizer.peek_token();
470 const Operator *oper = 0;
471 for(const Operator *i=Operator::operators; (!oper && i->type); ++i)
472 if(token==i->token && (!left || i->type!=Operator::PREFIX) && (left || i->type!=Operator::POSTFIX))
475 bool lower_precedence = (oper && oper->type!=Operator::PREFIX && oper->precedence>=outer_precedence);
476 if(token==";" || token==")" || token=="]" || token=="," || token==":" || lower_precedence)
481 throw parse_error(tokenizer.get_location(), token, "an expression");
488 throw invalid_shader_source(tokenizer.get_location(), "Syntax error before '(': function name must be an identifier");
489 left = parse_function_call(*left_var);
493 RefPtr<MemberAccess> memacc = create_node<MemberAccess>();
496 tokenizer.parse_token();
497 memacc->member = expect_identifier();
500 else if(oper && oper->type==Operator::POSTFIX)
502 RefPtr<UnaryExpression> unary = create_node<UnaryExpression>();
504 tokenizer.parse_token();
505 unary->expression = left;
508 else if(oper && oper->type==Operator::BINARY)
509 left = parse_binary(left, *oper);
510 else if(oper && oper->type==Operator::TERNARY)
511 left = parse_ternary(left, *oper);
513 throw parse_error(tokenizer.get_location(), token, "an operator");
520 tokenizer.parse_token();
521 left = parse_expression();
522 tokenizer.expect(")");
524 else if(isdigit(token[0]) || token=="true" || token=="false")
525 left = parse_literal();
526 else if(is_identifier(token))
528 RefPtr<VariableReference> var = create_node<VariableReference>();
529 var->name = expect_identifier();
531 left_var = var.get();
533 else if(oper && oper->type==Operator::PREFIX)
535 RefPtr<UnaryExpression> unary = create_node<UnaryExpression>();
537 tokenizer.parse_token();
538 unary->expression = parse_expression(oper);
542 throw parse_error(tokenizer.get_location(), token, "an expression");
547 RefPtr<Literal> Parser::parse_literal()
549 RefPtr<Literal> literal = create_node<Literal>();
550 literal->token = tokenizer.parse_token();
551 if(isdigit(literal->token[0]))
553 // TODO have the tokenizer return the type of the token
554 if(literal->token.back()=='u')
555 literal->value = lexical_cast<unsigned>(literal->token.substr(0, literal->token.size()-1));
556 else if(literal->token.back()=='f')
557 literal->value = lexical_cast<float>(literal->token.substr(0, literal->token.size()-1));
558 else if(literal->token.find('.')!=string::npos)
559 literal->value = lexical_cast<float>(literal->token);
561 literal->value = lexical_cast<int>(literal->token);
563 else if(literal->token=="true" || literal->token=="false")
564 literal->value = (literal->token=="true");
566 throw parse_error(tokenizer.get_location(), literal->token, "a literal");
571 RefPtr<BinaryExpression> Parser::parse_binary(const RefPtr<Expression> &left, const Operator &oper)
573 RefPtr<BinaryExpression> binary = (oper.precedence==16 ?
574 static_cast<RefPtr<BinaryExpression> >(create_node<Assignment>()) : create_node<BinaryExpression>());
576 binary->oper = &oper;
577 tokenizer.expect(oper.token);
580 binary->right = parse_expression();
581 tokenizer.expect(oper.token2);
584 binary->right = parse_expression(&oper);
588 RefPtr<TernaryExpression> Parser::parse_ternary(const RefPtr<Expression> &cond, const Operator &oper)
590 RefPtr<TernaryExpression> ternary = create_node<TernaryExpression>();
591 ternary->condition = cond;
592 ternary->oper = &oper;
593 tokenizer.expect("?");
594 ternary->true_expr = parse_expression(&oper);
595 tokenizer.expect(":");
596 ternary->false_expr = parse_expression(&oper);
600 RefPtr<FunctionCall> Parser::parse_function_call(const VariableReference &var)
602 RefPtr<FunctionCall> call = create_node<FunctionCall>();
603 call->name = var.name;
604 call->oper = &Operator::get_operator("(", Operator::POSTFIX);
605 tokenizer.expect("(");
606 while(tokenizer.peek_token()!=")")
608 if(!call->arguments.empty())
609 tokenizer.expect(",");
610 call->arguments.push_back(parse_expression());
612 tokenizer.expect(")");
616 RefPtr<TypeDeclaration> Parser::parse_type_declaration()
618 tokenizer.expect("typedef");
620 RefPtr<TypeDeclaration> type;
621 if(tokenizer.peek_token()=="image")
622 type = parse_image_type_declaration();
624 type = parse_basic_type_declaration();
626 tokenizer.expect(";");
627 cur_stage->types[type->name] = type.get();
628 if(cur_stage->type==Stage::SHARED)
629 global_types.insert(type->name);
631 stage_types.insert(type->name);
635 RefPtr<BasicTypeDeclaration> Parser::parse_basic_type_declaration()
637 RefPtr<BasicTypeDeclaration> type = create_node<BasicTypeDeclaration>();
639 if(tokenizer.peek_token()=="vector")
641 type->kind = BasicTypeDeclaration::VECTOR;
643 tokenizer.parse_token();
644 tokenizer.expect("(");
645 type->size = expect_integer();
646 tokenizer.expect(")");
649 type->base = expect_type();
650 type->name = expect_identifier();
652 if(type->kind==BasicTypeDeclaration::ALIAS && check("["))
654 type->kind = BasicTypeDeclaration::ARRAY;
655 tokenizer.expect("]");
661 RefPtr<ImageTypeDeclaration> Parser::parse_image_type_declaration()
663 tokenizer.expect("image");
664 tokenizer.expect("(");
666 RefPtr<ImageTypeDeclaration> type = create_node<ImageTypeDeclaration>();
669 string token = tokenizer.parse_token();
670 if(token=="dimensions")
672 tokenizer.expect("=");
673 token = tokenizer.parse_token();
675 type->dimensions = ImageTypeDeclaration::ONE;
677 type->dimensions = ImageTypeDeclaration::TWO;
679 type->dimensions = ImageTypeDeclaration::THREE;
680 else if(token=="cube")
681 type->dimensions = ImageTypeDeclaration::CUBE;
683 throw parse_error(tokenizer.get_location(), token, "dimensions");
688 tokenizer.expect("]");
691 else if(token=="sampled")
692 type->sampled = true;
693 else if(token=="shadow")
696 throw parse_error(tokenizer.get_location(), token, "image type attribute");
698 token = tokenizer.peek_token();
702 tokenizer.expect(",");
704 tokenizer.expect(")");
706 type->base = expect_type();
707 type->name = expect_identifier();
712 RefPtr<StructDeclaration> Parser::parse_struct_declaration()
714 tokenizer.expect("struct");
715 RefPtr<StructDeclaration> strct = create_node<StructDeclaration>();
717 strct->name = expect_identifier();
718 parse_block(strct->members, true, &Parser::parse_variable_declaration);
719 tokenizer.expect(";");
721 cur_stage->types[strct->name] = strct.get();
722 if(cur_stage->type==Stage::SHARED)
723 global_types.insert(strct->name);
725 stage_types.insert(strct->name);
729 RefPtr<VariableDeclaration> Parser::parse_variable_declaration()
731 RefPtr<VariableDeclaration> var = create_node<VariableDeclaration>();
733 string token = tokenizer.peek_token();
734 while(is_qualifier(token))
736 tokenizer.parse_token();
737 if(is_interface_qualifier(token))
738 var->interface = token;
739 else if(is_sampling_qualifier(token))
740 var->sampling = token;
741 else if(is_interpolation_qualifier(token))
742 var->interpolation = token;
743 else if(is_precision_qualifier(token))
744 var->precision = token;
745 else if(token=="const")
746 var->constant = true;
747 token = tokenizer.peek_token();
750 var->type = expect_type();
751 var->name = expect_identifier();
758 var->array_size = parse_expression();
759 tokenizer.expect("]");
764 var->init_expression = parse_expression();
766 tokenizer.expect(";");
770 RefPtr<VariableDeclaration> Parser::parse_variable_declaration_with_layout()
772 RefPtr<Layout> layout;
773 if(tokenizer.peek_token()=="layout")
774 layout = parse_layout();
776 RefPtr<VariableDeclaration> var = parse_variable_declaration();
777 var->layout = layout;
782 RefPtr<FunctionDeclaration> Parser::parse_function_declaration()
784 RefPtr<FunctionDeclaration> func = create_node<FunctionDeclaration>();
786 func->virtua = check("virtual");
787 func->return_type = expect_type();
788 func->name = expect_identifier();
789 tokenizer.expect("(");
790 while(tokenizer.peek_token()!=")")
792 if(!func->parameters.empty())
793 tokenizer.expect(",");
795 RefPtr<VariableDeclaration> var = create_node<VariableDeclaration>();
796 string token = tokenizer.peek_token();
797 if(token=="in" || token=="out" || token=="inout")
798 var->interface = tokenizer.parse_token();
799 var->type = expect_type();
800 var->name = expect_identifier();
801 func->parameters.push_back(var);
803 tokenizer.expect(")");
805 func->overrd = check("override");
807 string token = tokenizer.peek_token();
810 func->definition = func.get();
811 parse_block(func->body, true, &Parser::parse_statement);
814 tokenizer.parse_token();
816 throw parse_error(tokenizer.get_location(), token, "'{' or ';'");
821 RefPtr<InterfaceBlock> Parser::parse_interface_block()
823 RefPtr<InterfaceBlock> iface = create_node<InterfaceBlock>();
825 iface->interface = tokenizer.parse_token();
826 if(!is_interface_qualifier(iface->interface))
827 throw parse_error(tokenizer.get_location(), iface->interface, "an interface qualifier");
829 iface->block_name = expect_identifier();
830 iface->members = new Block;
831 parse_block(*iface->members, true, &Parser::parse_variable_declaration_with_layout);
834 iface->instance_name = expect_identifier();
838 tokenizer.expect("]");
840 tokenizer.expect(";");
846 RefPtr<Conditional> Parser::parse_conditional()
848 tokenizer.expect("if");
849 RefPtr<Conditional> cond = create_node<Conditional>();
850 tokenizer.expect("(");
851 cond->condition = parse_expression();
852 tokenizer.expect(")");
854 parse_block(cond->body, false, &Parser::parse_statement);
856 string token = tokenizer.peek_token();
859 tokenizer.parse_token();
860 parse_block(cond->else_body, false, &Parser::parse_statement);
866 RefPtr<Iteration> Parser::parse_for()
868 tokenizer.expect("for");
869 RefPtr<Iteration> loop = create_node<Iteration>();
870 tokenizer.expect("(");
871 string token = tokenizer.peek_token();
873 loop->init_statement = parse_statement();
878 RefPtr<ExpressionStatement> expr = create_node<ExpressionStatement>();
879 expr->expression = parse_expression();
880 loop->init_statement = expr;
882 tokenizer.expect(";");
884 if(tokenizer.peek_token()!=";")
885 loop->condition = parse_expression();
886 tokenizer.expect(";");
887 if(tokenizer.peek_token()!=")")
888 loop->loop_expression = parse_expression();
889 tokenizer.expect(")");
891 parse_block(loop->body, false, &Parser::parse_statement);
896 RefPtr<Iteration> Parser::parse_while()
898 tokenizer.expect("while");
899 RefPtr<Iteration> loop = create_node<Iteration>();
900 tokenizer.expect("(");
901 loop->condition = parse_expression();
902 tokenizer.expect(")");
904 parse_block(loop->body, false, &Parser::parse_statement);
909 RefPtr<Passthrough> Parser::parse_passthrough()
911 tokenizer.expect("passthrough");
912 RefPtr<Passthrough> pass = create_node<Passthrough>();
913 if(cur_stage->type==Stage::GEOMETRY)
915 tokenizer.expect("[");
916 pass->subscript = parse_expression();
917 tokenizer.expect("]");
919 tokenizer.expect(";");
923 RefPtr<Return> Parser::parse_return()
925 tokenizer.expect("return");
926 RefPtr<Return> ret = create_node<Return>();
927 if(tokenizer.peek_token()!=";")
928 ret->expression = parse_expression();
929 tokenizer.expect(";");