1 #include <msp/core/raii.h>
2 #include <msp/strings/format.h>
3 #include <msp/strings/regex.h>
4 #include <msp/strings/utils.h>
6 #include "glsl_error.h"
18 preprocessor(tokenizer),
21 tokenizer.signal_preprocess.connect(sigc::mem_fun(&preprocessor, &Preprocessor::preprocess));
22 preprocessor.signal_version.connect(sigc::mem_fun(this, &Parser::set_required_version));
23 preprocessor.signal_source.connect(sigc::mem_fun(this, &Parser::source_reference));
24 preprocessor.signal_stage_change.connect(sigc::mem_fun(this, &Parser::stage_change));
25 preprocessor.signal_line.connect(sigc::mem_fun(this, &Parser::line_change));
33 Module &Parser::parse(const string &s, const string &n, int i)
40 Module &Parser::parse(IO::Base &io, const string &n, int i)
46 unsigned len = io.read(buffer, sizeof(buffer));
47 source.append(buffer, len);
53 void Parser::parse_source(const string &name, int index)
58 cur_stage = &module->shared;
62 source_reference(1, name);
64 // TODO Need to somehow get type names from imports
65 if(const Stage *builtin = get_builtins(Stage::SHARED))
67 for(map<string, TypeDeclaration *>::const_iterator i=builtin->types.begin(); i!=builtin->types.end(); ++i)
68 declared_types.insert(i->first);
72 declared_types.insert("void");
73 declared_types.insert("bool");
74 declared_types.insert("int");
75 declared_types.insert("float");
78 tokenizer.begin(source, name);
79 allow_stage_change = true;
80 while(!tokenizer.peek_token().empty())
81 if(RefPtr<Statement> statement = parse_with_recovery(&Parser::parse_global_declaration))
82 cur_stage->content.body.push_back(statement);
85 throw invalid_shader_source(join(errors.begin(), errors.end(), "\n"));
88 void Parser::set_required_version(const Version &ver)
90 cur_stage->required_features.glsl_version = ver;
93 void Parser::source_reference(unsigned index, const string &name)
95 if(index<1 || base_index<0)
96 throw invalid_shader_source(tokenizer.get_location(), "Invalid source reference");
98 module->source_map.set_name(base_index+index-1, name);
101 void Parser::stage_change(Stage::Type stage)
103 if(!allow_stage_change)
104 throw invalid_shader_source(tokenizer.get_location(), "Changing stage not allowed here");
105 else if(stage<=cur_stage->type)
106 throw invalid_shader_source(tokenizer.get_location(), "Stage '%s' not allowed here", Stage::get_stage_name(stage));
108 module->stages.push_back(stage);
110 if(cur_stage->type!=Stage::SHARED)
111 module->stages.back().previous = cur_stage;
112 cur_stage = &module->stages.back();
115 void Parser::line_change(int index, unsigned line)
118 source_index = base_index+index-1;
122 index = source_index;
124 string name = module->source_map.get_name(index);
126 name = format("<%d>", index);
127 tokenizer.set_location(Location(name, line));
130 string Parser::expect_type()
132 string token = tokenizer.parse_token();
134 throw parse_error(tokenizer.get_location(), token, "a type");
138 string Parser::expect_identifier()
140 string token = tokenizer.parse_token();
141 if(!is_identifier(token))
142 throw parse_error(tokenizer.get_location(), token, "an identifier");
146 int Parser::expect_integer()
148 string token = tokenizer.parse_token();
150 throw parse_error(tokenizer.get_location(), token, "an integer literal");
151 return lexical_cast<int>(token);
154 bool Parser::check(const string &token)
156 bool result = (tokenizer.peek_token()==token);
158 tokenizer.parse_token();
162 bool Parser::is_interface_qualifier(const string &token)
164 return (token=="uniform" || token=="in" || token=="out");
167 bool Parser::is_sampling_qualifier(const string &token)
169 return (token=="centroid" || token=="sample");
172 bool Parser::is_interpolation_qualifier(const string &token)
174 return (token=="smooth" || token=="flat" || token=="noperspective");
177 bool Parser::is_precision_qualifier(const string &token)
179 return (token=="highp" || token=="mediump" || token=="lowp");
182 bool Parser::is_qualifier(const string &token)
184 return (token=="const" ||
185 is_interface_qualifier(token) ||
186 is_sampling_qualifier(token) ||
187 is_interpolation_qualifier(token) ||
188 is_precision_qualifier(token));
191 bool Parser::is_type(const string &token)
193 return declared_types.count(token);
196 bool Parser::is_identifier(const string &token)
198 static Regex re("^[a-zA-Z_][a-zA-Z0-9_]*$");
199 return re.match(token);
203 RefPtr<T> Parser::create_node()
205 RefPtr<T> node = new T;
206 node->source = source_index;
207 node->line = tokenizer.get_location().line;
212 RefPtr<T> Parser::parse_with_recovery(RefPtr<T> (Parser::*parse_func)())
214 tokenizer.clear_progress_mark();
217 return (this->*parse_func)();
219 catch(const invalid_shader_source &exc)
221 errors.push_back(exc.what());
224 if(tokenizer.get_last_token()!=";" || !tokenizer.get_progress_mark())
226 unsigned scope_level = 0;
229 if(tokenizer.peek_token()=="}" && scope_level==0)
231 if(!tokenizer.get_progress_mark())
232 tokenizer.parse_token();
236 string token = tokenizer.parse_token();
245 else if(token==";" && scope_level==0)
247 else if(token.empty())
255 RefPtr<Statement> Parser::parse_global_declaration()
257 string token = tokenizer.peek_token();
258 SetFlag disallow(allow_stage_change, false);
261 return parse_import();
262 else if(token=="precision")
263 return parse_precision();
264 else if(token=="layout")
266 RefPtr<Layout> layout = parse_layout();
267 token = tokenizer.peek_token();
268 if(is_interface_qualifier(token) && tokenizer.peek_token(1)==";")
270 RefPtr<InterfaceLayout> iface_lo = create_node<InterfaceLayout>();
271 iface_lo->layout.qualifiers = layout->qualifiers;
272 iface_lo->interface = tokenizer.parse_token();
273 tokenizer.expect(";");
278 RefPtr<VariableDeclaration> var = parse_variable_declaration();
279 var->layout = layout;
283 else if(token=="typedef")
284 return parse_type_declaration();
285 else if(token=="struct")
286 return parse_struct_declaration();
287 else if(is_interface_qualifier(token))
289 string next = tokenizer.peek_token(1);
290 if(is_type(next) || is_qualifier(next))
291 return parse_variable_declaration();
293 return parse_interface_block();
295 else if(is_qualifier(token))
296 return parse_variable_declaration();
297 else if(token=="virtual")
298 return parse_function_declaration();
299 else if(is_type(token))
301 if(tokenizer.peek_token(2)=="(")
302 return parse_function_declaration();
304 return parse_variable_declaration();
306 else if(token.empty())
309 throw parse_error(tokenizer.get_location(), token, "a global declaration");
312 RefPtr<Statement> Parser::parse_statement()
314 string token = tokenizer.peek_token();
316 return parse_conditional();
317 else if(token=="for")
319 else if(token=="while")
320 return parse_while();
321 else if(token=="passthrough")
322 return parse_passthrough();
323 else if(token=="return")
324 return parse_return();
325 else if(token=="break" || token=="continue" || token=="discard")
327 RefPtr<Jump> jump = create_node<Jump>();
328 jump->keyword = tokenizer.parse_token();
329 tokenizer.expect(";");
333 else if(is_qualifier(token) || is_type(token))
334 return parse_variable_declaration();
337 tokenizer.parse_token();
338 throw invalid_shader_source(tokenizer.get_location(), "Empty statement not allowed");
340 else if(!token.empty())
342 RefPtr<ExpressionStatement> expr = create_node<ExpressionStatement>();
343 expr->expression = parse_expression();
344 tokenizer.expect(";");
349 throw parse_error(tokenizer.get_location(), token, "a statement");
352 RefPtr<Import> Parser::parse_import()
354 if(cur_stage->type!=Stage::SHARED)
355 throw invalid_shader_source(tokenizer.get_location(), "Imports are only allowed in the shared section");
357 tokenizer.expect("import");
358 RefPtr<Import> import = create_node<Import>();
359 import->module = expect_identifier();
360 tokenizer.expect(";");
364 RefPtr<Precision> Parser::parse_precision()
366 tokenizer.expect("precision");
367 RefPtr<Precision> precision = create_node<Precision>();
369 precision->precision = tokenizer.parse_token();
370 if(!is_precision_qualifier(precision->precision))
371 throw parse_error(tokenizer.get_location(), precision->precision, "a precision qualifier");
373 // TODO Add validation for this
374 precision->type = expect_type();
376 tokenizer.expect(";");
381 RefPtr<Layout> Parser::parse_layout()
383 tokenizer.expect("layout");
384 tokenizer.expect("(");
385 RefPtr<Layout> layout = create_node<Layout>();
388 string token = tokenizer.parse_token();
390 throw parse_error(tokenizer.get_location(), token, "a layout qualifier name");
392 layout->qualifiers.push_back(Layout::Qualifier());
393 Layout::Qualifier &qual = layout->qualifiers.back();
396 if((qual.has_value = check("=")))
398 if(qual.name=="constant_id" && tokenizer.peek_token()=="auto")
401 tokenizer.parse_token();
404 qual.value = expect_integer();
407 if(tokenizer.peek_token()==")")
410 tokenizer.expect(",");
412 tokenizer.expect(")");
418 void Parser::parse_block(Block &block, bool require_braces, RefPtr<T> (Parser::*parse_content)())
420 bool have_braces = (require_braces || tokenizer.peek_token()=="{");
422 tokenizer.expect("{");
426 while(tokenizer.peek_token()!="}")
427 if(RefPtr<Statement> node = parse_with_recovery(parse_content))
428 block.body.push_back(node);
431 block.body.push_back((this->*parse_content)());
433 block.use_braces = (require_braces || block.body.size()!=1);
436 tokenizer.expect("}");
439 RefPtr<Expression> Parser::parse_expression(const Operator *outer_oper)
441 unsigned outer_precedence = (outer_oper ? outer_oper->precedence+(outer_oper->assoc==Operator::RIGHT_TO_LEFT) : 20);
442 RefPtr<Expression> left;
443 VariableReference *left_var = 0;
446 string token = tokenizer.peek_token();
448 const Operator *oper = 0;
449 for(const Operator *i=Operator::operators; (!oper && i->type); ++i)
450 if(token==i->token && (!left || i->type!=Operator::PREFIX) && (left || i->type!=Operator::POSTFIX))
453 bool lower_precedence = (oper && oper->type!=Operator::PREFIX && oper->precedence>=outer_precedence);
454 if(token==";" || token==")" || token=="]" || token=="," || token==":" || lower_precedence)
459 throw parse_error(tokenizer.get_location(), token, "an expression");
466 throw invalid_shader_source(tokenizer.get_location(), "Syntax error before '(': function name must be an identifier");
467 left = parse_function_call(*left_var);
471 RefPtr<MemberAccess> memacc = create_node<MemberAccess>();
474 tokenizer.parse_token();
475 memacc->member = expect_identifier();
478 else if(oper && oper->type==Operator::POSTFIX)
480 RefPtr<UnaryExpression> unary = create_node<UnaryExpression>();
482 tokenizer.parse_token();
483 unary->expression = left;
486 else if(oper && oper->type==Operator::BINARY)
487 left = parse_binary(left, *oper);
488 else if(oper && oper->type==Operator::TERNARY)
489 left = parse_ternary(left, *oper);
491 throw parse_error(tokenizer.get_location(), token, "an operator");
498 tokenizer.parse_token();
499 RefPtr<ParenthesizedExpression> parexpr = create_node<ParenthesizedExpression>();
500 parexpr->expression = parse_expression();
501 tokenizer.expect(")");
504 else if(isdigit(token[0]) || token=="true" || token=="false")
505 left = parse_literal();
506 else if(is_identifier(token))
508 RefPtr<VariableReference> var = create_node<VariableReference>();
509 var->name = expect_identifier();
511 left_var = var.get();
513 else if(oper && oper->type==Operator::PREFIX)
515 RefPtr<UnaryExpression> unary = create_node<UnaryExpression>();
517 tokenizer.parse_token();
518 unary->expression = parse_expression(oper);
522 throw parse_error(tokenizer.get_location(), token, "an expression");
527 RefPtr<Literal> Parser::parse_literal()
529 RefPtr<Literal> literal = create_node<Literal>();
530 literal->token = tokenizer.parse_token();
531 if(isdigit(literal->token[0]))
533 // TODO have the tokenizer return the type of the token
534 if(isnumrc(literal->token))
535 literal->value = lexical_cast<int>(literal->token);
537 literal->value = lexical_cast<float>(literal->token);
539 else if(literal->token=="true" || literal->token=="false")
540 literal->value = (literal->token=="true");
542 throw parse_error(tokenizer.get_location(), literal->token, "a literal");
547 RefPtr<BinaryExpression> Parser::parse_binary(const RefPtr<Expression> &left, const Operator &oper)
549 RefPtr<BinaryExpression> binary = (oper.precedence==16 ?
550 static_cast<RefPtr<BinaryExpression> >(create_node<Assignment>()) : create_node<BinaryExpression>());
552 binary->oper = &oper;
553 tokenizer.expect(oper.token);
554 if(oper.token[0]=='[')
556 binary->right = parse_expression();
557 tokenizer.expect("]");
560 binary->right = parse_expression(&oper);
564 RefPtr<TernaryExpression> Parser::parse_ternary(const RefPtr<Expression> &cond, const Operator &oper)
566 RefPtr<TernaryExpression> ternary = create_node<TernaryExpression>();
567 ternary->condition = cond;
568 ternary->oper = &oper;
569 tokenizer.expect("?");
570 ternary->true_expr = parse_expression(&oper);
571 tokenizer.expect(":");
572 ternary->false_expr = parse_expression(&oper);
576 RefPtr<FunctionCall> Parser::parse_function_call(const VariableReference &var)
578 RefPtr<FunctionCall> call = create_node<FunctionCall>();
579 call->name = var.name;
580 call->constructor = is_type(call->name);
581 call->oper = &Operator::get_operator("(", Operator::POSTFIX);
582 tokenizer.expect("(");
583 while(tokenizer.peek_token()!=")")
585 if(!call->arguments.empty())
586 tokenizer.expect(",");
587 call->arguments.push_back(parse_expression());
589 tokenizer.expect(")");
593 RefPtr<TypeDeclaration> Parser::parse_type_declaration()
595 tokenizer.expect("typedef");
597 RefPtr<TypeDeclaration> type;
598 if(tokenizer.peek_token()=="image")
599 type = parse_image_type_declaration();
601 type = parse_basic_type_declaration();
603 tokenizer.expect(";");
604 declared_types.insert(type->name);
608 RefPtr<BasicTypeDeclaration> Parser::parse_basic_type_declaration()
610 RefPtr<BasicTypeDeclaration> type = create_node<BasicTypeDeclaration>();
612 if(tokenizer.peek_token()=="vector")
614 type->kind = BasicTypeDeclaration::VECTOR;
616 tokenizer.parse_token();
617 tokenizer.expect("(");
618 type->size = expect_integer();
619 tokenizer.expect(")");
622 type->base = expect_type();
623 type->name = expect_identifier();
625 if(type->kind==BasicTypeDeclaration::ALIAS && check("["))
627 type->kind = BasicTypeDeclaration::ARRAY;
628 tokenizer.expect("]");
634 RefPtr<ImageTypeDeclaration> Parser::parse_image_type_declaration()
636 tokenizer.expect("image");
637 tokenizer.expect("(");
639 RefPtr<ImageTypeDeclaration> type = create_node<ImageTypeDeclaration>();
642 string token = tokenizer.parse_token();
643 if(token=="dimensions")
645 tokenizer.expect("=");
646 token = tokenizer.parse_token();
648 type->dimensions = ImageTypeDeclaration::ONE;
650 type->dimensions = ImageTypeDeclaration::TWO;
652 type->dimensions = ImageTypeDeclaration::THREE;
653 else if(token=="cube")
654 type->dimensions = ImageTypeDeclaration::CUBE;
656 throw parse_error(tokenizer.get_location(), token, "dimensions");
661 tokenizer.expect("]");
664 else if(token=="sampled")
665 type->sampled = true;
666 else if(token=="shadow")
669 throw parse_error(tokenizer.get_location(), token, "image type attribute");
671 token = tokenizer.peek_token();
675 tokenizer.expect(",");
677 tokenizer.expect(")");
679 type->base = expect_type();
680 type->name = expect_identifier();
685 RefPtr<StructDeclaration> Parser::parse_struct_declaration()
687 tokenizer.expect("struct");
688 RefPtr<StructDeclaration> strct = create_node<StructDeclaration>();
690 strct->name = expect_identifier();
691 parse_block(strct->members, true, &Parser::parse_variable_declaration);
692 tokenizer.expect(";");
694 declared_types.insert(strct->name);
698 RefPtr<VariableDeclaration> Parser::parse_variable_declaration()
700 RefPtr<VariableDeclaration> var = create_node<VariableDeclaration>();
702 string token = tokenizer.peek_token();
703 while(is_qualifier(token))
705 tokenizer.parse_token();
706 if(is_interface_qualifier(token))
707 var->interface = token;
708 else if(is_sampling_qualifier(token))
709 var->sampling = token;
710 else if(is_interpolation_qualifier(token))
711 var->interpolation = token;
712 else if(is_precision_qualifier(token))
713 var->precision = token;
714 else if(token=="const")
715 var->constant = true;
716 token = tokenizer.peek_token();
719 var->type = expect_type();
720 var->name = expect_identifier();
727 var->array_size = parse_expression();
728 tokenizer.expect("]");
733 var->init_expression = parse_expression();
735 tokenizer.expect(";");
739 RefPtr<VariableDeclaration> Parser::parse_variable_declaration_with_layout()
741 RefPtr<Layout> layout;
742 if(tokenizer.peek_token()=="layout")
743 layout = parse_layout();
745 RefPtr<VariableDeclaration> var = parse_variable_declaration();
746 var->layout = layout;
751 RefPtr<FunctionDeclaration> Parser::parse_function_declaration()
753 RefPtr<FunctionDeclaration> func = create_node<FunctionDeclaration>();
755 func->virtua = check("virtual");
756 func->return_type = expect_type();
757 func->name = expect_identifier();
758 tokenizer.expect("(");
759 while(tokenizer.peek_token()!=")")
761 if(!func->parameters.empty())
762 tokenizer.expect(",");
764 RefPtr<VariableDeclaration> var = create_node<VariableDeclaration>();
765 string token = tokenizer.peek_token();
766 if(token=="in" || token=="out" || token=="inout")
767 var->interface = tokenizer.parse_token();
768 var->type = expect_type();
769 var->name = expect_identifier();
770 func->parameters.push_back(var);
772 tokenizer.expect(")");
774 func->overrd = check("override");
776 string token = tokenizer.peek_token();
779 func->definition = func.get();
780 parse_block(func->body, true, &Parser::parse_statement);
783 tokenizer.parse_token();
785 throw parse_error(tokenizer.get_location(), token, "'{' or ';'");
790 RefPtr<InterfaceBlock> Parser::parse_interface_block()
792 RefPtr<InterfaceBlock> iface = create_node<InterfaceBlock>();
794 iface->interface = tokenizer.parse_token();
795 if(!is_interface_qualifier(iface->interface))
796 throw parse_error(tokenizer.get_location(), iface->interface, "an interface qualifier");
798 iface->name = expect_identifier();
799 iface->members = new Block;
800 parse_block(*iface->members, true, &Parser::parse_variable_declaration_with_layout);
803 iface->instance_name = expect_identifier();
807 tokenizer.expect("]");
809 tokenizer.expect(";");
815 RefPtr<Conditional> Parser::parse_conditional()
817 tokenizer.expect("if");
818 RefPtr<Conditional> cond = create_node<Conditional>();
819 tokenizer.expect("(");
820 cond->condition = parse_expression();
821 tokenizer.expect(")");
823 parse_block(cond->body, false, &Parser::parse_statement);
825 string token = tokenizer.peek_token();
828 tokenizer.parse_token();
829 parse_block(cond->else_body, false, &Parser::parse_statement);
835 RefPtr<Iteration> Parser::parse_for()
837 tokenizer.expect("for");
838 RefPtr<Iteration> loop = create_node<Iteration>();
839 tokenizer.expect("(");
840 string token = tokenizer.peek_token();
842 loop->init_statement = parse_statement();
847 RefPtr<ExpressionStatement> expr = create_node<ExpressionStatement>();
848 expr->expression = parse_expression();
849 loop->init_statement = expr;
851 tokenizer.expect(";");
853 if(tokenizer.peek_token()!=";")
854 loop->condition = parse_expression();
855 tokenizer.expect(";");
856 if(tokenizer.peek_token()!=")")
857 loop->loop_expression = parse_expression();
858 tokenizer.expect(")");
860 parse_block(loop->body, false, &Parser::parse_statement);
865 RefPtr<Iteration> Parser::parse_while()
867 tokenizer.expect("while");
868 RefPtr<Iteration> loop = create_node<Iteration>();
869 tokenizer.expect("(");
870 loop->condition = parse_expression();
871 tokenizer.expect(")");
873 parse_block(loop->body, false, &Parser::parse_statement);
878 RefPtr<Passthrough> Parser::parse_passthrough()
880 tokenizer.expect("passthrough");
881 RefPtr<Passthrough> pass = create_node<Passthrough>();
882 if(cur_stage->type==Stage::GEOMETRY)
884 tokenizer.expect("[");
885 pass->subscript = parse_expression();
886 tokenizer.expect("]");
888 tokenizer.expect(";");
892 RefPtr<Return> Parser::parse_return()
894 tokenizer.expect("return");
895 RefPtr<Return> ret = create_node<Return>();
896 if(tokenizer.peek_token()!=";")
897 ret->expression = parse_expression();
898 tokenizer.expect(";");