]> git.tdb.fi Git - libs/gl.git/blobdiff - source/glsl/parser.cpp
Split tokenizer and preprocessor out of the GLSL parser
[libs/gl.git] / source / glsl / parser.cpp
index 3126fae627a69941b1489ca969babf22e90658b9..eaf5706f25a6fd8b30935835efb32272943427e4 100644 (file)
@@ -1,4 +1,3 @@
-#include <msp/core/raii.h>
 #include <msp/strings/format.h>
 #include <msp/strings/regex.h>
 #include "glsl_error.h"
@@ -12,58 +11,14 @@ namespace Msp {
 namespace GL {
 namespace SL {
 
-Parser::Operator Parser::operators[] =
-{
-       { "[", 2, BINARY, LEFT_TO_RIGHT },
-       { "(", 2, BINARY, LEFT_TO_RIGHT },
-       { ".", 2, BINARY, LEFT_TO_RIGHT },
-       { "++", 2, POSTFIX, LEFT_TO_RIGHT },
-       { "--", 2, POSTFIX, LEFT_TO_RIGHT },
-       { "++", 3, PREFIX, RIGHT_TO_LEFT },
-       { "--", 3, PREFIX, RIGHT_TO_LEFT },
-       { "+", 3, PREFIX, RIGHT_TO_LEFT },
-       { "-", 3, PREFIX, RIGHT_TO_LEFT },
-       { "~", 3, PREFIX, RIGHT_TO_LEFT },
-       { "!", 3, PREFIX, RIGHT_TO_LEFT },
-       { "*", 4, BINARY, LEFT_TO_RIGHT },
-       { "/", 4, BINARY, LEFT_TO_RIGHT },
-       { "%", 4, BINARY, LEFT_TO_RIGHT },
-       { "+", 5, BINARY, LEFT_TO_RIGHT },
-       { "-", 5, BINARY, LEFT_TO_RIGHT },
-       { "<<", 6, BINARY, LEFT_TO_RIGHT },
-       { ">>", 6, BINARY, LEFT_TO_RIGHT },
-       { "<", 7, BINARY, LEFT_TO_RIGHT },
-       { ">", 7, BINARY, LEFT_TO_RIGHT },
-       { "<=", 7, BINARY, LEFT_TO_RIGHT },
-       { ">=", 7, BINARY, LEFT_TO_RIGHT },
-       { "==", 8, BINARY, LEFT_TO_RIGHT },
-       { "!=", 8, BINARY, LEFT_TO_RIGHT },
-       { "&", 9, BINARY, LEFT_TO_RIGHT },
-       { "^", 10, BINARY, LEFT_TO_RIGHT },
-       { "|", 11, BINARY, LEFT_TO_RIGHT },
-       { "&&", 12, BINARY, LEFT_TO_RIGHT },
-       { "^^", 13, BINARY, LEFT_TO_RIGHT },
-       { "||", 14, BINARY, LEFT_TO_RIGHT },
-       { "?", 15, BINARY, RIGHT_TO_LEFT },
-       { ":", 15, BINARY, RIGHT_TO_LEFT },
-       { "=", 16, BINARY, RIGHT_TO_LEFT },
-       { "+=", 16, BINARY, RIGHT_TO_LEFT },
-       { "-=", 16, BINARY, RIGHT_TO_LEFT },
-       { "*=", 16, BINARY, RIGHT_TO_LEFT },
-       { "/=", 16, BINARY, RIGHT_TO_LEFT },
-       { "%=", 16, BINARY, RIGHT_TO_LEFT },
-       { "<<=", 16, BINARY, RIGHT_TO_LEFT },
-       { ">>=", 16, BINARY, RIGHT_TO_LEFT },
-       { "&=", 16, BINARY, RIGHT_TO_LEFT },
-       { "^=", 16, BINARY, RIGHT_TO_LEFT },
-       { "|=", 16, BINARY, RIGHT_TO_LEFT },
-       { ",", 17, BINARY, LEFT_TO_RIGHT },
-       { { 0 }, 18, NO_OPERATOR, LEFT_TO_RIGHT }
-};
-
 Parser::Parser():
+       preprocessor(tokenizer),
        module(0)
-{ }
+{
+       tokenizer.signal_preprocess.connect(sigc::mem_fun(&preprocessor, &Preprocessor::preprocess));
+       preprocessor.signal_version.connect(sigc::mem_fun(this, &Parser::set_required_version));
+       preprocessor.signal_stage_change.connect(sigc::mem_fun(this, &Parser::stage_change));
+}
 
 Parser::~Parser()
 {
@@ -73,16 +28,14 @@ Parser::~Parser()
 Module &Parser::parse(const string &s, const string &n, unsigned i)
 {
        source = s;
-       location.name = n;
        source_index = i;
-       parse_source();
+       parse_source(n);
        return *module;
 }
 
 Module &Parser::parse(IO::Base &io, const string &n, unsigned i)
 {
        source = string();
-       location.name = n;
        source_index = i;
        while(!io.eof())
        {
@@ -90,11 +43,11 @@ Module &Parser::parse(IO::Base &io, const string &n, unsigned i)
                unsigned len = io.read(buffer, sizeof(buffer));
                source.append(buffer, len);
        }
-       parse_source();
+       parse_source(n);
        return *module;
 }
 
-void Parser::parse_source()
+void Parser::parse_source(const string &name)
 {
        while(1)
        {
@@ -110,194 +63,51 @@ void Parser::parse_source()
        delete module;
        module = new Module;
        cur_stage = &module->shared;
-       iter = source.begin();
-       source_end = source.end();
-       location.line = 1;
-       allow_preprocess = true;
+       tokenizer.begin(name, source);
        while(RefPtr<Statement> statement = parse_global_declaration())
                cur_stage->content.body.push_back(statement);
 }
 
-const string &Parser::peek_token(unsigned index)
-{
-       while(next_tokens.size()<=index)
-               next_tokens.push_back(parse_token_());
-       return (last_token = next_tokens[index]);
-}
-
-const string &Parser::parse_token()
-{
-       if(!next_tokens.empty())
-       {
-               last_token = next_tokens.front();
-               next_tokens.pop_front();
-               return last_token;
-       }
-
-       return (last_token = parse_token_());
-}
-
-string Parser::parse_token_()
-{
-       while(1)
-       {
-               skip_comment_and_whitespace();
-               if(iter==source_end)
-                       return string();
-               else if(allow_preprocess && *iter=='#')
-               {
-                       allow_preprocess = false;
-                       SetForScope<deque<string> > clear_tokens(next_tokens, deque<string>());
-                       preprocess();
-               }
-               else if(isalpha(*iter) || *iter=='_')
-                       return parse_identifier();
-               else if(isdigit(*iter))
-                       return parse_number();
-               else
-                       return parse_other();
-       }
-}
-
-string Parser::parse_identifier()
-{
-       string ident;
-       while(iter!=source_end)
-       {
-               if(isalnum(*iter) || *iter=='_')
-                       ident += *iter++;
-               else
-                       break;
-       }
-
-       return ident;
-}
-
-string Parser::parse_number()
-{
-       bool accept_sign = false;
-       string number;
-       while(iter!=source_end)
-       {
-               if(isdigit(*iter) || *iter=='.')
-                       number += *iter++;
-               else if(*iter=='e' || *iter=='E')
-               {
-                       number += *iter++;
-                       accept_sign = true;
-               }
-               else if(accept_sign && (*iter=='+' || *iter=='-'))
-                       number += *iter++;
-               else
-                       break;
-       }
-
-       return number;
-}
-
-string Parser::parse_other()
+void Parser::set_required_version(const Version &ver)
 {
-       if(iter==source_end)
-               return string();
-
-       string token(1, *iter++);
-       for(unsigned i=1; (i<3 && iter!=source_end); ++i)
-       {
-               bool matched = false;
-               for(const Operator *j=operators; (!matched && j->type); ++j)
-               {
-                       matched = (j->token[i]==*iter);
-                       for(unsigned k=0; (matched && k<i && j->token[k]); ++k)
-                               matched = (j->token[k]==token[k]);
-               }
-
-               if(!matched)
-                       break;
-
-               token += *iter++;
-       }
-
-       return token;
+       cur_stage->required_version = ver;
 }
 
-void Parser::skip_comment_and_whitespace()
+void Parser::stage_change(StageType stage)
 {
-       unsigned comment = 0;
-       while(iter!=source_end)
-       {
-               if(comment==0)
-               {
-                       if(*iter=='/')
-                               comment = 1;
-                       else if(!isspace(*iter))
-                               break;
-               }
-               else if(comment==1)
-               {
-                       if(*iter=='/')
-                               comment = 2;
-                       else if(*iter=='*')
-                               comment = 3;
-                       else
-                       {
-                               comment = 0;
-                               --iter;
-                               break;
-                       }
-               }
-               else if(comment==2)
-               {
-                       if(*iter=='\n')
-                               comment = 0;
-               }
-               else if(comment==3 && *iter=='*')
-                       comment = 4;
-               else if(comment==4)
-               {
-                       if(*iter=='/')
-                               comment = 0;
-                       else if(*iter!='*')
-                               comment = 3;
-               }
-
-               if(*iter=='\n')
-               {
-                       ++location.line;
-                       allow_preprocess = (comment<3);
-               }
+       if(!allow_stage_change)
+               throw invalid_shader_source(tokenizer.get_location(), "Changing stage not allowed here");
+       else if(stage<=cur_stage->type)
+               throw invalid_shader_source(tokenizer.get_location(), "Stage '%s' not allowed here", stage);
 
-               ++iter;
-       }
-}
+       module->stages.push_back(stage);
 
-void Parser::expect(const string &token)
-{
-       string parsed = parse_token();
-       if(parsed!=token)
-               throw parse_error(location, parsed, format("'%s'", token));
+       if(cur_stage->type!=SHARED)
+               module->stages.back().previous = cur_stage;
+       cur_stage = &module->stages.back();
 }
 
 string Parser::expect_type()
 {
-       string token = parse_token();
+       string token = tokenizer.parse_token();
        if(!is_type(token))
-               throw parse_error(location, token, "a type");
+               throw parse_error(tokenizer.get_location(), token, "a type");
        return token;
 }
 
 string Parser::expect_identifier()
 {
-       string token = parse_token();
+       string token = tokenizer.parse_token();
        if(!is_identifier(token))
-               throw parse_error(location, token, "an identifier");
+               throw parse_error(tokenizer.get_location(), token, "an identifier");
        return token;
 }
 
 bool Parser::check(const string &token)
 {
-       bool result = (peek_token()==token);
+       bool result = (tokenizer.peek_token()==token);
        if(result)
-               parse_token();
+               tokenizer.parse_token();
        return result;
 }
 
@@ -347,94 +157,10 @@ bool Parser::is_identifier(const string &token)
        return re.match(token);
 }
 
-void Parser::preprocess()
-{
-       expect("#");
-
-       string::const_iterator line_end = iter;
-       for(; (line_end!=source_end && *line_end!='\n'); ++line_end) ;
-       SetForScope<string::const_iterator> stop_at_line_end(source_end, line_end);
-
-       string token = peek_token();
-       if(token=="pragma")
-               preprocess_pragma();
-       else if(token=="version")
-               preprocess_version();
-       else if(token=="define" || token=="undef" || token=="if" || token=="ifdef" || token=="ifndef" || token=="else" ||
-               token=="elif" || token=="endif" || token=="error" || token=="extension" || token=="line")
-               throw invalid_shader_source(location, "Unsupported preprocessor directive '%s'", token);
-       else if(!token.empty())
-               throw parse_error(location, token, "a preprocessor directive");
-
-       iter = line_end;
-}
-
-void Parser::preprocess_version()
-{
-       expect("version");
-       string token = parse_token();
-       unsigned version = lexical_cast<unsigned>(token);
-       cur_stage->required_version = Version(version/100, version%100);
-
-       token = parse_token();
-       if(!token.empty())
-               throw parse_error(location, token, "end of line");
-}
-
-void Parser::preprocess_pragma()
-{
-       expect("pragma");
-       string token = parse_token();
-       if(token=="MSP")
-               preprocess_pragma_msp();
-}
-
-void Parser::preprocess_pragma_msp()
-{
-       string token = peek_token();
-       if(token=="stage")
-               preprocess_stage();
-       else
-               throw invalid_shader_source(location, "Unrecognized MSP pragma '%s'", token);
-
-       token = parse_token();
-       if(!token.empty())
-               throw parse_error(location, token, "end of line");
-}
-
-void Parser::preprocess_stage()
-{
-       if(!allow_stage_change)
-               throw invalid_shader_source(location, "Changing stage not allowed here");
-
-       expect("stage");
-       expect("(");
-       string token = expect_identifier();
-       StageType stage = SHARED;
-       if(token=="vertex")
-               stage = VERTEX;
-       else if(token=="geometry")
-               stage = GEOMETRY;
-       else if(token=="fragment")
-               stage = FRAGMENT;
-       else
-               throw parse_error(location, token, "stage identifier");
-       expect(")");
-
-       if(stage<=cur_stage->type)
-               throw invalid_shader_source(location, "Stage '%s' not allowed here", token);
-
-       module->stages.push_back(stage);
-
-       if(cur_stage->type!=SHARED)
-               module->stages.back().previous = cur_stage;
-       cur_stage = &module->stages.back();
-}
-
 RefPtr<Statement> Parser::parse_global_declaration()
 {
        allow_stage_change = true;
-       string token = peek_token();
+       string token = tokenizer.peek_token();
        allow_stage_change = false;
 
        if(token=="import")
@@ -444,15 +170,15 @@ RefPtr<Statement> Parser::parse_global_declaration()
        else if(token=="layout")
        {
                RefPtr<Layout> layout = parse_layout();
-               token = peek_token();
-               if(is_interface_qualifier(token) && peek_token(1)==";")
+               token = tokenizer.peek_token();
+               if(is_interface_qualifier(token) && tokenizer.peek_token(1)==";")
                {
                        RefPtr<InterfaceLayout> iface_lo = new InterfaceLayout;
                        iface_lo->source = source_index;
-                       iface_lo->line = location.line;
+                       iface_lo->line = tokenizer.get_location().line;
                        iface_lo->layout.qualifiers = layout->qualifiers;
-                       iface_lo->interface = parse_token();
-                       expect(";");
+                       iface_lo->interface = tokenizer.parse_token();
+                       tokenizer.expect(";");
                        return iface_lo;
                }
                else
@@ -466,7 +192,7 @@ RefPtr<Statement> Parser::parse_global_declaration()
                return parse_struct_declaration();
        else if(is_interface_qualifier(token))
        {
-               string next = peek_token(1);
+               string next = tokenizer.peek_token(1);
                if(is_type(next) || is_qualifier(next))
                        return parse_variable_declaration();
                else
@@ -476,7 +202,7 @@ RefPtr<Statement> Parser::parse_global_declaration()
                return parse_variable_declaration();
        else if(is_type(token))
        {
-               if(peek_token(2)=="(")
+               if(tokenizer.peek_token(2)=="(")
                        return parse_function_declaration();
                else
                        return parse_variable_declaration();
@@ -484,12 +210,12 @@ RefPtr<Statement> Parser::parse_global_declaration()
        else if(token.empty())
                return 0;
        else
-               throw parse_error(location, token, "a global declaration");
+               throw parse_error(tokenizer.get_location(), token, "a global declaration");
 }
 
 RefPtr<Statement> Parser::parse_statement()
 {
-       string token = peek_token();
+       string token = tokenizer.peek_token();
        if(token=="if")
                return parse_conditional();
        else if(token=="for")
@@ -504,9 +230,9 @@ RefPtr<Statement> Parser::parse_statement()
        {
                RefPtr<Jump> jump = new Jump;
                jump->source = source_index;
-               jump->line = location.line;
-               jump->keyword = parse_token();
-               expect(";");
+               jump->line = tokenizer.get_location().line;
+               jump->keyword = tokenizer.parse_token();
+               tokenizer.expect(";");
 
                return jump;
        }
@@ -516,88 +242,88 @@ RefPtr<Statement> Parser::parse_statement()
        {
                RefPtr<ExpressionStatement> expr = new ExpressionStatement;
                expr->source = source_index;
-               expr->line = location.line;
+               expr->line = tokenizer.get_location().line;
                expr->expression = parse_expression();
-               expect(";");
+               tokenizer.expect(";");
 
                return expr;
        }
        else
-               throw parse_error(location, token, "a statement");
+               throw parse_error(tokenizer.get_location(), token, "a statement");
 }
 
 RefPtr<Import> Parser::parse_import()
 {
        if(cur_stage->type!=SHARED)
-               throw invalid_shader_source(location, "Imports are only allowed in the shared section");
+               throw invalid_shader_source(tokenizer.get_location(), "Imports are only allowed in the shared section");
 
-       expect("import");
+       tokenizer.expect("import");
        RefPtr<Import> import = new Import;
        import->source = source_index;
-       import->line = location.line;
+       import->line = tokenizer.get_location().line;
        import->module = expect_identifier();
-       expect(";");
+       tokenizer.expect(";");
        return import;
 }
 
 RefPtr<Precision> Parser::parse_precision()
 {
-       expect("precision");
+       tokenizer.expect("precision");
        RefPtr<Precision> precision = new Precision;
        precision->source = source_index;
-       precision->line = location.line;
+       precision->line = tokenizer.get_location().line;
 
-       precision->precision = parse_token();
+       precision->precision = tokenizer.parse_token();
        if(!is_precision_qualifier(precision->precision))
-               throw parse_error(location, precision->precision, "a precision qualifier");
+               throw parse_error(tokenizer.get_location(), precision->precision, "a precision qualifier");
 
-       precision->type = parse_token();
+       precision->type = tokenizer.parse_token();
        // Not entirely accurate; only float, int and sampler types are allowed
        if(!is_builtin_type(precision->type))
-               throw parse_error(location, precision->type, "a builtin type");
+               throw parse_error(tokenizer.get_location(), precision->type, "a builtin type");
 
-       expect(";");
+       tokenizer.expect(";");
 
        return precision;
 }
 
 RefPtr<Layout> Parser::parse_layout()
 {
-       expect("layout");
-       expect("(");
+       tokenizer.expect("layout");
+       tokenizer.expect("(");
        RefPtr<Layout> layout = new Layout;
        while(1)
        {
-               string token = parse_token();
+               string token = tokenizer.parse_token();
                if(token==")")
-                       throw parse_error(location, token, "a layout qualifier name");
+                       throw parse_error(tokenizer.get_location(), token, "a layout qualifier name");
 
                layout->qualifiers.push_back(Layout::Qualifier());
                Layout::Qualifier &qual = layout->qualifiers.back();
                qual.identifier = token;
 
                if(check("="))
-                       qual.value = parse_token();
+                       qual.value = tokenizer.parse_token();
 
-               if(peek_token()==")")
+               if(tokenizer.peek_token()==")")
                        break;
 
-               expect(",");
+               tokenizer.expect(",");
        }
-       expect(")");
+       tokenizer.expect(")");
 
        return layout;
 }
 
 void Parser::parse_block(Block &block, bool require_braces)
 {
-       bool have_braces = (require_braces || peek_token()=="{");
+       bool have_braces = (require_braces || tokenizer.peek_token()=="{");
        if(have_braces)
-               expect("{");
+               tokenizer.expect("{");
 
        if(have_braces)
        {
-               while(peek_token()!="}")
+               while(tokenizer.peek_token()!="}")
                        block.body.push_back(parse_statement());
        }
        else
@@ -606,7 +332,7 @@ void Parser::parse_block(Block &block, bool require_braces)
        block.use_braces = (require_braces || block.body.size()!=1);
 
        if(have_braces)
-               expect("}");
+               tokenizer.expect("}");
 }
 
 RefPtr<Expression> Parser::parse_expression(unsigned precedence)
@@ -615,11 +341,11 @@ RefPtr<Expression> Parser::parse_expression(unsigned precedence)
        VariableReference *left_var = 0;
        while(1)
        {
-               string token = peek_token();
+               string token = tokenizer.peek_token();
 
                const Operator *oper = 0;
-               for(Operator *i=operators; (!oper && i->type); ++i)
-                       if(token==i->token && (!left || i->type!=PREFIX) && (left || i->type!=POSTFIX))
+               for(const Operator *i=Operator::operators; (!oper && i->type); ++i)
+                       if(token==i->token && (!left || i->type!=Operator::PREFIX) && (left || i->type!=Operator::POSTFIX))
                                oper = i;
 
                if(token==";" || token==")" || token=="]" || token=="," || (oper && precedence && oper->precedence>=precedence))
@@ -627,52 +353,52 @@ RefPtr<Expression> Parser::parse_expression(unsigned precedence)
                        if(left)
                                return left;
                        else
-                               throw parse_error(location, token, "an expression");
+                               throw parse_error(tokenizer.get_location(), token, "an expression");
                }
                else if(left)
                {
                        if(token=="(")
                        {
                                if(!left_var)
-                                       throw invalid_shader_source(location, "Syntax error before '(': function name must be an identifier");
+                                       throw invalid_shader_source(tokenizer.get_location(), "Syntax error before '(': function name must be an identifier");
                                left = parse_function_call(*left_var);
                        }
                        else if(token==".")
                        {
                                RefPtr<MemberAccess> memacc = new MemberAccess;
                                memacc->left = left;
-                               parse_token();
+                               tokenizer.parse_token();
                                memacc->member = expect_identifier();
                                left = memacc;
                        }
-                       else if(oper && oper->type==POSTFIX)
+                       else if(oper && oper->type==Operator::POSTFIX)
                        {
                                RefPtr<UnaryExpression> unary = new UnaryExpression;
-                               unary->oper = parse_token();
+                               unary->oper = tokenizer.parse_token();
                                unary->prefix = false;
                                unary->expression = left;
                                left = unary;
                        }
-                       else if(oper && oper->type==BINARY)
+                       else if(oper && oper->type==Operator::BINARY)
                                left = parse_binary(left, oper);
                        else
-                               throw parse_error(location, token, "an operator");
+                               throw parse_error(tokenizer.get_location(), token, "an operator");
                        left_var = 0;
                }
                else
                {
                        if(token=="(")
                        {
-                               parse_token();
+                               tokenizer.parse_token();
                                RefPtr<ParenthesizedExpression> parexpr = new ParenthesizedExpression;
                                parexpr->expression = parse_expression();
-                               expect(")");
+                               tokenizer.expect(")");
                                left = parexpr;
                        }
                        else if(isdigit(token[0]) || token=="true" || token=="false")
                        {
                                RefPtr<Literal> literal = new Literal;
-                               literal->token = parse_token();
+                               literal->token = tokenizer.parse_token();
                                left = literal;
                        }
                        else if(is_identifier(token))
@@ -682,16 +408,16 @@ RefPtr<Expression> Parser::parse_expression(unsigned precedence)
                                left = var;
                                left_var = var.get();
                        }
-                       else if(oper && oper->type==PREFIX)
+                       else if(oper && oper->type==Operator::PREFIX)
                        {
                                RefPtr<UnaryExpression> unary = new UnaryExpression;
-                               unary->oper = parse_token();
+                               unary->oper = tokenizer.parse_token();
                                unary->prefix = true;
                                unary->expression = parse_expression(oper->precedence);
                                left = unary;
                        }
                        else
-                               throw parse_error(location, token, "an expression");
+                               throw parse_error(tokenizer.get_location(), token, "an expression");
                }
        }
 }
@@ -700,15 +426,15 @@ RefPtr<BinaryExpression> Parser::parse_binary(const RefPtr<Expression> &left, co
 {
        RefPtr<BinaryExpression> binary = (oper->precedence==16 ? new Assignment : new BinaryExpression);
        binary->left = left;
-       binary->oper = parse_token();
+       binary->oper = tokenizer.parse_token();
        if(binary->oper=="[")
        {
                binary->right = parse_expression();
-               expect("]");
+               tokenizer.expect("]");
                binary->after = "]";
        }
        else
-               binary->right = parse_expression(oper->precedence+(oper->assoc==RIGHT_TO_LEFT));
+               binary->right = parse_expression(oper->precedence+(oper->assoc==Operator::RIGHT_TO_LEFT));
        return binary;
 }
 
@@ -717,27 +443,27 @@ RefPtr<FunctionCall> Parser::parse_function_call(const VariableReference &var)
        RefPtr<FunctionCall> call = new FunctionCall;
        call->name = var.name;
        call->constructor = is_type(call->name);
-       expect("(");
-       while(peek_token()!=")")
+       tokenizer.expect("(");
+       while(tokenizer.peek_token()!=")")
        {
                if(!call->arguments.empty())
-                       expect(",");
+                       tokenizer.expect(",");
                call->arguments.push_back(parse_expression());
        }
-       expect(")");
+       tokenizer.expect(")");
        return call;
 }
 
 RefPtr<StructDeclaration> Parser::parse_struct_declaration()
 {
-       expect("struct");
+       tokenizer.expect("struct");
        RefPtr<StructDeclaration> strct = new StructDeclaration;
        strct->source = source_index;
-       strct->line = location.line;
+       strct->line = tokenizer.get_location().line;
 
        strct->name = expect_identifier();
        parse_block(strct->members, true);
-       expect(";");
+       tokenizer.expect(";");
 
        declared_types.insert(strct->name);
        return strct;
@@ -747,12 +473,12 @@ RefPtr<VariableDeclaration> Parser::parse_variable_declaration()
 {
        RefPtr<VariableDeclaration> var = new VariableDeclaration;
        var->source = source_index;
-       var->line = location.line;
+       var->line = tokenizer.get_location().line;
 
-       string token = peek_token();
+       string token = tokenizer.peek_token();
        while(is_qualifier(token))
        {
-               parse_token();
+               tokenizer.parse_token();
                if(is_interface_qualifier(token))
                        var->interface = token;
                else if(is_sampling_qualifier(token))
@@ -763,7 +489,7 @@ RefPtr<VariableDeclaration> Parser::parse_variable_declaration()
                        var->precision = token;
                else if(token=="const")
                        var->constant = true;
-               token = peek_token();
+               token = tokenizer.peek_token();
        }
 
        var->type = expect_type();
@@ -775,14 +501,14 @@ RefPtr<VariableDeclaration> Parser::parse_variable_declaration()
                if(!check("]"))
                {
                        var->array_size = parse_expression();
-                       expect("]");
+                       tokenizer.expect("]");
                }
        }
 
        if(check("="))
                var->init_expression = parse_expression();
 
-       expect(";");
+       tokenizer.expect(";");
        return var;
 }
 
@@ -790,36 +516,36 @@ RefPtr<FunctionDeclaration> Parser::parse_function_declaration()
 {
        RefPtr<FunctionDeclaration> func = new FunctionDeclaration;
        func->source = source_index;
-       func->line = location.line;
+       func->line = tokenizer.get_location().line;
 
        func->return_type = expect_type();
        func->name = expect_identifier();
-       expect("(");
-       while(peek_token()!=")")
+       tokenizer.expect("(");
+       while(tokenizer.peek_token()!=")")
        {
                if(!func->parameters.empty())
-                       expect(",");
+                       tokenizer.expect(",");
 
                RefPtr<VariableDeclaration> var = new VariableDeclaration;
-               string token = peek_token();
+               string token = tokenizer.peek_token();
                if(token=="in" || token=="out" || token=="inout")
-                       var->interface = parse_token();
+                       var->interface = tokenizer.parse_token();
                var->type = expect_type();
                var->name = expect_identifier();
                func->parameters.push_back(var);
        }
-       expect(")");
+       tokenizer.expect(")");
 
-       string token = peek_token();
+       string token = tokenizer.peek_token();
        if(token=="{")
        {
                func->definition = func.get();
                parse_block(func->body, true);
        }
        else if(token==";")
-               parse_token();
+               tokenizer.parse_token();
        else
-               throw parse_error(location, token, "'{' or ';'");
+               throw parse_error(tokenizer.get_location(), token, "'{' or ';'");
 
        return func;
 }
@@ -828,11 +554,11 @@ RefPtr<InterfaceBlock> Parser::parse_interface_block()
 {
        RefPtr<InterfaceBlock> iface = new InterfaceBlock;
        iface->source = source_index;
-       iface->line = location.line;
+       iface->line = tokenizer.get_location().line;
 
-       iface->interface = parse_token();
+       iface->interface = tokenizer.parse_token();
        if(!is_interface_qualifier(iface->interface))
-               throw parse_error(location, iface->interface, "an interface qualifier");
+               throw parse_error(tokenizer.get_location(), iface->interface, "an interface qualifier");
 
        iface->name = expect_identifier();
        parse_block(iface->members, true);
@@ -842,9 +568,9 @@ RefPtr<InterfaceBlock> Parser::parse_interface_block()
                if(check("["))
                {
                        iface->array = true;
-                       expect("]");
+                       tokenizer.expect("]");
                }
-               expect(";");
+               tokenizer.expect(";");
        }
 
        return iface;
@@ -852,20 +578,20 @@ RefPtr<InterfaceBlock> Parser::parse_interface_block()
 
 RefPtr<Conditional> Parser::parse_conditional()
 {
-       expect("if");
+       tokenizer.expect("if");
        RefPtr<Conditional> cond = new Conditional;
        cond->source = source_index;
-       cond->line = location.line;
-       expect("(");
+       cond->line = tokenizer.get_location().line;
+       tokenizer.expect("(");
        cond->condition = parse_expression();
-       expect(")");
+       tokenizer.expect(")");
 
        parse_block(cond->body, false);
 
-       string token = peek_token();
+       string token = tokenizer.peek_token();
        if(token=="else")
        {
-               parse_token();
+               tokenizer.parse_token();
                parse_block(cond->else_body, false);
        }
 
@@ -874,12 +600,12 @@ RefPtr<Conditional> Parser::parse_conditional()
 
 RefPtr<Iteration> Parser::parse_for()
 {
-       expect("for");
+       tokenizer.expect("for");
        RefPtr<Iteration> loop = new Iteration;
        loop->source = source_index;
-       loop->line = location.line;
-       expect("(");
-       string token = peek_token();
+       loop->line = tokenizer.get_location().line;
+       tokenizer.expect("(");
+       string token = tokenizer.peek_token();
        if(is_type(token))
                loop->init_statement = parse_statement();
        else
@@ -890,14 +616,14 @@ RefPtr<Iteration> Parser::parse_for()
                        expr->expression = parse_expression();
                        loop->init_statement = expr;
                }
-               expect(";");
+               tokenizer.expect(";");
        }
-       if(peek_token()!=";")
+       if(tokenizer.peek_token()!=";")
                loop->condition = parse_expression();
-       expect(";");
-       if(peek_token()!=")")
+       tokenizer.expect(";");
+       if(tokenizer.peek_token()!=")")
                loop->loop_expression = parse_expression();
-       expect(")");
+       tokenizer.expect(")");
 
        parse_block(loop->body, false);
 
@@ -906,13 +632,13 @@ RefPtr<Iteration> Parser::parse_for()
 
 RefPtr<Iteration> Parser::parse_while()
 {
-       expect("while");
+       tokenizer.expect("while");
        RefPtr<Iteration> loop = new Iteration;
        loop->source = source_index;
-       loop->line = location.line;
-       expect("(");
+       loop->line = tokenizer.get_location().line;
+       tokenizer.expect("(");
        loop->condition = parse_expression();
-       expect(")");
+       tokenizer.expect(")");
 
        parse_block(loop->body, false);
 
@@ -921,29 +647,29 @@ RefPtr<Iteration> Parser::parse_while()
 
 RefPtr<Passthrough> Parser::parse_passthrough()
 {
-       expect("passthrough");
+       tokenizer.expect("passthrough");
        RefPtr<Passthrough> pass = new Passthrough;
        pass->source = source_index;
-       pass->line = location.line;
+       pass->line = tokenizer.get_location().line;
        if(cur_stage->type==GEOMETRY)
        {
-               expect("[");
+               tokenizer.expect("[");
                pass->subscript = parse_expression();
-               expect("]");
+               tokenizer.expect("]");
        }
-       expect(";");
+       tokenizer.expect(";");
        return pass;
 }
 
 RefPtr<Return> Parser::parse_return()
 {
-       expect("return");
+       tokenizer.expect("return");
        RefPtr<Return> ret = new Return;
        ret->source = source_index;
-       ret->line = location.line;
-       if(peek_token()!=";")
+       ret->line = tokenizer.get_location().line;
+       if(tokenizer.peek_token()!=";")
                ret->expression = parse_expression();
-       expect(";");
+       tokenizer.expect(";");
        return ret;
 }