X-Git-Url: http://git.tdb.fi/?a=blobdiff_plain;f=source%2Fglsl%2Fparser.cpp;h=4b6bd7ed08992271f5ba73318a253b8a4ec2d6ab;hb=ffdb126a55467245da08a9e6f5669f86e1899bb0;hp=d55cdab3abc5ff426fbf89b3edeab1d56109ddff;hpb=c1b0303f65ee966a973197cbdbf177c571478674;p=libs%2Fgl.git

diff --git a/source/glsl/parser.cpp b/source/glsl/parser.cpp
index d55cdab3..4b6bd7ed 100644
--- a/source/glsl/parser.cpp
+++ b/source/glsl/parser.cpp
@@ -1,6 +1,7 @@
-#include <msp/core/raii.h>
 #include <msp/strings/format.h>
 #include <msp/strings/regex.h>
+#include <msp/strings/utils.h>
+#include "glsl_error.h"
 #include "parser.h"
 
 #undef interface
@@ -11,58 +12,16 @@ namespace Msp {
 namespace GL {
 namespace SL {
 
-Parser::Operator Parser::operators[] =
-{
-	{ "[", 2, BINARY, LEFT_TO_RIGHT },
-	{ "(", 2, BINARY, LEFT_TO_RIGHT },
-	{ ".", 2, BINARY, LEFT_TO_RIGHT },
-	{ "++", 2, POSTFIX, LEFT_TO_RIGHT },
-	{ "--", 2, POSTFIX, LEFT_TO_RIGHT },
-	{ "++", 3, PREFIX, RIGHT_TO_LEFT },
-	{ "--", 3, PREFIX, RIGHT_TO_LEFT },
-	{ "+", 3, PREFIX, RIGHT_TO_LEFT },
-	{ "-", 3, PREFIX, RIGHT_TO_LEFT },
-	{ "~", 3, PREFIX, RIGHT_TO_LEFT },
-	{ "!", 3, PREFIX, RIGHT_TO_LEFT },
-	{ "*", 4, BINARY, LEFT_TO_RIGHT },
-	{ "/", 4, BINARY, LEFT_TO_RIGHT },
-	{ "%", 4, BINARY, LEFT_TO_RIGHT },
-	{ "+", 5, BINARY, LEFT_TO_RIGHT },
-	{ "-", 5, BINARY, LEFT_TO_RIGHT },
-	{ "<<", 6, BINARY, LEFT_TO_RIGHT },
-	{ ">>", 6, BINARY, LEFT_TO_RIGHT },
-	{ "<", 7, BINARY, LEFT_TO_RIGHT },
-	{ ">", 7, BINARY, LEFT_TO_RIGHT },
-	{ "<=", 7, BINARY, LEFT_TO_RIGHT },
-	{ ">=", 7, BINARY, LEFT_TO_RIGHT },
-	{ "==", 8, BINARY, LEFT_TO_RIGHT },
-	{ "!=", 8, BINARY, LEFT_TO_RIGHT },
-	{ "&", 9, BINARY, LEFT_TO_RIGHT },
-	{ "^", 10, BINARY, LEFT_TO_RIGHT },
-	{ "|", 11, BINARY, LEFT_TO_RIGHT },
-	{ "&&", 12, BINARY, LEFT_TO_RIGHT },
-	{ "^^", 13, BINARY, LEFT_TO_RIGHT },
-	{ "||", 14, BINARY, LEFT_TO_RIGHT },
-	{ "?", 15, BINARY, RIGHT_TO_LEFT },
-	{ ":", 15, BINARY, RIGHT_TO_LEFT },
-	{ "=", 16, BINARY, RIGHT_TO_LEFT },
-	{ "+=", 16, BINARY, RIGHT_TO_LEFT },
-	{ "-=", 16, BINARY, RIGHT_TO_LEFT },
-	{ "*=", 16, BINARY, RIGHT_TO_LEFT },
-	{ "/=", 16, BINARY, RIGHT_TO_LEFT },
-	{ "%=", 16, BINARY, RIGHT_TO_LEFT },
-	{ "<<=", 16, BINARY, RIGHT_TO_LEFT },
-	{ ">>=", 16, BINARY, RIGHT_TO_LEFT },
-	{ "&=", 16, BINARY, RIGHT_TO_LEFT },
-	{ "^=", 16, BINARY, RIGHT_TO_LEFT },
-	{ "|=", 16, BINARY, RIGHT_TO_LEFT },
-	{ ",", 17, BINARY, LEFT_TO_RIGHT },
-	{ { 0 }, 18, NO_OPERATOR, LEFT_TO_RIGHT }
-};
-
 Parser::Parser():
+	preprocessor(tokenizer),
 	module(0)
-{ }
+{
+	tokenizer.signal_preprocess.connect(sigc::mem_fun(&preprocessor, &Preprocessor::preprocess));
+	preprocessor.signal_version.connect(sigc::mem_fun(this, &Parser::set_required_version));
+	preprocessor.signal_source.connect(sigc::mem_fun(this, &Parser::source_reference));
+	preprocessor.signal_stage_change.connect(sigc::mem_fun(this, &Parser::stage_change));
+	preprocessor.signal_line.connect(sigc::mem_fun(this, &Parser::line_change));
+}
 
 Parser::~Parser()
 {
@@ -72,242 +31,107 @@ Parser::~Parser()
 Module &Parser::parse(const string &s, const string &n, unsigned i)
 {
 	source = s;
-	source_name = n;
-	source_index = i;
-	parse_source();
+	parse_source(n, i);
 	return *module;
 }
 
 Module &Parser::parse(IO::Base &io, const string &n, unsigned i)
 {
 	source = string();
-	source_name = n;
-	source_index = i;
 	while(!io.eof())
 	{
 		char buffer[4096];
 		unsigned len = io.read(buffer, sizeof(buffer));
 		source.append(buffer, len);
 	}
-	parse_source();
+	parse_source(n, i);
 	return *module;
 }
 
-void Parser::parse_source()
+void Parser::parse_source(const string &name, unsigned index)
 {
-	while(1)
-	{
-		string::size_type slashes = source.find("//////");
-		if(slashes==string::npos)
-			break;
-
-		string::size_type newline = source.find('\n', slashes);
-		string pragma = format("#pragma MSP stage(%s)", source.substr(slashes+6, newline-slashes-6));
-		source.replace(slashes, newline-slashes, pragma);
-	}
-
 	delete module;
 	module = new Module;
 	cur_stage = &module->shared;
-	iter = source.begin();
-	source_end = source.end();
-	current_line = 1;
-	allow_preprocess = true;
+	base_index = index;
+	source_index = index;
+	source_reference(1, name);
+	tokenizer.begin(name, source);
 	while(RefPtr<Statement> statement = parse_global_declaration())
 		cur_stage->content.body.push_back(statement);
 }
 
-string Parser::format_error(const std::string &message)
-{
-	string location = format("%s:%d: ", source_name, current_line);
-	return location+message;
-}
-
-string Parser::format_syntax_error(const std::string &expected)
-{
-	return format_error(format("Syntax error at '%s': expected %s", last_token, expected));
-}
-
-const string &Parser::peek_token(unsigned index)
+void Parser::set_required_version(const Version &ver)
 {
-	while(next_tokens.size()<=index)
-		next_tokens.push_back(parse_token_());
-	return (last_token = next_tokens[index]);
+	cur_stage->required_features.glsl_version = ver;
 }
 
-const string &Parser::parse_token()
+void Parser::source_reference(unsigned index, const string &name)
 {
-	if(!next_tokens.empty())
-	{
-		last_token = next_tokens.front();
-		next_tokens.pop_front();
-		return last_token;
-	}
+	if(index<1)
+		throw invalid_shader_source(tokenizer.get_location(), "Invalid source reference");
 
-	return (last_token = parse_token_());
-}
-
-string Parser::parse_token_()
-{
-	while(1)
-	{
-		skip_comment_and_whitespace();
-		if(iter==source_end)
-			return string();
-		else if(allow_preprocess && *iter=='#')
-		{
-			allow_preprocess = false;
-			SetForScope<deque<string> > clear_tokens(next_tokens, deque<string>());
-			preprocess();
-		}
-		else if(isalpha(*iter) || *iter=='_')
-			return parse_identifier();
-		else if(isdigit(*iter))
-			return parse_number();
-		else
-			return parse_other();
-	}
+	module->source_map.set_name(base_index+index-1, name);
 }
 
-string Parser::parse_identifier()
+void Parser::stage_change(Stage::Type stage)
 {
-	string ident;
-	while(iter!=source_end)
-	{
-		if(isalnum(*iter) || *iter=='_')
-			ident += *iter++;
-		else
-			break;
-	}
-
-	return ident;
-}
-
-string Parser::parse_number()
-{
-	bool accept_sign = false;
-	string number;
-	while(iter!=source_end)
-	{
-		if(isdigit(*iter) || *iter=='.')
-			number += *iter++;
-		else if(*iter=='e' || *iter=='E')
-		{
-			number += *iter++;
-			accept_sign = true;
-		}
-		else if(accept_sign && (*iter=='+' || *iter=='-'))
-			number += *iter++;
-		else
-			break;
-	}
-
-	return number;
-}
-
-string Parser::parse_other()
-{
-	if(iter==source_end)
-		return string();
-
-	string token(1, *iter++);
-	for(unsigned i=1; (i<3 && iter!=source_end); ++i)
-	{
-		bool matched = false;
-		for(const Operator *j=operators; (!matched && j->type); ++j)
-		{
-			matched = (j->token[i]==*iter);
-			for(unsigned k=0; (matched && k<i && j->token[k]); ++k)
-				matched = (j->token[k]==token[k]);
-		}
-
-		if(!matched)
-			break;
+	if(!allow_stage_change)
+		throw invalid_shader_source(tokenizer.get_location(), "Changing stage not allowed here");
+	else if(stage<=cur_stage->type)
+		throw invalid_shader_source(tokenizer.get_location(), "Stage '%s' not allowed here", Stage::get_stage_name(stage));
 
-		token += *iter++;
-	}
+	module->stages.push_back(stage);
 
-	return token;
+	if(cur_stage->type!=Stage::SHARED)
+		module->stages.back().previous = cur_stage;
+	cur_stage = &module->stages.back();
 }
 
-void Parser::skip_comment_and_whitespace()
+void Parser::line_change(int index, unsigned line)
 {
-	unsigned comment = 0;
-	while(iter!=source_end)
-	{
-		if(comment==0)
-		{
-			if(*iter=='/')
-				comment = 1;
-			else if(!isspace(*iter))
-				break;
-		}
-		else if(comment==1)
-		{
-			if(*iter=='/')
-				comment = 2;
-			else if(*iter=='*')
-				comment = 3;
-			else
-			{
-				comment = 0;
-				--iter;
-				break;
-			}
-		}
-		else if(comment==2)
-		{
-			if(*iter=='\n')
-				comment = 0;
-		}
-		else if(comment==3 && *iter=='*')
-			comment = 4;
-		else if(comment==4)
-		{
-			if(*iter=='/')
-				comment = 0;
-			else if(*iter!='*')
-				comment = 3;
-		}
-
-		if(*iter=='\n')
-		{
-			++current_line;
-			allow_preprocess = (comment<3);
-		}
-
-		++iter;
-	}
-}
+	if(index>0)
+		source_index = base_index+index-1;
+	else if(index==0)
+		source_index = 0;
+	else
+		index = source_index;
 
-void Parser::expect(const string &token)
-{
-	string parsed = parse_token();
-	if(parsed!=token)
-		throw runtime_error(format_syntax_error(format("'%s'", token)));
+	string name = module->source_map.get_name(index);
+	if(name.empty())
+		name = format("<%d>", index);
+	tokenizer.set_location(Location(name, line));
 }
 
 string Parser::expect_type()
 {
-	string token = parse_token();
+	string token = tokenizer.parse_token();
 	if(!is_type(token))
-		throw runtime_error(format_syntax_error("a type"));
+		throw parse_error(tokenizer.get_location(), token, "a type");
 	return token;
 }
 
 string Parser::expect_identifier()
 {
-	string token = parse_token();
+	string token = tokenizer.parse_token();
 	if(!is_identifier(token))
-		throw runtime_error(format_syntax_error("an identifier"));
+		throw parse_error(tokenizer.get_location(), token, "an identifier");
 	return token;
 }
 
+int Parser::expect_integer()
+{
+	string token = tokenizer.parse_token();
+	if(!isnumrc(token))
+		throw parse_error(tokenizer.get_location(), token, "an integer literal");
+	return lexical_cast<int>(token);
+}
+
 bool Parser::check(const string &token)
 {
-	bool result = (peek_token()==token);
+	bool result = (tokenizer.peek_token()==token);
 	if(result)
-		parse_token();
+		tokenizer.parse_token();
 	return result;
 }
 
@@ -357,94 +181,10 @@ bool Parser::is_identifier(const string &token)
 	return re.match(token);
 }
 
-void Parser::preprocess()
-{
-	expect("#");
-
-	string::const_iterator line_end = iter;
-	for(; (line_end!=source_end && *line_end!='\n'); ++line_end) ;
-	SetForScope<string::const_iterator> stop_at_line_end(source_end, line_end);
-
-	string token = peek_token();
-	if(token=="pragma")
-		preprocess_pragma();
-	else if(token=="version")
-		preprocess_version();
-	else if(token=="define" || token=="undef" || token=="if" || token=="ifdef" || token=="ifndef" || token=="else" ||
-		token=="elif" || token=="endif" || token=="error" || token=="extension" || token=="line")
-		throw runtime_error(format_error(format("Unsupported preprocessor directive '%s'", token)));
-	else if(!token.empty())
-		throw runtime_error(format_syntax_error("a preprocessor directive"));
-
-	iter = line_end;
-}
-
-void Parser::preprocess_version()
-{
-	expect("version");
-	string token = parse_token();
-	unsigned version = lexical_cast<unsigned>(token);
-	cur_stage->required_version = Version(version/100, version%100);
-
-	token = parse_token();
-	if(!token.empty())
-		throw runtime_error(format_syntax_error("end of line"));
-}
-
-void Parser::preprocess_pragma()
-{
-	expect("pragma");
-	string token = parse_token();
-	if(token=="MSP")
-		preprocess_pragma_msp();
-}
-
-void Parser::preprocess_pragma_msp()
-{
-	string token = peek_token();
-	if(token=="stage")
-		preprocess_stage();
-	else
-		throw runtime_error(format_error(format("Unrecognized MSP pragma '%s'", token)));
-
-	token = parse_token();
-	if(!token.empty())
-		throw runtime_error(format_syntax_error("end of line"));
-}
-
-void Parser::preprocess_stage()
-{
-	if(!allow_stage_change)
-		throw runtime_error(format_error("Changing stage not allowed here"));
-
-	expect("stage");
-	expect("(");
-	string token = expect_identifier();
-	StageType stage = SHARED;
-	if(token=="vertex")
-		stage = VERTEX;
-	else if(token=="geometry")
-		stage = GEOMETRY;
-	else if(token=="fragment")
-		stage = FRAGMENT;
-	else
-		throw runtime_error(format_syntax_error("stage identifier"));
-	expect(")");
-
-	if(stage<=cur_stage->type)
-		throw runtime_error(format_error(format("Stage '%s' not allowed here", token)));
-
-	module->stages.push_back(stage);
-
-	if(cur_stage->type!=SHARED)
-		module->stages.back().previous = cur_stage;
-	cur_stage = &module->stages.back();
-}
-
 RefPtr<Statement> Parser::parse_global_declaration()
 {
 	allow_stage_change = true;
-	string token = peek_token();
+	string token = tokenizer.peek_token();
 	allow_stage_change = false;
 
 	if(token=="import")
@@ -454,15 +194,15 @@ RefPtr<Statement> Parser::parse_global_declaration()
 	else if(token=="layout")
 	{
 		RefPtr<Layout> layout = parse_layout();
-		token = peek_token();
-		if(is_interface_qualifier(token) && peek_token(1)==";")
+		token = tokenizer.peek_token();
+		if(is_interface_qualifier(token) && tokenizer.peek_token(1)==";")
 		{
 			RefPtr<InterfaceLayout> iface_lo = new InterfaceLayout;
 			iface_lo->source = source_index;
-			iface_lo->line = current_line;
+			iface_lo->line = tokenizer.get_location().line;
 			iface_lo->layout.qualifiers = layout->qualifiers;
-			iface_lo->interface = parse_token();
-			expect(";");
+			iface_lo->interface = tokenizer.parse_token();
+			tokenizer.expect(";");
 			return iface_lo;
 		}
 		else
@@ -476,7 +216,7 @@ RefPtr<Statement> Parser::parse_global_declaration()
 		return parse_struct_declaration();
 	else if(is_interface_qualifier(token))
 	{
-		string next = peek_token(1);
+		string next = tokenizer.peek_token(1);
 		if(is_type(next) || is_qualifier(next))
 			return parse_variable_declaration();
 		else
@@ -486,7 +226,7 @@ RefPtr<Statement> Parser::parse_global_declaration()
 		return parse_variable_declaration();
 	else if(is_type(token))
 	{
-		if(peek_token(2)=="(")
+		if(tokenizer.peek_token(2)=="(")
 			return parse_function_declaration();
 		else
 			return parse_variable_declaration();
@@ -494,12 +234,12 @@ RefPtr<Statement> Parser::parse_global_declaration()
 	else if(token.empty())
 		return 0;
 	else
-		throw runtime_error(format_syntax_error("a global declaration"));
+		throw parse_error(tokenizer.get_location(), token, "a global declaration");
 }
 
 RefPtr<Statement> Parser::parse_statement()
 {
-	string token = peek_token();
+	string token = tokenizer.peek_token();
 	if(token=="if")
 		return parse_conditional();
 	else if(token=="for")
@@ -514,9 +254,9 @@ RefPtr<Statement> Parser::parse_statement()
 	{
 		RefPtr<Jump> jump = new Jump;
 		jump->source = source_index;
-		jump->line = current_line;
-		jump->keyword = parse_token();
-		expect(";");
+		jump->line = tokenizer.get_location().line;
+		jump->keyword = tokenizer.parse_token();
+		tokenizer.expect(";");
 
 		return jump;
 	}
@@ -526,88 +266,88 @@ RefPtr<Statement> Parser::parse_statement()
 	{
 		RefPtr<ExpressionStatement> expr = new ExpressionStatement;
 		expr->source = source_index;
-		expr->line = current_line;
+		expr->line = tokenizer.get_location().line;
 		expr->expression = parse_expression();
-		expect(";");
+		tokenizer.expect(";");
 
 		return expr;
 	}
 	else
-		throw runtime_error(format_syntax_error("a statement"));
+		throw parse_error(tokenizer.get_location(), token, "a statement");
 }
 
 RefPtr<Import> Parser::parse_import()
 {
-	if(cur_stage->type!=SHARED)
-		throw runtime_error(format_error("Imports are only allowed in the shared section"));
+	if(cur_stage->type!=Stage::SHARED)
+		throw invalid_shader_source(tokenizer.get_location(), "Imports are only allowed in the shared section");
 
-	expect("import");
+	tokenizer.expect("import");
 	RefPtr<Import> import = new Import;
 	import->source = source_index;
-	import->line = current_line;
+	import->line = tokenizer.get_location().line;
 	import->module = expect_identifier();
-	expect(";");
+	tokenizer.expect(";");
 	return import;
 }
 
 RefPtr<Precision> Parser::parse_precision()
 {
-	expect("precision");
+	tokenizer.expect("precision");
 	RefPtr<Precision> precision = new Precision;
 	precision->source = source_index;
-	precision->line = current_line;
+	precision->line = tokenizer.get_location().line;
 
-	precision->precision = parse_token();
+	precision->precision = tokenizer.parse_token();
 	if(!is_precision_qualifier(precision->precision))
-		throw runtime_error(format_syntax_error("a precision qualifier"));
+		throw parse_error(tokenizer.get_location(), precision->precision, "a precision qualifier");
 
-	precision->type = parse_token();
+	precision->type = tokenizer.parse_token();
 	// Not entirely accurate; only float, int and sampler types are allowed
 	if(!is_builtin_type(precision->type))
-		throw runtime_error(format_syntax_error("a builtin type"));
+		throw parse_error(tokenizer.get_location(), precision->type, "a builtin type");
 
-	expect(";");
+	tokenizer.expect(";");
 
 	return precision;
 }
 
 RefPtr<Layout> Parser::parse_layout()
 {
-	expect("layout");
-	expect("(");
+	tokenizer.expect("layout");
+	tokenizer.expect("(");
 	RefPtr<Layout> layout = new Layout;
 	while(1)
 	{
-		string token = parse_token();
+		string token = tokenizer.parse_token();
 		if(token==")")
-			throw runtime_error(format_syntax_error("a layout qualifier name"));
+			throw parse_error(tokenizer.get_location(), token, "a layout qualifier name");
 
 		layout->qualifiers.push_back(Layout::Qualifier());
 		Layout::Qualifier &qual = layout->qualifiers.back();
-		qual.identifier = token;
+		qual.name = token;
 
-		if(check("="))
-			qual.value = parse_token();
+		if((qual.has_value = check("=")))
+			qual.value = expect_integer();
 
-		if(peek_token()==")")
+		if(tokenizer.peek_token()==")")
 			break;
 
-		expect(",");
+		tokenizer.expect(",");
 	}
-	expect(")");
+	tokenizer.expect(")");
 
 	return layout;
 }
 
 void Parser::parse_block(Block &block, bool require_braces)
 {
-	bool have_braces = (require_braces || peek_token()=="{");
+	bool have_braces = (require_braces || tokenizer.peek_token()=="{");
 	if(have_braces)
-		expect("{");
+		tokenizer.expect("{");
 
 	if(have_braces)
 	{
-		while(peek_token()!="}")
+		while(tokenizer.peek_token()!="}")
 			block.body.push_back(parse_statement());
 	}
 	else
@@ -616,7 +356,7 @@ void Parser::parse_block(Block &block, bool require_braces)
 	block.use_braces = (require_braces || block.body.size()!=1);
 
 	if(have_braces)
-		expect("}");
+		tokenizer.expect("}");
 }
 
 RefPtr<Expression> Parser::parse_expression(unsigned precedence)
@@ -625,11 +365,11 @@ RefPtr<Expression> Parser::parse_expression(unsigned precedence)
 	VariableReference *left_var = 0;
 	while(1)
 	{
-		string token = peek_token();
+		string token = tokenizer.peek_token();
 
 		const Operator *oper = 0;
-		for(Operator *i=operators; (!oper && i->type); ++i)
-			if(token==i->token && (!left || i->type!=PREFIX) && (left || i->type!=POSTFIX))
+		for(const Operator *i=Operator::operators; (!oper && i->type); ++i)
+			if(token==i->token && (!left || i->type!=Operator::PREFIX) && (left || i->type!=Operator::POSTFIX))
 				oper = i;
 
 		if(token==";" || token==")" || token=="]" || token=="," || (oper && precedence && oper->precedence>=precedence))
@@ -637,52 +377,52 @@ RefPtr<Expression> Parser::parse_expression(unsigned precedence)
 			if(left)
 				return left;
 			else
-				throw runtime_error(format_syntax_error("an expression"));
+				throw parse_error(tokenizer.get_location(), token, "an expression");
 		}
 		else if(left)
 		{
 			if(token=="(")
 			{
 				if(!left_var)
-					throw runtime_error(format_error("Syntax error before '(': function name must be an identifier"));
+					throw invalid_shader_source(tokenizer.get_location(), "Syntax error before '(': function name must be an identifier");
 				left = parse_function_call(*left_var);
 			}
 			else if(token==".")
 			{
 				RefPtr<MemberAccess> memacc = new MemberAccess;
 				memacc->left = left;
-				parse_token();
+				tokenizer.parse_token();
 				memacc->member = expect_identifier();
 				left = memacc;
 			}
-			else if(oper && oper->type==POSTFIX)
+			else if(oper && oper->type==Operator::POSTFIX)
 			{
 				RefPtr<UnaryExpression> unary = new UnaryExpression;
-				unary->oper = parse_token();
+				unary->oper = tokenizer.parse_token();
 				unary->prefix = false;
 				unary->expression = left;
 				left = unary;
 			}
-			else if(oper && oper->type==BINARY)
+			else if(oper && oper->type==Operator::BINARY)
 				left = parse_binary(left, oper);
 			else
-				throw runtime_error(format_syntax_error("an operator"));
+				throw parse_error(tokenizer.get_location(), token, "an operator");
 			left_var = 0;
 		}
 		else
 		{
 			if(token=="(")
 			{
-				parse_token();
+				tokenizer.parse_token();
 				RefPtr<ParenthesizedExpression> parexpr = new ParenthesizedExpression;
 				parexpr->expression = parse_expression();
-				expect(")");
+				tokenizer.expect(")");
 				left = parexpr;
 			}
 			else if(isdigit(token[0]) || token=="true" || token=="false")
 			{
 				RefPtr<Literal> literal = new Literal;
-				literal->token = parse_token();
+				literal->token = tokenizer.parse_token();
 				left = literal;
 			}
 			else if(is_identifier(token))
@@ -692,16 +432,16 @@ RefPtr<Expression> Parser::parse_expression(unsigned precedence)
 				left = var;
 				left_var = var.get();
 			}
-			else if(oper && oper->type==PREFIX)
+			else if(oper && oper->type==Operator::PREFIX)
 			{
 				RefPtr<UnaryExpression> unary = new UnaryExpression;
-				unary->oper = parse_token();
+				unary->oper = tokenizer.parse_token();
 				unary->prefix = true;
 				unary->expression = parse_expression(oper->precedence);
 				left = unary;
 			}
 			else
-				throw runtime_error(format_syntax_error("an expression"));
+				throw parse_error(tokenizer.get_location(), token, "an expression");
 		}
 	}
 }
@@ -710,15 +450,15 @@ RefPtr<BinaryExpression> Parser::parse_binary(const RefPtr<Expression> &left, co
 {
 	RefPtr<BinaryExpression> binary = (oper->precedence==16 ? new Assignment : new BinaryExpression);
 	binary->left = left;
-	binary->oper = parse_token();
+	binary->oper = tokenizer.parse_token();
 	if(binary->oper=="[")
 	{
 		binary->right = parse_expression();
-		expect("]");
+		tokenizer.expect("]");
 		binary->after = "]";
 	}
 	else
-		binary->right = parse_expression(oper->precedence+(oper->assoc==RIGHT_TO_LEFT));
+		binary->right = parse_expression(oper->precedence+(oper->assoc==Operator::RIGHT_TO_LEFT));
 	return binary;
 }
 
@@ -727,27 +467,27 @@ RefPtr<FunctionCall> Parser::parse_function_call(const VariableReference &var)
 	RefPtr<FunctionCall> call = new FunctionCall;
 	call->name = var.name;
 	call->constructor = is_type(call->name);
-	expect("(");
-	while(peek_token()!=")")
+	tokenizer.expect("(");
+	while(tokenizer.peek_token()!=")")
 	{
 		if(!call->arguments.empty())
-			expect(",");
+			tokenizer.expect(",");
 		call->arguments.push_back(parse_expression());
 	}
-	expect(")");
+	tokenizer.expect(")");
 	return call;
 }
 
 RefPtr<StructDeclaration> Parser::parse_struct_declaration()
 {
-	expect("struct");
+	tokenizer.expect("struct");
 	RefPtr<StructDeclaration> strct = new StructDeclaration;
 	strct->source = source_index;
-	strct->line = current_line;
+	strct->line = tokenizer.get_location().line;
 
 	strct->name = expect_identifier();
 	parse_block(strct->members, true);
-	expect(";");
+	tokenizer.expect(";");
 
 	declared_types.insert(strct->name);
 	return strct;
@@ -757,12 +497,12 @@ RefPtr<VariableDeclaration> Parser::parse_variable_declaration()
 {
 	RefPtr<VariableDeclaration> var = new VariableDeclaration;
 	var->source = source_index;
-	var->line = current_line;
+	var->line = tokenizer.get_location().line;
 
-	string token = peek_token();
+	string token = tokenizer.peek_token();
 	while(is_qualifier(token))
 	{
-		parse_token();
+		tokenizer.parse_token();
 		if(is_interface_qualifier(token))
 			var->interface = token;
 		else if(is_sampling_qualifier(token))
@@ -773,7 +513,7 @@ RefPtr<VariableDeclaration> Parser::parse_variable_declaration()
 			var->precision = token;
 		else if(token=="const")
 			var->constant = true;
-		token = peek_token();
+		token = tokenizer.peek_token();
 	}
 
 	var->type = expect_type();
@@ -785,14 +525,14 @@ RefPtr<VariableDeclaration> Parser::parse_variable_declaration()
 		if(!check("]"))
 		{
 			var->array_size = parse_expression();
-			expect("]");
+			tokenizer.expect("]");
 		}
 	}
 
 	if(check("="))
 		var->init_expression = parse_expression();
 
-	expect(";");
+	tokenizer.expect(";");
 	return var;
 }
 
@@ -800,36 +540,36 @@ RefPtr<FunctionDeclaration> Parser::parse_function_declaration()
 {
 	RefPtr<FunctionDeclaration> func = new FunctionDeclaration;
 	func->source = source_index;
-	func->line = current_line;
+	func->line = tokenizer.get_location().line;
 
 	func->return_type = expect_type();
 	func->name = expect_identifier();
-	expect("(");
-	while(peek_token()!=")")
+	tokenizer.expect("(");
+	while(tokenizer.peek_token()!=")")
 	{
 		if(!func->parameters.empty())
-			expect(",");
+			tokenizer.expect(",");
 
 		RefPtr<VariableDeclaration> var = new VariableDeclaration;
-		string token = peek_token();
+		string token = tokenizer.peek_token();
 		if(token=="in" || token=="out" || token=="inout")
-			var->interface = parse_token();
+			var->interface = tokenizer.parse_token();
 		var->type = expect_type();
 		var->name = expect_identifier();
 		func->parameters.push_back(var);
 	}
-	expect(")");
+	tokenizer.expect(")");
 
-	string token = peek_token();
+	string token = tokenizer.peek_token();
 	if(token=="{")
 	{
 		func->definition = func.get();
 		parse_block(func->body, true);
 	}
 	else if(token==";")
-		parse_token();
+		tokenizer.parse_token();
 	else
-		throw runtime_error(format_syntax_error("'{' or ';'"));
+		throw parse_error(tokenizer.get_location(), token, "'{' or ';'");
 
 	return func;
 }
@@ -838,11 +578,11 @@ RefPtr<InterfaceBlock> Parser::parse_interface_block()
 {
 	RefPtr<InterfaceBlock> iface = new InterfaceBlock;
 	iface->source = source_index;
-	iface->line = current_line;
+	iface->line = tokenizer.get_location().line;
 
-	iface->interface = parse_token();
+	iface->interface = tokenizer.parse_token();
 	if(!is_interface_qualifier(iface->interface))
-		throw runtime_error(format_syntax_error("an interface qualifier"));
+		throw parse_error(tokenizer.get_location(), iface->interface, "an interface qualifier");
 
 	iface->name = expect_identifier();
 	parse_block(iface->members, true);
@@ -852,9 +592,9 @@ RefPtr<InterfaceBlock> Parser::parse_interface_block()
 		if(check("["))
 		{
 			iface->array = true;
-			expect("]");
+			tokenizer.expect("]");
 		}
-		expect(";");
+		tokenizer.expect(";");
 	}
 
 	return iface;
@@ -862,20 +602,20 @@ RefPtr<InterfaceBlock> Parser::parse_interface_block()
 
 RefPtr<Conditional> Parser::parse_conditional()
 {
-	expect("if");
+	tokenizer.expect("if");
 	RefPtr<Conditional> cond = new Conditional;
 	cond->source = source_index;
-	cond->line = current_line;
-	expect("(");
+	cond->line = tokenizer.get_location().line;
+	tokenizer.expect("(");
 	cond->condition = parse_expression();
-	expect(")");
+	tokenizer.expect(")");
 
 	parse_block(cond->body, false);
 
-	string token = peek_token();
+	string token = tokenizer.peek_token();
 	if(token=="else")
 	{
-		parse_token();
+		tokenizer.parse_token();
 		parse_block(cond->else_body, false);
 	}
 
@@ -884,12 +624,12 @@ RefPtr<Conditional> Parser::parse_conditional()
 
 RefPtr<Iteration> Parser::parse_for()
 {
-	expect("for");
+	tokenizer.expect("for");
 	RefPtr<Iteration> loop = new Iteration;
 	loop->source = source_index;
-	loop->line = current_line;
-	expect("(");
-	string token = peek_token();
+	loop->line = tokenizer.get_location().line;
+	tokenizer.expect("(");
+	string token = tokenizer.peek_token();
 	if(is_type(token))
 		loop->init_statement = parse_statement();
 	else
@@ -900,14 +640,14 @@ RefPtr<Iteration> Parser::parse_for()
 			expr->expression = parse_expression();
 			loop->init_statement = expr;
 		}
-		expect(";");
+		tokenizer.expect(";");
 	}
-	if(peek_token()!=";")
+	if(tokenizer.peek_token()!=";")
 		loop->condition = parse_expression();
-	expect(";");
-	if(peek_token()!=")")
+	tokenizer.expect(";");
+	if(tokenizer.peek_token()!=")")
 		loop->loop_expression = parse_expression();
-	expect(")");
+	tokenizer.expect(")");
 
 	parse_block(loop->body, false);
 
@@ -916,13 +656,13 @@ RefPtr<Iteration> Parser::parse_for()
 
 RefPtr<Iteration> Parser::parse_while()
 {
-	expect("while");
+	tokenizer.expect("while");
 	RefPtr<Iteration> loop = new Iteration;
 	loop->source = source_index;
-	loop->line = current_line;
-	expect("(");
+	loop->line = tokenizer.get_location().line;
+	tokenizer.expect("(");
 	loop->condition = parse_expression();
-	expect(")");
+	tokenizer.expect(")");
 
 	parse_block(loop->body, false);
 
@@ -931,29 +671,29 @@ RefPtr<Iteration> Parser::parse_while()
 
 RefPtr<Passthrough> Parser::parse_passthrough()
 {
-	expect("passthrough");
+	tokenizer.expect("passthrough");
 	RefPtr<Passthrough> pass = new Passthrough;
 	pass->source = source_index;
-	pass->line = current_line;
-	if(cur_stage->type==GEOMETRY)
+	pass->line = tokenizer.get_location().line;
+	if(cur_stage->type==Stage::GEOMETRY)
 	{
-		expect("[");
+		tokenizer.expect("[");
 		pass->subscript = parse_expression();
-		expect("]");
+		tokenizer.expect("]");
 	}
-	expect(";");
+	tokenizer.expect(";");
 	return pass;
 }
 
 RefPtr<Return> Parser::parse_return()
 {
-	expect("return");
+	tokenizer.expect("return");
 	RefPtr<Return> ret = new Return;
 	ret->source = source_index;
-	ret->line = current_line;
-	if(peek_token()!=";")
+	ret->line = tokenizer.get_location().line;
+	if(tokenizer.peek_token()!=";")
 		ret->expression = parse_expression();
-	expect(";");
+	tokenizer.expect(";");
 	return ret;
 }