X-Git-Url: http://git.tdb.fi/?p=libs%2Fgl.git;a=blobdiff_plain;f=source%2Fglsl%2Ftokenizer.cpp;h=4064d805e732dc8adaf5dbb912df1ce0824909bb;hp=be432143d34e2c6f21c570287bb2ac483c2515a3;hb=38712d8ecc57d043a2419ffbaeeb57f7a6586f14;hpb=d3ceb2186dc79130508093b3d0c944771a53534f

diff --git a/source/glsl/tokenizer.cpp b/source/glsl/tokenizer.cpp
index be432143..4064d805 100644
--- a/source/glsl/tokenizer.cpp
+++ b/source/glsl/tokenizer.cpp
@@ -10,16 +10,14 @@ namespace Msp {
 namespace GL {
 namespace SL {
 
-Tokenizer::Tokenizer():
-	allow_preprocess(true)
+Tokenizer::Tokenizer()
 {
 	static string empty;
 	iter = empty.begin();
 	source_end = empty.end();
-	location.line = 0;
 }
 
-void Tokenizer::begin(const string &name, const string &src)
+void Tokenizer::begin(const string &src, const string &name)
 {
 	iter = src.begin();
 	source_end = src.end();
@@ -34,11 +32,13 @@ const string &Tokenizer::peek_token(unsigned index)
 {
 	while(next_tokens.size()<=index)
 		next_tokens.push_back(parse_token_());
-	return (last_token = next_tokens[index]);
+	return next_tokens[index];
 }
 
 const string &Tokenizer::parse_token()
 {
+	progress_mark = true;
+
 	if(!next_tokens.empty())
 	{
 		last_token = next_tokens.front();
@@ -56,22 +56,34 @@ void Tokenizer::expect(const string &token)
 		throw parse_error(location, parsed, format("'%s'", token));
 }
 
+void Tokenizer::set_location(const Location &loc)
+{
+	location = loc;
+	suppress_line_advance = true;
+}
+
 string Tokenizer::parse_token_()
 {
 	while(1)
 	{
 		skip_comment_and_whitespace();
+		bool allow_preproc = allow_preprocess;
+		allow_preprocess = false;
 		if(iter==source_end)
 			return string();
-		else if(allow_preprocess && *iter=='#')
+		else if(allow_preproc && *iter=='#')
 		{
-			allow_preprocess = false;
+			++iter;
 			preprocess();
 		}
 		else if(isalpha(*iter) || *iter=='_')
 			return parse_identifier();
 		else if(isdigit(*iter))
 			return parse_number();
+		else if(*iter=='"')
+			return parse_string();
+		else if(*iter=='#' || *iter=='$' || *iter=='\'' || *iter=='@' || *iter=='\\' || *iter=='`')
+			throw syntax_error(location, string(1, *iter), "Invalid character in source");
 		else
 			return parse_other();
 	}
@@ -81,7 +93,7 @@ void Tokenizer::preprocess()
 {
 	SetForScope<deque<string> > clear_tokens(next_tokens, deque<string>());
 
-	string::const_iterator line_end = iter;
+	auto line_end = iter;
 	for(; (line_end!=source_end && *line_end!='\n'); ++line_end) ;
 	SetForScope<string::const_iterator> stop_at_line_end(source_end, line_end);
 
@@ -106,26 +118,73 @@ string Tokenizer::parse_identifier()
 
 string Tokenizer::parse_number()
 {
-	bool accept_sign = false;
+	bool got_fract = false;
 	string number;
 	while(iter!=source_end)
 	{
-		if(isdigit(*iter) || *iter=='.')
+		if(isdigit(*iter))
 			number += *iter++;
-		else if(*iter=='e' || *iter=='E')
+		else if(!got_fract && *iter=='.')
 		{
 			number += *iter++;
-			accept_sign = true;
+			got_fract = true;
 		}
-		else if(accept_sign && (*iter=='+' || *iter=='-'))
-			number += *iter++;
 		else
 			break;
 	}
 
+	bool require_digit = false;
+	if(iter==source_end)
+		;
+	else if(*iter=='e' || *iter=='E')
+	{
+		number += *iter++;
+		if(iter!=source_end && (*iter=='-' || *iter=='+'))
+			number += *iter++;
+		require_digit = true;
+		while(iter!=source_end)
+		{
+			if(isdigit(*iter))
+			{
+				number += *iter++;
+				require_digit = false;
+			}
+			else
+				break;
+		}
+		if(!require_digit && iter!=source_end && *iter=='f')
+			number += *iter++;
+	}
+	else if(*iter=='u' || *iter=='f')
+		number += *iter++;
+
+	if(require_digit)
+		throw syntax_error(location, number, "Incomplete numeric literal");
+	else if(iter!=source_end && (isalnum(*iter) || *iter=='_'))
+		throw syntax_error(location, number, "Garbage at end of numeric literal");
+
 	return number;
 }
 
+string Tokenizer::parse_string()
+{
+	string str(1, *iter++);
+	bool escape = false;
+	while(iter!=source_end)
+	{
+		char c = *iter++;
+		str += c;
+		if(c=='\\')
+			escape = true;
+		else if(c=='"' && !escape)
+			break;
+		else
+			escape = false;
+	}
+
+	return str;
+}
+
 string Tokenizer::parse_other()
 {
 	if(iter==source_end)
@@ -193,12 +252,15 @@ void Tokenizer::skip_comment_and_whitespace()
 
 		if(*iter=='\n')
 		{
-			++location.line;
+			if(!suppress_line_advance)
+				++location.line;
 			allow_preprocess = (comment<3);
 		}
 
 		++iter;
 	}
+
+	suppress_line_advance = false;
 }
 
 } // namespace SL