From 1083ad150c902bd153f88f7fb28660b4f31cf9fe Mon Sep 17 00:00:00 2001
From: Mikko Rasa <tdb@tdb.fi>
Date: Sat, 20 Feb 2021 14:59:44 +0200
Subject: [PATCH] Recognize the #line directive in the GLSL parser

Also add a new pragma to specify source names.
---
 source/glsl/parser.cpp       | 28 +++++++++++++++++++++++++-
 source/glsl/parser.h         |  3 +++
 source/glsl/preprocessor.cpp | 39 +++++++++++++++++++++++++++++++++++-
 source/glsl/preprocessor.h   |  4 ++++
 source/glsl/sourcemap.cpp    | 10 +++++++++
 source/glsl/sourcemap.h      |  1 +
 source/glsl/tokenizer.cpp    | 37 ++++++++++++++++++++++++++++++----
 source/glsl/tokenizer.h      |  6 ++++++
 8 files changed, 122 insertions(+), 6 deletions(-)

diff --git a/source/glsl/parser.cpp b/source/glsl/parser.cpp
index eee62129..7044a8c3 100644
--- a/source/glsl/parser.cpp
+++ b/source/glsl/parser.cpp
@@ -17,7 +17,9 @@ Parser::Parser():
 {
 	tokenizer.signal_preprocess.connect(sigc::mem_fun(&preprocessor, &Preprocessor::preprocess));
 	preprocessor.signal_version.connect(sigc::mem_fun(this, &Parser::set_required_version));
+	preprocessor.signal_source.connect(sigc::mem_fun(this, &Parser::source_reference));
 	preprocessor.signal_stage_change.connect(sigc::mem_fun(this, &Parser::stage_change));
+	preprocessor.signal_line.connect(sigc::mem_fun(this, &Parser::line_change));
 }
 
 Parser::~Parser()
@@ -50,8 +52,9 @@ void Parser::parse_source(const string &name, unsigned index)
 	delete module;
 	module = new Module;
 	cur_stage = &module->shared;
+	base_index = index;
 	source_index = index;
-	module->source_map.set_name(source_index, name);
+	source_reference(1, name);
 	tokenizer.begin(name, source);
 	while(RefPtr<Statement> statement = parse_global_declaration())
 		cur_stage->content.body.push_back(statement);
@@ -62,6 +65,14 @@ void Parser::set_required_version(const Version &ver)
 	cur_stage->required_version = ver;
 }
 
+void Parser::source_reference(unsigned index, const string &name)
+{
+	if(index<1)
+		throw invalid_shader_source(tokenizer.get_location(), "Invalid source reference");
+
+	module->source_map.set_name(base_index+index-1, name);
+}
+
 void Parser::stage_change(Stage::Type stage)
 {
 	if(!allow_stage_change)
@@ -76,6 +87,21 @@ void Parser::stage_change(Stage::Type stage)
 	cur_stage = &module->stages.back();
 }
 
+void Parser::line_change(int index, unsigned line)
+{
+	if(index>0)
+		source_index = base_index+index-1;
+	else if(index==0)
+		source_index = 0;
+	else
+		index = source_index;
+
+	string name = module->source_map.get_name(index);
+	if(name.empty())
+		name = format("<%d>", index);
+	tokenizer.set_location(Location(name, line));
+}
+
 string Parser::expect_type()
 {
 	string token = tokenizer.parse_token();
diff --git a/source/glsl/parser.h b/source/glsl/parser.h
index 58b14b50..cdb57ebc 100644
--- a/source/glsl/parser.h
+++ b/source/glsl/parser.h
@@ -16,6 +16,7 @@ class Parser
 {
 private:
 	std::string source;
+	unsigned base_index;
 	unsigned source_index;
 	Tokenizer tokenizer;
 	Preprocessor preprocessor;
@@ -34,7 +35,9 @@ public:
 private:
 	void parse_source(const std::string &, unsigned);
 	void set_required_version(const Version &);
+	void source_reference(unsigned, const std::string &);
 	void stage_change(Stage::Type);
+	void line_change(int, unsigned);
 
 	std::string expect_type();
 	std::string expect_identifier();
diff --git a/source/glsl/preprocessor.cpp b/source/glsl/preprocessor.cpp
index 354b2a7a..ea6dd6de 100644
--- a/source/glsl/preprocessor.cpp
+++ b/source/glsl/preprocessor.cpp
@@ -1,3 +1,4 @@
+#include <msp/strings/utils.h>
 #include "glsl_error.h"
 #include "preprocessor.h"
 #include "tokenizer.h"
@@ -19,8 +20,10 @@ void Preprocessor::preprocess()
 		preprocess_pragma();
 	else if(token=="version")
 		preprocess_version();
+	else if(token=="line")
+		preprocess_line();
 	else if(token=="define" || token=="undef" || token=="if" || token=="ifdef" || token=="ifndef" || token=="else" ||
-		token=="elif" || token=="endif" || token=="error" || token=="extension" || token=="line")
+		token=="elif" || token=="endif" || token=="error" || token=="extension")
 		throw invalid_shader_source(tokenizer.get_location(), "Unsupported preprocessor directive '%s'", token);
 	else if(!token.empty())
 		throw parse_error(tokenizer.get_location(), token, "a preprocessor directive");
@@ -38,6 +41,23 @@ void Preprocessor::preprocess_version()
 		throw parse_error(tokenizer.get_location(), token, "end of line");
 }
 
+void Preprocessor::preprocess_line()
+{
+	tokenizer.expect("line");
+	unsigned line_number = lexical_cast<unsigned>(tokenizer.parse_token());
+
+	int source_index = -1;
+	string token = tokenizer.parse_token();
+	if(!token.empty())
+		source_index = lexical_cast<unsigned>(token);
+
+	token = tokenizer.parse_token();
+	if(!token.empty())
+		throw parse_error(tokenizer.get_location(), token, "end of line");
+
+	signal_line.emit(source_index, line_number);
+}
+
 void Preprocessor::preprocess_pragma()
 {
 	tokenizer.expect("pragma");
@@ -51,6 +71,8 @@ void Preprocessor::preprocess_pragma_msp()
 	string token = tokenizer.peek_token();
 	if(token=="stage")
 		preprocess_stage();
+	else if(token=="source")
+		preprocess_source();
 	else
 		throw invalid_shader_source(tokenizer.get_location(), "Unrecognized MSP pragma '%s'", token);
 
@@ -78,6 +100,21 @@ void Preprocessor::preprocess_stage()
 	signal_stage_change.emit(stage);
 }
 
+void Preprocessor::preprocess_source()
+{
+	tokenizer.expect("source");
+	tokenizer.expect("(");
+	unsigned source_index = lexical_cast<unsigned>(tokenizer.parse_token());
+	tokenizer.expect(",");
+	string filename = tokenizer.parse_token();
+	if(filename[0]!='"')
+		throw parse_error(tokenizer.get_location(), filename, "a string literal");
+	filename = c_unescape(filename.substr(1, filename.size()-2));
+	tokenizer.expect(")");
+
+	signal_source.emit(source_index, filename);
+}
+
 } // namespace SL
 } // namespace GL
 } // namespace Msp
diff --git a/source/glsl/preprocessor.h b/source/glsl/preprocessor.h
index d812a3fa..5f6430c7 100644
--- a/source/glsl/preprocessor.h
+++ b/source/glsl/preprocessor.h
@@ -14,7 +14,9 @@ class Preprocessor
 {
 public:
 	sigc::signal<void, const Version &> signal_version;
+	sigc::signal<void, unsigned, const std::string &> signal_source;
 	sigc::signal<void, Stage::Type> signal_stage_change;
+	sigc::signal<void, int, unsigned> signal_line;
 
 private:
 	Tokenizer &tokenizer;
@@ -25,9 +27,11 @@ public:
 	void preprocess();
 private:
 	void preprocess_version();
+	void preprocess_line();
 	void preprocess_pragma();
 	void preprocess_pragma_msp();
 	void preprocess_stage();
+	void preprocess_source();
 };
 
 } // namespace SL
diff --git a/source/glsl/sourcemap.cpp b/source/glsl/sourcemap.cpp
index e74f1b67..9b341fe4 100644
--- a/source/glsl/sourcemap.cpp
+++ b/source/glsl/sourcemap.cpp
@@ -29,6 +29,16 @@ void SourceMap::set_name(unsigned i, const std::string &n)
 	source_names[i] = n;
 }
 
+const string &SourceMap::get_name(unsigned i) const
+{
+	i -= base_index;
+	if(i<source_names.size())
+		return source_names[i];
+
+	static string empty;
+	return empty;
+}
+
 void SourceMap::merge_from(const SourceMap &other)
 {
 	if(other.base_index<base_index)
diff --git a/source/glsl/sourcemap.h b/source/glsl/sourcemap.h
index 6df6418a..da1f9b76 100644
--- a/source/glsl/sourcemap.h
+++ b/source/glsl/sourcemap.h
@@ -19,6 +19,7 @@ public:
 
 	void set_name(unsigned, const std::string &);
 	unsigned get_count() const { return base_index+source_names.size(); }
+	const std::string &get_name(unsigned) const;
 	void merge_from(const SourceMap &);
 	std::string translate_errors(const std::string &) const;
 };
diff --git a/source/glsl/tokenizer.cpp b/source/glsl/tokenizer.cpp
index e73afd52..b7efb8f8 100644
--- a/source/glsl/tokenizer.cpp
+++ b/source/glsl/tokenizer.cpp
@@ -11,12 +11,12 @@ namespace GL {
 namespace SL {
 
 Tokenizer::Tokenizer():
-	allow_preprocess(true)
+	allow_preprocess(true),
+	suppress_line_advance(false)
 {
 	static string empty;
 	iter = empty.begin();
 	source_end = empty.end();
-	location.line = 0;
 }
 
 void Tokenizer::begin(const string &name, const string &src)
@@ -56,6 +56,11 @@ void Tokenizer::expect(const string &token)
 		throw parse_error(location, parsed, format("'%s'", token));
 }
 
+void Tokenizer::set_location(const Location &loc)
+{
+	location = loc;
+}
+
 string Tokenizer::parse_token_()
 {
 	while(1)
@@ -74,7 +79,9 @@ string Tokenizer::parse_token_()
 			return parse_identifier();
 		else if(isdigit(*iter))
 			return parse_number();
-		else if(*iter=='"' || *iter=='#' || *iter=='$' || *iter=='\'' || *iter=='@' || *iter=='\\' || *iter=='`')
+		else if(*iter=='"')
+			return parse_string();
+		else if(*iter=='#' || *iter=='$' || *iter=='\'' || *iter=='@' || *iter=='\\' || *iter=='`')
 			throw syntax_error(location, string(1, *iter), "Invalid character in source");
 		else
 			return parse_other();
@@ -152,6 +159,25 @@ string Tokenizer::parse_number()
 	return number;
 }
 
+string Tokenizer::parse_string()
+{
+	string str(1, *iter++);
+	bool escape = false;
+	while(iter!=source_end)
+	{
+		char c = *iter++;
+		str += c;
+		if(c=='\\')
+			escape = true;
+		else if(c=='"' && !escape)
+			break;
+		else
+			escape = false;
+	}
+
+	return str;
+}
+
 string Tokenizer::parse_other()
 {
 	if(iter==source_end)
@@ -219,12 +245,15 @@ void Tokenizer::skip_comment_and_whitespace()
 
 		if(*iter=='\n')
 		{
-			++location.line;
+			if(!suppress_line_advance)
+				++location.line;
 			allow_preprocess = (comment<3);
 		}
 
 		++iter;
 	}
+
+	suppress_line_advance = false;
 }
 
 } // namespace SL
diff --git a/source/glsl/tokenizer.h b/source/glsl/tokenizer.h
index a7329182..20ea756a 100644
--- a/source/glsl/tokenizer.h
+++ b/source/glsl/tokenizer.h
@@ -15,6 +15,9 @@ struct Location
 {
 	std::string name;
 	unsigned line;
+
+	Location(): line(0) { }
+	Location(const std::string &n, unsigned l): name(n), line(l) { }
 };
 
 class Tokenizer
@@ -27,6 +30,7 @@ private:
 	std::string::const_iterator source_end;
 	Location location;
 	bool allow_preprocess;
+	bool suppress_line_advance;
 	std::string last_token;
 	std::deque<std::string> next_tokens;
 
@@ -37,12 +41,14 @@ public:
 	const std::string &peek_token(unsigned = 0);
 	const std::string &parse_token();
 	void expect(const std::string &);
+	void set_location(const Location &);
 	const Location &get_location() const { return location; }
 private:
 	std::string parse_token_();
 	void preprocess();
 	std::string parse_identifier();
 	std::string parse_number();
+	std::string parse_string();
 	std::string parse_other();
 	void skip_comment_and_whitespace();
 };
-- 
2.43.0