1 #include <msp/core/raii.h>
2 #include "glsl_error.h"
3 #include "preprocessor.h"
13 Tokenizer::Tokenizer():
14 allow_preprocess(true),
15 suppress_line_advance(false)
19 source_end = empty.end();
22 void Tokenizer::begin(const string &name, const string &src)
25 source_end = src.end();
28 allow_preprocess = true;
33 const string &Tokenizer::peek_token(unsigned index)
35 while(next_tokens.size()<=index)
36 next_tokens.push_back(parse_token_());
37 return next_tokens[index];
40 const string &Tokenizer::parse_token()
44 if(!next_tokens.empty())
46 last_token = next_tokens.front();
47 next_tokens.pop_front();
51 return (last_token = parse_token_());
54 void Tokenizer::expect(const string &token)
56 string parsed = parse_token();
58 throw parse_error(location, parsed, format("'%s'", token));
61 void Tokenizer::set_location(const Location &loc)
66 string Tokenizer::parse_token_()
70 skip_comment_and_whitespace();
71 bool allow_preproc = allow_preprocess;
72 allow_preprocess = false;
75 else if(allow_preproc && *iter=='#')
80 else if(isalpha(*iter) || *iter=='_')
81 return parse_identifier();
82 else if(isdigit(*iter))
83 return parse_number();
85 return parse_string();
86 else if(*iter=='#' || *iter=='$' || *iter=='\'' || *iter=='@' || *iter=='\\' || *iter=='`')
87 throw syntax_error(location, string(1, *iter), "Invalid character in source");
93 void Tokenizer::preprocess()
95 SetForScope<deque<string> > clear_tokens(next_tokens, deque<string>());
97 string::const_iterator line_end = iter;
98 for(; (line_end!=source_end && *line_end!='\n'); ++line_end) ;
99 SetForScope<string::const_iterator> stop_at_line_end(source_end, line_end);
101 signal_preprocess.emit();
106 string Tokenizer::parse_identifier()
109 while(iter!=source_end)
111 if(isalnum(*iter) || *iter=='_')
120 string Tokenizer::parse_number()
122 bool got_fract = false;
124 while(iter!=source_end)
128 else if(!got_fract && *iter=='.')
137 bool require_digit = false;
138 if(iter!=source_end && (*iter=='e' || *iter=='E'))
141 if(iter!=source_end && (*iter=='-' || *iter=='+'))
143 require_digit = true;
144 while(iter!=source_end)
149 require_digit = false;
157 throw syntax_error(location, number, "Incomplete numeric literal");
158 else if(isalnum(*iter) || *iter=='_')
159 throw syntax_error(location, number, "Garbage at end of numeric literal");
164 string Tokenizer::parse_string()
166 string str(1, *iter++);
168 while(iter!=source_end)
174 else if(c=='"' && !escape)
183 string Tokenizer::parse_other()
188 string token(1, *iter++);
189 for(unsigned i=1; (i<3 && iter!=source_end); ++i)
191 bool matched = false;
192 for(const Operator *j=Operator::operators; (!matched && j->type); ++j)
194 matched = (j->token[i]==*iter);
195 for(unsigned k=0; (matched && k<i && j->token[k]); ++k)
196 matched = (j->token[k]==token[k]);
208 void Tokenizer::skip_comment_and_whitespace()
210 unsigned comment = 0;
211 while(iter!=source_end)
217 else if(!isspace(*iter))
238 else if(comment==3 && *iter=='*')
250 if(!suppress_line_advance)
252 allow_preprocess = (comment<3);
258 suppress_line_advance = false;