1 #include <msp/core/raii.h>
2 #include "glsl_error.h"
3 #include "preprocessor.h"
13 Tokenizer::Tokenizer()
17 source_end = empty.end();
20 void Tokenizer::begin(const string &src, const string &name)
23 source_end = src.end();
26 allow_preprocess = true;
31 const string &Tokenizer::peek_token(unsigned index)
33 while(next_tokens.size()<=index)
34 next_tokens.push_back(parse_token_());
35 return next_tokens[index];
38 const string &Tokenizer::parse_token()
42 if(!next_tokens.empty())
44 last_token = next_tokens.front();
45 next_tokens.pop_front();
49 return (last_token = parse_token_());
52 void Tokenizer::expect(const string &token)
54 string parsed = parse_token();
56 throw parse_error(location, parsed, format("'%s'", token));
59 void Tokenizer::set_location(const Location &loc)
62 suppress_line_advance = true;
65 string Tokenizer::parse_token_()
69 skip_comment_and_whitespace();
70 bool allow_preproc = allow_preprocess;
71 allow_preprocess = false;
74 else if(allow_preproc && *iter=='#')
79 else if(isalpha(*iter) || *iter=='_')
80 return parse_identifier();
81 else if(isdigit(*iter))
82 return parse_number();
84 return parse_string();
85 else if(*iter=='#' || *iter=='$' || *iter=='\'' || *iter=='@' || *iter=='\\' || *iter=='`')
86 throw syntax_error(location, string(1, *iter), "Invalid character in source");
92 void Tokenizer::preprocess()
94 SetForScope<deque<string> > clear_tokens(next_tokens, deque<string>());
97 for(; (line_end!=source_end && *line_end!='\n'); ++line_end) ;
98 SetForScope<string::const_iterator> stop_at_line_end(source_end, line_end);
100 signal_preprocess.emit();
105 string Tokenizer::parse_identifier()
108 while(iter!=source_end)
110 if(isalnum(*iter) || *iter=='_')
119 string Tokenizer::parse_number()
121 bool got_fract = false;
123 while(iter!=source_end)
127 else if(!got_fract && *iter=='.')
136 bool require_digit = false;
139 else if(*iter=='e' || *iter=='E')
142 if(iter!=source_end && (*iter=='-' || *iter=='+'))
144 require_digit = true;
145 while(iter!=source_end)
150 require_digit = false;
155 if(!require_digit && iter!=source_end && *iter=='f')
158 else if(*iter=='u' || *iter=='f')
162 throw syntax_error(location, number, "Incomplete numeric literal");
163 else if(iter!=source_end && (isalnum(*iter) || *iter=='_'))
164 throw syntax_error(location, number, "Garbage at end of numeric literal");
169 string Tokenizer::parse_string()
171 string str(1, *iter++);
173 while(iter!=source_end)
179 else if(c=='"' && !escape)
188 string Tokenizer::parse_other()
193 string token(1, *iter++);
194 for(unsigned i=1; (i<3 && iter!=source_end); ++i)
196 bool matched = false;
197 for(const Operator *j=Operator::operators; (!matched && j->type); ++j)
199 matched = (j->token[i]==*iter);
200 for(unsigned k=0; (matched && k<i && j->token[k]); ++k)
201 matched = (j->token[k]==token[k]);
213 void Tokenizer::skip_comment_and_whitespace()
215 unsigned comment = 0;
216 while(iter!=source_end)
222 else if(!isspace(*iter))
243 else if(comment==3 && *iter=='*')
255 if(!suppress_line_advance)
257 allow_preprocess = (comment<3);
263 suppress_line_advance = false;