1 #include <msp/core/raii.h>
2 #include "glsl_error.h"
3 #include "preprocessor.h"
13 Tokenizer::Tokenizer():
14 allow_preprocess(true)
18 source_end = empty.end();
22 void Tokenizer::begin(const string &name, const string &src)
25 source_end = src.end();
28 allow_preprocess = true;
33 const string &Tokenizer::peek_token(unsigned index)
35 while(next_tokens.size()<=index)
36 next_tokens.push_back(parse_token_());
37 return (last_token = next_tokens[index]);
40 const string &Tokenizer::parse_token()
42 if(!next_tokens.empty())
44 last_token = next_tokens.front();
45 next_tokens.pop_front();
49 return (last_token = parse_token_());
52 void Tokenizer::expect(const string &token)
54 string parsed = parse_token();
56 throw parse_error(location, parsed, format("'%s'", token));
59 string Tokenizer::parse_token_()
63 skip_comment_and_whitespace();
66 else if(allow_preprocess && *iter=='#')
68 allow_preprocess = false;
71 else if(isalpha(*iter) || *iter=='_')
72 return parse_identifier();
73 else if(isdigit(*iter))
74 return parse_number();
80 void Tokenizer::preprocess()
82 SetForScope<deque<string> > clear_tokens(next_tokens, deque<string>());
84 string::const_iterator line_end = iter;
85 for(; (line_end!=source_end && *line_end!='\n'); ++line_end) ;
86 SetForScope<string::const_iterator> stop_at_line_end(source_end, line_end);
88 signal_preprocess.emit();
93 string Tokenizer::parse_identifier()
96 while(iter!=source_end)
98 if(isalnum(*iter) || *iter=='_')
107 string Tokenizer::parse_number()
109 bool accept_sign = false;
111 while(iter!=source_end)
113 if(isdigit(*iter) || *iter=='.')
115 else if(*iter=='e' || *iter=='E')
120 else if(accept_sign && (*iter=='+' || *iter=='-'))
129 string Tokenizer::parse_other()
134 string token(1, *iter++);
135 for(unsigned i=1; (i<3 && iter!=source_end); ++i)
137 bool matched = false;
138 for(const Operator *j=Operator::operators; (!matched && j->type); ++j)
140 matched = (j->token[i]==*iter);
141 for(unsigned k=0; (matched && k<i && j->token[k]); ++k)
142 matched = (j->token[k]==token[k]);
154 void Tokenizer::skip_comment_and_whitespace()
156 unsigned comment = 0;
157 while(iter!=source_end)
163 else if(!isspace(*iter))
184 else if(comment==3 && *iter=='*')
197 allow_preprocess = (comment<3);