]> git.tdb.fi Git - libs/gl.git/blob - source/glsl/tokenizer.cpp
Refactor interface management
[libs/gl.git] / source / glsl / tokenizer.cpp
1 #include <msp/core/raii.h>
2 #include "glsl_error.h"
3 #include "preprocessor.h"
4 #include "syntax.h"
5 #include "tokenizer.h"
6
7 using namespace std;
8
9 namespace Msp {
10 namespace GL {
11 namespace SL {
12
13 Tokenizer::Tokenizer():
14         allow_preprocess(true),
15         suppress_line_advance(false)
16 {
17         static string empty;
18         iter = empty.begin();
19         source_end = empty.end();
20 }
21
22 void Tokenizer::begin(const string &name, const string &src)
23 {
24         iter = src.begin();
25         source_end = src.end();
26         location.name = name;
27         location.line = 1;
28         allow_preprocess = true;
29         last_token.clear();
30         next_tokens.clear();
31 }
32
33 const string &Tokenizer::peek_token(unsigned index)
34 {
35         while(next_tokens.size()<=index)
36                 next_tokens.push_back(parse_token_());
37         return (last_token = next_tokens[index]);
38 }
39
40 const string &Tokenizer::parse_token()
41 {
42         if(!next_tokens.empty())
43         {
44                 last_token = next_tokens.front();
45                 next_tokens.pop_front();
46                 return last_token;
47         }
48
49         return (last_token = parse_token_());
50 }
51
52 void Tokenizer::expect(const string &token)
53 {
54         string parsed = parse_token();
55         if(parsed!=token)
56                 throw parse_error(location, parsed, format("'%s'", token));
57 }
58
59 void Tokenizer::set_location(const Location &loc)
60 {
61         location = loc;
62 }
63
64 string Tokenizer::parse_token_()
65 {
66         while(1)
67         {
68                 skip_comment_and_whitespace();
69                 bool allow_preproc = allow_preprocess;
70                 allow_preprocess = false;
71                 if(iter==source_end)
72                         return string();
73                 else if(allow_preproc && *iter=='#')
74                 {
75                         ++iter;
76                         preprocess();
77                 }
78                 else if(isalpha(*iter) || *iter=='_')
79                         return parse_identifier();
80                 else if(isdigit(*iter))
81                         return parse_number();
82                 else if(*iter=='"')
83                         return parse_string();
84                 else if(*iter=='#' || *iter=='$' || *iter=='\'' || *iter=='@' || *iter=='\\' || *iter=='`')
85                         throw syntax_error(location, string(1, *iter), "Invalid character in source");
86                 else
87                         return parse_other();
88         }
89 }
90
91 void Tokenizer::preprocess()
92 {
93         SetForScope<deque<string> > clear_tokens(next_tokens, deque<string>());
94
95         string::const_iterator line_end = iter;
96         for(; (line_end!=source_end && *line_end!='\n'); ++line_end) ;
97         SetForScope<string::const_iterator> stop_at_line_end(source_end, line_end);
98
99         signal_preprocess.emit();
100
101         iter = line_end;
102 }
103
104 string Tokenizer::parse_identifier()
105 {
106         string ident;
107         while(iter!=source_end)
108         {
109                 if(isalnum(*iter) || *iter=='_')
110                         ident += *iter++;
111                 else
112                         break;
113         }
114
115         return ident;
116 }
117
118 string Tokenizer::parse_number()
119 {
120         bool got_fract = false;
121         string number;
122         while(iter!=source_end)
123         {
124                 if(isdigit(*iter))
125                         number += *iter++;
126                 else if(!got_fract && *iter=='.')
127                 {
128                         number += *iter++;
129                         got_fract = true;
130                 }
131                 else
132                         break;
133         }
134
135         bool require_digit = false;
136         if(iter!=source_end && (*iter=='e' || *iter=='E'))
137         {
138                 number += *iter++;
139                 if(iter!=source_end && (*iter=='-' || *iter=='+'))
140                         number += *iter++;
141                 require_digit = true;
142                 while(iter!=source_end)
143                 {
144                         if(isdigit(*iter))
145                         {
146                                 number += *iter++;
147                                 require_digit = false;
148                         }
149                         else
150                                 break;
151                 }
152         }
153
154         if(require_digit)
155                 throw syntax_error(location, number, "Incomplete numeric literal");
156         else if(isalnum(*iter) || *iter=='_')
157                 throw syntax_error(location, number, "Garbage at end of numeric literal");
158
159         return number;
160 }
161
162 string Tokenizer::parse_string()
163 {
164         string str(1, *iter++);
165         bool escape = false;
166         while(iter!=source_end)
167         {
168                 char c = *iter++;
169                 str += c;
170                 if(c=='\\')
171                         escape = true;
172                 else if(c=='"' && !escape)
173                         break;
174                 else
175                         escape = false;
176         }
177
178         return str;
179 }
180
181 string Tokenizer::parse_other()
182 {
183         if(iter==source_end)
184                 return string();
185
186         string token(1, *iter++);
187         for(unsigned i=1; (i<3 && iter!=source_end); ++i)
188         {
189                 bool matched = false;
190                 for(const Operator *j=Operator::operators; (!matched && j->type); ++j)
191                 {
192                         matched = (j->token[i]==*iter);
193                         for(unsigned k=0; (matched && k<i && j->token[k]); ++k)
194                                 matched = (j->token[k]==token[k]);
195                 }
196
197                 if(!matched)
198                         break;
199
200                 token += *iter++;
201         }
202
203         return token;
204 }
205
206 void Tokenizer::skip_comment_and_whitespace()
207 {
208         unsigned comment = 0;
209         while(iter!=source_end)
210         {
211                 if(comment==0)
212                 {
213                         if(*iter=='/')
214                                 comment = 1;
215                         else if(!isspace(*iter))
216                                 break;
217                 }
218                 else if(comment==1)
219                 {
220                         if(*iter=='/')
221                                 comment = 2;
222                         else if(*iter=='*')
223                                 comment = 3;
224                         else
225                         {
226                                 comment = 0;
227                                 --iter;
228                                 break;
229                         }
230                 }
231                 else if(comment==2)
232                 {
233                         if(*iter=='\n')
234                                 comment = 0;
235                 }
236                 else if(comment==3 && *iter=='*')
237                         comment = 4;
238                 else if(comment==4)
239                 {
240                         if(*iter=='/')
241                                 comment = 0;
242                         else if(*iter!='*')
243                                 comment = 3;
244                 }
245
246                 if(*iter=='\n')
247                 {
248                         if(!suppress_line_advance)
249                                 ++location.line;
250                         allow_preprocess = (comment<3);
251                 }
252
253                 ++iter;
254         }
255
256         suppress_line_advance = false;
257 }
258
259 } // namespace SL
260 } // namespace GL
261 } // namespace Msp