]> git.tdb.fi Git - libs/gl.git/blob - source/glsl/tokenizer.cpp
Remove unused constructors
[libs/gl.git] / source / glsl / tokenizer.cpp
1 #include <msp/core/raii.h>
2 #include "glsl_error.h"
3 #include "preprocessor.h"
4 #include "syntax.h"
5 #include "tokenizer.h"
6
7 using namespace std;
8
9 namespace Msp {
10 namespace GL {
11 namespace SL {
12
13 Tokenizer::Tokenizer():
14         allow_preprocess(true)
15 {
16         static string empty;
17         iter = empty.begin();
18         source_end = empty.end();
19         location.line = 0;
20 }
21
22 void Tokenizer::begin(const string &name, const string &src)
23 {
24         iter = src.begin();
25         source_end = src.end();
26         location.name = name;
27         location.line = 1;
28         allow_preprocess = true;
29         last_token.clear();
30         next_tokens.clear();
31 }
32
33 const string &Tokenizer::peek_token(unsigned index)
34 {
35         while(next_tokens.size()<=index)
36                 next_tokens.push_back(parse_token_());
37         return (last_token = next_tokens[index]);
38 }
39
40 const string &Tokenizer::parse_token()
41 {
42         if(!next_tokens.empty())
43         {
44                 last_token = next_tokens.front();
45                 next_tokens.pop_front();
46                 return last_token;
47         }
48
49         return (last_token = parse_token_());
50 }
51
52 void Tokenizer::expect(const string &token)
53 {
54         string parsed = parse_token();
55         if(parsed!=token)
56                 throw parse_error(location, parsed, format("'%s'", token));
57 }
58
59 string Tokenizer::parse_token_()
60 {
61         while(1)
62         {
63                 skip_comment_and_whitespace();
64                 bool allow_preproc = allow_preprocess;
65                 allow_preprocess = false;
66                 if(iter==source_end)
67                         return string();
68                 else if(allow_preproc && *iter=='#')
69                 {
70                         ++iter;
71                         preprocess();
72                 }
73                 else if(isalpha(*iter) || *iter=='_')
74                         return parse_identifier();
75                 else if(isdigit(*iter))
76                         return parse_number();
77                 else if(*iter=='"' || *iter=='#' || *iter=='$' || *iter=='\'' || *iter=='@' || *iter=='\\' || *iter=='`')
78                         throw syntax_error(location, string(1, *iter), "Invalid character in source");
79                 else
80                         return parse_other();
81         }
82 }
83
84 void Tokenizer::preprocess()
85 {
86         SetForScope<deque<string> > clear_tokens(next_tokens, deque<string>());
87
88         string::const_iterator line_end = iter;
89         for(; (line_end!=source_end && *line_end!='\n'); ++line_end) ;
90         SetForScope<string::const_iterator> stop_at_line_end(source_end, line_end);
91
92         signal_preprocess.emit();
93
94         iter = line_end;
95 }
96
97 string Tokenizer::parse_identifier()
98 {
99         string ident;
100         while(iter!=source_end)
101         {
102                 if(isalnum(*iter) || *iter=='_')
103                         ident += *iter++;
104                 else
105                         break;
106         }
107
108         return ident;
109 }
110
111 string Tokenizer::parse_number()
112 {
113         bool got_fract = false;
114         string number;
115         while(iter!=source_end)
116         {
117                 if(isdigit(*iter))
118                         number += *iter++;
119                 else if(!got_fract && *iter=='.')
120                 {
121                         number += *iter++;
122                         got_fract = true;
123                 }
124                 else
125                         break;
126         }
127
128         bool require_digit = false;
129         if(iter!=source_end && (*iter=='e' || *iter=='E'))
130         {
131                 number += *iter++;
132                 if(iter!=source_end && (*iter=='-' || *iter=='+'))
133                         number += *iter++;
134                 require_digit = true;
135                 while(iter!=source_end)
136                 {
137                         if(isdigit(*iter))
138                         {
139                                 number += *iter++;
140                                 require_digit = false;
141                         }
142                         else
143                                 break;
144                 }
145         }
146
147         if(require_digit)
148                 throw syntax_error(location, number, "Incomplete numeric literal");
149         else if(isalnum(*iter) || *iter=='_')
150                 throw syntax_error(location, number, "Garbage at end of numeric literal");
151
152         return number;
153 }
154
155 string Tokenizer::parse_other()
156 {
157         if(iter==source_end)
158                 return string();
159
160         string token(1, *iter++);
161         for(unsigned i=1; (i<3 && iter!=source_end); ++i)
162         {
163                 bool matched = false;
164                 for(const Operator *j=Operator::operators; (!matched && j->type); ++j)
165                 {
166                         matched = (j->token[i]==*iter);
167                         for(unsigned k=0; (matched && k<i && j->token[k]); ++k)
168                                 matched = (j->token[k]==token[k]);
169                 }
170
171                 if(!matched)
172                         break;
173
174                 token += *iter++;
175         }
176
177         return token;
178 }
179
180 void Tokenizer::skip_comment_and_whitespace()
181 {
182         unsigned comment = 0;
183         while(iter!=source_end)
184         {
185                 if(comment==0)
186                 {
187                         if(*iter=='/')
188                                 comment = 1;
189                         else if(!isspace(*iter))
190                                 break;
191                 }
192                 else if(comment==1)
193                 {
194                         if(*iter=='/')
195                                 comment = 2;
196                         else if(*iter=='*')
197                                 comment = 3;
198                         else
199                         {
200                                 comment = 0;
201                                 --iter;
202                                 break;
203                         }
204                 }
205                 else if(comment==2)
206                 {
207                         if(*iter=='\n')
208                                 comment = 0;
209                 }
210                 else if(comment==3 && *iter=='*')
211                         comment = 4;
212                 else if(comment==4)
213                 {
214                         if(*iter=='/')
215                                 comment = 0;
216                         else if(*iter!='*')
217                                 comment = 3;
218                 }
219
220                 if(*iter=='\n')
221                 {
222                         ++location.line;
223                         allow_preprocess = (comment<3);
224                 }
225
226                 ++iter;
227         }
228 }
229
230 } // namespace SL
231 } // namespace GL
232 } // namespace Msp