source/glsl/tokenizer.cpp

   1 #include <msp/core/raii.h>
   2 #include "glsl_error.h"
   3 #include "preprocessor.h"
   4 #include "syntax.h"
   5 #include "tokenizer.h"
   6
   7 using namespace std;
   8
   9 namespace Msp {
  10 namespace GL {
  11 namespace SL {
  12
  13 Tokenizer::Tokenizer()
  14 {
  15         static string empty;
  16         iter = empty.begin();
  17         source_end = empty.end();
  18 }
  19
  20 void Tokenizer::begin(const string &src, const string &name)
  21 {
  22         iter = src.begin();
  23         source_end = src.end();
  24         location.name = name;
  25         location.line = 1;
  26         allow_preprocess = true;
  27         last_token.clear();
  28         next_tokens.clear();
  29 }
  30
  31 const string &Tokenizer::peek_token(unsigned index)
  32 {
  33         while(next_tokens.size()<=index)
  34                 next_tokens.push_back(parse_token_());
  35         return next_tokens[index];
  36 }
  37
  38 const string &Tokenizer::parse_token()
  39 {
  40         progress_mark = true;
  41
  42         if(!next_tokens.empty())
  43         {
  44                 last_token = next_tokens.front();
  45                 next_tokens.pop_front();
  46                 return last_token;
  47         }
  48
  49         return (last_token = parse_token_());
  50 }
  51
  52 void Tokenizer::expect(const string &token)
  53 {
  54         string parsed = parse_token();
  55         if(parsed!=token)
  56                 throw parse_error(location, parsed, format("'%s'", token));
  57 }
  58
  59 void Tokenizer::set_location(const Location &loc)
  60 {
  61         location = loc;
  62         suppress_line_advance = true;
  63 }
  64
  65 string Tokenizer::parse_token_()
  66 {
  67         while(1)
  68         {
  69                 skip_comment_and_whitespace();
  70                 bool allow_preproc = allow_preprocess;
  71                 allow_preprocess = false;
  72                 if(iter==source_end)
  73                         return string();
  74                 else if(allow_preproc && *iter=='#')
  75                 {
  76                         ++iter;
  77                         preprocess();
  78                 }
  79                 else if(isalpha(*iter) || *iter=='_')
  80                         return parse_identifier();
  81                 else if(isdigit(*iter))
  82                         return parse_number();
  83                 else if(*iter=='"')
  84                         return parse_string();
  85                 else if(*iter=='#' || *iter=='$' || *iter=='\'' || *iter=='@' || *iter=='\\' || *iter=='`')
  86                         throw syntax_error(location, string(1, *iter), "Invalid character in source");
  87                 else
  88                         return parse_other();
  89         }
  90 }
  91
  92 void Tokenizer::preprocess()
  93 {
  94         SetForScope<deque<string> > clear_tokens(next_tokens, deque<string>());
  95
  96         auto line_end = iter;
  97         for(; (line_end!=source_end && *line_end!='\n'); ++line_end) ;
  98         SetForScope<string::const_iterator> stop_at_line_end(source_end, line_end);
  99
 100         signal_preprocess.emit();
 101
 102         iter = line_end;
 103 }
 104
 105 string Tokenizer::parse_identifier()
 106 {
 107         string ident;
 108         while(iter!=source_end)
 109         {
 110                 if(isalnum(*iter) || *iter=='_')
 111                         ident += *iter++;
 112                 else
 113                         break;
 114         }
 115
 116         return ident;
 117 }
 118
 119 string Tokenizer::parse_number()
 120 {
 121         bool got_fract = false;
 122         string number;
 123         while(iter!=source_end)
 124         {
 125                 if(isdigit(*iter))
 126                         number += *iter++;
 127                 else if(!got_fract && *iter=='.')
 128                 {
 129                         number += *iter++;
 130                         got_fract = true;
 131                 }
 132                 else
 133                         break;
 134         }
 135
 136         bool require_digit = false;
 137         if(iter==source_end)
 138                 ;
 139         else if(*iter=='e' || *iter=='E')
 140         {
 141                 number += *iter++;
 142                 if(iter!=source_end && (*iter=='-' || *iter=='+'))
 143                         number += *iter++;
 144                 require_digit = true;
 145                 while(iter!=source_end)
 146                 {
 147                         if(isdigit(*iter))
 148                         {
 149                                 number += *iter++;
 150                                 require_digit = false;
 151                         }
 152                         else
 153                                 break;
 154                 }
 155                 if(!require_digit && iter!=source_end && *iter=='f')
 156                         number += *iter++;
 157         }
 158         else if(*iter=='u' || *iter=='f')
 159                 number += *iter++;
 160
 161         if(require_digit)
 162                 throw syntax_error(location, number, "Incomplete numeric literal");
 163         else if(iter!=source_end && (isalnum(*iter) || *iter=='_'))
 164                 throw syntax_error(location, number, "Garbage at end of numeric literal");
 165
 166         return number;
 167 }
 168
 169 string Tokenizer::parse_string()
 170 {
 171         string str(1, *iter++);
 172         bool escape = false;
 173         while(iter!=source_end)
 174         {
 175                 char c = *iter++;
 176                 str += c;
 177                 if(c=='\\')
 178                         escape = true;
 179                 else if(c=='"' && !escape)
 180                         break;
 181                 else
 182                         escape = false;
 183         }
 184
 185         return str;
 186 }
 187
 188 string Tokenizer::parse_other()
 189 {
 190         if(iter==source_end)
 191                 return string();
 192
 193         string token(1, *iter++);
 194         for(unsigned i=1; (i<3 && iter!=source_end); ++i)
 195         {
 196                 bool matched = false;
 197                 for(const Operator *j=Operator::operators; (!matched && j->type); ++j)
 198                 {
 199                         matched = (j->token[i]==*iter);
 200                         for(unsigned k=0; (matched && k<i && j->token[k]); ++k)
 201                                 matched = (j->token[k]==token[k]);
 202                 }
 203
 204                 if(!matched)
 205                         break;
 206
 207                 token += *iter++;
 208         }
 209
 210         return token;
 211 }
 212
 213 void Tokenizer::skip_comment_and_whitespace()
 214 {
 215         unsigned comment = 0;
 216         while(iter!=source_end)
 217         {
 218                 if(comment==0)
 219                 {
 220                         if(*iter=='/')
 221                                 comment = 1;
 222                         else if(!isspace(*iter))
 223                                 break;
 224                 }
 225                 else if(comment==1)
 226                 {
 227                         if(*iter=='/')
 228                                 comment = 2;
 229                         else if(*iter=='*')
 230                                 comment = 3;
 231                         else
 232                         {
 233                                 comment = 0;
 234                                 --iter;
 235                                 break;
 236                         }
 237                 }
 238                 else if(comment==2)
 239                 {
 240                         if(*iter=='\n')
 241                                 comment = 0;
 242                 }
 243                 else if(comment==3 && *iter=='*')
 244                         comment = 4;
 245                 else if(comment==4)
 246                 {
 247                         if(*iter=='/')
 248                                 comment = 0;
 249                         else if(*iter!='*')
 250                                 comment = 3;
 251                 }
 252
 253                 if(*iter=='\n')
 254                 {
 255                         if(!suppress_line_advance)
 256                                 ++location.line;
 257                         allow_preprocess = (comment<3);
 258                 }
 259
 260                 ++iter;
 261         }
 262
 263         suppress_line_advance = false;
 264 }
 265
 266 } // namespace SL
 267 } // namespace GL
 268 } // namespace Msp