source/glsl/tokenizer.cpp

   1 #include <msp/core/raii.h>
   2 #include "glsl_error.h"
   3 #include "preprocessor.h"
   4 #include "syntax.h"
   5 #include "tokenizer.h"
   6
   7 using namespace std;
   8
   9 namespace Msp {
  10 namespace GL {
  11 namespace SL {
  12
  13 Tokenizer::Tokenizer():
  14         allow_preprocess(true)
  15 {
  16         static string empty;
  17         iter = empty.begin();
  18         source_end = empty.end();
  19         location.line = 0;
  20 }
  21
  22 void Tokenizer::begin(const string &name, const string &src)
  23 {
  24         iter = src.begin();
  25         source_end = src.end();
  26         location.name = name;
  27         location.line = 1;
  28         allow_preprocess = true;
  29         last_token.clear();
  30         next_tokens.clear();
  31 }
  32
  33 const string &Tokenizer::peek_token(unsigned index)
  34 {
  35         while(next_tokens.size()<=index)
  36                 next_tokens.push_back(parse_token_());
  37         return (last_token = next_tokens[index]);
  38 }
  39
  40 const string &Tokenizer::parse_token()
  41 {
  42         if(!next_tokens.empty())
  43         {
  44                 last_token = next_tokens.front();
  45                 next_tokens.pop_front();
  46                 return last_token;
  47         }
  48
  49         return (last_token = parse_token_());
  50 }
  51
  52 void Tokenizer::expect(const string &token)
  53 {
  54         string parsed = parse_token();
  55         if(parsed!=token)
  56                 throw parse_error(location, parsed, format("'%s'", token));
  57 }
  58
  59 string Tokenizer::parse_token_()
  60 {
  61         while(1)
  62         {
  63                 skip_comment_and_whitespace();
  64                 bool allow_preproc = allow_preprocess;
  65                 allow_preprocess = false;
  66                 if(iter==source_end)
  67                         return string();
  68                 else if(allow_preproc && *iter=='#')
  69                 {
  70                         ++iter;
  71                         preprocess();
  72                 }
  73                 else if(isalpha(*iter) || *iter=='_')
  74                         return parse_identifier();
  75                 else if(isdigit(*iter))
  76                         return parse_number();
  77                 else if(*iter=='"' || *iter=='#' || *iter=='$' || *iter=='\'' || *iter=='@' || *iter=='\\' || *iter=='`')
  78                         throw syntax_error(location, string(1, *iter), "Invalid character in source");
  79                 else
  80                         return parse_other();
  81         }
  82 }
  83
  84 void Tokenizer::preprocess()
  85 {
  86         SetForScope<deque<string> > clear_tokens(next_tokens, deque<string>());
  87
  88         string::const_iterator line_end = iter;
  89         for(; (line_end!=source_end && *line_end!='\n'); ++line_end) ;
  90         SetForScope<string::const_iterator> stop_at_line_end(source_end, line_end);
  91
  92         signal_preprocess.emit();
  93
  94         iter = line_end;
  95 }
  96
  97 string Tokenizer::parse_identifier()
  98 {
  99         string ident;
 100         while(iter!=source_end)
 101         {
 102                 if(isalnum(*iter) || *iter=='_')
 103                         ident += *iter++;
 104                 else
 105                         break;
 106         }
 107
 108         return ident;
 109 }
 110
 111 string Tokenizer::parse_number()
 112 {
 113         bool got_fract = false;
 114         string number;
 115         while(iter!=source_end)
 116         {
 117                 if(isdigit(*iter))
 118                         number += *iter++;
 119                 else if(!got_fract && *iter=='.')
 120                 {
 121                         number += *iter++;
 122                         got_fract = true;
 123                 }
 124                 else
 125                         break;
 126         }
 127
 128         bool require_digit = false;
 129         if(iter!=source_end && (*iter=='e' || *iter=='E'))
 130         {
 131                 number += *iter++;
 132                 if(iter!=source_end && (*iter=='-' || *iter=='+'))
 133                         number += *iter++;
 134                 require_digit = true;
 135                 while(iter!=source_end)
 136                 {
 137                         if(isdigit(*iter))
 138                         {
 139                                 number += *iter++;
 140                                 require_digit = false;
 141                         }
 142                         else
 143                                 break;
 144                 }
 145         }
 146
 147         if(require_digit)
 148                 throw syntax_error(location, number, "Incomplete numeric literal");
 149         else if(isalnum(*iter) || *iter=='_')
 150                 throw syntax_error(location, number, "Garbage at end of numeric literal");
 151
 152         return number;
 153 }
 154
 155 string Tokenizer::parse_other()
 156 {
 157         if(iter==source_end)
 158                 return string();
 159
 160         string token(1, *iter++);
 161         for(unsigned i=1; (i<3 && iter!=source_end); ++i)
 162         {
 163                 bool matched = false;
 164                 for(const Operator *j=Operator::operators; (!matched && j->type); ++j)
 165                 {
 166                         matched = (j->token[i]==*iter);
 167                         for(unsigned k=0; (matched && k<i && j->token[k]); ++k)
 168                                 matched = (j->token[k]==token[k]);
 169                 }
 170
 171                 if(!matched)
 172                         break;
 173
 174                 token += *iter++;
 175         }
 176
 177         return token;
 178 }
 179
 180 void Tokenizer::skip_comment_and_whitespace()
 181 {
 182         unsigned comment = 0;
 183         while(iter!=source_end)
 184         {
 185                 if(comment==0)
 186                 {
 187                         if(*iter=='/')
 188                                 comment = 1;
 189                         else if(!isspace(*iter))
 190                                 break;
 191                 }
 192                 else if(comment==1)
 193                 {
 194                         if(*iter=='/')
 195                                 comment = 2;
 196                         else if(*iter=='*')
 197                                 comment = 3;
 198                         else
 199                         {
 200                                 comment = 0;
 201                                 --iter;
 202                                 break;
 203                         }
 204                 }
 205                 else if(comment==2)
 206                 {
 207                         if(*iter=='\n')
 208                                 comment = 0;
 209                 }
 210                 else if(comment==3 && *iter=='*')
 211                         comment = 4;
 212                 else if(comment==4)
 213                 {
 214                         if(*iter=='/')
 215                                 comment = 0;
 216                         else if(*iter!='*')
 217                                 comment = 3;
 218                 }
 219
 220                 if(*iter=='\n')
 221                 {
 222                         ++location.line;
 223                         allow_preprocess = (comment<3);
 224                 }
 225
 226                 ++iter;
 227         }
 228 }
 229
 230 } // namespace SL
 231 } // namespace GL
 232 } // namespace Msp