source/glsl/tokenizer.cpp

   1 #include <msp/core/raii.h>
   2 #include "glsl_error.h"
   3 #include "preprocessor.h"
   4 #include "syntax.h"
   5 #include "tokenizer.h"
   6
   7 using namespace std;
   8
   9 namespace Msp {
  10 namespace GL {
  11 namespace SL {
  12
  13 Tokenizer::Tokenizer():
  14         allow_preprocess(true)
  15 {
  16         static string empty;
  17         iter = empty.begin();
  18         source_end = empty.end();
  19         location.line = 0;
  20 }
  21
  22 void Tokenizer::begin(const string &name, const string &src)
  23 {
  24         iter = src.begin();
  25         source_end = src.end();
  26         location.name = name;
  27         location.line = 1;
  28         allow_preprocess = true;
  29         last_token.clear();
  30         next_tokens.clear();
  31 }
  32
  33 const string &Tokenizer::peek_token(unsigned index)
  34 {
  35         while(next_tokens.size()<=index)
  36                 next_tokens.push_back(parse_token_());
  37         return (last_token = next_tokens[index]);
  38 }
  39
  40 const string &Tokenizer::parse_token()
  41 {
  42         if(!next_tokens.empty())
  43         {
  44                 last_token = next_tokens.front();
  45                 next_tokens.pop_front();
  46                 return last_token;
  47         }
  48
  49         return (last_token = parse_token_());
  50 }
  51
  52 void Tokenizer::expect(const string &token)
  53 {
  54         string parsed = parse_token();
  55         if(parsed!=token)
  56                 throw parse_error(location, parsed, format("'%s'", token));
  57 }
  58
  59 string Tokenizer::parse_token_()
  60 {
  61         while(1)
  62         {
  63                 skip_comment_and_whitespace();
  64                 if(iter==source_end)
  65                         return string();
  66                 else if(allow_preprocess && *iter=='#')
  67                 {
  68                         allow_preprocess = false;
  69                         preprocess();
  70                 }
  71                 else if(isalpha(*iter) || *iter=='_')
  72                         return parse_identifier();
  73                 else if(isdigit(*iter))
  74                         return parse_number();
  75                 else
  76                         return parse_other();
  77         }
  78 }
  79
  80 void Tokenizer::preprocess()
  81 {
  82         SetForScope<deque<string> > clear_tokens(next_tokens, deque<string>());
  83
  84         string::const_iterator line_end = iter;
  85         for(; (line_end!=source_end && *line_end!='\n'); ++line_end) ;
  86         SetForScope<string::const_iterator> stop_at_line_end(source_end, line_end);
  87
  88         signal_preprocess.emit();
  89
  90         iter = line_end;
  91 }
  92
  93 string Tokenizer::parse_identifier()
  94 {
  95         string ident;
  96         while(iter!=source_end)
  97         {
  98                 if(isalnum(*iter) || *iter=='_')
  99                         ident += *iter++;
 100                 else
 101                         break;
 102         }
 103
 104         return ident;
 105 }
 106
 107 string Tokenizer::parse_number()
 108 {
 109         bool accept_sign = false;
 110         string number;
 111         while(iter!=source_end)
 112         {
 113                 if(isdigit(*iter) || *iter=='.')
 114                         number += *iter++;
 115                 else if(*iter=='e' || *iter=='E')
 116                 {
 117                         number += *iter++;
 118                         accept_sign = true;
 119                 }
 120                 else if(accept_sign && (*iter=='+' || *iter=='-'))
 121                         number += *iter++;
 122                 else
 123                         break;
 124         }
 125
 126         return number;
 127 }
 128
 129 string Tokenizer::parse_other()
 130 {
 131         if(iter==source_end)
 132                 return string();
 133
 134         string token(1, *iter++);
 135         for(unsigned i=1; (i<3 && iter!=source_end); ++i)
 136         {
 137                 bool matched = false;
 138                 for(const Operator *j=Operator::operators; (!matched && j->type); ++j)
 139                 {
 140                         matched = (j->token[i]==*iter);
 141                         for(unsigned k=0; (matched && k<i && j->token[k]); ++k)
 142                                 matched = (j->token[k]==token[k]);
 143                 }
 144
 145                 if(!matched)
 146                         break;
 147
 148                 token += *iter++;
 149         }
 150
 151         return token;
 152 }
 153
 154 void Tokenizer::skip_comment_and_whitespace()
 155 {
 156         unsigned comment = 0;
 157         while(iter!=source_end)
 158         {
 159                 if(comment==0)
 160                 {
 161                         if(*iter=='/')
 162                                 comment = 1;
 163                         else if(!isspace(*iter))
 164                                 break;
 165                 }
 166                 else if(comment==1)
 167                 {
 168                         if(*iter=='/')
 169                                 comment = 2;
 170                         else if(*iter=='*')
 171                                 comment = 3;
 172                         else
 173                         {
 174                                 comment = 0;
 175                                 --iter;
 176                                 break;
 177                         }
 178                 }
 179                 else if(comment==2)
 180                 {
 181                         if(*iter=='\n')
 182                                 comment = 0;
 183                 }
 184                 else if(comment==3 && *iter=='*')
 185                         comment = 4;
 186                 else if(comment==4)
 187                 {
 188                         if(*iter=='/')
 189                                 comment = 0;
 190                         else if(*iter!='*')
 191                                 comment = 3;
 192                 }
 193
 194                 if(*iter=='\n')
 195                 {
 196                         ++location.line;
 197                         allow_preprocess = (comment<3);
 198                 }
 199
 200                 ++iter;
 201         }
 202 }
 203
 204 } // namespace SL
 205 } // namespace GL
 206 } // namespace Msp