namespace SL {
Tokenizer::Tokenizer():
- allow_preprocess(true)
+ allow_preprocess(true),
+ suppress_line_advance(false)
{
static string empty;
iter = empty.begin();
source_end = empty.end();
- location.line = 0;
}
void Tokenizer::begin(const string &name, const string &src)
throw parse_error(location, parsed, format("'%s'", token));
}
+void Tokenizer::set_location(const Location &loc)
+{
+ location = loc;
+}
+
string Tokenizer::parse_token_()
{
while(1)
return parse_identifier();
else if(isdigit(*iter))
return parse_number();
- else if(*iter=='"' || *iter=='#' || *iter=='$' || *iter=='\'' || *iter=='@' || *iter=='\\' || *iter=='`')
+ else if(*iter=='"')
+ return parse_string();
+ else if(*iter=='#' || *iter=='$' || *iter=='\'' || *iter=='@' || *iter=='\\' || *iter=='`')
throw syntax_error(location, string(1, *iter), "Invalid character in source");
else
return parse_other();
return number;
}
+string Tokenizer::parse_string()
+{
+ string str(1, *iter++);
+ bool escape = false;
+ while(iter!=source_end)
+ {
+ char c = *iter++;
+ str += c;
+ if(c=='\\')
+ escape = true;
+ else if(c=='"' && !escape)
+ break;
+ else
+ escape = false;
+ }
+
+ return str;
+}
+
string Tokenizer::parse_other()
{
if(iter==source_end)
if(*iter=='\n')
{
- ++location.line;
+ if(!suppress_line_advance)
+ ++location.line;
allow_preprocess = (comment<3);
}
++iter;
}
+
+ suppress_line_advance = false;
}
} // namespace SL