Fix creating Symbols from arbitary types

[libs/datafile.git] / source / textparser.cpp

diff --git a/source/textparser.cpp b/source/textparser.cpp
index 0076c1b4bab954f8a893abf9f2e6a4ff7da5039a..00192ccc147ce24f4c181a751a741d8fc1481b3d 100644 (file)
--- a/source/textparser.cpp
+++ b/source/textparser.cpp
@@ -144,6 +144,7 @@ Token TextParser::parse_token()
                OCTAL,
                FLOAT,
                FLOATEXP,
+               STRING_END,
                IDENTIFIER
        };
 
@@ -153,7 +154,7 @@ Token TextParser::parse_token()
                Token::SPECIAL,
                Token::SPECIAL,
                Token::SPECIAL,
-               Token::STRING,
+               Token::SPECIAL,
                Token::SPECIAL,
                Token::INTEGER,
                Token::INTEGER,
@@ -161,6 +162,7 @@ Token TextParser::parse_token()
                Token::INTEGER,
                Token::FLOAT,
                Token::FLOAT,
+               Token::STRING,
                Token::IDENTIFIER
        };
 
@@ -191,10 +193,10 @@ Token TextParser::parse_token()
                                return Token(Token::SPECIAL, string(1, c));
                        else if(isdigit(c))
                                state = DECIMAL;
-                       else if(isalpha(c) || c=='_')
+                       else if(isalpha(c) || c=='_' || c=='\\')
                                state = IDENTIFIER;
                        else
-                               parse_error(c, "0-9A-Za-z_.\"{};+-");
+                               parse_error(c, "0-9A-Za-z_\\.\"{};+-");
                        break;
 
                case SIGN:
@@ -268,6 +270,28 @@ Token TextParser::parse_token()
                        if(c=='\\')
                                escape = !escape;
                        else if(c=='"' && !escape)
+                               state = STRING_END;
+                       else
+                               escape = false;
+                       break;
+
+               case IDENTIFIER:
+                       if(!isalpha(c) && !isdigit(c) && c!='_' && c!='-' && c!='/')
+                               parse_error(c, "0-9A-Za-z_/-");
+                       break;
+
+               case STRING_END:
+                       throw_at(ParseError("Garbage after string"), get_location());
+
+               default:
+                       throw_at(InvalidState("Internal error (bad state)"), get_location());
+               }
+
+               if(is_delimiter(next) && state>=ACCEPT)
+               {
+                       if(state==IDENTIFIER && buf[0]=='\\')
+                               return Token(Token::IDENTIFIER, buf.substr(1));
+                       else if(state==STRING_END)
                        {
                                try
                                {
@@ -280,20 +304,8 @@ Token TextParser::parse_token()
                                }
                        }
                        else
-                               escape = false;
-                       break;
-
-               case IDENTIFIER:
-                       if(!isalpha(c) && !isdigit(c) && c!='_')
-                               parse_error(c, "0-9A-Za-z_");
-                       break;
-
-               default:
-                       throw_at(InvalidState("Internal error (bad state)"), get_location());
+                               return Token(token_type[state], buf);
                }
-
-               if(is_delimiter(next) && state>=ACCEPT)
-                       return Token(token_type[state], buf);
        }
 
        return Token(Token::SPECIAL, "");