From: Mikko Rasa <tdb@tdb.fi>
Date: Wed, 27 Oct 2010 20:24:17 +0000 (+0000)
Subject: Fix creating Symbols from arbitary types
X-Git-Url: http://git.tdb.fi/?p=libs%2Fdatafile.git;a=commitdiff_plain;h=e5d760ccfaaa01884be2424b62e47a24466e0c4b

Fix creating Symbols from arbitary types
Allow a few other delimiters in symbols
Allow symbols to begin with a non-alphanumeric if escaped
Fix a bug where another token was accepted immediately after a string
---

diff --git a/source/textparser.cpp b/source/textparser.cpp
index 0076c1b..00192cc 100644
--- a/source/textparser.cpp
+++ b/source/textparser.cpp
@@ -144,6 +144,7 @@ Token TextParser::parse_token()
 		OCTAL,
 		FLOAT,
 		FLOATEXP,
+		STRING_END,
 		IDENTIFIER
 	};
 
@@ -153,7 +154,7 @@ Token TextParser::parse_token()
 		Token::SPECIAL,
 		Token::SPECIAL,
 		Token::SPECIAL,
-		Token::STRING,
+		Token::SPECIAL,
 		Token::SPECIAL,
 		Token::INTEGER,
 		Token::INTEGER,
@@ -161,6 +162,7 @@ Token TextParser::parse_token()
 		Token::INTEGER,
 		Token::FLOAT,
 		Token::FLOAT,
+		Token::STRING,
 		Token::IDENTIFIER
 	};
 
@@ -191,10 +193,10 @@ Token TextParser::parse_token()
 				return Token(Token::SPECIAL, string(1, c));
 			else if(isdigit(c))
 				state = DECIMAL;
-			else if(isalpha(c) || c=='_')
+			else if(isalpha(c) || c=='_' || c=='\\')
 				state = IDENTIFIER;
 			else
-				parse_error(c, "0-9A-Za-z_.\"{};+-");
+				parse_error(c, "0-9A-Za-z_\\.\"{};+-");
 			break;
 
 		case SIGN:
@@ -268,6 +270,28 @@ Token TextParser::parse_token()
 			if(c=='\\')
 				escape = !escape;
 			else if(c=='"' && !escape)
+				state = STRING_END;
+			else
+				escape = false;
+			break;
+
+		case IDENTIFIER:
+			if(!isalpha(c) && !isdigit(c) && c!='_' && c!='-' && c!='/')
+				parse_error(c, "0-9A-Za-z_/-");
+			break;
+
+		case STRING_END:
+			throw_at(ParseError("Garbage after string"), get_location());
+
+		default:
+			throw_at(InvalidState("Internal error (bad state)"), get_location());
+		}
+
+		if(is_delimiter(next) && state>=ACCEPT)
+		{
+			if(state==IDENTIFIER && buf[0]=='\\')
+				return Token(Token::IDENTIFIER, buf.substr(1));
+			else if(state==STRING_END)
 			{
 				try
 				{
@@ -280,20 +304,8 @@ Token TextParser::parse_token()
 				}
 			}
 			else
-				escape = false;
-			break;
-
-		case IDENTIFIER:
-			if(!isalpha(c) && !isdigit(c) && c!='_')
-				parse_error(c, "0-9A-Za-z_");
-			break;
-
-		default:
-			throw_at(InvalidState("Internal error (bad state)"), get_location());
+				return Token(token_type[state], buf);
 		}
-
-		if(is_delimiter(next) && state>=ACCEPT)
-			return Token(token_type[state], buf);
 	}
 
 	return Token(Token::SPECIAL, "");
diff --git a/source/textwriter.cpp b/source/textwriter.cpp
index 08909a1..eb3aa2d 100644
--- a/source/textwriter.cpp
+++ b/source/textwriter.cpp
@@ -41,7 +41,13 @@ void TextWriter::write_(const Statement &st, unsigned level)
 		else if(i->get_signature()==FloatType::signature)
 			out.write(format("%15g", (i->get<FloatType::Store>())));
 		else if(i->get_signature()==SymbolType::signature)
-			out.write(i->get<SymbolType::Store>().name);
+		{
+			string name = i->get<SymbolType::Store>().name;
+			if(isdigit(name[0]))
+				out.write("\\"+name);
+			else
+				out.write(name);
+		}
 	}
 	if(!st.sub.empty())
 	{
diff --git a/source/type.h b/source/type.h
index 5a87a10..242a9e1 100644
--- a/source/type.h
+++ b/source/type.h
@@ -17,7 +17,8 @@ struct Symbol
 {
 	std::string name;
 
-	Symbol(const std::string &n): name(n) { }
+	template<typename T>
+	Symbol(const T &n): name(lexical_cast(n)) { }
 
 	template<typename T> operator T() const { return lexical_cast<T>(name); }
 };