From a4091f14dfbf4d5f822fa26740a7b3f305c449e6 Mon Sep 17 00:00:00 2001 From: Mikko Rasa Date: Fri, 20 Jan 2012 22:28:06 +0200 Subject: [PATCH 01/16] Add DirectoryCollection class for creating directory-backed collections --- source/directorycollection.cpp | 41 +++++++++++++++++++++++++++ source/directorycollection.h | 52 ++++++++++++++++++++++++++++++++++ 2 files changed, 93 insertions(+) create mode 100644 source/directorycollection.cpp create mode 100644 source/directorycollection.h diff --git a/source/directorycollection.cpp b/source/directorycollection.cpp new file mode 100644 index 0000000..25808d6 --- /dev/null +++ b/source/directorycollection.cpp @@ -0,0 +1,41 @@ +#include +#include "directorycollection.h" + +using namespace std; + +namespace Msp { +namespace DataFile { + +DirectoryCollection::DirectoryCollection() +{ + set_directory("."); +} + +void DirectoryCollection::set_directory(const FS::Path &d) +{ + dirs.clear(); + add_directory(d); +} + +void DirectoryCollection::add_directory(const FS::Path &d) +{ + dirs.push_back(d); +} + +bool DirectoryCollection::lookup_file(const string &name, FS::Path &result) const +{ + for(list::const_iterator i=dirs.begin(); i!=dirs.end(); ++i) + { + FS::Path file_path = *i/name; + if(FS::exists(file_path)) + { + result = file_path; + return true; + } + } + + return false; +} + +} // namespace DataFile +} // namespace Msp diff --git a/source/directorycollection.h b/source/directorycollection.h new file mode 100644 index 0000000..93f9354 --- /dev/null +++ b/source/directorycollection.h @@ -0,0 +1,52 @@ +#ifndef MSP_DATAFILE_DIRECTORYCOLLECTION_H_ +#define MSP_DATAFILE_DIRECTORYCOLLECTION_H_ + +#include +#include "collection.h" + +namespace Msp { +namespace DataFile { + +/** +A Collection that can automatically load items from files in a directory. +*/ +class DirectoryCollection: public Collection +{ +private: + std::list dirs; + +public: + DirectoryCollection(); + +protected: + void set_directory(const FS::Path &); + void add_directory(const FS::Path &); + + template + CollectionItemType &add_type() + { + return Collection::add_type().creator(&DirectoryCollection::create); + } + +private: + template + T *create(const std::string &name) + { + FS::Path file; + if(lookup_file(name, file)) + { + RefPtr item = new T; + load(*item, file.str()); + return item.release(); + } + else + return 0; + } + + bool lookup_file(const std::string &, FS::Path &) const; +}; + +} // namespace DataFile +} // namespace Msp + +#endif -- 2.43.0 From 61f6f66a7db1a074143cd0e9948a262d30876dea Mon Sep 17 00:00:00 2001 From: Mikko Rasa Date: Tue, 17 Apr 2012 12:00:03 +0300 Subject: [PATCH 02/16] Style fixes --- source/collection.h | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/source/collection.h b/source/collection.h index 91f8f07..b223990 100644 --- a/source/collection.h +++ b/source/collection.h @@ -58,7 +58,7 @@ public: template void coll_item(const std::string &n) { - RefPtr it=new T; + RefPtr it = new T; load_sub(*it, dynamic_cast(coll)); coll.add(n, it.get()); it.release(); @@ -67,13 +67,11 @@ public: template void item(const std::string &n) { - RefPtr it=new T; + RefPtr it = new T; load_sub(*it); coll.add(n, it.get()); it.release(); } - - template friend class ItemKeyword; }; private: -- 2.43.0 From f51adf2b95f32bdda4d90eb52431405c3ecf8db1 Mon Sep 17 00:00:00 2001 From: Mikko Rasa Date: Tue, 10 Jul 2012 18:28:11 +0300 Subject: [PATCH 03/16] Add DerivedObjectLoader class --- source/objectloader.h | 20 ++++++++++++++++++++ 1 file changed, 20 insertions(+) diff --git a/source/objectloader.h b/source/objectloader.h index f1cb6ff..5928714 100644 --- a/source/objectloader.h +++ b/source/objectloader.h @@ -35,6 +35,26 @@ public: }; +/** +Convenience class for loading derived objects. Inherits from the base class +loader and shadows its members with ones for the derived type. +*/ +template +class DerivedObjectLoader: public B::Loader +{ +public: + typedef O Object; + +protected: + O &obj; + + DerivedObjectLoader(O &o): B::Loader(o), obj(o) { } + +public: + O &get_object() const { return obj; } +}; + + /** Provides functionality for loading objects with a Collection. Deriving from this allows loading pointers to objects in the collection automatically. -- 2.43.0 From b0b9af7216560da2a46ea38fe2df959f4dfb126f Mon Sep 17 00:00:00 2001 From: Mikko Rasa Date: Wed, 18 Jul 2012 13:09:37 +0300 Subject: [PATCH 04/16] Convenience function for reporting errors in loaded data --- source/loader.cpp | 7 +++++++ source/loader.h | 3 +++ 2 files changed, 10 insertions(+) diff --git a/source/loader.cpp b/source/loader.cpp index 2bf90ee..ad62445 100644 --- a/source/loader.cpp +++ b/source/loader.cpp @@ -176,6 +176,13 @@ const string &Loader::get_source() const return cur_st->source; } +void Loader::error(const string &msg) const +{ + if(!cur_st) + throw logic_error("!cur_st"); + throw data_error(cur_st->source, cur_st->line, msg); +} + Loader::ActionKey::ActionKey(const string &k, const string &s): keyword(k), diff --git a/source/loader.h b/source/loader.h index 6048861..4a1faa3 100644 --- a/source/loader.h +++ b/source/loader.h @@ -139,6 +139,9 @@ protected: source may not necessarily be a file. */ const std::string &get_source() const; + /** Throws a data_error from the current line. */ + void error(const std::string &) const; + virtual void finish() { } }; -- 2.43.0 From 6f94aaece716a31e75166e261cc47579288892b4 Mon Sep 17 00:00:00 2001 From: Mikko Rasa Date: Wed, 18 Jul 2012 15:00:36 +0300 Subject: [PATCH 05/16] Refactor symbol handling in binary format --- source/binaryparser.cpp | 4 ++-- source/binaryparser.h | 3 ++- source/binarywriter.cpp | 6 +++--- source/binarywriter.h | 3 ++- 4 files changed, 9 insertions(+), 7 deletions(-) diff --git a/source/binaryparser.cpp b/source/binaryparser.cpp index 550319f..73bb5cb 100644 --- a/source/binaryparser.cpp +++ b/source/binaryparser.cpp @@ -89,7 +89,7 @@ Statement BinaryParser::parse_statement() result.args.push_back(parse_bool()); break; case SymbolType::signature: - result.args.push_back(Symbol(parse_enum())); + result.args.push_back(parse_symbol()); break; } } @@ -164,7 +164,7 @@ string BinaryParser::parse_string() return get_item(strings, -len); } -string BinaryParser::parse_enum() +Symbol BinaryParser::parse_symbol() { return get_item(strings, parse_int()); } diff --git a/source/binaryparser.h b/source/binaryparser.h index 2cdfbbe..4f33d95 100644 --- a/source/binaryparser.h +++ b/source/binaryparser.h @@ -4,6 +4,7 @@ #include #include "binarydict.h" #include "parsermode.h" +#include "type.h" namespace Msp { namespace DataFile { @@ -31,7 +32,7 @@ private: float parse_float(); std::string parse_string(); bool parse_bool(); - std::string parse_enum(); + Symbol parse_symbol(); }; } // namespace DataFile diff --git a/source/binarywriter.cpp b/source/binarywriter.cpp index 2ff96a4..a8306c2 100644 --- a/source/binarywriter.cpp +++ b/source/binarywriter.cpp @@ -34,7 +34,7 @@ void BinaryWriter::write_(const Statement &st) case StringType::signature: write_string(j->get()); break; case BoolType::signature: write_int (j->get()); break; case FloatType::signature: write_float (j->get()); break; - case SymbolType::signature: write_enum (j->get()); break; + case SymbolType::signature: write_symbol(j->get()); break; } write_int(st.sub.size()); @@ -132,9 +132,9 @@ void BinaryWriter::write_float(float f) #endif } -void BinaryWriter::write_enum(const string &e) +void BinaryWriter::write_symbol(const Symbol &s) { - write_int(get_item(strings, e)); + write_int(get_item(strings, s.name)); } } // namespace DataFile diff --git a/source/binarywriter.h b/source/binarywriter.h index f028f39..6ad2904 100644 --- a/source/binarywriter.h +++ b/source/binarywriter.h @@ -3,6 +3,7 @@ #include #include "binarydict.h" +#include "type.h" #include "writermode.h" namespace Msp { @@ -32,7 +33,7 @@ private: void write_int(long long n); void write_string(const std::string &s); void write_float(float f); - void write_enum(const std::string &e); + void write_symbol(const Symbol &s); }; } // namespace DataFile -- 2.43.0 From 5fd2c13c3036ea6a767802fdc9a2ab809ef8ec17 Mon Sep 17 00:00:00 2001 From: Mikko Rasa Date: Wed, 18 Jul 2012 15:56:42 +0300 Subject: [PATCH 06/16] Use the metadata for type.h in binary write/parse functions --- source/binaryparser.cpp | 14 +++++++------- source/binaryparser.h | 10 +++++----- source/binarywriter.cpp | 10 +++++----- source/binarywriter.h | 8 ++++---- 4 files changed, 21 insertions(+), 21 deletions(-) diff --git a/source/binaryparser.cpp b/source/binaryparser.cpp index 73bb5cb..5313128 100644 --- a/source/binaryparser.cpp +++ b/source/binaryparser.cpp @@ -103,9 +103,9 @@ Statement BinaryParser::parse_statement() return result; } -long long BinaryParser::parse_int() +IntType::Store BinaryParser::parse_int() { - long long result = 0; + IntType::Store result = 0; unsigned bits = 0; while(in) @@ -119,13 +119,13 @@ long long BinaryParser::parse_int() break; } - const long long mask = 1LL<<(bits-1); + const IntType::Store mask = 1LL<<(bits-1); result = (result^mask)-mask; return result; } -float BinaryParser::parse_float() +FloatType::Store BinaryParser::parse_float() { union { @@ -144,12 +144,12 @@ float BinaryParser::parse_float() return f; } -bool BinaryParser::parse_bool() +BoolType::Store BinaryParser::parse_bool() { return in.get(); } -string BinaryParser::parse_string() +StringType::Store BinaryParser::parse_string() { int len = parse_int(); if(len>=0) @@ -164,7 +164,7 @@ string BinaryParser::parse_string() return get_item(strings, -len); } -Symbol BinaryParser::parse_symbol() +SymbolType::Store BinaryParser::parse_symbol() { return get_item(strings, parse_int()); } diff --git a/source/binaryparser.h b/source/binaryparser.h index 4f33d95..f9b8e54 100644 --- a/source/binaryparser.h +++ b/source/binaryparser.h @@ -28,11 +28,11 @@ public: virtual Statement parse(); private: Statement parse_statement(); - long long parse_int(); - float parse_float(); - std::string parse_string(); - bool parse_bool(); - Symbol parse_symbol(); + IntType::Store parse_int(); + FloatType::Store parse_float(); + StringType::Store parse_string(); + BoolType::Store parse_bool(); + SymbolType::Store parse_symbol(); }; } // namespace DataFile diff --git a/source/binarywriter.cpp b/source/binarywriter.cpp index a8306c2..c1e53fb 100644 --- a/source/binarywriter.cpp +++ b/source/binarywriter.cpp @@ -89,9 +89,9 @@ void BinaryWriter::collect_keywords(const Statement &st) collect_keywords(*i); } -void BinaryWriter::write_int(long long n) +void BinaryWriter::write_int(IntType::Store n) { - unsigned i = sizeof(long long)-1; + unsigned i = sizeof(IntType::Store)-1; if(n>=0) for(; (i>0 && (n>>(i*7-1))==0); --i) ; @@ -102,7 +102,7 @@ void BinaryWriter::write_int(long long n) out.put((n>>(i*7) & 0x7F) | (i?0x80:0)); } -void BinaryWriter::write_string(const string &s) +void BinaryWriter::write_string(const StringType::Store &s) { StringMap::const_iterator i = strings.find(s); if(i!=strings.end()) @@ -114,7 +114,7 @@ void BinaryWriter::write_string(const string &s) } } -void BinaryWriter::write_float(float f) +void BinaryWriter::write_float(FloatType::Store f) { union { @@ -132,7 +132,7 @@ void BinaryWriter::write_float(float f) #endif } -void BinaryWriter::write_symbol(const Symbol &s) +void BinaryWriter::write_symbol(const SymbolType::Store &s) { write_int(get_item(strings, s.name)); } diff --git a/source/binarywriter.h b/source/binarywriter.h index 6ad2904..4d18196 100644 --- a/source/binarywriter.h +++ b/source/binarywriter.h @@ -30,10 +30,10 @@ public: private: void write_(const Statement &st); void collect_keywords(const Statement &st); - void write_int(long long n); - void write_string(const std::string &s); - void write_float(float f); - void write_symbol(const Symbol &s); + void write_int(IntType::Store n); + void write_string(const StringType::Store &s); + void write_float(FloatType::Store f); + void write_symbol(const SymbolType::Store &s); }; } // namespace DataFile -- 2.43.0 From 9bd3968eaaabb278d22f182365d022704d2a2cf1 Mon Sep 17 00:00:00 2001 From: Mikko Rasa Date: Wed, 18 Jul 2012 16:01:50 +0300 Subject: [PATCH 07/16] Don't use long long on MSVC --- source/type.h | 13 +++++++++++++ 1 file changed, 13 insertions(+) diff --git a/source/type.h b/source/type.h index 6959e98..a7c57c4 100644 --- a/source/type.h +++ b/source/type.h @@ -19,7 +19,12 @@ struct Symbol struct IntType { static const char signature = 'i'; +#ifdef MSVC + typedef __int64 Store; +#else typedef long long int Store; +#endif + typedef Store Load; }; struct FloatType @@ -68,11 +73,19 @@ struct TypeInfo: IntType { }; template<> struct TypeInfo: IntType { }; +#ifdef MSVC +template<> +struct TypeInfo<__int64>: IntType { }; + +template<> +struct TypeInfo: IntType { }; +#else template<> struct TypeInfo: IntType { }; template<> struct TypeInfo: IntType { }; +#endif template<> struct TypeInfo: FloatType { }; -- 2.43.0 From 818ead7b6ccef1e4d2435cc959bc07f910fcde46 Mon Sep 17 00:00:00 2001 From: Mikko Rasa Date: Wed, 18 Jul 2012 17:43:19 +0300 Subject: [PATCH 08/16] Add facility for classes to specify what type they should be loaded as --- source/type.h | 59 +++++++++++++++++++++++++++++++++++--------------- source/value.h | 2 +- 2 files changed, 43 insertions(+), 18 deletions(-) diff --git a/source/type.h b/source/type.h index a7c57c4..72d148a 100644 --- a/source/type.h +++ b/source/type.h @@ -31,18 +31,21 @@ struct FloatType { static const char signature = 'f'; typedef double Store; + typedef Store Load; }; struct BoolType { static const char signature = 'b'; typedef bool Store; + typedef Store Load; }; struct StringType { static const char signature = 's'; typedef std::string Store; + typedef Store Load; }; struct SymbolType @@ -53,57 +56,79 @@ struct SymbolType }; template -struct TypeInfo: SymbolType { }; +struct HasLoadType +{ + struct Yes { char c[2]; }; + struct No { char c; }; + + template + static Yes f(typename U::LoadType *); + template + static No f(...); + + enum { value = (sizeof(f(0))==sizeof(Yes)) }; +}; + +template::value> +struct TypeInfo; template<> -struct TypeInfo: IntType { }; +struct TypeInfo: IntType { }; template<> -struct TypeInfo: IntType { }; +struct TypeInfo: IntType { }; template<> -struct TypeInfo: IntType { }; +struct TypeInfo: IntType { }; template<> -struct TypeInfo: IntType { }; +struct TypeInfo: IntType { }; template<> -struct TypeInfo: IntType { }; +struct TypeInfo: IntType { }; template<> -struct TypeInfo: IntType { }; +struct TypeInfo: IntType { }; #ifdef MSVC template<> -struct TypeInfo<__int64>: IntType { }; +struct TypeInfo<__int64, false>: IntType { }; template<> -struct TypeInfo: IntType { }; +struct TypeInfo: IntType { }; #else template<> -struct TypeInfo: IntType { }; +struct TypeInfo: IntType { }; template<> -struct TypeInfo: IntType { }; +struct TypeInfo: IntType { }; #endif template<> -struct TypeInfo: FloatType { }; +struct TypeInfo: FloatType { }; template<> -struct TypeInfo: FloatType { }; +struct TypeInfo: FloatType { }; template<> -struct TypeInfo: BoolType { }; +struct TypeInfo: BoolType { }; template<> -struct TypeInfo: StringType { }; +struct TypeInfo: StringType { }; + +template +struct TypeInfo: TypeInfo { }; + +template +struct TypeInfo: TypeInfo { }; template -struct TypeInfo: TypeInfo { }; +struct TypeInfo: TypeInfo +{ typedef typename T::LoadType Load; }; template -struct TypeInfo: TypeInfo { }; +struct TypeInfo: SymbolType +{ typedef T Load; }; } // namespace DataFile } // namespace Msp diff --git a/source/value.h b/source/value.h index 78ec54a..e2b9fb1 100644 --- a/source/value.h +++ b/source/value.h @@ -25,7 +25,7 @@ public: Value(Symbol d): sig(TypeInfo::signature), data(d) { } template - typename RemoveReference::Type get() const + typename TypeInfo::Load get() const { return get_::Store>(); } char get_signature() const { return sig; } -- 2.43.0 From 75defe799ac0eea231c2d85696f9c92bbf8dc662 Mon Sep 17 00:00:00 2001 From: Mikko Rasa Date: Fri, 20 Jul 2012 23:55:26 +0300 Subject: [PATCH 09/16] Fix tests/Build --- tests/Build | 5 +---- 1 file changed, 1 insertion(+), 4 deletions(-) diff --git a/tests/Build b/tests/Build index 2ab2369..3da53b8 100644 --- a/tests/Build +++ b/tests/Build @@ -1,14 +1,11 @@ package "mspdatafile-tests" { + require "mspcore"; require "mspdatafile"; require "msptest"; program "test" { source "."; - build_info - { - ldflag "-Wl,-E"; - }; }; }; -- 2.43.0 From b83eea8073a4f9228e386eafd6df2e3dcf9f9ccc Mon Sep 17 00:00:00 2001 From: Mikko Rasa Date: Wed, 1 Aug 2012 11:08:08 +0300 Subject: [PATCH 10/16] Update the Build file with new Builder features --- Build | 14 +++++--------- 1 file changed, 5 insertions(+), 9 deletions(-) diff --git a/Build b/Build index 62aa92b..43599d9 100644 --- a/Build +++ b/Build @@ -1,5 +1,3 @@ -/* $Id$ */ - package "mspdatafile" { version "1.1.1"; @@ -7,16 +5,14 @@ package "mspdatafile" require "mspcore"; - headers "msp/datafile" - { - source "source"; - install true; - }; - library "mspdatafile" { source "source"; install true; + install_map + { + map "source" "include/msp/datafile"; + }; }; program "mspdatatool" @@ -29,7 +25,7 @@ package "mspdatafile" }; }; - tarball "@src" + source_tarball { source "Readme.txt"; source "License.txt"; -- 2.43.0 From a1575d9c2a4c233f84b2d814e9054ea807f4e640 Mon Sep 17 00:00:00 2001 From: Mikko Rasa Date: Wed, 1 Aug 2012 11:47:11 +0300 Subject: [PATCH 11/16] Bump version to 2.0 due to incompatible changes --- Build | 2 +- Changelog.txt | 8 ++++++++ 2 files changed, 9 insertions(+), 1 deletion(-) diff --git a/Build b/Build index 43599d9..7debad7 100644 --- a/Build +++ b/Build @@ -1,6 +1,6 @@ package "mspdatafile" { - version "1.1.1"; + version "2.0"; description "Structured datafile library"; require "mspcore"; diff --git a/Changelog.txt b/Changelog.txt index c0a8236..85d36d6 100644 --- a/Changelog.txt +++ b/Changelog.txt @@ -1,3 +1,11 @@ +2.0 +* Keyword overloading +* More efficient type/value system +* Improved classes for loading objects +* Redesigned Collection interface +* Bugfixes + - Empty files no longer cause an exception + 1.1.1 * Enhancements - Use lexical_cast instead of istringstream -- 2.43.0 From 019446fa25ee6338ede32cf722f833a307ecc841 Mon Sep 17 00:00:00 2001 From: Mikko Rasa Date: Wed, 1 Aug 2012 11:54:14 +0300 Subject: [PATCH 12/16] Update .gitignore --- .gitignore | 11 ++--------- 1 file changed, 2 insertions(+), 9 deletions(-) diff --git a/.gitignore b/.gitignore index fe5376f..7119609 100644 --- a/.gitignore +++ b/.gitignore @@ -1,14 +1,7 @@ -.deps -.options.* -.profile -/debug +.config +temp /libmspdatafile.a /libmspdatafile.so /mspdatafile.pc /mspdatatool -/pc-32-windows -/release -/temp -/tests/debug -/tests/temp /tests/test -- 2.43.0 From eec12aaf350ef69b79d149adbc3585cf784552ad Mon Sep 17 00:00:00 2001 From: Mikko Rasa Date: Wed, 1 Aug 2012 18:06:31 +0300 Subject: [PATCH 13/16] Recognize floating-point literals with an exponent but no decimal point --- source/textparser.cpp | 2 ++ 1 file changed, 2 insertions(+) diff --git a/source/textparser.cpp b/source/textparser.cpp index c4eb03c..9bed08a 100644 --- a/source/textparser.cpp +++ b/source/textparser.cpp @@ -237,6 +237,8 @@ Token TextParser::parse_token() case DECIMAL: if(c=='.') state = FLOAT; + else if(c=='e' || c=='E') + state = FLOATEXPINIT; else if(!isdigit(c)) throw parse_error(buf); break; -- 2.43.0 From 10c7c4157f02e3abbb7a0505bfc2985b67e04043 Mon Sep 17 00:00:00 2001 From: Mikko Rasa Date: Wed, 1 Aug 2012 18:46:49 +0300 Subject: [PATCH 14/16] Use Int64 from inttypes.h, and proper defines --- source/type.h | 11 ++++------- 1 file changed, 4 insertions(+), 7 deletions(-) diff --git a/source/type.h b/source/type.h index 72d148a..701a5dc 100644 --- a/source/type.h +++ b/source/type.h @@ -1,6 +1,7 @@ #ifndef MSP_DATAFILE_TYPE_H_ #define MSP_DATAFILE_TYPE_H_ +#include #include namespace Msp { @@ -19,11 +20,7 @@ struct Symbol struct IntType { static const char signature = 'i'; -#ifdef MSVC - typedef __int64 Store; -#else - typedef long long int Store; -#endif + typedef Int64 Store; typedef Store Load; }; @@ -90,13 +87,13 @@ struct TypeInfo: IntType { }; template<> struct TypeInfo: IntType { }; -#ifdef MSVC +#if defined(_MSC_VER) template<> struct TypeInfo<__int64, false>: IntType { }; template<> struct TypeInfo: IntType { }; -#else +#elif defined(__GNUC__) template<> struct TypeInfo: IntType { }; -- 2.43.0 From e0af585ed57bdb5b1ea4f4a415fda13b5d99d2dc Mon Sep 17 00:00:00 2001 From: Mikko Rasa Date: Wed, 1 Aug 2012 18:52:25 +0300 Subject: [PATCH 15/16] Use negative integers for built-in statements for better extensibility --- source/binaryparser.cpp | 8 ++++---- source/binaryparser.h | 2 +- source/binarywriter.cpp | 8 ++++---- source/binarywriter.h | 2 +- 4 files changed, 10 insertions(+), 10 deletions(-) diff --git a/source/binaryparser.cpp b/source/binaryparser.cpp index 5313128..81a86ea 100644 --- a/source/binaryparser.cpp +++ b/source/binaryparser.cpp @@ -24,8 +24,8 @@ BinaryParser::BinaryParser(Input &i, const string &s): ParserMode(i, s), first(true) { - dict[1] = DictEntry("__kwd", "iss"); - dict[2] = DictEntry("__str", "is"); + dict[-1] = DictEntry("__kwd", "iss"); + dict[-2] = DictEntry("__str", "is"); } Statement BinaryParser::parse() @@ -38,7 +38,7 @@ Statement BinaryParser::parse() if(st.args.size()!=3) throw bad_definition("__kwd"); - const unsigned id = st.args[0].get(); + const int id = st.args[0].get(); const string &kw = st.args[1].get(); const string &args = st.args[2].get(); dict[id] = DictEntry(kw, args); @@ -62,7 +62,7 @@ Statement BinaryParser::parse_statement() in.get(); first = false; - unsigned id = parse_int(); + int id = parse_int(); if(!in) return Statement(); diff --git a/source/binaryparser.h b/source/binaryparser.h index f9b8e54..b10d7b8 100644 --- a/source/binaryparser.h +++ b/source/binaryparser.h @@ -15,7 +15,7 @@ Parses data in binary format. class BinaryParser: public ParserMode { private: - typedef std::map Dictionary; + typedef std::map Dictionary; typedef std::map StringMap; Dictionary dict; diff --git a/source/binarywriter.cpp b/source/binarywriter.cpp index c1e53fb..9c9c893 100644 --- a/source/binarywriter.cpp +++ b/source/binarywriter.cpp @@ -9,11 +9,11 @@ namespace DataFile { BinaryWriter::BinaryWriter(IO::Base &o): WriterMode(o), - next_kwd_id(3), + next_kwd_id(1), next_str_id(1) { - dict[DictEntry("__kwd", "iss")] = 1; - dict[DictEntry("__str", "is")] = 2; + dict[DictEntry("__kwd", "iss")] = -1; + dict[DictEntry("__str", "is")] = -2; } void BinaryWriter::write(const Statement &st) @@ -24,7 +24,7 @@ void BinaryWriter::write(const Statement &st) void BinaryWriter::write_(const Statement &st) { - unsigned id = get_item(dict, DictEntry(st.keyword, st.get_signature())); + int id = get_item(dict, DictEntry(st.keyword, st.get_signature())); write_int(id); for(Statement::Arguments::const_iterator j = st.args.begin(); j!=st.args.end(); ++j) diff --git a/source/binarywriter.h b/source/binarywriter.h index 4d18196..1a11ae5 100644 --- a/source/binarywriter.h +++ b/source/binarywriter.h @@ -15,7 +15,7 @@ Writes data in binary format. class BinaryWriter: public WriterMode { private: - typedef std::map Dictionary; + typedef std::map Dictionary; typedef std::map StringMap; Dictionary dict; -- 2.43.0 From 19179a622c1de88de5ed7047643eec79f285bf2a Mon Sep 17 00:00:00 2001 From: Mikko Rasa Date: Thu, 2 Aug 2012 11:11:40 +0300 Subject: [PATCH 16/16] Use custom encoding for floats in binary format This makes the binary format fully machine-independent, and provides control over the precision of floating point values. --- source/binaryparser.cpp | 62 ++++++++++++++++++++++++++--------- source/binaryparser.h | 1 + source/binarywriter.cpp | 67 ++++++++++++++++++++++++++++++-------- source/binarywriter.h | 2 ++ source/binfloat.cpp | 72 +++++++++++++++++++++++++++++++++++++++++ source/binfloat.h | 63 ++++++++++++++++++++++++++++++++++++ source/textwriter.cpp | 10 ++++-- source/textwriter.h | 4 +++ source/writer.cpp | 5 +++ source/writer.h | 5 +++ source/writermode.h | 1 + tool/tool.cpp | 6 +++- tool/tool.h | 1 + 13 files changed, 267 insertions(+), 32 deletions(-) create mode 100644 source/binfloat.cpp create mode 100644 source/binfloat.h diff --git a/source/binaryparser.cpp b/source/binaryparser.cpp index 81a86ea..fa99a7e 100644 --- a/source/binaryparser.cpp +++ b/source/binaryparser.cpp @@ -1,7 +1,9 @@ +#include #include #include #include #include "binaryparser.h" +#include "binfloat.h" #include "input.h" using namespace std; @@ -22,10 +24,12 @@ public: BinaryParser::BinaryParser(Input &i, const string &s): ParserMode(i, s), - first(true) + first(true), + float_precision(32) { dict[-1] = DictEntry("__kwd", "iss"); dict[-2] = DictEntry("__str", "is"); + dict[-3] = DictEntry("__flt", "i"); } Statement BinaryParser::parse() @@ -51,6 +55,8 @@ Statement BinaryParser::parse() const unsigned id = st.args[0].get(); strings[id] = st.args[1].get(); } + else if(st.keyword=="__flt") + float_precision = st.args[0].get(); else return st; } @@ -127,21 +133,47 @@ IntType::Store BinaryParser::parse_int() FloatType::Store BinaryParser::parse_float() { - union + UInt64 encoded = 0; + for(unsigned i=0; i::is_iec559) + return bf.compose_iec559(); + else + { + /* Put the float together with arithmetic since we don't know its + internal layout */ + FloatType::Store f = 0; + if(bf.infinity) + { + if(numeric_limits::has_infinity) + f = numeric_limits::infinity(); + else + f = numeric_limits::max(); + } + else + { + for(unsigned i=0; i<64; ++i) + { + f /= 2; + if(bf.mantissa&1) + f += 1; + bf.mantissa >>= 1; + } + for(int i=0; ibf.exponent; --i) + f /= 2; + } + if(bf.sign) + f = -f; + return f; + } } BoolType::Store BinaryParser::parse_bool() diff --git a/source/binaryparser.h b/source/binaryparser.h index b10d7b8..b4fec6b 100644 --- a/source/binaryparser.h +++ b/source/binaryparser.h @@ -21,6 +21,7 @@ private: Dictionary dict; StringMap strings; bool first; + unsigned float_precision; public: BinaryParser(Input &i, const std::string &s); diff --git a/source/binarywriter.cpp b/source/binarywriter.cpp index 9c9c893..ffdc040 100644 --- a/source/binarywriter.cpp +++ b/source/binarywriter.cpp @@ -1,5 +1,7 @@ +#include #include #include "binarywriter.h" +#include "binfloat.h" #include "statement.h" using namespace std; @@ -10,10 +12,23 @@ namespace DataFile { BinaryWriter::BinaryWriter(IO::Base &o): WriterMode(o), next_kwd_id(1), - next_str_id(1) + next_str_id(1), + float_precision(32) { dict[DictEntry("__kwd", "iss")] = -1; dict[DictEntry("__str", "is")] = -2; + dict[DictEntry("__flt", "i")] = -3; +} + +void BinaryWriter::set_float_precision(unsigned fp) +{ + if(fp<16 || fp>64 || fp%8) + throw invalid_argument("BinaryWriter::set_float_precision"); + float_precision = fp; + Statement fst; + fst.keyword = "__flt"; + fst.args.push_back(float_precision); + write_(fst); } void BinaryWriter::write(const Statement &st) @@ -116,20 +131,44 @@ void BinaryWriter::write_string(const StringType::Store &s) void BinaryWriter::write_float(FloatType::Store f) { - union + BinFloat bf; + + if(numeric_limits::is_iec559) + bf = BinFloat::explode_iec559(f); + else { - float v; - char d[sizeof(float)]; - }; - - v = f; -#if BYTE_ORDER == LITTLE_ENDIAN - for(unsigned i = sizeof(float); i--;) - out.put(d[i]); -#else - for(unsigned i = 0; if)) + bf.infinity = true; + else if(f!=0) + { + for(; f<1; f*=2) + --bf.exponent; + for(; f>=2; f/=2) + ++bf.exponent; + for(unsigned i=0; i<64; ++i) + { + bf.mantissa <<= 1; + if(f>=1) + { + bf.mantissa |= 1; + f -= 1; + } + f *= 2; + } + } + } + + UInt64 encoded = bf.compose(float_precision); + for(unsigned i=float_precision/8; i--; ) + out.put((encoded>>(i*8))&0xFF); } void BinaryWriter::write_symbol(const SymbolType::Store &s) diff --git a/source/binarywriter.h b/source/binarywriter.h index 1a11ae5..2f67bbd 100644 --- a/source/binarywriter.h +++ b/source/binarywriter.h @@ -22,10 +22,12 @@ private: unsigned next_kwd_id; StringMap strings; unsigned next_str_id; + unsigned float_precision; public: BinaryWriter(IO::Base &o); + virtual void set_float_precision(unsigned); virtual void write(const Statement &st); private: void write_(const Statement &st); diff --git a/source/binfloat.cpp b/source/binfloat.cpp new file mode 100644 index 0000000..fa4be73 --- /dev/null +++ b/source/binfloat.cpp @@ -0,0 +1,72 @@ +#include +#include "binfloat.h" + +using namespace std; + + +namespace Msp { +namespace DataFile { + +BinFloat BinFloat::explode(UInt64 value, const Bits &bits) +{ + UInt64 mantissa_mask = (UInt64(1)<>bits.mantissa)&exponent_mask; + bf.sign = value>>(bits.mantissa+bits.exponent); + bf.infinity = (bf.exponent==exponent_mask); + + if(bf.exponent==0 || bf.infinity) + // Zeroes and infinities have zero mantissa + bf.mantissa = 0; + else + { + // Extract mantissa, add the implied one and align it to high bits + bf.mantissa = (value&mantissa_mask) | (UInt64(1)<>1; + + return bf; +} + +UInt64 BinFloat::compose(const Bits &bits) +{ + UInt64 mantissa_mask = (UInt64(1)<>1); + // Shift down and round the mantissa + UInt64 rounded_mantissa = ((mantissa>>(62-bits.mantissa))+1)>>1; + // If the integer part is greater than one, we need to use a higher exponent + if((rounded_mantissa>>bits.mantissa)>1) + ++biased_exponent; + + if(biased_exponent>=exponent_mask || infinity) + // Overflow, return infinity + return UInt64(sign< + union Conversion + { + T f; + typename MatchingInt::UnsignedType i; + }; + + bool sign; + bool infinity; + int exponent; + UInt64 mantissa; + + static BinFloat explode(UInt64, const Bits &); + + template + static BinFloat explode_iec559(T v) + { + Conversion c; + c.f = v; + return explode(c.i, sizeof(T)*CHAR_BIT); + } + + UInt64 compose(const Bits &); + + template + T compose_iec559() + { + Conversion c; + c.i = compose(sizeof(T)*CHAR_BIT); + return c.f; + } +}; + +} // namespace DataFile +} // namespace Msp + +#endif diff --git a/source/textwriter.cpp b/source/textwriter.cpp index 0e5b674..65f4870 100644 --- a/source/textwriter.cpp +++ b/source/textwriter.cpp @@ -9,9 +9,15 @@ namespace Msp { namespace DataFile { TextWriter::TextWriter(IO::Base &o): - WriterMode(o) + WriterMode(o), + float_format("%#.7g") { } +void TextWriter::set_float_precision(unsigned fp) +{ + float_format = format("%%#.%dg", fp/4-1); +} + void TextWriter::write(const Statement &st) { write_(st, 0); @@ -32,7 +38,7 @@ void TextWriter::write_(const Statement &st, unsigned level) else if(i->get_signature()==IntType::signature) out.write(lexical_cast(i->get())); else if(i->get_signature()==FloatType::signature) - out.write(format("%15g", (i->get()))); + out.write(format(float_format, i->get())); else if(i->get_signature()==SymbolType::signature) { string name = i->get().name; diff --git a/source/textwriter.h b/source/textwriter.h index ad127de..a0ed1b7 100644 --- a/source/textwriter.h +++ b/source/textwriter.h @@ -8,9 +8,13 @@ namespace DataFile { class TextWriter: public WriterMode { +private: + std::string float_format; + public: TextWriter(IO::Base &o); + virtual void set_float_precision(unsigned); virtual void write(const Statement &st); private: void write_(const Statement &st, unsigned); diff --git a/source/writer.cpp b/source/writer.cpp index c6a1e12..6091409 100644 --- a/source/writer.cpp +++ b/source/writer.cpp @@ -46,5 +46,10 @@ void Writer::set_binary(bool b) mode = new TextWriter(out); } +void Writer::set_float_precision(unsigned fp) +{ + mode->set_float_precision(fp); +} + } // namespace DataFile } // namespace Msp diff --git a/source/writer.h b/source/writer.h index 159bf95..c8d75e3 100644 --- a/source/writer.h +++ b/source/writer.h @@ -38,6 +38,11 @@ public: @param b true for binary mode, false for text */ void set_binary(bool b); + + /** Sets the precision of floating point numbers in bits. Depending on the + mode not all values may be valid, but any value between 16 and 64 that is + divisible by 8 is guaranteed to work. */ + void set_float_precision(unsigned); }; } // namespace DataFile diff --git a/source/writermode.h b/source/writermode.h index 3778db6..21df83f 100644 --- a/source/writermode.h +++ b/source/writermode.h @@ -17,6 +17,7 @@ protected: public: virtual ~WriterMode() { } + virtual void set_float_precision(unsigned) = 0; virtual void write(const Statement &st) = 0; }; diff --git a/tool/tool.cpp b/tool/tool.cpp index 4e66087..5f9febe 100644 --- a/tool/tool.cpp +++ b/tool/tool.cpp @@ -15,11 +15,13 @@ DataTool::DataTool(int argc, char **argv): in_fn("-"), out_fn("-"), binary(false), - compile(false) + compile(false), + float_size(0) { GetOpt getopt; getopt.add_option('b', "binary", binary, GetOpt::NO_ARG); getopt.add_option('c', "compile", compile, GetOpt::NO_ARG); + getopt.add_option('f', "float-size", float_size, GetOpt::REQUIRED_ARG); getopt.add_option('o', "output", out_fn, GetOpt::REQUIRED_ARG); getopt(argc, argv); @@ -49,6 +51,8 @@ int DataTool::main() DataFile::Writer writer(out_buf); if(binary) writer.set_binary(true); + if(float_size) + writer.set_float_precision(float_size); if(compile) { diff --git a/tool/tool.h b/tool/tool.h index a43fbfb..43c82e1 100644 --- a/tool/tool.h +++ b/tool/tool.h @@ -11,6 +11,7 @@ private: std::string out_fn; bool binary; bool compile; + unsigned float_size; public: DataTool(int argc, char **argv); -- 2.43.0