Add an intelligent packed collection class

[libs/datafile.git] / tool / packer.cpp

diff --git a/tool/packer.cpp b/tool/packer.cpp
new file mode 100644 (file)
index 0000000..51b7d8f
--- /dev/null
+++ b/tool/packer.cpp
@@ -0,0 +1,140 @@
+#include <msp/datafile/parser.h>
+#include <msp/datafile/statement.h>
+#include <msp/datafile/writer.h>
+#include <msp/fs/utils.h>
+#include <msp/io/memory.h>
+#include <msp/strings/format.h>
+#include "packer.h"
+#include "tool.h"
+
+using namespace std;
+using namespace Msp;
+
+Packer::Packer(DataTool &t):
+       tool(t),
+       tmp_file(tempfile()),
+       tmp_buf(new IO::Buffered(*tmp_file)),
+       dir_alloc(0)
+{ }
+
+IO::File *Packer::tempfile()
+{
+       for(unsigned i=0;; ++i)
+       {
+               try
+               {
+                       std::string filename = format("/tmp/mspdatatool.%d", i);
+                       /*filename.reserve(25);
+                       filename.append("/tmp/mspdatatool.");
+                       for(unsigned i=0; i<8; ++i)
+                               filename.append(rand());*/
+                       IO::File *file = new IO::File(filename, IO::M_RDWR, IO::File::C_NEW);
+                       FS::unlink(filename);
+                       return file;
+               }
+               catch(const IO::file_already_exists &)
+               {
+                       continue;
+               }
+       }
+}
+
+Packer::~Packer()
+{
+       delete tmp_buf;
+       delete tmp_file;
+}
+
+void Packer::pack_file(const string &fn)
+{
+       if(!tmp_file)
+               throw logic_error("Packer::pack_file");
+
+       unsigned offset = tmp_file->tell();
+
+       IO::BufferedFile in(fn);
+       DataFile::Parser parser(in, fn);
+
+       DataFile::Writer *writer = tool.create_writer(*tmp_buf);
+
+       bool collection = FS::extpart(fn)==".mdc";
+       std::list<Object> objects;
+       while(parser)
+       {
+               DataFile::Statement st = parser.parse(true);
+               if(st.valid)
+               {
+                       bool sys = !st.keyword.compare(0, 2, "__");
+                       if(collection && !sys)
+                       {
+                               if(st.get_signature()=="s")
+                               {
+                                       Object obj;
+                                       obj.name = st.args[0].get<string>();
+                                       obj.keyword = st.keyword;
+                                       objects.push_back(obj);
+                               }
+                               else
+                                       collection = false;
+                       }
+                       if(!sys || st.keyword=="__src")
+                               writer->write(st);
+               }
+       }
+       writer->write(DataFile::Statement("__end"));
+       delete writer;
+
+       tmp_buf->flush();
+       unsigned length = tmp_file->tell()-offset;
+
+       DataFile::Statement st("file");
+       st.append(FS::basename(fn));
+       st.sub.push_back((DataFile::Statement("slice"), offset, length));
+       if(collection)
+       {
+               for(list<Object>::const_iterator i=objects.begin(); i!=objects.end(); ++i)
+                       st.sub.push_back((DataFile::Statement("object"), i->name, i->keyword));
+               dir_alloc += objects.size()*100;
+       }
+       directory.push_back(st);
+       dir_alloc += 100;
+}
+
+void Packer::create_pack(const string &fn)
+{
+       vector<char> dir_buffer(dir_alloc);
+       IO::Memory mem(&dir_buffer[0], dir_buffer.size(), IO::M_WRITE);
+
+       unsigned base_offset = 0;
+       while(1)
+       {
+               mem.seek(0, IO::S_BEG);
+
+               DataFile::Writer *writer = tool.create_writer(mem);
+
+               for(list<DataFile::Statement>::const_iterator i=directory.begin(); i!=directory.end(); ++i)
+                       writer->write(*i);
+               if(base_offset==0)
+                       base_offset = mem.tell();
+               writer->write((DataFile::Statement("base_offset"), base_offset));
+               writer->write(DataFile::Statement("__end"));
+               delete writer;
+
+               unsigned dir_size = mem.tell();
+               if(dir_size<=base_offset)
+                       break;
+               base_offset = dir_size;
+       }
+
+       IO::File out(fn, IO::M_WRITE);
+       out.write(&dir_buffer[0], base_offset);
+       tmp_file->seek(0, IO::S_BEG);
+       while(!tmp_file->eof())
+       {
+               char buf[16384];
+               unsigned len = tmp_file->read(buf, sizeof(buf));
+               if(!len)
+                       break;
+               out.write(buf, len);
+       }
+}