tool/packer.cpp

   1 #include <msp/datafile/parser.h>
   2 #include <msp/datafile/statement.h>
   3 #include <msp/datafile/writer.h>
   4 #include <msp/fs/utils.h>
   5 #include <msp/io/memory.h>
   6 #include <msp/strings/format.h>
   7 #include "packer.h"
   8 #include "tool.h"
   9
  10 using namespace std;
  11 using namespace Msp;
  12
  13 Packer::Packer(DataTool &t):
  14         tool(t),
  15         tmp_file(tempfile()),
  16         tmp_buf(new IO::Buffered(*tmp_file)),
  17         dir_alloc(0)
  18 { }
  19
  20 IO::File *Packer::tempfile()
  21 {
  22         for(unsigned i=0;; ++i)
  23         {
  24                 try
  25                 {
  26                         std::string filename = format("/tmp/mspdatatool.%d", i);
  27                         /*filename.reserve(25);
  28                         filename.append("/tmp/mspdatatool.");
  29                         for(unsigned i=0; i<8; ++i)
  30                                 filename.append(rand());*/
  31                         IO::File *file = new IO::File(filename, IO::M_RDWR, IO::File::C_NEW);
  32                         FS::unlink(filename);
  33                         return file;
  34                 }
  35                 catch(const IO::file_already_exists &)
  36                 {
  37                         continue;
  38                 }
  39         }
  40 }
  41
  42 Packer::~Packer()
  43 {
  44         delete tmp_buf;
  45         delete tmp_file;
  46 }
  47
  48 void Packer::pack_file(const string &fn)
  49 {
  50         if(!tmp_file)
  51                 throw logic_error("Packer::pack_file");
  52
  53         IO::SeekOffset offset = tmp_file->tell();
  54
  55         IO::BufferedFile in(fn);
  56         ObjectList objects;
  57         bool raw = detect_raw(in);
  58         if(raw)
  59                 transfer_raw(in, *tmp_file);
  60         else
  61         {
  62                 transfer_datafile(in, fn, *tmp_buf, objects);
  63                 tmp_buf->flush();
  64         }
  65
  66         unsigned length = tmp_file->tell()-offset;
  67
  68         DataFile::Statement st("file");
  69         st.append(FS::basename(fn));
  70         st.sub.push_back((DataFile::Statement("slice"), offset, length));
  71         for(list<Object>::const_iterator i=objects.begin(); i!=objects.end(); ++i)
  72                 st.sub.push_back((DataFile::Statement("object"), i->name, i->keyword));
  73         directory.push_back(st);
  74         dir_alloc += 100+objects.size()*100;
  75 }
  76
  77 bool Packer::detect_raw(IO::Seekable &in)
  78 {
  79         IO::SeekOffset offset = in.tell();
  80         bool raw = false;
  81         DataFile::Parser parser(in, "-");
  82         while(!raw && !in.eof() && in.tell()<10240)
  83         {
  84                 try
  85                 {
  86                         parser.parse();
  87                 }
  88                 catch(...)
  89                 {
  90                         raw = true;
  91                 }
  92         }
  93         in.seek(offset, IO::S_BEG);
  94         return raw;
  95 }
  96
  97 void Packer::transfer_datafile(IO::Base &in, const string &fn, IO::Base &out, ObjectList &objects)
  98 {
  99         DataFile::Parser parser(in, fn);
 100
 101         DataFile::Writer *writer = tool.create_writer(out);
 102
 103         bool collection = FS::extpart(fn)==".mdc";
 104         while(parser)
 105         {
 106                 DataFile::Statement st = parser.parse(true);
 107                 if(st.valid)
 108                 {
 109                         bool sys = !st.keyword.compare(0, 2, "__");
 110                         if(collection && !sys)
 111                         {
 112                                 if(st.get_signature()=="s")
 113                                 {
 114                                         Object obj;
 115                                         obj.name = st.args[0].get<string>();
 116                                         obj.keyword = st.keyword;
 117                                         objects.push_back(obj);
 118                                 }
 119                                 else
 120                                         collection = false;
 121                         }
 122                         if(!sys || st.keyword=="__src")
 123                                 writer->write(st);
 124                 }
 125         }
 126         writer->write(DataFile::Statement("__end"));
 127         delete writer;
 128 }
 129
 130 void Packer::transfer_raw(IO::Base &in, IO::Base &out)
 131 {
 132         while(!in.eof())
 133         {
 134                 char buf[4096];
 135                 unsigned len = in.read(buf, sizeof(buf));
 136                 if(len==0)
 137                         break;
 138                 out.write(buf, len);
 139         }
 140 }
 141
 142 void Packer::create_pack(const string &fn)
 143 {
 144         vector<char> dir_buffer(dir_alloc);
 145         IO::Memory mem(&dir_buffer[0], dir_buffer.size(), IO::M_WRITE);
 146
 147         unsigned base_offset = 0;
 148         while(1)
 149         {
 150                 mem.seek(0, IO::S_BEG);
 151
 152                 DataFile::Writer *writer = tool.create_writer(mem);
 153
 154                 for(list<DataFile::Statement>::const_iterator i=directory.begin(); i!=directory.end(); ++i)
 155                         writer->write(*i);
 156                 if(base_offset==0)
 157                         base_offset = mem.tell();
 158                 writer->write((DataFile::Statement("base_offset"), base_offset));
 159                 writer->write(DataFile::Statement("__end"));
 160                 delete writer;
 161
 162                 unsigned dir_size = mem.tell();
 163                 if(dir_size<=base_offset)
 164                         break;
 165                 base_offset = dir_size;
 166         }
 167
 168         IO::File out(fn, IO::M_WRITE);
 169         out.write(&dir_buffer[0], base_offset);
 170         tmp_file->seek(0, IO::S_BEG);
 171         while(!tmp_file->eof())
 172         {
 173                 char buf[16384];
 174                 unsigned len = tmp_file->read(buf, sizeof(buf));
 175                 if(!len)
 176                         break;
 177                 out.write(buf, len);
 178         }
 179 }