tool/packer.cpp

   1 #include <msp/datafile/parser.h>
   2 #include <msp/datafile/rawdata.h>
   3 #include <msp/datafile/statement.h>
   4 #include <msp/datafile/writer.h>
   5 #include <msp/fs/utils.h>
   6 #include <msp/io/memory.h>
   7 #include <msp/strings/format.h>
   8 #include "packer.h"
   9 #include "tool.h"
  10
  11 using namespace std;
  12 using namespace Msp;
  13
  14 Packer::Packer(DataTool &t):
  15         tool(t),
  16         tmp_file(tempfile()),
  17         dir_alloc(0)
  18 { }
  19
  20 IO::BufferedFile *Packer::tempfile()
  21 {
  22         FS::Path tmpdir;
  23         const char *tmp_env = getenv("TMPDIR");
  24         if(tmp_env)
  25                 tmpdir = tmp_env;
  26         else
  27                 tmpdir = "/tmp";
  28
  29         for(unsigned i=0;; ++i)
  30         {
  31                 try
  32                 {
  33                         FS::Path filename = tmpdir/format("mspdatatool.%d", i);
  34                         IO::BufferedFile *file = new IO::BufferedFile(filename.str(), IO::M_RDWR, IO::File::C_NEW);
  35                         FS::unlink(filename.str());
  36                         return file;
  37                 }
  38                 catch(const IO::file_already_exists &)
  39                 {
  40                         continue;
  41                 }
  42         }
  43 }
  44
  45 Packer::~Packer()
  46 {
  47         delete tmp_file;
  48 }
  49
  50 void Packer::pack_file(const string &fn)
  51 {
  52         if(!tmp_file)
  53                 throw logic_error("Packer::pack_file");
  54
  55         IO::SeekOffset offset = tmp_file->tell();
  56
  57         IO::BufferedFile in(fn);
  58         ObjectList objects;
  59         if(detect_data(in))
  60                 transfer_datafile(in, fn, *tmp_file, objects);
  61         else if(detect_raw(in))
  62                 transfer_raw_data(in, fn, *tmp_file);
  63         else
  64                 transfer_unknown(in, *tmp_file);
  65
  66         unsigned length = tmp_file->tell()-offset;
  67
  68         DataFile::Statement st("file");
  69         st.append(FS::basename(fn));
  70         st.sub.push_back((DataFile::Statement("slice"), offset, length));
  71         for(list<Object>::const_iterator i=objects.begin(); i!=objects.end(); ++i)
  72                 st.sub.push_back((DataFile::Statement("object"), i->name, i->keyword));
  73         directory.push_back(st);
  74         dir_alloc += 100+objects.size()*100;
  75 }
  76
  77 bool Packer::detect_data(IO::Seekable &in)
  78 {
  79         IO::SeekOffset offset = in.tell();
  80         bool data = true;
  81         DataFile::Parser parser(in, "-");
  82         while(data && parser && in.tell()<10240)
  83         {
  84                 try
  85                 {
  86                         parser.parse();
  87                 }
  88                 catch(...)
  89                 {
  90                         data = false;
  91                 }
  92         }
  93         in.seek(offset, IO::S_BEG);
  94         return data;
  95 }
  96
  97 bool Packer::detect_raw(IO::Seekable &in)
  98 {
  99         char header[4] = { };
 100         IO::SeekOffset offset = in.tell();
 101         in.read(header, sizeof(header));
 102         in.seek(offset, IO::S_BEG);
 103         return DataFile::RawData::detect_signature(string(header, sizeof(header)));
 104 }
 105
 106 void Packer::transfer_datafile(IO::Base &in, const string &fn, IO::Base &out, ObjectList &objects)
 107 {
 108         DataFile::Parser parser(in, fn);
 109
 110         DataFile::Writer *writer = tool.create_writer(out);
 111
 112         bool collection = FS::extpart(fn)==".mdc";
 113         while(parser)
 114         {
 115                 DataFile::Statement st = parser.parse(true);
 116                 if(st.valid)
 117                 {
 118                         if(collection && !st.control)
 119                         {
 120                                 if(st.get_signature()=="s")
 121                                 {
 122                                         Object obj;
 123                                         obj.name = st.args[0].get<string>();
 124                                         obj.keyword = st.keyword;
 125                                         objects.push_back(obj);
 126                                 }
 127                                 else
 128                                         collection = false;
 129                         }
 130                         if(!st.control || st.keyword=="__src")
 131                                 writer->write(st);
 132                 }
 133         }
 134         writer->write(DataFile::Statement("__end"));
 135         delete writer;
 136 }
 137
 138 void Packer::transfer_raw_data(IO::Base &in, const string &fn, IO::Base &out)
 139 {
 140         DataFile::RawData raw;
 141         raw.open_io(in, fn);
 142         raw.load();
 143         raw.write_io(out, tool.is_compressed());
 144 }
 145
 146 void Packer::transfer_unknown(IO::Base &in, IO::Base &out)
 147 {
 148         while(!in.eof())
 149         {
 150                 char buf[4096];
 151                 unsigned len = in.read(buf, sizeof(buf));
 152                 if(len==0)
 153                         break;
 154                 out.write(buf, len);
 155         }
 156 }
 157
 158 void Packer::create_pack(const string &fn)
 159 {
 160         vector<char> dir_buffer(dir_alloc);
 161         IO::Memory mem(&dir_buffer[0], dir_buffer.size(), IO::M_WRITE);
 162
 163         unsigned base_offset = 0;
 164         while(1)
 165         {
 166                 mem.seek(0, IO::S_BEG);
 167
 168                 DataFile::Writer *writer = tool.create_writer(mem);
 169
 170                 for(list<DataFile::Statement>::const_iterator i=directory.begin(); i!=directory.end(); ++i)
 171                         writer->write(*i);
 172                 if(base_offset==0)
 173                         base_offset = mem.tell();
 174                 writer->write((DataFile::Statement("base_offset"), base_offset));
 175                 writer->write(DataFile::Statement("__end"));
 176                 delete writer;
 177
 178                 unsigned dir_size = mem.tell();
 179                 if(dir_size<=base_offset)
 180                         break;
 181                 base_offset = dir_size;
 182         }
 183
 184         IO::File out(fn, IO::M_WRITE);
 185         out.write(&dir_buffer[0], base_offset);
 186         tmp_file->seek(0, IO::S_BEG);
 187         unsigned bufsize = 1048576;
 188         char *buf = new char[bufsize];
 189         while(!tmp_file->eof())
 190         {
 191                 unsigned len = tmp_file->read(buf, bufsize);
 192                 if(!len)
 193                         break;
 194                 out.write(buf, len);
 195         }
 196         delete[] buf;
 197 }