Reimplement datafile building

diff --git a/source/builder.cpp b/source/builder.cpp
index c599e269408adb38d4b38c75808d19ef71dc83a6..07007f61bd75c82ae9f80d5e35315e29b7374b23 100644 (file)
--- a/source/builder.cpp
+++ b/source/builder.cpp
@@ -16,6 +16,7 @@
 #include "binarypackage.h"
 #include "builder.h"
 #include "copy.h"
+#include "datatool.h"
 #include "gnuarchiver.h"
 #include "gnuccompiler.h"
 #include "gnucxxcompiler.h"
@@ -203,6 +204,7 @@ Builder::Builder(int argc, char **argv):
        toolchain.add_tool(new PkgConfigGenerator(*this));
        if(current_arch->get_system()=="windows")
                toolchain.add_tool(new MingwDllTool(*this, *current_arch));
+       toolchain.add_tool(new DataTool(*this));
 }
 
 Builder::~Builder()

diff --git a/source/component.cpp b/source/component.cpp
index 48675f7e5d7c0148cc2752f39f6946ff163a9f32..3b1f872ae92292529b845ab09e9b7b112763fa4f 100644 (file)
--- a/source/component.cpp
+++ b/source/component.cpp
@@ -7,7 +7,7 @@
 #include "builder.h"
 #include "component.h"
 #include "csourcefile.h"
-#include "datafile.h"
+#include "datapack.h"
 #include "executable.h"
 #include "file.h"
 #include "objectfile.h"
@@ -156,16 +156,26 @@ void Component::create_targets() const
                        inst->add_dependency(*copy.create_target(*ft, name));
                }
        }
-       else if(type==DATAFILE)
+       else if(type==DATAPACK)
        {
                const Tool &dcomp = toolchain.get_tool("DATA");
 
-               File *source;
-               if(Target *tgt = builder.get_vfs().get_target(source_filenames.front()))
-                       source = dynamic_cast<File *>(tgt);
-               else
-                       source = new File(builder, package, source_filenames.front());
-               Target *result = dcomp.create_target(*source);
+               list<Target *> files;
+               for(SourceList::const_iterator i=source_filenames.begin(); i!=source_filenames.end(); ++i)
+               {
+                       string ext = FS::extpart(FS::basename(*i));
+                       if(ext==".mdt")
+                       {
+                               Target *src = dcomp.create_source(*this, *i);
+                               files.push_back(dcomp.create_target(*src, "collection"));
+                       }
+                       else if(Target *tgt = builder.get_vfs().get_target(*i))
+                               files.push_back(tgt);
+                       else
+                               files.push_back(new File(builder, package, *i));
+               }
+
+               Target *result = dcomp.create_target(files, "pack");
 
                build_graph.add_primary_target(*result);
                if(install)

diff --git a/source/component.h b/source/component.h
index 044c5ba0a1343fb843e9d3e9ec54c98981ac8bb5..09767d37437fa8aa4f38a241e2c37a54d13307b8 100644 (file)
--- a/source/component.h
+++ b/source/component.h
@@ -38,7 +38,7 @@ public:
                LIBRARY,
                PROGRAM,
                MODULE,
-               DATAFILE,
+               DATAPACK,
                INSTALL,
                TARBALL
        };

diff --git a/source/datacollection.cpp b/source/datacollection.cpp
new file mode 100644 (file)
index 0000000..28f90c8
--- /dev/null
+++ b/source/datacollection.cpp
@@ -0,0 +1,20 @@
+#include <msp/fs/utils.h>
+#include "component.h"
+#include "datacollection.h"
+#include "datatransform.h"
+#include "sourcepackage.h"
+
+using namespace Msp;
+
+DataCollection::DataCollection(Builder &b, const Component &c, DataTransform &s):
+       FileTarget(b, c.get_package(), generate_target_path(c, s.get_path())),
+       source(s)
+{
+       component = &c;
+       add_dependency(source);
+}
+
+Msp::FS::Path DataCollection::generate_target_path(const Component &comp, const Msp::FS::Path &src)
+{
+       return comp.get_package().get_temp_directory()/comp.get_name()/(FS::basepart(FS::basename(src))+".mdc");
+}

diff --git a/source/datacollection.h b/source/datacollection.h
new file mode 100644 (file)
index 0000000..2f61b5f
--- /dev/null
+++ b/source/datacollection.h
@@ -0,0 +1,23 @@
+#ifndef DATACOLLECTION_H_
+#define DATACOLLECTION_H_
+
+#include "filetarget.h"
+
+class DataTransform;
+
+class DataCollection: public FileTarget
+{
+private:
+       DataTransform &source;
+
+public:
+       DataCollection(Builder &, const Component &, DataTransform &);
+private:
+       static Msp::FS::Path generate_target_path(const Component &, const Msp::FS::Path &);
+
+public:
+       virtual const char *get_type() const { return "DataCollection"; }
+       DataTransform &get_source() const { return source; }
+};
+
+#endif

diff --git a/source/datacompiler.cpp b/source/datacompiler.cpp
deleted file mode 100644 (file)
index 2f806bf..0000000
--- a/source/datacompiler.cpp
+++ /dev/null
@@ -1,23 +0,0 @@
-#include <stdexcept>
-#include "datacompiler.h"
-
-using namespace std;
-
-DataCompiler::DataCompiler(Builder &b):
-       Tool(b, "DATA")
-{
-}
-
-Target *DataCompiler::create_target(const list<Target *> &sources, const string &) const
-{
-       if(sources.size()!=1)
-               throw invalid_argument("DataCompiler::create_target");
-       throw runtime_error("Not implemented");
-       //File &source = dynamic_cast<File &>(sources.front());
-       //return new ::DataFile(
-}
-
-Task *DataCompiler::run(const Target &) const
-{
-       throw runtime_error("Not implemented");
-}

diff --git a/source/datacompiler.h b/source/datacompiler.h
deleted file mode 100644 (file)
index e4c8ea2..0000000
--- a/source/datacompiler.h
+++ /dev/null
@@ -1,15 +0,0 @@
-#ifndef DATACOMPILER_H_
-#define DATACOMPILER_H_
-
-#include "tool.h"
-
-class DataCompiler: public Tool
-{
-public:
-       DataCompiler(Builder &);
-
-       virtual Target *create_target(const std::list<Target *> &, const std::string &) const;
-       virtual Task *run(const Target &) const;
-};
-
-#endif

diff --git a/source/datafile.cpp b/source/datafile.cpp
deleted file mode 100644 (file)
index 17a47b6..0000000
--- a/source/datafile.cpp
+++ /dev/null
@@ -1,19 +0,0 @@
-#include "component.h"
-#include "datafile.h"
-#include "file.h"
-#include "sourcepackage.h"
-
-DataFile::DataFile(Builder &b, const Component &c, File &s):
-       FileTarget(b, c.get_package(), generate_target_path(c)),
-       component(c),
-       source(s)
-{
-       add_dependency(source);
-
-       install_location = Msp::FS::Path("share")/package->get_name();
-}
-
-Msp::FS::Path DataFile::generate_target_path(const Component &comp)
-{
-       return comp.get_package().get_output_directory()/(comp.get_name()+".dat");
-}

diff --git a/source/datafile.h b/source/datafile.h
deleted file mode 100644 (file)
index 0ba501c..0000000
--- a/source/datafile.h
+++ /dev/null
@@ -1,26 +0,0 @@
-#ifndef DATAFILE_H_
-#define DATAFILE_H_
-
-#include "filetarget.h"
-
-class Component;
-class File;
-
-class DataFile: public FileTarget
-{
-private:
-       const Component &component;
-       File &source;
-
-public:
-       DataFile(Builder &, const Component &, File &);
-private:
-       static Msp::FS::Path generate_target_path(const Component &);
-
-public:
-       virtual const char *get_type() const { return "DataFile"; }
-       const Component &get_component() const { return component; }
-       File &get_source() const { return source; }
-};
-
-#endif

diff --git a/source/datapack.cpp b/source/datapack.cpp
new file mode 100644 (file)
index 0000000..2b3d9f2
--- /dev/null
+++ b/source/datapack.cpp
@@ -0,0 +1,21 @@
+#include "component.h"
+#include "datapack.h"
+#include "sourcepackage.h"
+
+using namespace std;
+
+DataPack::DataPack(Builder &b, const Component &c, const list<FileTarget *> &f):
+       FileTarget(b, c.get_package(), generate_target_path(c)),
+       files(f)
+{
+       component = &c;
+       for(list<FileTarget *>::const_iterator i=files.begin(); i!=files.end(); ++i)
+               add_dependency(**i);
+
+       install_location = Msp::FS::Path("share")/package->get_name();
+}
+
+Msp::FS::Path DataPack::generate_target_path(const Component &comp)
+{
+       return comp.get_package().get_output_directory()/(comp.get_name()+".mdp");
+}

diff --git a/source/datapack.h b/source/datapack.h
new file mode 100644 (file)
index 0000000..d2f667a
--- /dev/null
+++ b/source/datapack.h
@@ -0,0 +1,24 @@
+#ifndef DATAPACK_H_
+#define DATAPACK_H_
+
+#include "filetarget.h"
+
+class DataPack: public FileTarget
+{
+public:
+       typedef std::list<FileTarget *> FileList;
+private:
+       FileList files;
+
+public:
+       DataPack(Builder &, const Component &, const std::list<FileTarget *> &);
+private:
+       static Msp::FS::Path generate_target_path(const Component &);
+
+public:
+       virtual const char *get_type() const { return "DataPack"; }
+
+       const FileList &get_files() const { return files; }
+};
+
+#endif

diff --git a/source/datatool.cpp b/source/datatool.cpp
new file mode 100644 (file)
index 0000000..a956984
--- /dev/null
+++ b/source/datatool.cpp
@@ -0,0 +1,90 @@
+#include <stdexcept>
+#include <msp/fs/utils.h>
+#include <msp/strings/format.h>
+#include "builder.h"
+#include "component.h"
+#include "datacollection.h"
+#include "datapack.h"
+#include "datatool.h"
+#include "datatransform.h"
+#include "externaltask.h"
+#include "sourcepackage.h"
+
+using namespace std;
+using namespace Msp;
+
+DataTool::DataTool(Builder &b):
+       Tool(b, "DATA")
+{
+       executable = builder.get_vfs().find_binary("mspdatatool");
+       if(!executable)
+               builder.problem(string(), format("Can't find executable mspdatatool for tool %s", tag));
+
+       input_suffixes.push_back(".mdt");
+}
+
+Target *DataTool::create_source(const Component &comp, const FS::Path &path) const
+{
+       return new DataTransform(builder, comp, path);
+}
+
+Target *DataTool::create_target(const list<Target *> &sources, const string &arg) const
+{
+       if(arg=="collection")
+       {
+               if(sources.size()!=1)
+                       throw invalid_argument("DataTool::create_target");
+               DataTransform &source = dynamic_cast<DataTransform &>(*sources.front());
+               DataCollection *coll = new DataCollection(builder, *source.get_component(), source);
+               coll->set_tool(*this);
+               return coll;
+       }
+       else if(arg=="pack")
+       {
+               list<FileTarget *> files;
+               for(list<Target *>::const_iterator i=sources.begin(); i!=sources.end(); ++i)
+                       files.push_back(&dynamic_cast<FileTarget &>(**i));
+               DataPack *pack = new DataPack(builder, *files.front()->get_component(), files);
+               pack->set_tool(*this);
+               return pack;
+       }
+       else
+               throw invalid_argument("DataTool::create_target");
+}
+
+Task *DataTool::run(const Target &tgt) const
+{
+       const Component &comp = *tgt.get_component();
+       FS::Path work_dir = comp.get_package().get_source_directory();
+
+       vector<string> argv;
+       argv.push_back(executable->get_path().str());
+
+       argv.push_back("-o");
+       argv.push_back(FS::relative(dynamic_cast<const FileTarget &>(tgt).get_path(), work_dir).str());
+
+       const BuildInfo &binfo = comp.get_build_info();
+       if(binfo.debug)
+               argv.push_back("-g");
+       if(binfo.optimize>0)
+       {
+               argv.push_back("-b");
+               if(binfo.optimize>1)
+                       argv.push_back("-z");
+       }
+
+       if(const DataCollection *coll = dynamic_cast<const DataCollection *>(&tgt))
+       {
+               argv.push_back("-c");
+               argv.push_back(FS::relative(coll->get_source().get_path(), work_dir).str());
+       }
+       else if(const DataPack *pack = dynamic_cast<const DataPack *>(&tgt))
+       {
+               argv.push_back("-p");
+               const DataPack::FileList &files = pack->get_files();
+               for(DataPack::FileList::const_iterator i=files.begin(); i!=files.end(); ++i)
+                       argv.push_back(FS::relative((*i)->get_path(), work_dir).str());
+       }
+
+       return new ExternalTask(argv, work_dir);
+}

diff --git a/source/datatool.h b/source/datatool.h
new file mode 100644 (file)
index 0000000..b6c4f5b
--- /dev/null
+++ b/source/datatool.h
@@ -0,0 +1,16 @@
+#ifndef DATACOMPILER_H_
+#define DATACOMPILER_H_
+
+#include "tool.h"
+
+class DataTool: public Tool
+{
+public:
+       DataTool(Builder &);
+
+       virtual Target *create_source(const Component &, const Msp::FS::Path &) const;
+       virtual Target *create_target(const std::list<Target *> &, const std::string &) const;
+       virtual Task *run(const Target &) const;
+};
+
+#endif

diff --git a/source/datatransform.cpp b/source/datatransform.cpp
new file mode 100644 (file)
index 0000000..67b892e
--- /dev/null
+++ b/source/datatransform.cpp
@@ -0,0 +1,63 @@
+#include <msp/fs/dir.h>
+#include <msp/fs/utils.h>
+#include <msp/strings/format.h>
+#include <msp/strings/regex.h>
+#include "builder.h"
+#include "cache.h"
+#include "component.h"
+#include "datatransform.h"
+#include "file.h"
+#include "sourcepackage.h"
+
+using namespace std;
+using namespace Msp;
+
+DataTransform::DataTransform(Builder &b, const Component &c, const FS::Path &p):
+       FileTarget(b, c.get_package(), p)
+{
+       component = &c;
+}
+
+void DataTransform::find_dependencies()
+{
+       list<string> files;
+       Cache &cache = component->get_package().get_cache();
+       // XXX Should check directory mtime as well
+       if(mtime<cache.get_mtime() && cache.has_key(this, "files"))
+               files = cache.get_values(this, "files");
+       else
+       {
+               builder.get_logger().log("files", format("Reading imports from %s", path.str()));
+
+               list<string> dir_files = list_files(FS::dirname(path));
+               IO::File in(path.str());
+               DataFile::Parser parser(in, path.str());
+               while(!in.eof())
+               {
+                       DataFile::Statement st = parser.parse();
+                       if(st.keyword=="for_each")
+                       {
+                               for(DataFile::Statement::Arguments::const_iterator i=st.args.begin(); i!=st.args.end(); ++i)
+                               {
+                                       Regex re(i->get<string>());
+                                       for(list<string>::const_iterator j=dir_files.begin(); j!=dir_files.end(); ++j)
+                                               if(re.match(*j))
+                                                       files.push_back(*j);
+                               }
+                       }
+                       else if(st.keyword=="file" && st.args.size()==1)
+                               files.push_back(st.args.front().get<string>());
+               }
+
+               cache.set_values(this, "files", files);
+       }
+
+       for(list<string>::iterator i=files.begin(); i!=files.end(); ++i)
+       {
+               FS::Path file_path = FS::dirname(path)/ *i;
+               if(Target *tgt = builder.get_vfs().get_target(file_path))
+                       add_dependency(*tgt);
+               else
+                       add_dependency(*new File(builder, *package, file_path));
+       }
+}

diff --git a/source/datatransform.h b/source/datatransform.h
new file mode 100644 (file)
index 0000000..38e559b
--- /dev/null
+++ b/source/datatransform.h
@@ -0,0 +1,17 @@
+#ifndef DATATRANSFORM_H_
+#define DATATRANSFORM_H_
+
+#include "filetarget.h"
+
+class DataTransform: public FileTarget
+{
+public:
+       DataTransform(Builder &, const Component &, const Msp::FS::Path &);
+
+       virtual const char *get_type() const { return "DataTransform"; }
+
+private:
+       virtual void find_dependencies();
+};
+
+#endif

diff --git a/source/file.h b/source/file.h
index a36259d29cf52ea9617b7337509cb0aaf8523e66..aa0035f47599cca906e4a6f4fedba142b14cb1fb 100644 (file)
--- a/source/file.h
+++ b/source/file.h
@@ -10,7 +10,7 @@ class File: public FileTarget
 {
 public:
        File(Builder &b, const Msp::FS::Path &t): FileTarget(b, t) { }
-       File(Builder &b, SourcePackage &p, const Msp::FS::Path &t): FileTarget(b, p, t) { }
+       File(Builder &b, const SourcePackage &p, const Msp::FS::Path &t): FileTarget(b, p, t) { }
 
        virtual const char *get_type() const { return "File"; }
 };

diff --git a/source/sourcepackage.cpp b/source/sourcepackage.cpp
index 2cefeb871065e76434d10642fa57dd7928f61b9f..faf3bfe3c062d1948af5dd57c7086f3697ae8465 100644 (file)
--- a/source/sourcepackage.cpp
+++ b/source/sourcepackage.cpp
@@ -174,7 +174,7 @@ void SourcePackage::Loader::init(const Config::InputOptions *o)
        add("headers",     &Loader::headers);
        add("install",     &Loader::component<Component::INSTALL>);
        add("interface_version", &Loader::interface_version);
-       add("datafile",    &Loader::component<Component::DATAFILE>);
+       add("datapack",    &Loader::component<Component::DATAPACK>);
        add("source_tarball", &Loader::source_tarball);
        add("tarball",     &Loader::tarball);
        add("tar_file",    &Loader::tar_file);