source/lib/builder.cpp

   1 #include <deque>
   2 #include <set>
   3 #include <msp/core/algorithm.h>
   4 #include <msp/core/except.h>
   5 #include <msp/core/maputils.h>
   6 #include <msp/datafile/parser.h>
   7 #include <msp/fs/dir.h>
   8 #include <msp/fs/utils.h>
   9 #include <msp/io/buffered.h>
  10 #include <msp/io/file.h>
  11 #include <msp/io/print.h>
  12 #include <msp/strings/format.h>
  13 #include <msp/time/timedelta.h>
  14 #include <msp/time/utils.h>
  15 #include "android/androidtools.h"
  16 #include "binarypackage.h"
  17 #include "builder.h"
  18 #include "datafile/datatool.h"
  19 #include "installedfile.h"
  20 #include "package.h"
  21 #include "plugin.h"
  22 #include "sharedlibrary.h"
  23 #include "sourcepackage.h"
  24 #include "task.h"
  25 #include "virtualtarget.h"
  26
  27 using namespace std;
  28 using namespace Msp;
  29
  30 Builder::Builder():
  31         package_manager(*this),
  32         native_arch(*this, string()),
  33         vfs(*this),
  34         build_graph(*this),
  35         logger(&default_logger)
  36 {
  37         set_architecture(string());
  38 }
  39
  40 Builder::~Builder()
  41 {
  42         if(current_arch!=&native_arch)
  43                 delete current_arch;
  44 }
  45
  46 void Builder::load_plugins()
  47 {
  48         using CreateFunc = Plugin *(Builder &);
  49
  50         FS::Path plugins_dir = FS::get_sys_lib_dir();
  51         logger->log("files", "Traversing %s", plugins_dir);
  52         for(const string &f: list_filtered(plugins_dir, "\\.dlm$"))
  53         {
  54                 LoadedPlugin plugin;
  55                 plugin.path = plugins_dir/f;
  56
  57                 try
  58                 {
  59                         plugin.module = new Module(plugin.path.str());
  60                 }
  61                 catch(const exception &exc)
  62                 {
  63                         logger->log("plugins", "Failed to load plugin %s: %s", f, exc.what());
  64                         continue;
  65                 }
  66
  67                 try
  68                 {
  69                         CreateFunc *create_func = reinterpret_cast<CreateFunc *>(plugin.module->get_symbol("create_plugin"));
  70                         plugin.plugin = create_func(*this);
  71                         if(plugin.plugin)
  72                         {
  73                                 logger->log("plugins", "Loaded plugin %s", f);
  74                                 plugins.emplace_back(move(plugin));
  75                                 continue;
  76                         }
  77                         else
  78                                 logger->log("plugins", "Plugin %s refused to initialize", f);
  79                 }
  80                 catch(const exception &exc)
  81                 {
  82                         logger->log("plugins", "Failed to initialize plugin %s: %s", f, exc.what());
  83                 }
  84
  85                 delete plugin.module;
  86         }
  87 }
  88
  89 void Builder::set_architecture(const string &name)
  90 {
  91         if(current_arch!=&native_arch)
  92                 delete current_arch;
  93
  94         if(name.empty())
  95                 current_arch = &native_arch;
  96         else
  97                 current_arch = new Architecture(*this, name);
  98
  99         if(auto_prefix)
 100                 update_auto_prefix();
 101 }
 102
 103 vector<string> Builder::get_build_types() const
 104 {
 105         vector<string> keys;
 106         keys.reserve(build_types.size());
 107         for(const auto &kvp: build_types)
 108                 keys.push_back(kvp.first);
 109         return keys;
 110 }
 111
 112 const BuildType &Builder::get_build_type() const
 113 {
 114         if(!build_type)
 115                 throw invalid_state("no build type");
 116         return *build_type;
 117 }
 118
 119 void Builder::set_build_type(const string &name)
 120 {
 121         build_type = &get_item(build_types, name);
 122 }
 123
 124 void Builder::set_prefix(const FS::Path &p)
 125 {
 126         auto_prefix = false;
 127         prefix = p;
 128 }
 129
 130 void Builder::set_temp_directory(const FS::Path &p)
 131 {
 132         tempdir = p;
 133 }
 134
 135 void Builder::update_auto_prefix()
 136 {
 137         if(current_arch->is_native())
 138                 prefix = FS::get_home_dir()/"local";
 139         else
 140                 prefix = FS::get_home_dir()/"local"/current_arch->get_name();
 141 }
 142
 143 void Builder::add_default_tools()
 144 {
 145         if(current_arch->get_system()=="android")
 146                 toolchain.add_toolchain(new AndroidTools(*this, *current_arch));
 147         toolchain.add_tool(new DataTool(*this));
 148         for(const LoadedPlugin &p: plugins)
 149                 p.plugin->add_tools(toolchain, *current_arch);
 150
 151         auto i = find_if(toolchain.get_toolchains(), [](const Toolchain *tc){ return (tc->has_tool("CC") || tc->has_tool("CXX")); });
 152         if(i!=toolchain.get_toolchains().end())
 153         {
 154                 current_arch->refine((*i)->get_name());
 155                 if(auto_prefix)
 156                         update_auto_prefix();
 157         }
 158 }
 159
 160 void Builder::set_logger(const Logger *l)
 161 {
 162         logger = (l ? l : &default_logger);
 163 }
 164
 165 vector<string> Builder::collect_problems() const
 166 {
 167         vector<string> problems;
 168         set<const Package *> broken_packages;
 169         set<const Component *> broken_components;
 170         set<const Tool *> broken_tools;
 171
 172         for(const auto &kvp: build_graph.get_targets())
 173                 if(kvp.second->is_broken())
 174                 {
 175                         for(const string &p: kvp.second->get_problems())
 176                                 problems.push_back(format("%s: %s", kvp.second->get_name(), p));
 177
 178                         const Package *package = kvp.second->get_package();
 179                         if(package && !package->get_problems().empty())
 180                                 broken_packages.insert(package);
 181
 182                         const Component *component = kvp.second->get_component();
 183                         if(component && !component->get_problems().empty())
 184                                 broken_components.insert(component);
 185
 186                         const Tool *tool = kvp.second->get_tool();
 187                         if(tool && !tool->get_problems().empty())
 188                                 broken_tools.insert(tool);
 189                 }
 190
 191         // TODO Sort components after their packages, and targets last
 192         for(const Package *p: broken_packages)
 193                 for(const string &b: p->get_problems())
 194                         problems.push_back(format("%s: %s", p->get_name(), b));
 195
 196         for(const Component *c: broken_components)
 197                 for(const string &b: c->get_problems())
 198                         problems.push_back(format("%s/%s: %s", c->get_package().get_name(), c->get_name(), b));
 199
 200         for(const Tool *t: broken_tools)
 201                 for(const string &b: t->get_problems())
 202                         problems.push_back(format("%s: %s", t->get_tag(), b));
 203
 204         return problems;
 205 }
 206
 207 void Builder::load_build_file(const FS::Path &fn, const Config::InputOptions *opts, bool all)
 208 {
 209         IO::BufferedFile in(fn.str());
 210
 211         get_logger().log("files", "Reading %s", fn);
 212
 213         DataFile::Parser parser(in, fn.str());
 214         Loader loader(*this, opts, all);
 215         loader.load(parser);
 216 }
 217
 218 void Builder::save_caches()
 219 {
 220         for(const auto &kvp: package_manager.get_packages())
 221                 kvp.second->save_caches();
 222 }
 223
 224 int Builder::build(unsigned jobs, bool dry_run, bool show_progress)
 225 {
 226         unsigned total = build_graph.count_rebuild_targets();
 227
 228         if(!total)
 229         {
 230                 get_logger().log("summary", "Already up to date");
 231                 return 0;
 232         }
 233         get_logger().log("summary", "Will build %d target%s", total, (total!=1 ? "s" : ""));
 234
 235         vector<Task *> tasks;
 236
 237         unsigned count = 0;
 238
 239         bool fail = false;
 240         bool finish = false;
 241         bool starved = false;
 242
 243         while(!finish)
 244         {
 245                 if(tasks.size()<jobs && !fail && !starved)
 246                 {
 247                         Target *tgt = build_graph.get_buildable_target();
 248                         if(tgt)
 249                         {
 250                                 if(tgt->get_tool())
 251                                 {
 252                                         if(show_progress)
 253                                                 IO::print("\033[K");
 254                                         get_logger().log("tasks", "%-4s  %s", tgt->get_tool()->get_tag(), tgt->get_name());
 255                                 }
 256                                 Task *task = tgt->build();
 257                                 if(task)
 258                                 {
 259                                         get_logger().log("commands", "%s", task->get_command());
 260                                         if(dry_run)
 261                                         {
 262                                                 task->signal_finished.emit(true);
 263                                                 delete task;
 264                                         }
 265                                         else
 266                                         {
 267                                                 task->start();
 268                                                 tasks.push_back(task);
 269                                         }
 270                                 }
 271
 272                                 if(show_progress)
 273                                         IO::print("%d of %d target%s built\033[1G", count, total, (total!=1 ? "s" : ""));
 274                         }
 275                         else if(tasks.empty())
 276                                 finish = true;
 277                         else
 278                                 starved = true;
 279                 }
 280                 else
 281                         Time::sleep(10*Time::msec);
 282
 283                 for(unsigned i=0; i<tasks.size();)
 284                 {
 285                         Task::Status status;
 286                         if(jobs==1 || (tasks.size()==1 && starved))
 287                                 status = tasks[i]->wait();
 288                         else
 289                                 status = tasks[i]->check();
 290
 291                         if(status!=Task::RUNNING)
 292                         {
 293                                 ++count;
 294
 295                                 delete tasks[i];
 296                                 tasks.erase(tasks.begin()+i);
 297                                 if(status==Task::ERROR)
 298                                         fail = true;
 299                                 if(tasks.empty() && fail)
 300                                         finish = true;
 301                                 starved = false;
 302                         }
 303                         else
 304                                 ++i;
 305                 }
 306         }
 307
 308         if(show_progress)
 309                 IO::print("\033[K");
 310         if(fail)
 311                 get_logger().log("summary", "Build failed");
 312         else if(show_progress)
 313                 get_logger().log("summary", "Build complete");
 314
 315         return fail;
 316 }
 317
 318 int Builder::clean(bool all, bool dry_run)
 319 {
 320         // Cleaning doesn't care about ordering, so a simpler method can be used
 321
 322         set<Target *> clean_tgts;
 323         deque<Target *> queue;
 324         queue.push_back(&build_graph.get_goals());
 325
 326         while(!queue.empty())
 327         {
 328                 Target *tgt = queue.front();
 329                 queue.pop_front();
 330
 331                 if(tgt->is_buildable() && (tgt->get_package()==&package_manager.get_main_package() || all))
 332                         clean_tgts.insert(tgt);
 333
 334                 for(Target *t: tgt->get_dependencies())
 335                         if(!clean_tgts.count(t))
 336                                 queue.push_back(t);
 337         }
 338
 339         for(Target *t: clean_tgts)
 340         {
 341                 get_logger().log("tasks", "RM    %s", t->get_name());
 342                 if(!dry_run)
 343                         t->clean();
 344         }
 345
 346         return 0;
 347 }
 348
 349
 350 Builder::LoadedPlugin::LoadedPlugin(LoadedPlugin &&other):
 351         path(move(other.path)),
 352         module(other.module),
 353         plugin(other.plugin)
 354 {
 355         other.module = 0;
 356         other.plugin = 0;
 357 }
 358
 359 Builder::LoadedPlugin::~LoadedPlugin()
 360 {
 361         delete plugin;
 362         delete module;
 363 }
 364
 365
 366 Builder::Loader::Loader(Builder &b, const Config::InputOptions *o, bool a):
 367         DataFile::ObjectLoader<Builder>(b),
 368         options(o),
 369         conf_all(a)
 370 {
 371         add("architecture", &Loader::architecture);
 372         add("binary_package", &Loader::binpkg);
 373         add("build_type", &Loader::build_type);
 374         add("package", &Loader::package);
 375
 376         if(!obj.top_loader)
 377                 obj.top_loader = this;
 378         else if(!options && obj.top_loader!=this && obj.top_loader->conf_all)
 379                 options = obj.top_loader->options;
 380 }
 381
 382 Builder::Loader::~Loader()
 383 {
 384         if(obj.top_loader==this)
 385                 obj.top_loader = 0;
 386 }
 387
 388 void Builder::Loader::architecture(const string &n)
 389 {
 390         if(obj.current_arch->match_name(n))
 391                 load_sub(*obj.current_arch);
 392 }
 393
 394 void Builder::Loader::binpkg(const string &n)
 395 {
 396         BinaryPackage *pkg = new BinaryPackage(obj, n);
 397         load_sub(*pkg);
 398 }
 399
 400 void Builder::Loader::build_type(const string &n)
 401 {
 402         BuildType btype(n);
 403         load_sub(btype);
 404         auto i = obj.build_types.insert({ n, btype }).first;
 405         if(!obj.build_type)
 406                 obj.build_type = &i->second;
 407 }
 408
 409 void Builder::Loader::package(const string &n)
 410 {
 411         SourcePackage *pkg = new SourcePackage(obj, n, get_source());
 412
 413         load_sub(*pkg, options);
 414
 415         if(obj.build_type)
 416                 pkg->set_build_type(*obj.build_type);
 417 }