source/lib/builder.cpp

   1 #include <deque>
   2 #include <set>
   3 #include <msp/core/algorithm.h>
   4 #include <msp/core/except.h>
   5 #include <msp/core/maputils.h>
   6 #include <msp/datafile/parser.h>
   7 #include <msp/fs/dir.h>
   8 #include <msp/fs/utils.h>
   9 #include <msp/io/buffered.h>
  10 #include <msp/io/file.h>
  11 #include <msp/io/print.h>
  12 #include <msp/strings/format.h>
  13 #include <msp/time/timedelta.h>
  14 #include <msp/time/utils.h>
  15 #include "android/androidtools.h"
  16 #include "binarypackage.h"
  17 #include "builder.h"
  18 #include "builtin/builtintools.h"
  19 #include "datafile/datatool.h"
  20 #include "installedfile.h"
  21 #include "package.h"
  22 #include "plugin.h"
  23 #include "sharedlibrary.h"
  24 #include "sourcepackage.h"
  25 #include "task.h"
  26 #include "virtualtarget.h"
  27
  28 using namespace std;
  29 using namespace Msp;
  30
  31 Builder::Builder():
  32         package_manager(*this),
  33         native_arch(*this, string()),
  34         vfs(*this),
  35         build_graph(*this),
  36         logger(&default_logger)
  37 {
  38         set_architecture(string());
  39 }
  40
  41 Builder::~Builder()
  42 {
  43         if(current_arch!=&native_arch)
  44                 delete current_arch;
  45 }
  46
  47 void Builder::load_plugins()
  48 {
  49         using CreateFunc = Plugin *(Builder &);
  50
  51         FS::Path plugins_dir = FS::get_sys_lib_dir();
  52         logger->log("files", "Traversing %s", plugins_dir);
  53         for(const string &f: list_filtered(plugins_dir, "\\.dlm$"))
  54         {
  55                 LoadedPlugin plugin;
  56                 plugin.path = plugins_dir/f;
  57
  58                 try
  59                 {
  60                         plugin.module = new Module(plugin.path.str());
  61                 }
  62                 catch(const exception &exc)
  63                 {
  64                         logger->log("plugins", "Failed to load plugin %s: %s", f, exc.what());
  65                         continue;
  66                 }
  67
  68                 try
  69                 {
  70                         CreateFunc *create_func = reinterpret_cast<CreateFunc *>(plugin.module->get_symbol("create_plugin"));
  71                         plugin.plugin = create_func(*this);
  72                         if(plugin.plugin)
  73                         {
  74                                 logger->log("plugins", "Loaded plugin %s", f);
  75                                 plugins.emplace_back(move(plugin));
  76                                 continue;
  77                         }
  78                         else
  79                                 logger->log("plugins", "Plugin %s refused to initialize", f);
  80                 }
  81                 catch(const exception &exc)
  82                 {
  83                         logger->log("plugins", "Failed to initialize plugin %s: %s", f, exc.what());
  84                 }
  85
  86                 delete plugin.module;
  87         }
  88 }
  89
  90 void Builder::set_architecture(const string &name)
  91 {
  92         if(current_arch!=&native_arch)
  93                 delete current_arch;
  94
  95         if(name.empty())
  96                 current_arch = &native_arch;
  97         else
  98                 current_arch = new Architecture(*this, name);
  99
 100         if(auto_prefix)
 101                 update_auto_prefix();
 102 }
 103
 104 vector<string> Builder::get_build_types() const
 105 {
 106         vector<string> keys;
 107         keys.reserve(build_types.size());
 108         for(const auto &kvp: build_types)
 109                 keys.push_back(kvp.first);
 110         return keys;
 111 }
 112
 113 const BuildType &Builder::get_build_type() const
 114 {
 115         if(!build_type)
 116                 throw invalid_state("no build type");
 117         return *build_type;
 118 }
 119
 120 void Builder::set_build_type(const string &name)
 121 {
 122         build_type = &get_item(build_types, name);
 123 }
 124
 125 void Builder::set_prefix(const FS::Path &p)
 126 {
 127         auto_prefix = false;
 128         prefix = p;
 129 }
 130
 131 void Builder::set_temp_directory(const FS::Path &p)
 132 {
 133         tempdir = p;
 134 }
 135
 136 void Builder::update_auto_prefix()
 137 {
 138         if(current_arch->is_native())
 139                 prefix = FS::get_home_dir()/"local";
 140         else
 141                 prefix = FS::get_home_dir()/"local"/current_arch->get_name();
 142 }
 143
 144 void Builder::add_default_tools()
 145 {
 146         if(current_arch->get_system()=="android")
 147                 toolchain.add_toolchain(new AndroidTools(*this, *current_arch));
 148         toolchain.add_toolchain(new BuiltinTools(*this));
 149         toolchain.add_tool(new DataTool(*this));
 150         for(const LoadedPlugin &p: plugins)
 151                 p.plugin->add_tools(toolchain, *current_arch);
 152
 153         auto i = find_if(toolchain.get_toolchains(), [](const Toolchain *tc){ return (tc->has_tool("CC") || tc->has_tool("CXX")); });
 154         if(i!=toolchain.get_toolchains().end())
 155         {
 156                 current_arch->refine((*i)->get_name());
 157                 if(auto_prefix)
 158                         update_auto_prefix();
 159         }
 160 }
 161
 162 void Builder::set_logger(const Logger *l)
 163 {
 164         logger = (l ? l : &default_logger);
 165 }
 166
 167 vector<string> Builder::collect_problems() const
 168 {
 169         vector<string> problems;
 170         set<const Package *> broken_packages;
 171         set<const Component *> broken_components;
 172         set<const Tool *> broken_tools;
 173
 174         for(const auto &kvp: build_graph.get_targets())
 175                 if(kvp.second->is_broken())
 176                 {
 177                         for(const string &p: kvp.second->get_problems())
 178                                 problems.push_back(format("%s: %s", kvp.second->get_name(), p));
 179
 180                         const Package *package = kvp.second->get_package();
 181                         if(package && !package->get_problems().empty())
 182                                 broken_packages.insert(package);
 183
 184                         const Component *component = kvp.second->get_component();
 185                         if(component && !component->get_problems().empty())
 186                                 broken_components.insert(component);
 187
 188                         const Tool *tool = kvp.second->get_tool();
 189                         if(tool && !tool->get_problems().empty())
 190                                 broken_tools.insert(tool);
 191                 }
 192
 193         // TODO Sort components after their packages, and targets last
 194         for(const Package *p: broken_packages)
 195                 for(const string &b: p->get_problems())
 196                         problems.push_back(format("%s: %s", p->get_name(), b));
 197
 198         for(const Component *c: broken_components)
 199                 for(const string &b: c->get_problems())
 200                         problems.push_back(format("%s/%s: %s", c->get_package().get_name(), c->get_name(), b));
 201
 202         for(const Tool *t: broken_tools)
 203                 for(const string &b: t->get_problems())
 204                         problems.push_back(format("%s: %s", t->get_tag(), b));
 205
 206         return problems;
 207 }
 208
 209 void Builder::load_build_file(const FS::Path &fn, const Config::InputOptions *opts, bool all)
 210 {
 211         IO::BufferedFile in(fn.str());
 212
 213         get_logger().log("files", "Reading %s", fn);
 214
 215         DataFile::Parser parser(in, fn.str());
 216         Loader loader(*this, opts, all);
 217         loader.load(parser);
 218 }
 219
 220 void Builder::save_caches()
 221 {
 222         for(const auto &kvp: package_manager.get_packages())
 223                 kvp.second->save_caches();
 224 }
 225
 226 int Builder::build(unsigned jobs, bool dry_run, bool show_progress)
 227 {
 228         unsigned total = build_graph.count_rebuild_targets();
 229
 230         if(!total)
 231         {
 232                 get_logger().log("summary", "Already up to date");
 233                 return 0;
 234         }
 235         get_logger().log("summary", "Will build %d target%s", total, (total!=1 ? "s" : ""));
 236
 237         vector<Task *> tasks;
 238
 239         unsigned count = 0;
 240
 241         bool fail = false;
 242         bool finish = false;
 243         bool starved = false;
 244
 245         while(!finish)
 246         {
 247                 if(tasks.size()<jobs && !fail && !starved)
 248                 {
 249                         Target *tgt = build_graph.get_buildable_target();
 250                         if(tgt)
 251                         {
 252                                 if(tgt->get_tool())
 253                                 {
 254                                         if(show_progress)
 255                                                 IO::print("\033[K");
 256                                         get_logger().log("tasks", "%-4s  %s", tgt->get_tool()->get_tag(), tgt->get_name());
 257                                 }
 258                                 Task *task = tgt->build();
 259                                 if(task)
 260                                 {
 261                                         get_logger().log("commands", "%s", task->get_command());
 262                                         if(dry_run)
 263                                         {
 264                                                 task->signal_finished.emit(true);
 265                                                 delete task;
 266                                         }
 267                                         else
 268                                         {
 269                                                 task->start();
 270                                                 tasks.push_back(task);
 271                                         }
 272                                 }
 273
 274                                 if(show_progress)
 275                                         IO::print("%d of %d target%s built\033[1G", count, total, (total!=1 ? "s" : ""));
 276                         }
 277                         else if(tasks.empty())
 278                                 finish = true;
 279                         else
 280                                 starved = true;
 281                 }
 282                 else
 283                         Time::sleep(10*Time::msec);
 284
 285                 for(unsigned i=0; i<tasks.size();)
 286                 {
 287                         Task::Status status;
 288                         if(jobs==1 || (tasks.size()==1 && starved))
 289                                 status = tasks[i]->wait();
 290                         else
 291                                 status = tasks[i]->check();
 292
 293                         if(status!=Task::RUNNING)
 294                         {
 295                                 ++count;
 296
 297                                 delete tasks[i];
 298                                 tasks.erase(tasks.begin()+i);
 299                                 if(status==Task::ERROR)
 300                                         fail = true;
 301                                 if(tasks.empty() && fail)
 302                                         finish = true;
 303                                 starved = false;
 304                         }
 305                         else
 306                                 ++i;
 307                 }
 308         }
 309
 310         if(show_progress)
 311                 IO::print("\033[K");
 312         if(fail)
 313                 get_logger().log("summary", "Build failed");
 314         else if(show_progress)
 315                 get_logger().log("summary", "Build complete");
 316
 317         return fail;
 318 }
 319
 320 int Builder::clean(bool all, bool dry_run)
 321 {
 322         // Cleaning doesn't care about ordering, so a simpler method can be used
 323
 324         set<Target *> clean_tgts;
 325         deque<Target *> queue;
 326         queue.push_back(&build_graph.get_goals());
 327
 328         while(!queue.empty())
 329         {
 330                 Target *tgt = queue.front();
 331                 queue.pop_front();
 332
 333                 if(tgt->is_buildable() && (tgt->get_package()==&package_manager.get_main_package() || all))
 334                         clean_tgts.insert(tgt);
 335
 336                 for(Target *t: tgt->get_dependencies())
 337                         if(!clean_tgts.count(t))
 338                                 queue.push_back(t);
 339         }
 340
 341         for(Target *t: clean_tgts)
 342         {
 343                 get_logger().log("tasks", "RM    %s", t->get_name());
 344                 if(!dry_run)
 345                         t->clean();
 346         }
 347
 348         return 0;
 349 }
 350
 351
 352 Builder::LoadedPlugin::LoadedPlugin(LoadedPlugin &&other):
 353         path(move(other.path)),
 354         module(other.module),
 355         plugin(other.plugin)
 356 {
 357         other.module = 0;
 358         other.plugin = 0;
 359 }
 360
 361 Builder::LoadedPlugin::~LoadedPlugin()
 362 {
 363         delete plugin;
 364         delete module;
 365 }
 366
 367
 368 Builder::Loader::Loader(Builder &b, const Config::InputOptions *o, bool a):
 369         DataFile::ObjectLoader<Builder>(b),
 370         options(o),
 371         conf_all(a)
 372 {
 373         add("architecture", &Loader::architecture);
 374         add("binary_package", &Loader::binpkg);
 375         add("build_type", &Loader::build_type);
 376         add("package", &Loader::package);
 377
 378         if(!obj.top_loader)
 379                 obj.top_loader = this;
 380         else if(!options && obj.top_loader!=this && obj.top_loader->conf_all)
 381                 options = obj.top_loader->options;
 382 }
 383
 384 Builder::Loader::~Loader()
 385 {
 386         if(obj.top_loader==this)
 387                 obj.top_loader = 0;
 388 }
 389
 390 void Builder::Loader::architecture(const string &n)
 391 {
 392         if(obj.current_arch->match_name(n))
 393                 load_sub(*obj.current_arch);
 394 }
 395
 396 void Builder::Loader::binpkg(const string &n)
 397 {
 398         BinaryPackage *pkg = new BinaryPackage(obj, n);
 399         load_sub(*pkg);
 400 }
 401
 402 void Builder::Loader::build_type(const string &n)
 403 {
 404         BuildType btype(n);
 405         load_sub(btype);
 406         auto i = obj.build_types.insert({ n, btype }).first;
 407         if(!obj.build_type)
 408                 obj.build_type = &i->second;
 409 }
 410
 411 void Builder::Loader::package(const string &n)
 412 {
 413         SourcePackage *pkg = new SourcePackage(obj, n, get_source());
 414
 415         load_sub(*pkg, options);
 416
 417         if(obj.build_type)
 418                 pkg->set_build_type(*obj.build_type);
 419 }