source/lib/builder.cpp

   1 #include <deque>
   2 #include <set>
   3 #include <msp/core/algorithm.h>
   4 #include <msp/core/except.h>
   5 #include <msp/core/maputils.h>
   6 #include <msp/datafile/parser.h>
   7 #include <msp/fs/dir.h>
   8 #include <msp/fs/utils.h>
   9 #include <msp/io/buffered.h>
  10 #include <msp/io/file.h>
  11 #include <msp/io/print.h>
  12 #include <msp/strings/format.h>
  13 #include <msp/strings/utils.h>
  14 #include <msp/time/timedelta.h>
  15 #include <msp/time/utils.h>
  16 #include "binarypackage.h"
  17 #include "builder.h"
  18 #include "installedfile.h"
  19 #include "package.h"
  20 #include "plugin.h"
  21 #include "sharedlibrary.h"
  22 #include "task.h"
  23 #include "tool.h"
  24 #include "virtualtarget.h"
  25
  26 using namespace std;
  27 using namespace Msp;
  28
  29 Builder::Builder():
  30         package_manager(*this),
  31         native_arch(*this, string()),
  32         vfs(*this),
  33         build_graph(*this),
  34         logger(&default_logger)
  35 {
  36         set_architecture(string());
  37 }
  38
  39 Builder::~Builder()
  40 {
  41         if(current_arch!=&native_arch)
  42                 delete current_arch;
  43 }
  44
  45 void Builder::load_plugins()
  46 {
  47         using CreateFunc = Plugin *(Builder &);
  48
  49         FS::Path plugins_dir = FS::get_sys_lib_dir();
  50         logger->log("files", "Traversing %s", plugins_dir);
  51         vector<LoadedPlugin> unordered_plugins;
  52         for(const string &f: list_filtered(plugins_dir, "\\.dlm$"))
  53         {
  54                 LoadedPlugin plugin;
  55                 plugin.path = plugins_dir/f;
  56
  57                 try
  58                 {
  59                         plugin.module = new Module(plugin.path.str());
  60                 }
  61                 catch(const exception &exc)
  62                 {
  63                         logger->log("plugins", "Failed to load plugin %s: %s", f, exc.what());
  64                         continue;
  65                 }
  66
  67                 try
  68                 {
  69                         CreateFunc *create_func = reinterpret_cast<CreateFunc *>(plugin.module->get_symbol("create_plugin"));
  70                         plugin.plugin = create_func(*this);
  71                         if(plugin.plugin)
  72                         {
  73                                 logger->log("plugins", "Loaded plugin %s", f);
  74                                 unordered_plugins.emplace_back(move(plugin));
  75                                 continue;
  76                         }
  77                         else
  78                                 logger->log("plugins", "Plugin %s refused to initialize", f);
  79                 }
  80                 catch(const exception &exc)
  81                 {
  82                         logger->log("plugins", "Failed to initialize plugin %s: %s", f, exc.what());
  83                 }
  84         }
  85
  86         auto have_plugin = [this](const string &r){
  87                 return any_of(plugins.begin(), plugins.end(), [&r](const LoadedPlugin &p){ return FS::basepart(FS::basename(p.path))==r; });
  88         };
  89
  90         while(!unordered_plugins.empty())
  91         {
  92                 bool any_added = false;
  93                 for(auto i=unordered_plugins.begin(); i!=unordered_plugins.end(); )
  94                 {
  95                         const vector<string> &required = i->plugin->get_required_plugins();
  96                         if(all_of(required.begin(), required.end(), have_plugin))
  97                         {
  98                                 plugins.push_back(move(*i));
  99                                 i = unordered_plugins.erase(i);
 100                                 any_added = true;
 101                         }
 102                         else
 103                                 ++i;
 104                 }
 105
 106                 if(!any_added)
 107                         break;
 108         }
 109
 110         for(const LoadedPlugin &p: unordered_plugins)
 111         {
 112                 vector<string> missing;
 113                 for(const string &r: p.plugin->get_required_plugins())
 114                         if(!have_plugin(r))
 115                                 missing.push_back(r);
 116                 logger->log("plugins", "Missing required plugins for plugin %s: %s",
 117                         FS::basename(p.path), join(missing.begin(), missing.end()));
 118         }
 119 }
 120
 121 void Builder::set_architecture(const string &name)
 122 {
 123         if(current_arch!=&native_arch)
 124                 delete current_arch;
 125
 126         if(name.empty())
 127                 current_arch = &native_arch;
 128         else
 129                 current_arch = new Architecture(*this, name);
 130
 131         if(auto_prefix)
 132                 update_auto_prefix();
 133 }
 134
 135 vector<string> Builder::get_build_types() const
 136 {
 137         vector<string> keys;
 138         keys.reserve(build_types.size());
 139         for(const auto &kvp: build_types)
 140                 keys.push_back(kvp.first);
 141         return keys;
 142 }
 143
 144 const BuildType &Builder::get_build_type() const
 145 {
 146         if(!build_type)
 147                 throw invalid_state("no build type");
 148         return *build_type;
 149 }
 150
 151 void Builder::set_build_type(const string &name)
 152 {
 153         build_type = &get_item(build_types, name);
 154 }
 155
 156 void Builder::set_prefix(const FS::Path &p)
 157 {
 158         auto_prefix = false;
 159         prefix = p;
 160 }
 161
 162 void Builder::set_temp_directory(const FS::Path &p)
 163 {
 164         tempdir = p;
 165 }
 166
 167 void Builder::update_auto_prefix()
 168 {
 169         if(current_arch->is_native())
 170                 prefix = FS::get_home_dir()/"local";
 171         else
 172                 prefix = FS::get_home_dir()/"local"/current_arch->get_name();
 173 }
 174
 175 void Builder::add_default_tools()
 176 {
 177         for(const LoadedPlugin &p: plugins)
 178                 p.plugin->add_tools(toolchain, *current_arch);
 179
 180         auto i = find_if(toolchain.get_toolchains(), [](const Toolchain *tc){ return (tc->has_tool("CC") || tc->has_tool("CXX")); });
 181         if(i!=toolchain.get_toolchains().end())
 182         {
 183                 current_arch->refine((*i)->get_name());
 184                 if(auto_prefix)
 185                         update_auto_prefix();
 186         }
 187 }
 188
 189 void Builder::set_logger(const Logger *l)
 190 {
 191         logger = (l ? l : &default_logger);
 192 }
 193
 194 vector<string> Builder::collect_problems() const
 195 {
 196         vector<string> target_problems;
 197         vector<string> problems;
 198         vector<const Package *> broken_packages;
 199         vector<const Component *> broken_components;
 200         vector<const Tool *> broken_tools;
 201
 202         for(const auto &kvp: build_graph.get_targets())
 203                 if(kvp.second->is_broken())
 204                 {
 205                         const Package *package = kvp.second->get_package();
 206                         if(package && package->is_broken())
 207                                 collect_broken_packages(*package, broken_packages);
 208
 209                         const Component *component = kvp.second->get_component();
 210                         if(component && component->is_broken() && !any_equals(broken_components, component))
 211                         {
 212                                 broken_components.push_back(component);
 213                                 collect_broken_packages(component->get_package(), broken_packages);
 214                                 for(const Package *r: component->get_required_packages())
 215                                         if(r->is_broken())
 216                                                 collect_broken_packages(*r, broken_packages);
 217                         }
 218
 219                         const Tool *tool = kvp.second->get_tool();
 220                         if(tool && tool->is_broken() && !any_equals(broken_tools, tool))
 221                         {
 222                                 broken_tools.push_back(tool);
 223                                 for(const string &p: tool->get_problems())
 224                                         problems.push_back(format("%s: %s", tool->get_tag(), p));
 225                         }
 226
 227                         for(const string &p: kvp.second->get_problems())
 228                                 target_problems.push_back(format("%s: %s", kvp.second->get_name(), p));
 229                 }
 230
 231         for(const Package *p: broken_packages)
 232         {
 233                 for(const string &b: p->get_problems())
 234                         problems.push_back(format("%s: %s", p->get_name(), b));
 235
 236                 for(const Component *c: broken_components)
 237                         if(&c->get_package()==p)
 238                         {
 239                                 for(const string &b: c->get_problems())
 240                                         problems.push_back(format("%s/%s: %s", p->get_name(), c->get_name(), b));
 241                         }
 242         }
 243
 244         problems.insert(problems.end(), make_move_iterator(target_problems.begin()), make_move_iterator(target_problems.end()));
 245
 246         return problems;
 247 }
 248
 249 void Builder::collect_broken_packages(const Package &pkg, vector<const Package *> &broken) const
 250 {
 251         if(any_equals(broken, &pkg))
 252                 return;
 253
 254         broken.push_back(&pkg);
 255
 256         for(const Package *r: pkg.get_required_packages())
 257                 if(r->is_broken())
 258                         collect_broken_packages(*r, broken);
 259 }
 260
 261 void Builder::load_build_file(const FS::Path &fn, const Config::InputOptions *opts, bool all)
 262 {
 263         IO::BufferedFile in(fn.str());
 264
 265         get_logger().log("files", "Reading %s", fn);
 266
 267         DataFile::Parser parser(in, fn.str());
 268         Loader loader(*this, opts, all);
 269         loader.load(parser);
 270 }
 271
 272 void Builder::save_caches()
 273 {
 274         for(const auto &kvp: package_manager.get_packages())
 275                 kvp.second->save_caches();
 276 }
 277
 278 int Builder::build(unsigned jobs, bool dry_run, bool show_progress)
 279 {
 280         unsigned total = build_graph.count_rebuild_targets();
 281
 282         if(!total)
 283         {
 284                 get_logger().log("summary", "Already up to date");
 285                 return 0;
 286         }
 287         get_logger().log("summary", "Will build %d target%s", total, (total!=1 ? "s" : ""));
 288
 289         vector<Task *> tasks;
 290
 291         unsigned count = 0;
 292
 293         bool fail = false;
 294         bool finish = false;
 295         bool starved = false;
 296
 297         while(!finish)
 298         {
 299                 if(tasks.size()<jobs && !fail && !starved)
 300                 {
 301                         Target *tgt = build_graph.get_buildable_target();
 302                         if(tgt)
 303                         {
 304                                 if(tgt->get_tool())
 305                                 {
 306                                         if(show_progress)
 307                                                 IO::print("\033[K");
 308                                         get_logger().log("tasks", "%-4s  %s", tgt->get_tool()->get_tag(), tgt->get_name());
 309                                 }
 310                                 Task *task = tgt->build();
 311                                 if(task)
 312                                 {
 313                                         get_logger().log("commands", "%s", task->get_command());
 314                                         if(dry_run)
 315                                         {
 316                                                 task->signal_finished.emit(true);
 317                                                 delete task;
 318                                         }
 319                                         else
 320                                         {
 321                                                 task->start();
 322                                                 tasks.push_back(task);
 323                                         }
 324                                 }
 325
 326                                 if(show_progress)
 327                                         IO::print("%d of %d target%s built\033[1G", count, total, (total!=1 ? "s" : ""));
 328                         }
 329                         else if(tasks.empty())
 330                                 finish = true;
 331                         else
 332                                 starved = true;
 333                 }
 334                 else
 335                         Time::sleep(10*Time::msec);
 336
 337                 for(unsigned i=0; i<tasks.size();)
 338                 {
 339                         Task::Status status;
 340                         if(jobs==1 || (tasks.size()==1 && starved))
 341                                 status = tasks[i]->wait();
 342                         else
 343                                 status = tasks[i]->check();
 344
 345                         if(status!=Task::RUNNING)
 346                         {
 347                                 ++count;
 348
 349                                 delete tasks[i];
 350                                 tasks.erase(tasks.begin()+i);
 351                                 if(status==Task::ERROR)
 352                                         fail = true;
 353                                 if(tasks.empty() && fail)
 354                                         finish = true;
 355                                 starved = false;
 356                         }
 357                         else
 358                                 ++i;
 359                 }
 360         }
 361
 362         if(show_progress)
 363                 IO::print("\033[K");
 364         if(fail)
 365                 get_logger().log("summary", "Build failed");
 366         else if(show_progress)
 367                 get_logger().log("summary", "Build complete");
 368
 369         return fail;
 370 }
 371
 372 int Builder::clean(bool all, bool dry_run)
 373 {
 374         // Cleaning doesn't care about ordering, so a simpler method can be used
 375
 376         set<Target *> clean_tgts;
 377         deque<Target *> queue;
 378         queue.push_back(&build_graph.get_goals());
 379
 380         while(!queue.empty())
 381         {
 382                 Target *tgt = queue.front();
 383                 queue.pop_front();
 384
 385                 if(tgt->is_buildable() && (tgt->get_package()==&package_manager.get_main_package() || all))
 386                         clean_tgts.insert(tgt);
 387
 388                 for(Target *t: tgt->get_dependencies())
 389                         if(!clean_tgts.count(t))
 390                                 queue.push_back(t);
 391         }
 392
 393         for(Target *t: clean_tgts)
 394         {
 395                 get_logger().log("tasks", "RM    %s", t->get_name());
 396                 if(!dry_run)
 397                         t->clean();
 398         }
 399
 400         return 0;
 401 }
 402
 403
 404 Builder::LoadedPlugin::LoadedPlugin(LoadedPlugin &&other):
 405         path(move(other.path)),
 406         module(other.module),
 407         plugin(other.plugin)
 408 {
 409         other.module = 0;
 410         other.plugin = 0;
 411 }
 412
 413 Builder::LoadedPlugin &Builder::LoadedPlugin::operator=(LoadedPlugin &&other)
 414 {
 415         delete plugin;
 416         delete module;
 417         path = move(other.path);
 418         module = other.module;
 419         plugin = other.plugin;
 420         other.module = 0;
 421         other.plugin = 0;
 422         return *this;
 423 }
 424
 425 Builder::LoadedPlugin::~LoadedPlugin()
 426 {
 427         delete plugin;
 428         delete module;
 429 }
 430
 431
 432 Builder::Loader::Loader(Builder &b, const Config::InputOptions *o, bool a):
 433         DataFile::ObjectLoader<Builder>(b),
 434         options(o),
 435         conf_all(a)
 436 {
 437         add("architecture", &Loader::architecture);
 438         add("binary_package", &Loader::binpkg);
 439         add("build_type", &Loader::build_type);
 440         add("package", &Loader::package);
 441
 442         if(!obj.top_loader)
 443                 obj.top_loader = this;
 444         else if(!options && obj.top_loader!=this && obj.top_loader->conf_all)
 445                 options = obj.top_loader->options;
 446 }
 447
 448 Builder::Loader::~Loader()
 449 {
 450         if(obj.top_loader==this)
 451                 obj.top_loader = 0;
 452 }
 453
 454 void Builder::Loader::architecture(const string &n)
 455 {
 456         if(obj.current_arch->match_name(n))
 457                 load_sub(*obj.current_arch);
 458 }
 459
 460 void Builder::Loader::binpkg(const string &n)
 461 {
 462         BinaryPackage *pkg = new BinaryPackage(obj, n);
 463         load_sub(*pkg);
 464 }
 465
 466 void Builder::Loader::build_type(const string &n)
 467 {
 468         BuildType btype(n);
 469         load_sub(btype);
 470         auto i = obj.build_types.insert({ n, btype }).first;
 471         if(!obj.build_type)
 472                 obj.build_type = &i->second;
 473 }
 474
 475 void Builder::Loader::package(const string &n)
 476 {
 477         SourcePackage *pkg = new SourcePackage(obj, n, get_source());
 478
 479         load_sub(*pkg, options);
 480
 481         if(obj.build_type)
 482                 pkg->set_build_type(*obj.build_type);
 483 }