source/lib/builder.cpp

   1 #include <deque>
   2 #include <set>
   3 #include <msp/core/algorithm.h>
   4 #include <msp/core/except.h>
   5 #include <msp/core/maputils.h>
   6 #include <msp/datafile/parser.h>
   7 #include <msp/fs/dir.h>
   8 #include <msp/fs/stat.h>
   9 #include <msp/fs/utils.h>
  10 #include <msp/io/buffered.h>
  11 #include <msp/io/file.h>
  12 #include <msp/io/print.h>
  13 #include <msp/strings/format.h>
  14 #include <msp/strings/utils.h>
  15 #include <msp/time/timedelta.h>
  16 #include <msp/time/utils.h>
  17 #include "binarypackage.h"
  18 #include "builder.h"
  19 #include "installedfile.h"
  20 #include "package.h"
  21 #include "plugin.h"
  22 #include "sharedlibrary.h"
  23 #include "task.h"
  24 #include "tool.h"
  25 #include "virtualtarget.h"
  26
  27 using namespace std;
  28 using namespace Msp;
  29
  30 Builder::Builder():
  31         package_manager(*this),
  32         native_arch(*this, string()),
  33         vfs(*this),
  34         build_graph(*this),
  35         logger(&default_logger)
  36 {
  37         set_architecture(string());
  38 }
  39
  40 Builder::~Builder()
  41 {
  42         if(current_arch!=&native_arch)
  43                 delete current_arch;
  44 }
  45
  46 void Builder::load_plugins()
  47 {
  48         using CreateFunc = Plugin *(Builder &);
  49
  50         FS::Path plugins_dir = FS::get_sys_lib_dir();
  51         if(!FS::exists(plugins_dir))
  52                 return;
  53
  54         logger->log("files", "Traversing %s", plugins_dir);
  55         vector<LoadedPlugin> unordered_plugins;
  56         for(const string &f: list_filtered(plugins_dir, "\\.dlm$"))
  57         {
  58                 LoadedPlugin plugin;
  59                 plugin.path = plugins_dir/f;
  60
  61                 try
  62                 {
  63                         plugin.module = new Module(plugin.path.str());
  64                 }
  65                 catch(const exception &exc)
  66                 {
  67                         logger->log("plugins", "Failed to load plugin %s: %s", f, exc.what());
  68                         continue;
  69                 }
  70
  71                 try
  72                 {
  73                         CreateFunc *create_func = reinterpret_cast<CreateFunc *>(plugin.module->get_symbol("create_plugin"));
  74                         plugin.plugin = create_func(*this);
  75                         if(plugin.plugin)
  76                         {
  77                                 logger->log("plugins", "Loaded plugin %s", f);
  78                                 unordered_plugins.emplace_back(move(plugin));
  79                                 continue;
  80                         }
  81                         else
  82                                 logger->log("plugins", "Plugin %s refused to initialize", f);
  83                 }
  84                 catch(const exception &exc)
  85                 {
  86                         logger->log("plugins", "Failed to initialize plugin %s: %s", f, exc.what());
  87                 }
  88         }
  89
  90         add_plugins(unordered_plugins);
  91 }
  92
  93 void Builder::add_plugins(vector<LoadedPlugin> &unordered_plugins)
  94 {
  95         auto have_plugin = [this](const string &r){
  96                 return any_of(plugins.begin(), plugins.end(), [&r](const LoadedPlugin &p){ return FS::basepart(FS::basename(p.path))==r; });
  97         };
  98
  99         while(!unordered_plugins.empty())
 100         {
 101                 bool any_added = false;
 102                 for(auto i=unordered_plugins.begin(); i!=unordered_plugins.end(); )
 103                 {
 104                         const vector<string> &required = i->plugin->get_required_plugins();
 105                         if(all_of(required.begin(), required.end(), have_plugin))
 106                         {
 107                                 plugins.push_back(move(*i));
 108                                 i = unordered_plugins.erase(i);
 109                                 any_added = true;
 110                         }
 111                         else
 112                                 ++i;
 113                 }
 114
 115                 if(!any_added)
 116                         break;
 117         }
 118
 119         for(const LoadedPlugin &p: unordered_plugins)
 120         {
 121                 vector<string> missing;
 122                 for(const string &r: p.plugin->get_required_plugins())
 123                         if(!have_plugin(r))
 124                                 missing.push_back(r);
 125                 logger->log("plugins", "Missing required plugins for plugin %s: %s",
 126                         FS::basename(p.path), join(missing.begin(), missing.end()));
 127         }
 128 }
 129
 130 void Builder::set_architecture(const string &name)
 131 {
 132         if(current_arch!=&native_arch)
 133                 delete current_arch;
 134
 135         if(name.empty())
 136                 current_arch = &native_arch;
 137         else
 138                 current_arch = new Architecture(*this, name);
 139
 140         if(auto_prefix)
 141                 update_auto_prefix();
 142 }
 143
 144 vector<string> Builder::get_build_types() const
 145 {
 146         vector<string> keys;
 147         keys.reserve(build_types.size());
 148         for(const auto &kvp: build_types)
 149                 keys.push_back(kvp.first);
 150         return keys;
 151 }
 152
 153 const BuildType &Builder::get_build_type() const
 154 {
 155         if(!build_type)
 156                 throw invalid_state("no build type");
 157         return *build_type;
 158 }
 159
 160 void Builder::set_build_type(const string &name)
 161 {
 162         build_type = &get_item(build_types, name);
 163 }
 164
 165 void Builder::set_prefix(const FS::Path &p)
 166 {
 167         auto_prefix = false;
 168         prefix = p;
 169 }
 170
 171 void Builder::set_temp_directory(const FS::Path &p)
 172 {
 173         tempdir = p;
 174 }
 175
 176 void Builder::update_auto_prefix()
 177 {
 178         if(current_arch->is_native())
 179                 prefix = FS::get_home_dir()/"local";
 180         else
 181                 prefix = FS::get_home_dir()/"local"/current_arch->get_name();
 182 }
 183
 184 void Builder::add_default_tools()
 185 {
 186         for(const LoadedPlugin &p: plugins)
 187                 p.plugin->add_tools(toolchain, *current_arch);
 188
 189         auto i = find_if(toolchain.get_toolchains(), [](const Toolchain *tc){ return (tc->has_tool("CC") || tc->has_tool("CXX")); });
 190         if(i!=toolchain.get_toolchains().end())
 191         {
 192                 current_arch->refine((*i)->get_name());
 193                 if(auto_prefix)
 194                         update_auto_prefix();
 195         }
 196 }
 197
 198 void Builder::set_logger(const Logger *l)
 199 {
 200         logger = (l ? l : &default_logger);
 201 }
 202
 203 vector<string> Builder::collect_problems() const
 204 {
 205         vector<string> target_problems;
 206         vector<string> problems;
 207         vector<const Package *> broken_packages;
 208         vector<const Component *> broken_components;
 209         vector<const Tool *> broken_tools;
 210
 211         for(const auto &kvp: build_graph.get_targets())
 212                 if(kvp.second->is_broken())
 213                 {
 214                         const Package *package = kvp.second->get_package();
 215                         if(package && package->is_broken())
 216                                 collect_broken_packages(*package, broken_packages);
 217
 218                         const Component *component = kvp.second->get_component();
 219                         if(component && component->is_broken() && !any_equals(broken_components, component))
 220                         {
 221                                 broken_components.push_back(component);
 222                                 collect_broken_packages(component->get_package(), broken_packages);
 223                                 for(const Package *r: component->get_required_packages())
 224                                         if(r->is_broken())
 225                                                 collect_broken_packages(*r, broken_packages);
 226                         }
 227
 228                         const Tool *tool = kvp.second->get_tool();
 229                         if(tool && tool->is_broken() && !any_equals(broken_tools, tool))
 230                         {
 231                                 broken_tools.push_back(tool);
 232                                 for(const string &p: tool->get_problems())
 233                                         problems.push_back(format("%s: %s", tool->get_tag(), p));
 234                         }
 235
 236                         for(const string &p: kvp.second->get_problems())
 237                                 target_problems.push_back(format("%s: %s", kvp.second->get_name(), p));
 238                 }
 239
 240         for(const Package *p: broken_packages)
 241         {
 242                 for(const string &b: p->get_problems())
 243                         problems.push_back(format("%s: %s", p->get_name(), b));
 244
 245                 for(const Component *c: broken_components)
 246                         if(&c->get_package()==p)
 247                         {
 248                                 for(const string &b: c->get_problems())
 249                                         problems.push_back(format("%s/%s: %s", p->get_name(), c->get_name(), b));
 250                         }
 251         }
 252
 253         problems.insert(problems.end(), make_move_iterator(target_problems.begin()), make_move_iterator(target_problems.end()));
 254
 255         return problems;
 256 }
 257
 258 void Builder::collect_broken_packages(const Package &pkg, vector<const Package *> &broken) const
 259 {
 260         if(any_equals(broken, &pkg))
 261                 return;
 262
 263         broken.push_back(&pkg);
 264
 265         for(const Package *r: pkg.get_required_packages())
 266                 if(r->is_broken())
 267                         collect_broken_packages(*r, broken);
 268 }
 269
 270 void Builder::load_build_file(const FS::Path &fn, const Config::InputOptions *opts, bool all)
 271 {
 272         IO::BufferedFile in(fn.str());
 273
 274         get_logger().log("files", "Reading %s", fn);
 275
 276         DataFile::Parser parser(in, fn.str());
 277         Loader loader(*this, opts, all);
 278         loader.load(parser);
 279 }
 280
 281 void Builder::save_caches()
 282 {
 283         for(const auto &kvp: package_manager.get_packages())
 284                 kvp.second->save_caches();
 285 }
 286
 287 int Builder::build(unsigned jobs, bool dry_run, bool show_progress)
 288 {
 289         unsigned total = build_graph.count_rebuild_targets();
 290         Time::TimeStamp start_time = Time::now();
 291
 292         if(!total)
 293         {
 294                 get_logger().log("summary", "Already up to date");
 295                 return 0;
 296         }
 297         get_logger().log("summary", "Will build %d target%s", total, (total!=1 ? "s" : ""));
 298
 299         vector<Task *> tasks;
 300
 301         unsigned count = 0;
 302         Time::TimeDelta sum_time;
 303
 304         bool fail = false;
 305         bool finish = false;
 306         bool starved = false;
 307
 308         while(!finish)
 309         {
 310                 if(tasks.size()<jobs && !fail && !starved)
 311                 {
 312                         Target *tgt = build_graph.get_buildable_target();
 313                         if(tgt)
 314                         {
 315                                 if(tgt->get_tool())
 316                                 {
 317                                         if(show_progress)
 318                                                 IO::print("\033[K");
 319                                         get_logger().log("tasks", "%-4s  %s", tgt->get_tool()->get_tag(), tgt->get_name());
 320                                 }
 321                                 Task *task = tgt->build();
 322                                 if(task)
 323                                 {
 324                                         get_logger().log("commands", "%s", task->get_command());
 325                                         if(dry_run)
 326                                         {
 327                                                 task->signal_finished.emit(true);
 328                                                 delete task;
 329                                         }
 330                                         else
 331                                         {
 332                                                 task->start();
 333                                                 tasks.push_back(task);
 334                                         }
 335                                 }
 336
 337                                 if(show_progress)
 338                                         IO::print("%d of %d target%s built\033[1G", count, total, (total!=1 ? "s" : ""));
 339                         }
 340                         else if(tasks.empty())
 341                                 finish = true;
 342                         else
 343                                 starved = true;
 344                 }
 345                 else
 346                         Time::sleep(10*Time::msec);
 347
 348                 for(unsigned i=0; i<tasks.size();)
 349                 {
 350                         Task::Status status;
 351                         if(jobs==1 || (tasks.size()==1 && starved))
 352                                 status = tasks[i]->wait();
 353                         else
 354                                 status = tasks[i]->check();
 355
 356                         if(status!=Task::RUNNING)
 357                         {
 358                                 ++count;
 359
 360                                 const vector<const FileTarget *> &targets = tasks[i]->get_targets();
 361                                 if(!targets.empty())
 362                                 {
 363                                         sum_time += tasks[i]->get_duration();
 364                                         get_logger().log("timings", "%s built in %s", targets.front()->get_name(), tasks[i]->get_duration());
 365                                 }
 366
 367                                 delete tasks[i];
 368                                 tasks.erase(tasks.begin()+i);
 369                                 if(status==Task::ERROR)
 370                                         fail = true;
 371                                 if(tasks.empty() && fail)
 372                                         finish = true;
 373                                 starved = false;
 374                         }
 375                         else
 376                                 ++i;
 377                 }
 378         }
 379
 380         if(show_progress)
 381                 IO::print("\033[K");
 382         if(fail)
 383                 get_logger().log("summary", "Build failed");
 384         else if(show_progress)
 385                 get_logger().log("summary", "Build complete");
 386
 387         Time::TimeStamp end_time = Time::now();
 388         get_logger().log("timings", "Build took %s, with a total %s spent on tasks", end_time-start_time, sum_time);
 389
 390         return fail;
 391 }
 392
 393 int Builder::clean(bool all, bool dry_run)
 394 {
 395         // Cleaning doesn't care about ordering, so a simpler method can be used
 396
 397         set<Target *> clean_tgts;
 398         deque<Target *> queue;
 399         queue.push_back(&build_graph.get_goals());
 400
 401         while(!queue.empty())
 402         {
 403                 Target *tgt = queue.front();
 404                 queue.pop_front();
 405
 406                 if(tgt->is_buildable() && (tgt->get_package()==&package_manager.get_main_package() || all))
 407                         clean_tgts.insert(tgt);
 408
 409                 for(Target *t: tgt->get_dependencies())
 410                         if(!clean_tgts.count(t))
 411                                 queue.push_back(t);
 412         }
 413
 414         for(Target *t: clean_tgts)
 415         {
 416                 get_logger().log("tasks", "RM    %s", t->get_name());
 417                 if(!dry_run)
 418                         t->clean();
 419         }
 420
 421         return 0;
 422 }
 423
 424
 425 Builder::LoadedPlugin::LoadedPlugin(LoadedPlugin &&other):
 426         path(move(other.path)),
 427         module(other.module),
 428         plugin(other.plugin)
 429 {
 430         other.module = 0;
 431         other.plugin = 0;
 432 }
 433
 434 Builder::LoadedPlugin &Builder::LoadedPlugin::operator=(LoadedPlugin &&other)
 435 {
 436         delete plugin;
 437         delete module;
 438         path = move(other.path);
 439         module = other.module;
 440         plugin = other.plugin;
 441         other.module = 0;
 442         other.plugin = 0;
 443         return *this;
 444 }
 445
 446 Builder::LoadedPlugin::~LoadedPlugin()
 447 {
 448         delete plugin;
 449         delete module;
 450 }
 451
 452
 453 Builder::Loader::Loader(Builder &b, const Config::InputOptions *o, bool a):
 454         DataFile::ObjectLoader<Builder>(b),
 455         options(o),
 456         conf_all(a)
 457 {
 458         add("architecture", &Loader::architecture);
 459         add("binary_package", &Loader::binpkg);
 460         add("build_type", &Loader::build_type);
 461         add("package", &Loader::package);
 462
 463         if(!obj.top_loader)
 464                 obj.top_loader = this;
 465         else if(!options && obj.top_loader!=this && obj.top_loader->conf_all)
 466                 options = obj.top_loader->options;
 467 }
 468
 469 Builder::Loader::~Loader()
 470 {
 471         if(obj.top_loader==this)
 472                 obj.top_loader = 0;
 473 }
 474
 475 void Builder::Loader::architecture(const string &n)
 476 {
 477         if(obj.current_arch->match_name(n))
 478                 load_sub(*obj.current_arch);
 479 }
 480
 481 void Builder::Loader::binpkg(const string &n)
 482 {
 483         BinaryPackage *pkg = new BinaryPackage(obj, n);
 484         load_sub(*pkg);
 485 }
 486
 487 void Builder::Loader::build_type(const string &n)
 488 {
 489         BuildType btype(n);
 490         load_sub(btype);
 491         auto i = obj.build_types.insert({ n, btype }).first;
 492         if(!obj.build_type)
 493                 obj.build_type = &i->second;
 494 }
 495
 496 void Builder::Loader::package(const string &n)
 497 {
 498         SourcePackage *pkg = new SourcePackage(obj, n, get_source());
 499
 500         load_sub(*pkg, options);
 501
 502         if(obj.build_type)
 503                 pkg->set_build_type(*obj.build_type);
 504 }