source/lib/builder.cpp

   1 #include <deque>
   2 #include <set>
   3 #include <msp/core/algorithm.h>
   4 #include <msp/core/except.h>
   5 #include <msp/core/maputils.h>
   6 #include <msp/datafile/parser.h>
   7 #include <msp/fs/dir.h>
   8 #include <msp/fs/utils.h>
   9 #include <msp/io/buffered.h>
  10 #include <msp/io/file.h>
  11 #include <msp/io/print.h>
  12 #include <msp/strings/format.h>
  13 #include <msp/time/timedelta.h>
  14 #include <msp/time/utils.h>
  15 #include "binarypackage.h"
  16 #include "builder.h"
  17 #include "installedfile.h"
  18 #include "package.h"
  19 #include "plugin.h"
  20 #include "sharedlibrary.h"
  21 #include "task.h"
  22 #include "tool.h"
  23 #include "virtualtarget.h"
  24
  25 using namespace std;
  26 using namespace Msp;
  27
  28 Builder::Builder():
  29         package_manager(*this),
  30         native_arch(*this, string()),
  31         vfs(*this),
  32         build_graph(*this),
  33         logger(&default_logger)
  34 {
  35         set_architecture(string());
  36 }
  37
  38 Builder::~Builder()
  39 {
  40         if(current_arch!=&native_arch)
  41                 delete current_arch;
  42 }
  43
  44 void Builder::load_plugins()
  45 {
  46         using CreateFunc = Plugin *(Builder &);
  47
  48         FS::Path plugins_dir = FS::get_sys_lib_dir();
  49         logger->log("files", "Traversing %s", plugins_dir);
  50         for(const string &f: list_filtered(plugins_dir, "\\.dlm$"))
  51         {
  52                 LoadedPlugin plugin;
  53                 plugin.path = plugins_dir/f;
  54
  55                 try
  56                 {
  57                         plugin.module = new Module(plugin.path.str());
  58                 }
  59                 catch(const exception &exc)
  60                 {
  61                         logger->log("plugins", "Failed to load plugin %s: %s", f, exc.what());
  62                         continue;
  63                 }
  64
  65                 try
  66                 {
  67                         CreateFunc *create_func = reinterpret_cast<CreateFunc *>(plugin.module->get_symbol("create_plugin"));
  68                         plugin.plugin = create_func(*this);
  69                         if(plugin.plugin)
  70                         {
  71                                 logger->log("plugins", "Loaded plugin %s", f);
  72                                 plugins.emplace_back(move(plugin));
  73                                 continue;
  74                         }
  75                         else
  76                                 logger->log("plugins", "Plugin %s refused to initialize", f);
  77                 }
  78                 catch(const exception &exc)
  79                 {
  80                         logger->log("plugins", "Failed to initialize plugin %s: %s", f, exc.what());
  81                 }
  82
  83                 delete plugin.module;
  84         }
  85 }
  86
  87 void Builder::set_architecture(const string &name)
  88 {
  89         if(current_arch!=&native_arch)
  90                 delete current_arch;
  91
  92         if(name.empty())
  93                 current_arch = &native_arch;
  94         else
  95                 current_arch = new Architecture(*this, name);
  96
  97         if(auto_prefix)
  98                 update_auto_prefix();
  99 }
 100
 101 vector<string> Builder::get_build_types() const
 102 {
 103         vector<string> keys;
 104         keys.reserve(build_types.size());
 105         for(const auto &kvp: build_types)
 106                 keys.push_back(kvp.first);
 107         return keys;
 108 }
 109
 110 const BuildType &Builder::get_build_type() const
 111 {
 112         if(!build_type)
 113                 throw invalid_state("no build type");
 114         return *build_type;
 115 }
 116
 117 void Builder::set_build_type(const string &name)
 118 {
 119         build_type = &get_item(build_types, name);
 120 }
 121
 122 void Builder::set_prefix(const FS::Path &p)
 123 {
 124         auto_prefix = false;
 125         prefix = p;
 126 }
 127
 128 void Builder::set_temp_directory(const FS::Path &p)
 129 {
 130         tempdir = p;
 131 }
 132
 133 void Builder::update_auto_prefix()
 134 {
 135         if(current_arch->is_native())
 136                 prefix = FS::get_home_dir()/"local";
 137         else
 138                 prefix = FS::get_home_dir()/"local"/current_arch->get_name();
 139 }
 140
 141 void Builder::add_default_tools()
 142 {
 143         for(const LoadedPlugin &p: plugins)
 144                 p.plugin->add_tools(toolchain, *current_arch);
 145
 146         auto i = find_if(toolchain.get_toolchains(), [](const Toolchain *tc){ return (tc->has_tool("CC") || tc->has_tool("CXX")); });
 147         if(i!=toolchain.get_toolchains().end())
 148         {
 149                 current_arch->refine((*i)->get_name());
 150                 if(auto_prefix)
 151                         update_auto_prefix();
 152         }
 153 }
 154
 155 void Builder::set_logger(const Logger *l)
 156 {
 157         logger = (l ? l : &default_logger);
 158 }
 159
 160 vector<string> Builder::collect_problems() const
 161 {
 162         vector<string> target_problems;
 163         vector<string> problems;
 164         vector<const Package *> broken_packages;
 165         vector<const Component *> broken_components;
 166         vector<const Tool *> broken_tools;
 167
 168         for(const auto &kvp: build_graph.get_targets())
 169                 if(kvp.second->is_broken())
 170                 {
 171                         const Package *package = kvp.second->get_package();
 172                         if(package && package->is_broken())
 173                                 collect_broken_packages(*package, broken_packages);
 174
 175                         const Component *component = kvp.second->get_component();
 176                         if(component && component->is_broken() && !any_equals(broken_components, component))
 177                         {
 178                                 broken_components.push_back(component);
 179                                 collect_broken_packages(component->get_package(), broken_packages);
 180                                 for(const Package *r: component->get_required_packages())
 181                                         if(r->is_broken())
 182                                                 collect_broken_packages(*r, broken_packages);
 183                         }
 184
 185                         const Tool *tool = kvp.second->get_tool();
 186                         if(tool && tool->is_broken() && !any_equals(broken_tools, tool))
 187                         {
 188                                 broken_tools.push_back(tool);
 189                                 for(const string &p: tool->get_problems())
 190                                         problems.push_back(format("%s: %s", tool->get_tag(), p));
 191                         }
 192
 193                         for(const string &p: kvp.second->get_problems())
 194                                 target_problems.push_back(format("%s: %s", kvp.second->get_name(), p));
 195                 }
 196
 197         for(const Package *p: broken_packages)
 198         {
 199                 for(const string &b: p->get_problems())
 200                         problems.push_back(format("%s: %s", p->get_name(), b));
 201
 202                 for(const Component *c: broken_components)
 203                         if(&c->get_package()==p)
 204                         {
 205                                 for(const string &b: c->get_problems())
 206                                         problems.push_back(format("%s/%s: %s", p->get_name(), c->get_name(), b));
 207                         }
 208         }
 209
 210         problems.insert(problems.end(), make_move_iterator(target_problems.begin()), make_move_iterator(target_problems.end()));
 211
 212         return problems;
 213 }
 214
 215 void Builder::collect_broken_packages(const Package &pkg, vector<const Package *> &broken) const
 216 {
 217         if(any_equals(broken, &pkg))
 218                 return;
 219
 220         broken.push_back(&pkg);
 221
 222         for(const Package *r: pkg.get_required_packages())
 223                 if(r->is_broken())
 224                         collect_broken_packages(*r, broken);
 225 }
 226
 227 void Builder::load_build_file(const FS::Path &fn, const Config::InputOptions *opts, bool all)
 228 {
 229         IO::BufferedFile in(fn.str());
 230
 231         get_logger().log("files", "Reading %s", fn);
 232
 233         DataFile::Parser parser(in, fn.str());
 234         Loader loader(*this, opts, all);
 235         loader.load(parser);
 236 }
 237
 238 void Builder::save_caches()
 239 {
 240         for(const auto &kvp: package_manager.get_packages())
 241                 kvp.second->save_caches();
 242 }
 243
 244 int Builder::build(unsigned jobs, bool dry_run, bool show_progress)
 245 {
 246         unsigned total = build_graph.count_rebuild_targets();
 247
 248         if(!total)
 249         {
 250                 get_logger().log("summary", "Already up to date");
 251                 return 0;
 252         }
 253         get_logger().log("summary", "Will build %d target%s", total, (total!=1 ? "s" : ""));
 254
 255         vector<Task *> tasks;
 256
 257         unsigned count = 0;
 258
 259         bool fail = false;
 260         bool finish = false;
 261         bool starved = false;
 262
 263         while(!finish)
 264         {
 265                 if(tasks.size()<jobs && !fail && !starved)
 266                 {
 267                         Target *tgt = build_graph.get_buildable_target();
 268                         if(tgt)
 269                         {
 270                                 if(tgt->get_tool())
 271                                 {
 272                                         if(show_progress)
 273                                                 IO::print("\033[K");
 274                                         get_logger().log("tasks", "%-4s  %s", tgt->get_tool()->get_tag(), tgt->get_name());
 275                                 }
 276                                 Task *task = tgt->build();
 277                                 if(task)
 278                                 {
 279                                         get_logger().log("commands", "%s", task->get_command());
 280                                         if(dry_run)
 281                                         {
 282                                                 task->signal_finished.emit(true);
 283                                                 delete task;
 284                                         }
 285                                         else
 286                                         {
 287                                                 task->start();
 288                                                 tasks.push_back(task);
 289                                         }
 290                                 }
 291
 292                                 if(show_progress)
 293                                         IO::print("%d of %d target%s built\033[1G", count, total, (total!=1 ? "s" : ""));
 294                         }
 295                         else if(tasks.empty())
 296                                 finish = true;
 297                         else
 298                                 starved = true;
 299                 }
 300                 else
 301                         Time::sleep(10*Time::msec);
 302
 303                 for(unsigned i=0; i<tasks.size();)
 304                 {
 305                         Task::Status status;
 306                         if(jobs==1 || (tasks.size()==1 && starved))
 307                                 status = tasks[i]->wait();
 308                         else
 309                                 status = tasks[i]->check();
 310
 311                         if(status!=Task::RUNNING)
 312                         {
 313                                 ++count;
 314
 315                                 delete tasks[i];
 316                                 tasks.erase(tasks.begin()+i);
 317                                 if(status==Task::ERROR)
 318                                         fail = true;
 319                                 if(tasks.empty() && fail)
 320                                         finish = true;
 321                                 starved = false;
 322                         }
 323                         else
 324                                 ++i;
 325                 }
 326         }
 327
 328         if(show_progress)
 329                 IO::print("\033[K");
 330         if(fail)
 331                 get_logger().log("summary", "Build failed");
 332         else if(show_progress)
 333                 get_logger().log("summary", "Build complete");
 334
 335         return fail;
 336 }
 337
 338 int Builder::clean(bool all, bool dry_run)
 339 {
 340         // Cleaning doesn't care about ordering, so a simpler method can be used
 341
 342         set<Target *> clean_tgts;
 343         deque<Target *> queue;
 344         queue.push_back(&build_graph.get_goals());
 345
 346         while(!queue.empty())
 347         {
 348                 Target *tgt = queue.front();
 349                 queue.pop_front();
 350
 351                 if(tgt->is_buildable() && (tgt->get_package()==&package_manager.get_main_package() || all))
 352                         clean_tgts.insert(tgt);
 353
 354                 for(Target *t: tgt->get_dependencies())
 355                         if(!clean_tgts.count(t))
 356                                 queue.push_back(t);
 357         }
 358
 359         for(Target *t: clean_tgts)
 360         {
 361                 get_logger().log("tasks", "RM    %s", t->get_name());
 362                 if(!dry_run)
 363                         t->clean();
 364         }
 365
 366         return 0;
 367 }
 368
 369
 370 Builder::LoadedPlugin::LoadedPlugin(LoadedPlugin &&other):
 371         path(move(other.path)),
 372         module(other.module),
 373         plugin(other.plugin)
 374 {
 375         other.module = 0;
 376         other.plugin = 0;
 377 }
 378
 379 Builder::LoadedPlugin::~LoadedPlugin()
 380 {
 381         delete plugin;
 382         delete module;
 383 }
 384
 385
 386 Builder::Loader::Loader(Builder &b, const Config::InputOptions *o, bool a):
 387         DataFile::ObjectLoader<Builder>(b),
 388         options(o),
 389         conf_all(a)
 390 {
 391         add("architecture", &Loader::architecture);
 392         add("binary_package", &Loader::binpkg);
 393         add("build_type", &Loader::build_type);
 394         add("package", &Loader::package);
 395
 396         if(!obj.top_loader)
 397                 obj.top_loader = this;
 398         else if(!options && obj.top_loader!=this && obj.top_loader->conf_all)
 399                 options = obj.top_loader->options;
 400 }
 401
 402 Builder::Loader::~Loader()
 403 {
 404         if(obj.top_loader==this)
 405                 obj.top_loader = 0;
 406 }
 407
 408 void Builder::Loader::architecture(const string &n)
 409 {
 410         if(obj.current_arch->match_name(n))
 411                 load_sub(*obj.current_arch);
 412 }
 413
 414 void Builder::Loader::binpkg(const string &n)
 415 {
 416         BinaryPackage *pkg = new BinaryPackage(obj, n);
 417         load_sub(*pkg);
 418 }
 419
 420 void Builder::Loader::build_type(const string &n)
 421 {
 422         BuildType btype(n);
 423         load_sub(btype);
 424         auto i = obj.build_types.insert({ n, btype }).first;
 425         if(!obj.build_type)
 426                 obj.build_type = &i->second;
 427 }
 428
 429 void Builder::Loader::package(const string &n)
 430 {
 431         SourcePackage *pkg = new SourcePackage(obj, n, get_source());
 432
 433         load_sub(*pkg, options);
 434
 435         if(obj.build_type)
 436                 pkg->set_build_type(*obj.build_type);
 437 }