source/lib/builder.cpp

   1 #include <deque>
   2 #include <set>
   3 #include <msp/core/algorithm.h>
   4 #include <msp/core/except.h>
   5 #include <msp/core/maputils.h>
   6 #include <msp/datafile/parser.h>
   7 #include <msp/fs/dir.h>
   8 #include <msp/fs/utils.h>
   9 #include <msp/io/buffered.h>
  10 #include <msp/io/file.h>
  11 #include <msp/io/print.h>
  12 #include <msp/strings/format.h>
  13 #include <msp/time/timedelta.h>
  14 #include <msp/time/utils.h>
  15 #include "android/androidtools.h"
  16 #include "binarypackage.h"
  17 #include "builder.h"
  18 #include "builtin/builtintools.h"
  19 #include "clang/clangtools.h"
  20 #include "datafile/datatool.h"
  21 #include "gnu/gnutools.h"
  22 #include "installedfile.h"
  23 #include "msvc/microsofttools.h"
  24 #include "package.h"
  25 #include "plugin.h"
  26 #include "sharedlibrary.h"
  27 #include "sourcepackage.h"
  28 #include "task.h"
  29 #include "virtualtarget.h"
  30
  31 using namespace std;
  32 using namespace Msp;
  33
  34 Builder::Builder():
  35         package_manager(*this),
  36         native_arch(*this, string()),
  37         vfs(*this),
  38         build_graph(*this),
  39         logger(&default_logger)
  40 {
  41         set_architecture(string());
  42 }
  43
  44 Builder::~Builder()
  45 {
  46         if(current_arch!=&native_arch)
  47                 delete current_arch;
  48 }
  49
  50 void Builder::load_plugins()
  51 {
  52         using CreateFunc = Plugin *(Builder &);
  53
  54         FS::Path plugins_dir = FS::get_sys_lib_dir();
  55         logger->log("files", "Traversing %s", plugins_dir);
  56         for(const string &f: list_filtered(plugins_dir, "\\.dlm$"))
  57         {
  58                 LoadedPlugin plugin;
  59                 plugin.path = plugins_dir/f;
  60
  61                 try
  62                 {
  63                         plugin.module = new Module(plugin.path.str());
  64                 }
  65                 catch(const exception &exc)
  66                 {
  67                         logger->log("plugins", "Failed to load plugin %s: %s", f, exc.what());
  68                         continue;
  69                 }
  70
  71                 try
  72                 {
  73                         CreateFunc *create_func = reinterpret_cast<CreateFunc *>(plugin.module->get_symbol("create_plugin"));
  74                         plugin.plugin = create_func(*this);
  75                         if(plugin.plugin)
  76                         {
  77                                 logger->log("plugins", "Loaded plugin %s", f);
  78                                 plugins.emplace_back(move(plugin));
  79                                 continue;
  80                         }
  81                         else
  82                                 logger->log("plugins", "Plugin %s refused to initialize", f);
  83                 }
  84                 catch(const exception &exc)
  85                 {
  86                         logger->log("plugins", "Failed to initialize plugin %s: %s", f, exc.what());
  87                 }
  88
  89                 delete plugin.module;
  90         }
  91 }
  92
  93 void Builder::set_architecture(const string &name)
  94 {
  95         if(current_arch!=&native_arch)
  96                 delete current_arch;
  97
  98         if(name.empty())
  99                 current_arch = &native_arch;
 100         else
 101                 current_arch = new Architecture(*this, name);
 102
 103         if(auto_prefix)
 104                 update_auto_prefix();
 105 }
 106
 107 vector<string> Builder::get_build_types() const
 108 {
 109         vector<string> keys;
 110         keys.reserve(build_types.size());
 111         for(const auto &kvp: build_types)
 112                 keys.push_back(kvp.first);
 113         return keys;
 114 }
 115
 116 const BuildType &Builder::get_build_type() const
 117 {
 118         if(!build_type)
 119                 throw invalid_state("no build type");
 120         return *build_type;
 121 }
 122
 123 void Builder::set_build_type(const string &name)
 124 {
 125         build_type = &get_item(build_types, name);
 126 }
 127
 128 void Builder::set_prefix(const FS::Path &p)
 129 {
 130         auto_prefix = false;
 131         prefix = p;
 132 }
 133
 134 void Builder::set_temp_directory(const FS::Path &p)
 135 {
 136         tempdir = p;
 137 }
 138
 139 void Builder::update_auto_prefix()
 140 {
 141         if(current_arch->is_native())
 142                 prefix = FS::get_home_dir()/"local";
 143         else
 144                 prefix = FS::get_home_dir()/"local"/current_arch->get_name();
 145 }
 146
 147 void Builder::add_default_tools()
 148 {
 149         toolchain.add_toolchain(new GnuTools(*this, *current_arch));
 150         toolchain.add_toolchain(new ClangTools(*this, *current_arch));
 151         if(current_arch->get_system()=="android")
 152                 toolchain.add_toolchain(new AndroidTools(*this, *current_arch));
 153         if(current_arch->get_system()=="windows")
 154                 toolchain.add_toolchain(new MicrosoftTools(*this, *current_arch));
 155         toolchain.add_toolchain(new BuiltinTools(*this));
 156         toolchain.add_tool(new DataTool(*this));
 157         for(const LoadedPlugin &p: plugins)
 158                 p.plugin->add_tools(toolchain, *current_arch);
 159
 160         auto i = find_if(toolchain.get_toolchains(), [](const Toolchain *tc){ return (tc->has_tool("CC") || tc->has_tool("CXX")); });
 161         if(i!=toolchain.get_toolchains().end())
 162         {
 163                 current_arch->refine((*i)->get_name());
 164                 if(auto_prefix)
 165                         update_auto_prefix();
 166         }
 167 }
 168
 169 void Builder::set_logger(const Logger *l)
 170 {
 171         logger = (l ? l : &default_logger);
 172 }
 173
 174 vector<string> Builder::collect_problems() const
 175 {
 176         vector<string> problems;
 177         set<const Package *> broken_packages;
 178         set<const Component *> broken_components;
 179         set<const Tool *> broken_tools;
 180
 181         for(const auto &kvp: build_graph.get_targets())
 182                 if(kvp.second->is_broken())
 183                 {
 184                         for(const string &p: kvp.second->get_problems())
 185                                 problems.push_back(format("%s: %s", kvp.second->get_name(), p));
 186
 187                         const Package *package = kvp.second->get_package();
 188                         if(package && !package->get_problems().empty())
 189                                 broken_packages.insert(package);
 190
 191                         const Component *component = kvp.second->get_component();
 192                         if(component && !component->get_problems().empty())
 193                                 broken_components.insert(component);
 194
 195                         const Tool *tool = kvp.second->get_tool();
 196                         if(tool && !tool->get_problems().empty())
 197                                 broken_tools.insert(tool);
 198                 }
 199
 200         // TODO Sort components after their packages, and targets last
 201         for(const Package *p: broken_packages)
 202                 for(const string &b: p->get_problems())
 203                         problems.push_back(format("%s: %s", p->get_name(), b));
 204
 205         for(const Component *c: broken_components)
 206                 for(const string &b: c->get_problems())
 207                         problems.push_back(format("%s/%s: %s", c->get_package().get_name(), c->get_name(), b));
 208
 209         for(const Tool *t: broken_tools)
 210                 for(const string &b: t->get_problems())
 211                         problems.push_back(format("%s: %s", t->get_tag(), b));
 212
 213         return problems;
 214 }
 215
 216 void Builder::load_build_file(const FS::Path &fn, const Config::InputOptions *opts, bool all)
 217 {
 218         IO::BufferedFile in(fn.str());
 219
 220         get_logger().log("files", "Reading %s", fn);
 221
 222         DataFile::Parser parser(in, fn.str());
 223         Loader loader(*this, opts, all);
 224         loader.load(parser);
 225 }
 226
 227 void Builder::save_caches()
 228 {
 229         for(const auto &kvp: package_manager.get_packages())
 230                 kvp.second->save_caches();
 231 }
 232
 233 int Builder::build(unsigned jobs, bool dry_run, bool show_progress)
 234 {
 235         unsigned total = build_graph.count_rebuild_targets();
 236
 237         if(!total)
 238         {
 239                 get_logger().log("summary", "Already up to date");
 240                 return 0;
 241         }
 242         get_logger().log("summary", "Will build %d target%s", total, (total!=1 ? "s" : ""));
 243
 244         vector<Task *> tasks;
 245
 246         unsigned count = 0;
 247
 248         bool fail = false;
 249         bool finish = false;
 250         bool starved = false;
 251
 252         while(!finish)
 253         {
 254                 if(tasks.size()<jobs && !fail && !starved)
 255                 {
 256                         Target *tgt = build_graph.get_buildable_target();
 257                         if(tgt)
 258                         {
 259                                 if(tgt->get_tool())
 260                                 {
 261                                         if(show_progress)
 262                                                 IO::print("\033[K");
 263                                         get_logger().log("tasks", "%-4s  %s", tgt->get_tool()->get_tag(), tgt->get_name());
 264                                 }
 265                                 Task *task = tgt->build();
 266                                 if(task)
 267                                 {
 268                                         get_logger().log("commands", "%s", task->get_command());
 269                                         if(dry_run)
 270                                         {
 271                                                 task->signal_finished.emit(true);
 272                                                 delete task;
 273                                         }
 274                                         else
 275                                         {
 276                                                 task->start();
 277                                                 tasks.push_back(task);
 278                                         }
 279                                 }
 280
 281                                 if(show_progress)
 282                                         IO::print("%d of %d target%s built\033[1G", count, total, (total!=1 ? "s" : ""));
 283                         }
 284                         else if(tasks.empty())
 285                                 finish = true;
 286                         else
 287                                 starved = true;
 288                 }
 289                 else
 290                         Time::sleep(10*Time::msec);
 291
 292                 for(unsigned i=0; i<tasks.size();)
 293                 {
 294                         Task::Status status;
 295                         if(jobs==1 || (tasks.size()==1 && starved))
 296                                 status = tasks[i]->wait();
 297                         else
 298                                 status = tasks[i]->check();
 299
 300                         if(status!=Task::RUNNING)
 301                         {
 302                                 ++count;
 303
 304                                 delete tasks[i];
 305                                 tasks.erase(tasks.begin()+i);
 306                                 if(status==Task::ERROR)
 307                                         fail = true;
 308                                 if(tasks.empty() && fail)
 309                                         finish = true;
 310                                 starved = false;
 311                         }
 312                         else
 313                                 ++i;
 314                 }
 315         }
 316
 317         if(show_progress)
 318                 IO::print("\033[K");
 319         if(fail)
 320                 get_logger().log("summary", "Build failed");
 321         else if(show_progress)
 322                 get_logger().log("summary", "Build complete");
 323
 324         return fail;
 325 }
 326
 327 int Builder::clean(bool all, bool dry_run)
 328 {
 329         // Cleaning doesn't care about ordering, so a simpler method can be used
 330
 331         set<Target *> clean_tgts;
 332         deque<Target *> queue;
 333         queue.push_back(&build_graph.get_goals());
 334
 335         while(!queue.empty())
 336         {
 337                 Target *tgt = queue.front();
 338                 queue.pop_front();
 339
 340                 if(tgt->is_buildable() && (tgt->get_package()==&package_manager.get_main_package() || all))
 341                         clean_tgts.insert(tgt);
 342
 343                 for(Target *t: tgt->get_dependencies())
 344                         if(!clean_tgts.count(t))
 345                                 queue.push_back(t);
 346         }
 347
 348         for(Target *t: clean_tgts)
 349         {
 350                 get_logger().log("tasks", "RM    %s", t->get_name());
 351                 if(!dry_run)
 352                         t->clean();
 353         }
 354
 355         return 0;
 356 }
 357
 358
 359 Builder::LoadedPlugin::LoadedPlugin(LoadedPlugin &&other):
 360         path(move(other.path)),
 361         module(other.module),
 362         plugin(other.plugin)
 363 {
 364         other.module = 0;
 365         other.plugin = 0;
 366 }
 367
 368 Builder::LoadedPlugin::~LoadedPlugin()
 369 {
 370         delete plugin;
 371         delete module;
 372 }
 373
 374
 375 Builder::Loader::Loader(Builder &b, const Config::InputOptions *o, bool a):
 376         DataFile::ObjectLoader<Builder>(b),
 377         options(o),
 378         conf_all(a)
 379 {
 380         add("architecture", &Loader::architecture);
 381         add("binary_package", &Loader::binpkg);
 382         add("build_type", &Loader::build_type);
 383         add("package", &Loader::package);
 384
 385         if(!obj.top_loader)
 386                 obj.top_loader = this;
 387         else if(!options && obj.top_loader!=this && obj.top_loader->conf_all)
 388                 options = obj.top_loader->options;
 389 }
 390
 391 Builder::Loader::~Loader()
 392 {
 393         if(obj.top_loader==this)
 394                 obj.top_loader = 0;
 395 }
 396
 397 void Builder::Loader::architecture(const string &n)
 398 {
 399         if(obj.current_arch->match_name(n))
 400                 load_sub(*obj.current_arch);
 401 }
 402
 403 void Builder::Loader::binpkg(const string &n)
 404 {
 405         BinaryPackage *pkg = new BinaryPackage(obj, n);
 406         load_sub(*pkg);
 407 }
 408
 409 void Builder::Loader::build_type(const string &n)
 410 {
 411         BuildType btype(n);
 412         load_sub(btype);
 413         auto i = obj.build_types.insert({ n, btype }).first;
 414         if(!obj.build_type)
 415                 obj.build_type = &i->second;
 416 }
 417
 418 void Builder::Loader::package(const string &n)
 419 {
 420         SourcePackage *pkg = new SourcePackage(obj, n, get_source());
 421
 422         load_sub(*pkg, options);
 423
 424         if(obj.build_type)
 425                 pkg->set_build_type(*obj.build_type);
 426 }