source/lib/builder.cpp

   1 #include <deque>
   2 #include <set>
   3 #include <msp/core/algorithm.h>
   4 #include <msp/core/except.h>
   5 #include <msp/core/maputils.h>
   6 #include <msp/datafile/parser.h>
   7 #include <msp/fs/dir.h>
   8 #include <msp/fs/utils.h>
   9 #include <msp/io/buffered.h>
  10 #include <msp/io/file.h>
  11 #include <msp/io/print.h>
  12 #include <msp/strings/format.h>
  13 #include <msp/time/timedelta.h>
  14 #include <msp/time/utils.h>
  15 #include "android/androidtools.h"
  16 #include "binarypackage.h"
  17 #include "builder.h"
  18 #include "datafile/datatool.h"
  19 #include "installedfile.h"
  20 #include "package.h"
  21 #include "plugin.h"
  22 #include "sharedlibrary.h"
  23 #include "task.h"
  24 #include "virtualtarget.h"
  25
  26 using namespace std;
  27 using namespace Msp;
  28
  29 Builder::Builder():
  30         package_manager(*this),
  31         native_arch(*this, string()),
  32         vfs(*this),
  33         build_graph(*this),
  34         logger(&default_logger)
  35 {
  36         set_architecture(string());
  37 }
  38
  39 Builder::~Builder()
  40 {
  41         if(current_arch!=&native_arch)
  42                 delete current_arch;
  43 }
  44
  45 void Builder::load_plugins()
  46 {
  47         using CreateFunc = Plugin *(Builder &);
  48
  49         FS::Path plugins_dir = FS::get_sys_lib_dir();
  50         logger->log("files", "Traversing %s", plugins_dir);
  51         for(const string &f: list_filtered(plugins_dir, "\\.dlm$"))
  52         {
  53                 LoadedPlugin plugin;
  54                 plugin.path = plugins_dir/f;
  55
  56                 try
  57                 {
  58                         plugin.module = new Module(plugin.path.str());
  59                 }
  60                 catch(const exception &exc)
  61                 {
  62                         logger->log("plugins", "Failed to load plugin %s: %s", f, exc.what());
  63                         continue;
  64                 }
  65
  66                 try
  67                 {
  68                         CreateFunc *create_func = reinterpret_cast<CreateFunc *>(plugin.module->get_symbol("create_plugin"));
  69                         plugin.plugin = create_func(*this);
  70                         if(plugin.plugin)
  71                         {
  72                                 logger->log("plugins", "Loaded plugin %s", f);
  73                                 plugins.emplace_back(move(plugin));
  74                                 continue;
  75                         }
  76                         else
  77                                 logger->log("plugins", "Plugin %s refused to initialize", f);
  78                 }
  79                 catch(const exception &exc)
  80                 {
  81                         logger->log("plugins", "Failed to initialize plugin %s: %s", f, exc.what());
  82                 }
  83
  84                 delete plugin.module;
  85         }
  86 }
  87
  88 void Builder::set_architecture(const string &name)
  89 {
  90         if(current_arch!=&native_arch)
  91                 delete current_arch;
  92
  93         if(name.empty())
  94                 current_arch = &native_arch;
  95         else
  96                 current_arch = new Architecture(*this, name);
  97
  98         if(auto_prefix)
  99                 update_auto_prefix();
 100 }
 101
 102 vector<string> Builder::get_build_types() const
 103 {
 104         vector<string> keys;
 105         keys.reserve(build_types.size());
 106         for(const auto &kvp: build_types)
 107                 keys.push_back(kvp.first);
 108         return keys;
 109 }
 110
 111 const BuildType &Builder::get_build_type() const
 112 {
 113         if(!build_type)
 114                 throw invalid_state("no build type");
 115         return *build_type;
 116 }
 117
 118 void Builder::set_build_type(const string &name)
 119 {
 120         build_type = &get_item(build_types, name);
 121 }
 122
 123 void Builder::set_prefix(const FS::Path &p)
 124 {
 125         auto_prefix = false;
 126         prefix = p;
 127 }
 128
 129 void Builder::set_temp_directory(const FS::Path &p)
 130 {
 131         tempdir = p;
 132 }
 133
 134 void Builder::update_auto_prefix()
 135 {
 136         if(current_arch->is_native())
 137                 prefix = FS::get_home_dir()/"local";
 138         else
 139                 prefix = FS::get_home_dir()/"local"/current_arch->get_name();
 140 }
 141
 142 void Builder::add_default_tools()
 143 {
 144         if(current_arch->get_system()=="android")
 145                 toolchain.add_toolchain(new AndroidTools(*this, *current_arch));
 146         toolchain.add_tool(new DataTool(*this));
 147         for(const LoadedPlugin &p: plugins)
 148                 p.plugin->add_tools(toolchain, *current_arch);
 149
 150         auto i = find_if(toolchain.get_toolchains(), [](const Toolchain *tc){ return (tc->has_tool("CC") || tc->has_tool("CXX")); });
 151         if(i!=toolchain.get_toolchains().end())
 152         {
 153                 current_arch->refine((*i)->get_name());
 154                 if(auto_prefix)
 155                         update_auto_prefix();
 156         }
 157 }
 158
 159 void Builder::set_logger(const Logger *l)
 160 {
 161         logger = (l ? l : &default_logger);
 162 }
 163
 164 vector<string> Builder::collect_problems() const
 165 {
 166         vector<string> problems;
 167         set<const Package *> broken_packages;
 168         set<const Component *> broken_components;
 169         set<const Tool *> broken_tools;
 170
 171         for(const auto &kvp: build_graph.get_targets())
 172                 if(kvp.second->is_broken())
 173                 {
 174                         for(const string &p: kvp.second->get_problems())
 175                                 problems.push_back(format("%s: %s", kvp.second->get_name(), p));
 176
 177                         const Package *package = kvp.second->get_package();
 178                         if(package && !package->get_problems().empty())
 179                                 broken_packages.insert(package);
 180
 181                         const Component *component = kvp.second->get_component();
 182                         if(component && !component->get_problems().empty())
 183                                 broken_components.insert(component);
 184
 185                         const Tool *tool = kvp.second->get_tool();
 186                         if(tool && !tool->get_problems().empty())
 187                                 broken_tools.insert(tool);
 188                 }
 189
 190         // TODO Sort components after their packages, and targets last
 191         for(const Package *p: broken_packages)
 192                 for(const string &b: p->get_problems())
 193                         problems.push_back(format("%s: %s", p->get_name(), b));
 194
 195         for(const Component *c: broken_components)
 196                 for(const string &b: c->get_problems())
 197                         problems.push_back(format("%s/%s: %s", c->get_package().get_name(), c->get_name(), b));
 198
 199         for(const Tool *t: broken_tools)
 200                 for(const string &b: t->get_problems())
 201                         problems.push_back(format("%s: %s", t->get_tag(), b));
 202
 203         return problems;
 204 }
 205
 206 void Builder::load_build_file(const FS::Path &fn, const Config::InputOptions *opts, bool all)
 207 {
 208         IO::BufferedFile in(fn.str());
 209
 210         get_logger().log("files", "Reading %s", fn);
 211
 212         DataFile::Parser parser(in, fn.str());
 213         Loader loader(*this, opts, all);
 214         loader.load(parser);
 215 }
 216
 217 void Builder::save_caches()
 218 {
 219         for(const auto &kvp: package_manager.get_packages())
 220                 kvp.second->save_caches();
 221 }
 222
 223 int Builder::build(unsigned jobs, bool dry_run, bool show_progress)
 224 {
 225         unsigned total = build_graph.count_rebuild_targets();
 226
 227         if(!total)
 228         {
 229                 get_logger().log("summary", "Already up to date");
 230                 return 0;
 231         }
 232         get_logger().log("summary", "Will build %d target%s", total, (total!=1 ? "s" : ""));
 233
 234         vector<Task *> tasks;
 235
 236         unsigned count = 0;
 237
 238         bool fail = false;
 239         bool finish = false;
 240         bool starved = false;
 241
 242         while(!finish)
 243         {
 244                 if(tasks.size()<jobs && !fail && !starved)
 245                 {
 246                         Target *tgt = build_graph.get_buildable_target();
 247                         if(tgt)
 248                         {
 249                                 if(tgt->get_tool())
 250                                 {
 251                                         if(show_progress)
 252                                                 IO::print("\033[K");
 253                                         get_logger().log("tasks", "%-4s  %s", tgt->get_tool()->get_tag(), tgt->get_name());
 254                                 }
 255                                 Task *task = tgt->build();
 256                                 if(task)
 257                                 {
 258                                         get_logger().log("commands", "%s", task->get_command());
 259                                         if(dry_run)
 260                                         {
 261                                                 task->signal_finished.emit(true);
 262                                                 delete task;
 263                                         }
 264                                         else
 265                                         {
 266                                                 task->start();
 267                                                 tasks.push_back(task);
 268                                         }
 269                                 }
 270
 271                                 if(show_progress)
 272                                         IO::print("%d of %d target%s built\033[1G", count, total, (total!=1 ? "s" : ""));
 273                         }
 274                         else if(tasks.empty())
 275                                 finish = true;
 276                         else
 277                                 starved = true;
 278                 }
 279                 else
 280                         Time::sleep(10*Time::msec);
 281
 282                 for(unsigned i=0; i<tasks.size();)
 283                 {
 284                         Task::Status status;
 285                         if(jobs==1 || (tasks.size()==1 && starved))
 286                                 status = tasks[i]->wait();
 287                         else
 288                                 status = tasks[i]->check();
 289
 290                         if(status!=Task::RUNNING)
 291                         {
 292                                 ++count;
 293
 294                                 delete tasks[i];
 295                                 tasks.erase(tasks.begin()+i);
 296                                 if(status==Task::ERROR)
 297                                         fail = true;
 298                                 if(tasks.empty() && fail)
 299                                         finish = true;
 300                                 starved = false;
 301                         }
 302                         else
 303                                 ++i;
 304                 }
 305         }
 306
 307         if(show_progress)
 308                 IO::print("\033[K");
 309         if(fail)
 310                 get_logger().log("summary", "Build failed");
 311         else if(show_progress)
 312                 get_logger().log("summary", "Build complete");
 313
 314         return fail;
 315 }
 316
 317 int Builder::clean(bool all, bool dry_run)
 318 {
 319         // Cleaning doesn't care about ordering, so a simpler method can be used
 320
 321         set<Target *> clean_tgts;
 322         deque<Target *> queue;
 323         queue.push_back(&build_graph.get_goals());
 324
 325         while(!queue.empty())
 326         {
 327                 Target *tgt = queue.front();
 328                 queue.pop_front();
 329
 330                 if(tgt->is_buildable() && (tgt->get_package()==&package_manager.get_main_package() || all))
 331                         clean_tgts.insert(tgt);
 332
 333                 for(Target *t: tgt->get_dependencies())
 334                         if(!clean_tgts.count(t))
 335                                 queue.push_back(t);
 336         }
 337
 338         for(Target *t: clean_tgts)
 339         {
 340                 get_logger().log("tasks", "RM    %s", t->get_name());
 341                 if(!dry_run)
 342                         t->clean();
 343         }
 344
 345         return 0;
 346 }
 347
 348
 349 Builder::LoadedPlugin::LoadedPlugin(LoadedPlugin &&other):
 350         path(move(other.path)),
 351         module(other.module),
 352         plugin(other.plugin)
 353 {
 354         other.module = 0;
 355         other.plugin = 0;
 356 }
 357
 358 Builder::LoadedPlugin::~LoadedPlugin()
 359 {
 360         delete plugin;
 361         delete module;
 362 }
 363
 364
 365 Builder::Loader::Loader(Builder &b, const Config::InputOptions *o, bool a):
 366         DataFile::ObjectLoader<Builder>(b),
 367         options(o),
 368         conf_all(a)
 369 {
 370         add("architecture", &Loader::architecture);
 371         add("binary_package", &Loader::binpkg);
 372         add("build_type", &Loader::build_type);
 373         add("package", &Loader::package);
 374
 375         if(!obj.top_loader)
 376                 obj.top_loader = this;
 377         else if(!options && obj.top_loader!=this && obj.top_loader->conf_all)
 378                 options = obj.top_loader->options;
 379 }
 380
 381 Builder::Loader::~Loader()
 382 {
 383         if(obj.top_loader==this)
 384                 obj.top_loader = 0;
 385 }
 386
 387 void Builder::Loader::architecture(const string &n)
 388 {
 389         if(obj.current_arch->match_name(n))
 390                 load_sub(*obj.current_arch);
 391 }
 392
 393 void Builder::Loader::binpkg(const string &n)
 394 {
 395         BinaryPackage *pkg = new BinaryPackage(obj, n);
 396         load_sub(*pkg);
 397 }
 398
 399 void Builder::Loader::build_type(const string &n)
 400 {
 401         BuildType btype(n);
 402         load_sub(btype);
 403         auto i = obj.build_types.insert({ n, btype }).first;
 404         if(!obj.build_type)
 405                 obj.build_type = &i->second;
 406 }
 407
 408 void Builder::Loader::package(const string &n)
 409 {
 410         SourcePackage *pkg = new SourcePackage(obj, n, get_source());
 411
 412         load_sub(*pkg, options);
 413
 414         if(obj.build_type)
 415                 pkg->set_build_type(*obj.build_type);
 416 }