source/builder.cpp

   1 #include <deque>
   2 #include <set>
   3 #include <msp/core/algorithm.h>
   4 #include <msp/core/except.h>
   5 #include <msp/core/maputils.h>
   6 #include <msp/datafile/parser.h>
   7 #include <msp/fs/dir.h>
   8 #include <msp/fs/utils.h>
   9 #include <msp/io/buffered.h>
  10 #include <msp/io/file.h>
  11 #include <msp/io/print.h>
  12 #include <msp/strings/format.h>
  13 #include <msp/time/timedelta.h>
  14 #include <msp/time/utils.h>
  15 #include "androidtools.h"
  16 #include "binarypackage.h"
  17 #include "builder.h"
  18 #include "builtintools.h"
  19 #include "clangtools.h"
  20 #include "datatool.h"
  21 #include "gnutools.h"
  22 #include "installedfile.h"
  23 #include "microsofttools.h"
  24 #include "package.h"
  25 #include "sharedlibrary.h"
  26 #include "sourcepackage.h"
  27 #include "task.h"
  28 #include "virtualtarget.h"
  29
  30 using namespace std;
  31 using namespace Msp;
  32
  33 Builder::Builder():
  34         package_manager(*this),
  35         native_arch(*this, string()),
  36         vfs(*this),
  37         build_graph(*this),
  38         logger(&default_logger)
  39 {
  40         set_architecture(string());
  41 }
  42
  43 Builder::~Builder()
  44 {
  45         if(current_arch!=&native_arch)
  46                 delete current_arch;
  47 }
  48
  49 void Builder::set_architecture(const string &name)
  50 {
  51         if(current_arch!=&native_arch)
  52                 delete current_arch;
  53
  54         if(name.empty())
  55                 current_arch = &native_arch;
  56         else
  57                 current_arch = new Architecture(*this, name);
  58
  59         if(auto_prefix)
  60                 update_auto_prefix();
  61 }
  62
  63 vector<string> Builder::get_build_types() const
  64 {
  65         vector<string> keys;
  66         keys.reserve(build_types.size());
  67         for(const auto &kvp: build_types)
  68                 keys.push_back(kvp.first);
  69         return keys;
  70 }
  71
  72 const BuildType &Builder::get_build_type() const
  73 {
  74         if(!build_type)
  75                 throw invalid_state("no build type");
  76         return *build_type;
  77 }
  78
  79 void Builder::set_build_type(const string &name)
  80 {
  81         build_type = &get_item(build_types, name);
  82 }
  83
  84 void Builder::set_prefix(const FS::Path &p)
  85 {
  86         auto_prefix = false;
  87         prefix = p;
  88 }
  89
  90 void Builder::set_temp_directory(const FS::Path &p)
  91 {
  92         tempdir = p;
  93 }
  94
  95 void Builder::update_auto_prefix()
  96 {
  97         if(current_arch->is_native())
  98                 prefix = FS::get_home_dir()/"local";
  99         else
 100                 prefix = FS::get_home_dir()/"local"/current_arch->get_name();
 101 }
 102
 103 void Builder::add_default_tools()
 104 {
 105         toolchain.add_toolchain(new GnuTools(*this, *current_arch));
 106         toolchain.add_toolchain(new ClangTools(*this, *current_arch));
 107         if(current_arch->get_system()=="android")
 108                 toolchain.add_toolchain(new AndroidTools(*this, *current_arch));
 109         if(current_arch->get_system()=="windows")
 110                 toolchain.add_toolchain(new MicrosoftTools(*this, *current_arch));
 111         toolchain.add_toolchain(new BuiltinTools(*this));
 112         toolchain.add_tool(new DataTool(*this));
 113
 114         auto i = find_if(toolchain.get_toolchains(), [](const Toolchain *tc){ return (tc->has_tool("CC") || tc->has_tool("CXX")); });
 115         if(i!=toolchain.get_toolchains().end())
 116         {
 117                 current_arch->refine((*i)->get_name());
 118                 if(auto_prefix)
 119                         update_auto_prefix();
 120         }
 121 }
 122
 123 void Builder::set_logger(const Logger *l)
 124 {
 125         logger = (l ? l : &default_logger);
 126 }
 127
 128 vector<string> Builder::collect_problems() const
 129 {
 130         vector<string> problems;
 131         set<const Package *> broken_packages;
 132         set<const Component *> broken_components;
 133         set<const Tool *> broken_tools;
 134
 135         for(const auto &kvp: build_graph.get_targets())
 136                 if(kvp.second->is_broken())
 137                 {
 138                         for(const string &p: kvp.second->get_problems())
 139                                 problems.push_back(format("%s: %s", kvp.second->get_name(), p));
 140
 141                         const Package *package = kvp.second->get_package();
 142                         if(package && !package->get_problems().empty())
 143                                 broken_packages.insert(package);
 144
 145                         const Component *component = kvp.second->get_component();
 146                         if(component && !component->get_problems().empty())
 147                                 broken_components.insert(component);
 148
 149                         const Tool *tool = kvp.second->get_tool();
 150                         if(tool && !tool->get_problems().empty())
 151                                 broken_tools.insert(tool);
 152                 }
 153
 154         // TODO Sort components after their packages, and targets last
 155         for(const Package *p: broken_packages)
 156                 for(const string &b: p->get_problems())
 157                         problems.push_back(format("%s: %s", p->get_name(), b));
 158
 159         for(const Component *c: broken_components)
 160                 for(const string &b: c->get_problems())
 161                         problems.push_back(format("%s/%s: %s", c->get_package().get_name(), c->get_name(), b));
 162
 163         for(const Tool *t: broken_tools)
 164                 for(const string &b: t->get_problems())
 165                         problems.push_back(format("%s: %s", t->get_tag(), b));
 166
 167         return problems;
 168 }
 169
 170 void Builder::load_build_file(const FS::Path &fn, const Config::InputOptions *opts, bool all)
 171 {
 172         IO::BufferedFile in(fn.str());
 173
 174         get_logger().log("files", "Reading %s", fn);
 175
 176         DataFile::Parser parser(in, fn.str());
 177         Loader loader(*this, opts, all);
 178         loader.load(parser);
 179 }
 180
 181 void Builder::save_caches()
 182 {
 183         for(const auto &kvp: package_manager.get_packages())
 184                 kvp.second->save_caches();
 185 }
 186
 187 int Builder::build(unsigned jobs, bool dry_run, bool show_progress)
 188 {
 189         unsigned total = build_graph.count_rebuild_targets();
 190
 191         if(!total)
 192         {
 193                 get_logger().log("summary", "Already up to date");
 194                 return 0;
 195         }
 196         get_logger().log("summary", "Will build %d target%s", total, (total!=1 ? "s" : ""));
 197
 198         vector<Task *> tasks;
 199
 200         unsigned count = 0;
 201
 202         bool fail = false;
 203         bool finish = false;
 204         bool starved = false;
 205
 206         while(!finish)
 207         {
 208                 if(tasks.size()<jobs && !fail && !starved)
 209                 {
 210                         Target *tgt = build_graph.get_buildable_target();
 211                         if(tgt)
 212                         {
 213                                 if(tgt->get_tool())
 214                                 {
 215                                         if(show_progress)
 216                                                 IO::print("\033[K");
 217                                         get_logger().log("tasks", "%-4s  %s", tgt->get_tool()->get_tag(), tgt->get_name());
 218                                 }
 219                                 Task *task = tgt->build();
 220                                 if(task)
 221                                 {
 222                                         get_logger().log("commands", "%s", task->get_command());
 223                                         if(dry_run)
 224                                         {
 225                                                 task->signal_finished.emit(true);
 226                                                 delete task;
 227                                         }
 228                                         else
 229                                         {
 230                                                 task->start();
 231                                                 tasks.push_back(task);
 232                                         }
 233                                 }
 234
 235                                 if(show_progress)
 236                                         IO::print("%d of %d target%s built\033[1G", count, total, (total!=1 ? "s" : ""));
 237                         }
 238                         else if(tasks.empty())
 239                                 finish = true;
 240                         else
 241                                 starved = true;
 242                 }
 243                 else
 244                         Time::sleep(10*Time::msec);
 245
 246                 for(unsigned i=0; i<tasks.size();)
 247                 {
 248                         Task::Status status;
 249                         if(jobs==1 || (tasks.size()==1 && starved))
 250                                 status = tasks[i]->wait();
 251                         else
 252                                 status = tasks[i]->check();
 253
 254                         if(status!=Task::RUNNING)
 255                         {
 256                                 ++count;
 257
 258                                 delete tasks[i];
 259                                 tasks.erase(tasks.begin()+i);
 260                                 if(status==Task::ERROR)
 261                                         fail = true;
 262                                 if(tasks.empty() && fail)
 263                                         finish = true;
 264                                 starved = false;
 265                         }
 266                         else
 267                                 ++i;
 268                 }
 269         }
 270
 271         if(show_progress)
 272                 IO::print("\033[K");
 273         if(fail)
 274                 get_logger().log("summary", "Build failed");
 275         else if(show_progress)
 276                 get_logger().log("summary", "Build complete");
 277
 278         return fail;
 279 }
 280
 281 int Builder::clean(bool all, bool dry_run)
 282 {
 283         // Cleaning doesn't care about ordering, so a simpler method can be used
 284
 285         set<Target *> clean_tgts;
 286         deque<Target *> queue;
 287         queue.push_back(&build_graph.get_goals());
 288
 289         while(!queue.empty())
 290         {
 291                 Target *tgt = queue.front();
 292                 queue.pop_front();
 293
 294                 if(tgt->is_buildable() && (tgt->get_package()==&package_manager.get_main_package() || all))
 295                         clean_tgts.insert(tgt);
 296
 297                 for(Target *t: tgt->get_dependencies())
 298                         if(!clean_tgts.count(t))
 299                                 queue.push_back(t);
 300         }
 301
 302         for(Target *t: clean_tgts)
 303         {
 304                 get_logger().log("tasks", "RM    %s", t->get_name());
 305                 if(!dry_run)
 306                         t->clean();
 307         }
 308
 309         return 0;
 310 }
 311
 312
 313 Builder::Loader::Loader(Builder &b, const Config::InputOptions *o, bool a):
 314         DataFile::ObjectLoader<Builder>(b),
 315         options(o),
 316         conf_all(a)
 317 {
 318         add("architecture", &Loader::architecture);
 319         add("binary_package", &Loader::binpkg);
 320         add("build_type", &Loader::build_type);
 321         add("package", &Loader::package);
 322
 323         if(!obj.top_loader)
 324                 obj.top_loader = this;
 325         else if(!options && obj.top_loader!=this && obj.top_loader->conf_all)
 326                 options = obj.top_loader->options;
 327 }
 328
 329 Builder::Loader::~Loader()
 330 {
 331         if(obj.top_loader==this)
 332                 obj.top_loader = 0;
 333 }
 334
 335 void Builder::Loader::architecture(const string &n)
 336 {
 337         if(obj.current_arch->match_name(n))
 338                 load_sub(*obj.current_arch);
 339 }
 340
 341 void Builder::Loader::binpkg(const string &n)
 342 {
 343         BinaryPackage *pkg = new BinaryPackage(obj, n);
 344         load_sub(*pkg);
 345 }
 346
 347 void Builder::Loader::build_type(const string &n)
 348 {
 349         BuildType btype(n);
 350         load_sub(btype);
 351         auto i = obj.build_types.insert({ n, btype }).first;
 352         if(!obj.build_type)
 353                 obj.build_type = &i->second;
 354 }
 355
 356 void Builder::Loader::package(const string &n)
 357 {
 358         SourcePackage *pkg = new SourcePackage(obj, n, get_source());
 359
 360         load_sub(*pkg, options);
 361
 362         if(obj.build_type)
 363                 pkg->set_build_type(*obj.build_type);
 364 }