source/builder.cpp

   1 #include <deque>
   2 #include <set>
   3 #include <msp/core/except.h>
   4 #include <msp/core/maputils.h>
   5 #include <msp/datafile/parser.h>
   6 #include <msp/fs/dir.h>
   7 #include <msp/fs/utils.h>
   8 #include <msp/io/buffered.h>
   9 #include <msp/io/file.h>
  10 #include <msp/io/print.h>
  11 #include <msp/strings/format.h>
  12 #include <msp/time/timedelta.h>
  13 #include <msp/time/utils.h>
  14 #include "androidtools.h"
  15 #include "binarypackage.h"
  16 #include "builder.h"
  17 #include "builtintools.h"
  18 #include "clangtools.h"
  19 #include "datatool.h"
  20 #include "gnutools.h"
  21 #include "installedfile.h"
  22 #include "microsofttools.h"
  23 #include "package.h"
  24 #include "sharedlibrary.h"
  25 #include "sourcepackage.h"
  26 #include "task.h"
  27 #include "virtualtarget.h"
  28
  29 using namespace std;
  30 using namespace Msp;
  31
  32 Builder::Builder():
  33         package_manager(*this),
  34         native_arch(*this, string()),
  35         current_arch(0),
  36         build_type(0),
  37         vfs(*this),
  38         build_graph(*this),
  39         logger(&default_logger),
  40         tempdir("temp"),
  41         top_loader(0)
  42 {
  43         set_architecture(string());
  44 }
  45
  46 Builder::~Builder()
  47 {
  48         if(current_arch!=&native_arch)
  49                 delete current_arch;
  50 }
  51
  52 void Builder::set_architecture(const string &name)
  53 {
  54         if(name.empty())
  55         {
  56                 current_arch = &native_arch;
  57                 prefix = FS::get_home_dir()/"local";
  58         }
  59         else
  60         {
  61                 current_arch = new Architecture(*this, name);
  62                 prefix = FS::get_home_dir()/"local"/current_arch->get_name();
  63         }
  64 }
  65
  66 vector<string> Builder::get_build_types() const
  67 {
  68         vector<string> keys;
  69         keys.reserve(build_types.size());
  70         for(const auto &kvp: build_types)
  71                 keys.push_back(kvp.first);
  72         return keys;
  73 }
  74
  75 const BuildType &Builder::get_build_type() const
  76 {
  77         if(!build_type)
  78                 throw invalid_state("no build type");
  79         return *build_type;
  80 }
  81
  82 void Builder::set_build_type(const string &name)
  83 {
  84         build_type = &get_item(build_types, name);
  85 }
  86
  87 void Builder::set_prefix(const FS::Path &p)
  88 {
  89         prefix = p;
  90 }
  91
  92 void Builder::set_temp_directory(const FS::Path &p)
  93 {
  94         tempdir = p;
  95 }
  96
  97 void Builder::add_default_tools()
  98 {
  99         const string &arch_tc = current_arch->get_toolchain();
 100         if(current_arch->get_system()=="android")
 101                 toolchain.add_toolchain(new AndroidTools(*this, *current_arch));
 102         else if(arch_tc=="msvc")
 103                 toolchain.add_toolchain(new MicrosoftTools(*this, *current_arch));
 104         else if(arch_tc=="clang")
 105                 toolchain.add_toolchain(new ClangTools(*this, *current_arch));
 106         else if(arch_tc=="gnu")
 107                 toolchain.add_toolchain(new GnuTools(*this, *current_arch));
 108         toolchain.add_toolchain(new BuiltinTools(*this));
 109         toolchain.add_tool(new DataTool(*this));
 110 }
 111
 112 void Builder::set_logger(const Logger *l)
 113 {
 114         logger = (l ? l : &default_logger);
 115 }
 116
 117 vector<string> Builder::collect_problems() const
 118 {
 119         vector<string> problems;
 120         set<const Package *> broken_packages;
 121         set<const Component *> broken_components;
 122         set<const Tool *> broken_tools;
 123
 124         for(const auto &kvp: build_graph.get_targets())
 125                 if(kvp.second->is_broken())
 126                 {
 127                         for(const string &p: kvp.second->get_problems())
 128                                 problems.push_back(format("%s: %s", kvp.second->get_name(), p));
 129
 130                         const Package *package = kvp.second->get_package();
 131                         if(package && !package->get_problems().empty())
 132                                 broken_packages.insert(package);
 133
 134                         const Component *component = kvp.second->get_component();
 135                         if(component && !component->get_problems().empty())
 136                                 broken_components.insert(component);
 137
 138                         const Tool *tool = kvp.second->get_tool();
 139                         if(tool && !tool->get_problems().empty())
 140                                 broken_tools.insert(tool);
 141                 }
 142
 143         // TODO Sort components after their packages, and targets last
 144         for(const Package *p: broken_packages)
 145                 for(const string &b: p->get_problems())
 146                         problems.push_back(format("%s: %s", p->get_name(), b));
 147
 148         for(const Component *c: broken_components)
 149                 for(const string &b: c->get_problems())
 150                         problems.push_back(format("%s/%s: %s", c->get_package().get_name(), c->get_name(), b));
 151
 152         for(const Tool *t: broken_tools)
 153                 for(const string &b: t->get_problems())
 154                         problems.push_back(format("%s: %s", t->get_tag(), b));
 155
 156         return problems;
 157 }
 158
 159 void Builder::load_build_file(const FS::Path &fn, const Config::InputOptions *opts, bool all)
 160 {
 161         IO::BufferedFile in(fn.str());
 162
 163         get_logger().log("files", "Reading %s", fn);
 164
 165         DataFile::Parser parser(in, fn.str());
 166         Loader loader(*this, opts, all);
 167         loader.load(parser);
 168 }
 169
 170 void Builder::save_caches()
 171 {
 172         for(const auto &kvp: package_manager.get_packages())
 173                 kvp.second->save_caches();
 174 }
 175
 176 int Builder::build(unsigned jobs, bool dry_run, bool show_progress)
 177 {
 178         unsigned total = build_graph.count_rebuild_targets();
 179
 180         if(!total)
 181         {
 182                 get_logger().log("summary", "Already up to date");
 183                 return 0;
 184         }
 185         get_logger().log("summary", "Will build %d target%s", total, (total!=1 ? "s" : ""));
 186
 187         vector<Task *> tasks;
 188
 189         unsigned count = 0;
 190
 191         bool fail = false;
 192         bool finish = false;
 193         bool starved = false;
 194
 195         while(!finish)
 196         {
 197                 if(tasks.size()<jobs && !fail && !starved)
 198                 {
 199                         Target *tgt = build_graph.get_buildable_target();
 200                         if(tgt)
 201                         {
 202                                 if(tgt->get_tool())
 203                                         get_logger().log("tasks", "%-4s  %s", tgt->get_tool()->get_tag(), tgt->get_name());
 204                                 Task *task = tgt->build();
 205                                 if(task)
 206                                 {
 207                                         get_logger().log("commands", "%s", task->get_command());
 208                                         if(dry_run)
 209                                         {
 210                                                 task->signal_finished.emit(true);
 211                                                 delete task;
 212                                         }
 213                                         else
 214                                         {
 215                                                 task->start();
 216                                                 tasks.push_back(task);
 217                                         }
 218                                 }
 219
 220                                 if(show_progress)
 221                                         IO::print("%d of %d target%s built\033[1G", count, total, (total!=1 ? "s" : ""));
 222                         }
 223                         else if(tasks.empty())
 224                                 finish = true;
 225                         else
 226                                 starved = true;
 227                 }
 228                 else
 229                         Time::sleep(10*Time::msec);
 230
 231                 for(unsigned i=0; i<tasks.size();)
 232                 {
 233                         Task::Status status;
 234                         if(jobs==1 || (tasks.size()==1 && starved))
 235                                 status = tasks[i]->wait();
 236                         else
 237                                 status = tasks[i]->check();
 238
 239                         if(status!=Task::RUNNING)
 240                         {
 241                                 ++count;
 242
 243                                 delete tasks[i];
 244                                 tasks.erase(tasks.begin()+i);
 245                                 if(status==Task::ERROR)
 246                                         fail = true;
 247                                 if(tasks.empty() && fail)
 248                                         finish = true;
 249                                 starved = false;
 250                         }
 251                         else
 252                                 ++i;
 253                 }
 254         }
 255
 256         if(show_progress)
 257                 IO::print("\033[K");
 258         if(fail)
 259                 get_logger().log("summary", "Build failed");
 260         else if(show_progress)
 261                 get_logger().log("summary", "Build complete");
 262
 263         return fail;
 264 }
 265
 266 int Builder::clean(bool all, bool dry_run)
 267 {
 268         // Cleaning doesn't care about ordering, so a simpler method can be used
 269
 270         set<Target *> clean_tgts;
 271         deque<Target *> queue;
 272         queue.push_back(&build_graph.get_goals());
 273
 274         while(!queue.empty())
 275         {
 276                 Target *tgt = queue.front();
 277                 queue.pop_front();
 278
 279                 if(tgt->is_buildable() && (tgt->get_package()==&package_manager.get_main_package() || all))
 280                         clean_tgts.insert(tgt);
 281
 282                 for(Target *t: tgt->get_dependencies())
 283                         if(!clean_tgts.count(t))
 284                                 queue.push_back(t);
 285         }
 286
 287         for(Target *t: clean_tgts)
 288         {
 289                 get_logger().log("tasks", "RM    %s", t->get_name());
 290                 if(!dry_run)
 291                         t->clean();
 292         }
 293
 294         return 0;
 295 }
 296
 297
 298 Builder::Loader::Loader(Builder &b, const Config::InputOptions *o, bool a):
 299         DataFile::ObjectLoader<Builder>(b),
 300         options(o),
 301         conf_all(a)
 302 {
 303         add("architecture", &Loader::architecture);
 304         add("binary_package", &Loader::binpkg);
 305         add("build_type", &Loader::build_type);
 306         add("package", &Loader::package);
 307
 308         if(!obj.top_loader)
 309                 obj.top_loader = this;
 310         else if(!options && obj.top_loader!=this && obj.top_loader->conf_all)
 311                 options = obj.top_loader->options;
 312 }
 313
 314 Builder::Loader::~Loader()
 315 {
 316         if(obj.top_loader==this)
 317                 obj.top_loader = 0;
 318 }
 319
 320 void Builder::Loader::architecture(const string &n)
 321 {
 322         if(obj.current_arch->match_name(n))
 323                 load_sub(*obj.current_arch);
 324 }
 325
 326 void Builder::Loader::binpkg(const string &n)
 327 {
 328         BinaryPackage *pkg = new BinaryPackage(obj, n);
 329         load_sub(*pkg);
 330 }
 331
 332 void Builder::Loader::build_type(const string &n)
 333 {
 334         BuildType btype(n);
 335         load_sub(btype);
 336         auto i = obj.build_types.insert({ n, btype }).first;
 337         if(!obj.build_type)
 338                 obj.build_type = &i->second;
 339 }
 340
 341 void Builder::Loader::package(const string &n)
 342 {
 343         SourcePackage *pkg = new SourcePackage(obj, n, get_source());
 344
 345         load_sub(*pkg, options);
 346
 347         if(obj.build_type)
 348                 pkg->set_build_type(*obj.build_type);
 349 }