source/builder.cpp

   1 #include <deque>
   2 #include <set>
   3 #include <msp/core/except.h>
   4 #include <msp/core/maputils.h>
   5 #include <msp/datafile/parser.h>
   6 #include <msp/fs/dir.h>
   7 #include <msp/fs/utils.h>
   8 #include <msp/io/buffered.h>
   9 #include <msp/io/file.h>
  10 #include <msp/io/print.h>
  11 #include <msp/strings/format.h>
  12 #include <msp/time/timedelta.h>
  13 #include <msp/time/utils.h>
  14 #include "androidtools.h"
  15 #include "binarypackage.h"
  16 #include "builder.h"
  17 #include "builtintools.h"
  18 #include "clangtools.h"
  19 #include "datatool.h"
  20 #include "gnutools.h"
  21 #include "installedfile.h"
  22 #include "microsofttools.h"
  23 #include "package.h"
  24 #include "sharedlibrary.h"
  25 #include "sourcepackage.h"
  26 #include "task.h"
  27 #include "virtualtarget.h"
  28
  29 using namespace std;
  30 using namespace Msp;
  31
  32 Builder::Builder():
  33         package_manager(*this),
  34         native_arch(*this, string()),
  35         vfs(*this),
  36         build_graph(*this),
  37         logger(&default_logger)
  38 {
  39         set_architecture(string());
  40 }
  41
  42 Builder::~Builder()
  43 {
  44         if(current_arch!=&native_arch)
  45                 delete current_arch;
  46 }
  47
  48 void Builder::set_architecture(const string &name)
  49 {
  50         if(name.empty())
  51         {
  52                 current_arch = &native_arch;
  53                 prefix = FS::get_home_dir()/"local";
  54         }
  55         else
  56         {
  57                 current_arch = new Architecture(*this, name);
  58                 prefix = FS::get_home_dir()/"local"/current_arch->get_name();
  59         }
  60 }
  61
  62 vector<string> Builder::get_build_types() const
  63 {
  64         vector<string> keys;
  65         keys.reserve(build_types.size());
  66         for(const auto &kvp: build_types)
  67                 keys.push_back(kvp.first);
  68         return keys;
  69 }
  70
  71 const BuildType &Builder::get_build_type() const
  72 {
  73         if(!build_type)
  74                 throw invalid_state("no build type");
  75         return *build_type;
  76 }
  77
  78 void Builder::set_build_type(const string &name)
  79 {
  80         build_type = &get_item(build_types, name);
  81 }
  82
  83 void Builder::set_prefix(const FS::Path &p)
  84 {
  85         prefix = p;
  86 }
  87
  88 void Builder::set_temp_directory(const FS::Path &p)
  89 {
  90         tempdir = p;
  91 }
  92
  93 void Builder::add_default_tools()
  94 {
  95         const string &arch_tc = current_arch->get_toolchain();
  96         if(current_arch->get_system()=="android")
  97                 toolchain.add_toolchain(new AndroidTools(*this, *current_arch));
  98         else if(arch_tc=="msvc")
  99                 toolchain.add_toolchain(new MicrosoftTools(*this, *current_arch));
 100         else if(arch_tc=="clang")
 101                 toolchain.add_toolchain(new ClangTools(*this, *current_arch));
 102         else if(arch_tc=="gnu")
 103                 toolchain.add_toolchain(new GnuTools(*this, *current_arch));
 104         toolchain.add_toolchain(new BuiltinTools(*this));
 105         toolchain.add_tool(new DataTool(*this));
 106 }
 107
 108 void Builder::set_logger(const Logger *l)
 109 {
 110         logger = (l ? l : &default_logger);
 111 }
 112
 113 vector<string> Builder::collect_problems() const
 114 {
 115         vector<string> problems;
 116         set<const Package *> broken_packages;
 117         set<const Component *> broken_components;
 118         set<const Tool *> broken_tools;
 119
 120         for(const auto &kvp: build_graph.get_targets())
 121                 if(kvp.second->is_broken())
 122                 {
 123                         for(const string &p: kvp.second->get_problems())
 124                                 problems.push_back(format("%s: %s", kvp.second->get_name(), p));
 125
 126                         const Package *package = kvp.second->get_package();
 127                         if(package && !package->get_problems().empty())
 128                                 broken_packages.insert(package);
 129
 130                         const Component *component = kvp.second->get_component();
 131                         if(component && !component->get_problems().empty())
 132                                 broken_components.insert(component);
 133
 134                         const Tool *tool = kvp.second->get_tool();
 135                         if(tool && !tool->get_problems().empty())
 136                                 broken_tools.insert(tool);
 137                 }
 138
 139         // TODO Sort components after their packages, and targets last
 140         for(const Package *p: broken_packages)
 141                 for(const string &b: p->get_problems())
 142                         problems.push_back(format("%s: %s", p->get_name(), b));
 143
 144         for(const Component *c: broken_components)
 145                 for(const string &b: c->get_problems())
 146                         problems.push_back(format("%s/%s: %s", c->get_package().get_name(), c->get_name(), b));
 147
 148         for(const Tool *t: broken_tools)
 149                 for(const string &b: t->get_problems())
 150                         problems.push_back(format("%s: %s", t->get_tag(), b));
 151
 152         return problems;
 153 }
 154
 155 void Builder::load_build_file(const FS::Path &fn, const Config::InputOptions *opts, bool all)
 156 {
 157         IO::BufferedFile in(fn.str());
 158
 159         get_logger().log("files", "Reading %s", fn);
 160
 161         DataFile::Parser parser(in, fn.str());
 162         Loader loader(*this, opts, all);
 163         loader.load(parser);
 164 }
 165
 166 void Builder::save_caches()
 167 {
 168         for(const auto &kvp: package_manager.get_packages())
 169                 kvp.second->save_caches();
 170 }
 171
 172 int Builder::build(unsigned jobs, bool dry_run, bool show_progress)
 173 {
 174         unsigned total = build_graph.count_rebuild_targets();
 175
 176         if(!total)
 177         {
 178                 get_logger().log("summary", "Already up to date");
 179                 return 0;
 180         }
 181         get_logger().log("summary", "Will build %d target%s", total, (total!=1 ? "s" : ""));
 182
 183         vector<Task *> tasks;
 184
 185         unsigned count = 0;
 186
 187         bool fail = false;
 188         bool finish = false;
 189         bool starved = false;
 190
 191         while(!finish)
 192         {
 193                 if(tasks.size()<jobs && !fail && !starved)
 194                 {
 195                         Target *tgt = build_graph.get_buildable_target();
 196                         if(tgt)
 197                         {
 198                                 if(tgt->get_tool())
 199                                         get_logger().log("tasks", "%-4s  %s", tgt->get_tool()->get_tag(), tgt->get_name());
 200                                 Task *task = tgt->build();
 201                                 if(task)
 202                                 {
 203                                         get_logger().log("commands", "%s", task->get_command());
 204                                         if(dry_run)
 205                                         {
 206                                                 task->signal_finished.emit(true);
 207                                                 delete task;
 208                                         }
 209                                         else
 210                                         {
 211                                                 task->start();
 212                                                 tasks.push_back(task);
 213                                         }
 214                                 }
 215
 216                                 if(show_progress)
 217                                         IO::print("%d of %d target%s built\033[1G", count, total, (total!=1 ? "s" : ""));
 218                         }
 219                         else if(tasks.empty())
 220                                 finish = true;
 221                         else
 222                                 starved = true;
 223                 }
 224                 else
 225                         Time::sleep(10*Time::msec);
 226
 227                 for(unsigned i=0; i<tasks.size();)
 228                 {
 229                         Task::Status status;
 230                         if(jobs==1 || (tasks.size()==1 && starved))
 231                                 status = tasks[i]->wait();
 232                         else
 233                                 status = tasks[i]->check();
 234
 235                         if(status!=Task::RUNNING)
 236                         {
 237                                 ++count;
 238
 239                                 delete tasks[i];
 240                                 tasks.erase(tasks.begin()+i);
 241                                 if(status==Task::ERROR)
 242                                         fail = true;
 243                                 if(tasks.empty() && fail)
 244                                         finish = true;
 245                                 starved = false;
 246                         }
 247                         else
 248                                 ++i;
 249                 }
 250         }
 251
 252         if(show_progress)
 253                 IO::print("\033[K");
 254         if(fail)
 255                 get_logger().log("summary", "Build failed");
 256         else if(show_progress)
 257                 get_logger().log("summary", "Build complete");
 258
 259         return fail;
 260 }
 261
 262 int Builder::clean(bool all, bool dry_run)
 263 {
 264         // Cleaning doesn't care about ordering, so a simpler method can be used
 265
 266         set<Target *> clean_tgts;
 267         deque<Target *> queue;
 268         queue.push_back(&build_graph.get_goals());
 269
 270         while(!queue.empty())
 271         {
 272                 Target *tgt = queue.front();
 273                 queue.pop_front();
 274
 275                 if(tgt->is_buildable() && (tgt->get_package()==&package_manager.get_main_package() || all))
 276                         clean_tgts.insert(tgt);
 277
 278                 for(Target *t: tgt->get_dependencies())
 279                         if(!clean_tgts.count(t))
 280                                 queue.push_back(t);
 281         }
 282
 283         for(Target *t: clean_tgts)
 284         {
 285                 get_logger().log("tasks", "RM    %s", t->get_name());
 286                 if(!dry_run)
 287                         t->clean();
 288         }
 289
 290         return 0;
 291 }
 292
 293
 294 Builder::Loader::Loader(Builder &b, const Config::InputOptions *o, bool a):
 295         DataFile::ObjectLoader<Builder>(b),
 296         options(o),
 297         conf_all(a)
 298 {
 299         add("architecture", &Loader::architecture);
 300         add("binary_package", &Loader::binpkg);
 301         add("build_type", &Loader::build_type);
 302         add("package", &Loader::package);
 303
 304         if(!obj.top_loader)
 305                 obj.top_loader = this;
 306         else if(!options && obj.top_loader!=this && obj.top_loader->conf_all)
 307                 options = obj.top_loader->options;
 308 }
 309
 310 Builder::Loader::~Loader()
 311 {
 312         if(obj.top_loader==this)
 313                 obj.top_loader = 0;
 314 }
 315
 316 void Builder::Loader::architecture(const string &n)
 317 {
 318         if(obj.current_arch->match_name(n))
 319                 load_sub(*obj.current_arch);
 320 }
 321
 322 void Builder::Loader::binpkg(const string &n)
 323 {
 324         BinaryPackage *pkg = new BinaryPackage(obj, n);
 325         load_sub(*pkg);
 326 }
 327
 328 void Builder::Loader::build_type(const string &n)
 329 {
 330         BuildType btype(n);
 331         load_sub(btype);
 332         auto i = obj.build_types.insert({ n, btype }).first;
 333         if(!obj.build_type)
 334                 obj.build_type = &i->second;
 335 }
 336
 337 void Builder::Loader::package(const string &n)
 338 {
 339         SourcePackage *pkg = new SourcePackage(obj, n, get_source());
 340
 341         load_sub(*pkg, options);
 342
 343         if(obj.build_type)
 344                 pkg->set_build_type(*obj.build_type);
 345 }