source/builder.cpp

   1 #include <set>
   2 #include <msp/core/except.h>
   3 #include <msp/core/maputils.h>
   4 #include <msp/datafile/parser.h>
   5 #include <msp/fs/dir.h>
   6 #include <msp/fs/utils.h>
   7 #include <msp/io/buffered.h>
   8 #include <msp/io/file.h>
   9 #include <msp/io/print.h>
  10 #include <msp/strings/format.h>
  11 #include <msp/time/timedelta.h>
  12 #include <msp/time/utils.h>
  13 #include "androidtools.h"
  14 #include "binarypackage.h"
  15 #include "builder.h"
  16 #include "builtintools.h"
  17 #include "clangtools.h"
  18 #include "datatool.h"
  19 #include "gnutools.h"
  20 #include "installedfile.h"
  21 #include "microsofttools.h"
  22 #include "package.h"
  23 #include "sharedlibrary.h"
  24 #include "sourcepackage.h"
  25 #include "task.h"
  26 #include "virtualtarget.h"
  27
  28 using namespace std;
  29 using namespace Msp;
  30
  31 Builder::Builder():
  32         package_manager(*this),
  33         native_arch(*this, string()),
  34         current_arch(0),
  35         build_type(0),
  36         vfs(*this),
  37         build_graph(*this),
  38         logger(&default_logger),
  39         tempdir("temp"),
  40         top_loader(0)
  41 {
  42         set_architecture(string());
  43 }
  44
  45 Builder::~Builder()
  46 {
  47         if(current_arch!=&native_arch)
  48                 delete current_arch;
  49 }
  50
  51 void Builder::set_architecture(const string &name)
  52 {
  53         if(name.empty())
  54         {
  55                 current_arch = &native_arch;
  56                 prefix = FS::get_home_dir()/"local";
  57         }
  58         else
  59         {
  60                 current_arch = new Architecture(*this, name);
  61                 prefix = FS::get_home_dir()/"local"/current_arch->get_name();
  62         }
  63 }
  64
  65 vector<string> Builder::get_build_types() const
  66 {
  67         vector<string> keys;
  68         keys.reserve(build_types.size());
  69         for(const auto &kvp: build_types)
  70                 keys.push_back(kvp.first);
  71         return keys;
  72 }
  73
  74 const BuildType &Builder::get_build_type() const
  75 {
  76         if(!build_type)
  77                 throw invalid_state("no build type");
  78         return *build_type;
  79 }
  80
  81 void Builder::set_build_type(const string &name)
  82 {
  83         build_type = &get_item(build_types, name);
  84 }
  85
  86 void Builder::set_prefix(const FS::Path &p)
  87 {
  88         prefix = p;
  89 }
  90
  91 void Builder::set_temp_directory(const FS::Path &p)
  92 {
  93         tempdir = p;
  94 }
  95
  96 void Builder::add_default_tools()
  97 {
  98         const string &arch_tc = current_arch->get_toolchain();
  99         if(current_arch->get_system()=="android")
 100                 toolchain.add_toolchain(new AndroidTools(*this, *current_arch));
 101         else if(arch_tc=="msvc")
 102                 toolchain.add_toolchain(new MicrosoftTools(*this, *current_arch));
 103         else if(arch_tc=="clang")
 104                 toolchain.add_toolchain(new ClangTools(*this, *current_arch));
 105         else if(arch_tc=="gnu")
 106                 toolchain.add_toolchain(new GnuTools(*this, *current_arch));
 107         toolchain.add_toolchain(new BuiltinTools(*this));
 108         toolchain.add_tool(new DataTool(*this));
 109 }
 110
 111 void Builder::set_logger(const Logger *l)
 112 {
 113         logger = (l ? l : &default_logger);
 114 }
 115
 116 list<string> Builder::collect_problems() const
 117 {
 118         list<string> problems;
 119         set<const Package *> broken_packages;
 120         set<const Component *> broken_components;
 121         set<const Tool *> broken_tools;
 122
 123         for(const auto &kvp: build_graph.get_targets())
 124                 if(kvp.second->is_broken())
 125                 {
 126                         for(const string &p: kvp.second->get_problems())
 127                                 problems.push_back(format("%s: %s", kvp.second->get_name(), p));
 128
 129                         const Package *package = kvp.second->get_package();
 130                         if(package && !package->get_problems().empty())
 131                                 broken_packages.insert(package);
 132
 133                         const Component *component = kvp.second->get_component();
 134                         if(component && !component->get_problems().empty())
 135                                 broken_components.insert(component);
 136
 137                         const Tool *tool = kvp.second->get_tool();
 138                         if(tool && !tool->get_problems().empty())
 139                                 broken_tools.insert(tool);
 140                 }
 141
 142         // TODO Sort components after their packages, and targets last
 143         for(const Package *p: broken_packages)
 144                 for(const string &b: p->get_problems())
 145                         problems.push_back(format("%s: %s", p->get_name(), b));
 146
 147         for(const Component *c: broken_components)
 148                 for(const string &b: c->get_problems())
 149                         problems.push_back(format("%s/%s: %s", c->get_package().get_name(), c->get_name(), b));
 150
 151         for(const Tool *t: broken_tools)
 152                 for(const string &b: t->get_problems())
 153                         problems.push_back(format("%s: %s", t->get_tag(), b));
 154
 155         return problems;
 156 }
 157
 158 void Builder::load_build_file(const FS::Path &fn, const Config::InputOptions *opts, bool all)
 159 {
 160         IO::BufferedFile in(fn.str());
 161
 162         get_logger().log("files", format("Reading %s", fn));
 163
 164         DataFile::Parser parser(in, fn.str());
 165         Loader loader(*this, opts, all);
 166         loader.load(parser);
 167 }
 168
 169 void Builder::save_caches()
 170 {
 171         for(const auto &kvp: package_manager.get_packages())
 172                 kvp.second->save_caches();
 173 }
 174
 175 int Builder::build(unsigned jobs, bool dry_run, bool show_progress)
 176 {
 177         unsigned total = build_graph.count_rebuild_targets();
 178
 179         if(!total)
 180         {
 181                 get_logger().log("summary", "Already up to date");
 182                 return 0;
 183         }
 184         get_logger().log("summary", format("Will build %d target%s", total, (total!=1 ? "s" : "")));
 185
 186         vector<Task *> tasks;
 187
 188         unsigned count = 0;
 189
 190         bool fail = false;
 191         bool finish = false;
 192         bool starved = false;
 193
 194         while(!finish)
 195         {
 196                 if(tasks.size()<jobs && !fail && !starved)
 197                 {
 198                         Target *tgt = build_graph.get_buildable_target();
 199                         if(tgt)
 200                         {
 201                                 if(tgt->get_tool())
 202                                         get_logger().log("tasks", format("%-4s  %s", tgt->get_tool()->get_tag(), tgt->get_name()));
 203                                 Task *task = tgt->build();
 204                                 if(task)
 205                                 {
 206                                         get_logger().log("commands", format("%s", task->get_command()));
 207                                         if(dry_run)
 208                                         {
 209                                                 task->signal_finished.emit(true);
 210                                                 delete task;
 211                                         }
 212                                         else
 213                                         {
 214                                                 task->start();
 215                                                 tasks.push_back(task);
 216                                         }
 217                                 }
 218
 219                                 if(show_progress)
 220                                         IO::print("%d of %d target%s built\033[1G", count, total, (total!=1 ? "s" : ""));
 221                         }
 222                         else if(tasks.empty())
 223                                 finish = true;
 224                         else
 225                                 starved = true;
 226                 }
 227                 else
 228                         Time::sleep(10*Time::msec);
 229
 230                 for(unsigned i=0; i<tasks.size();)
 231                 {
 232                         Task::Status status;
 233                         if(jobs==1 || (tasks.size()==1 && starved))
 234                                 status = tasks[i]->wait();
 235                         else
 236                                 status = tasks[i]->check();
 237
 238                         if(status!=Task::RUNNING)
 239                         {
 240                                 ++count;
 241
 242                                 delete tasks[i];
 243                                 tasks.erase(tasks.begin()+i);
 244                                 if(status==Task::ERROR)
 245                                         fail = true;
 246                                 if(tasks.empty() && fail)
 247                                         finish = true;
 248                                 starved = false;
 249                         }
 250                         else
 251                                 ++i;
 252                 }
 253         }
 254
 255         if(show_progress)
 256                 IO::print("\033[K");
 257         if(fail)
 258                 get_logger().log("summary", "Build failed");
 259         else if(show_progress)
 260                 get_logger().log("summary", "Build complete");
 261
 262         return fail;
 263 }
 264
 265 int Builder::clean(bool all, bool dry_run)
 266 {
 267         // Cleaning doesn't care about ordering, so a simpler method can be used
 268
 269         set<Target *> clean_tgts;
 270         list<Target *> queue;
 271         queue.push_back(&build_graph.get_goals());
 272
 273         while(!queue.empty())
 274         {
 275                 Target *tgt = queue.front();
 276                 queue.pop_front();
 277
 278                 if(tgt->is_buildable() && (tgt->get_package()==&package_manager.get_main_package() || all))
 279                         clean_tgts.insert(tgt);
 280
 281                 for(Target *t: tgt->get_dependencies())
 282                         if(!clean_tgts.count(t))
 283                                 queue.push_back(t);
 284         }
 285
 286         for(Target *t: clean_tgts)
 287         {
 288                 get_logger().log("tasks", format("RM    %s", t->get_name()));
 289                 if(!dry_run)
 290                         t->clean();
 291         }
 292
 293         return 0;
 294 }
 295
 296
 297 Builder::Loader::Loader(Builder &b, const Config::InputOptions *o, bool a):
 298         DataFile::ObjectLoader<Builder>(b),
 299         options(o),
 300         conf_all(a)
 301 {
 302         add("architecture", &Loader::architecture);
 303         add("binary_package", &Loader::binpkg);
 304         add("build_type", &Loader::build_type);
 305         add("package", &Loader::package);
 306
 307         if(!obj.top_loader)
 308                 obj.top_loader = this;
 309         else if(!options && obj.top_loader!=this && obj.top_loader->conf_all)
 310                 options = obj.top_loader->options;
 311 }
 312
 313 Builder::Loader::~Loader()
 314 {
 315         if(obj.top_loader==this)
 316                 obj.top_loader = 0;
 317 }
 318
 319 void Builder::Loader::architecture(const string &n)
 320 {
 321         if(obj.current_arch->match_name(n))
 322                 load_sub(*obj.current_arch);
 323 }
 324
 325 void Builder::Loader::binpkg(const string &n)
 326 {
 327         BinaryPackage *pkg = new BinaryPackage(obj, n);
 328         load_sub(*pkg);
 329 }
 330
 331 void Builder::Loader::build_type(const string &n)
 332 {
 333         BuildType btype(n);
 334         load_sub(btype);
 335         auto i = obj.build_types.insert({ n, btype }).first;
 336         if(!obj.build_type)
 337                 obj.build_type = &i->second;
 338 }
 339
 340 void Builder::Loader::package(const string &n)
 341 {
 342         SourcePackage *pkg = new SourcePackage(obj, n, get_source());
 343
 344         load_sub(*pkg, options);
 345
 346         if(obj.build_type)
 347                 pkg->set_build_type(*obj.build_type);
 348 }