source/builder.cpp

   1 #include <deque>
   2 #include <set>
   3 #include <msp/core/algorithm.h>
   4 #include <msp/core/except.h>
   5 #include <msp/core/maputils.h>
   6 #include <msp/datafile/parser.h>
   7 #include <msp/fs/dir.h>
   8 #include <msp/fs/utils.h>
   9 #include <msp/io/buffered.h>
  10 #include <msp/io/file.h>
  11 #include <msp/io/print.h>
  12 #include <msp/strings/format.h>
  13 #include <msp/time/timedelta.h>
  14 #include <msp/time/utils.h>
  15 #include "androidtools.h"
  16 #include "binarypackage.h"
  17 #include "builder.h"
  18 #include "builtintools.h"
  19 #include "clangtools.h"
  20 #include "datatool.h"
  21 #include "gnutools.h"
  22 #include "installedfile.h"
  23 #include "microsofttools.h"
  24 #include "package.h"
  25 #include "sharedlibrary.h"
  26 #include "sourcepackage.h"
  27 #include "task.h"
  28 #include "virtualtarget.h"
  29
  30 using namespace std;
  31 using namespace Msp;
  32
  33 Builder::Builder():
  34         package_manager(*this),
  35         native_arch(*this, string()),
  36         vfs(*this),
  37         build_graph(*this),
  38         logger(&default_logger)
  39 {
  40         set_architecture(string());
  41 }
  42
  43 Builder::~Builder()
  44 {
  45         if(current_arch!=&native_arch)
  46                 delete current_arch;
  47 }
  48
  49 void Builder::set_architecture(const string &name)
  50 {
  51         if(name.empty())
  52         {
  53                 current_arch = &native_arch;
  54                 prefix = FS::get_home_dir()/"local";
  55         }
  56         else
  57         {
  58                 current_arch = new Architecture(*this, name);
  59                 prefix = FS::get_home_dir()/"local"/current_arch->get_name();
  60         }
  61 }
  62
  63 vector<string> Builder::get_build_types() const
  64 {
  65         vector<string> keys;
  66         keys.reserve(build_types.size());
  67         for(const auto &kvp: build_types)
  68                 keys.push_back(kvp.first);
  69         return keys;
  70 }
  71
  72 const BuildType &Builder::get_build_type() const
  73 {
  74         if(!build_type)
  75                 throw invalid_state("no build type");
  76         return *build_type;
  77 }
  78
  79 void Builder::set_build_type(const string &name)
  80 {
  81         build_type = &get_item(build_types, name);
  82 }
  83
  84 void Builder::set_prefix(const FS::Path &p)
  85 {
  86         prefix = p;
  87 }
  88
  89 void Builder::set_temp_directory(const FS::Path &p)
  90 {
  91         tempdir = p;
  92 }
  93
  94 void Builder::add_default_tools()
  95 {
  96         toolchain.add_toolchain(new GnuTools(*this, *current_arch));
  97         toolchain.add_toolchain(new ClangTools(*this, *current_arch));
  98         if(current_arch->get_system()=="android")
  99                 toolchain.add_toolchain(new AndroidTools(*this, *current_arch));
 100         if(current_arch->get_system()=="windows")
 101                 toolchain.add_toolchain(new MicrosoftTools(*this, *current_arch));
 102         toolchain.add_toolchain(new BuiltinTools(*this));
 103         toolchain.add_tool(new DataTool(*this));
 104
 105         auto i = find_if(toolchain.get_toolchains(), [](const Toolchain *tc){ return (tc->has_tool("CC") || tc->has_tool("CXX")); });
 106         if(i!=toolchain.get_toolchains().end())
 107                 current_arch->refine((*i)->get_name());
 108 }
 109
 110 void Builder::set_logger(const Logger *l)
 111 {
 112         logger = (l ? l : &default_logger);
 113 }
 114
 115 vector<string> Builder::collect_problems() const
 116 {
 117         vector<string> problems;
 118         set<const Package *> broken_packages;
 119         set<const Component *> broken_components;
 120         set<const Tool *> broken_tools;
 121
 122         for(const auto &kvp: build_graph.get_targets())
 123                 if(kvp.second->is_broken())
 124                 {
 125                         for(const string &p: kvp.second->get_problems())
 126                                 problems.push_back(format("%s: %s", kvp.second->get_name(), p));
 127
 128                         const Package *package = kvp.second->get_package();
 129                         if(package && !package->get_problems().empty())
 130                                 broken_packages.insert(package);
 131
 132                         const Component *component = kvp.second->get_component();
 133                         if(component && !component->get_problems().empty())
 134                                 broken_components.insert(component);
 135
 136                         const Tool *tool = kvp.second->get_tool();
 137                         if(tool && !tool->get_problems().empty())
 138                                 broken_tools.insert(tool);
 139                 }
 140
 141         // TODO Sort components after their packages, and targets last
 142         for(const Package *p: broken_packages)
 143                 for(const string &b: p->get_problems())
 144                         problems.push_back(format("%s: %s", p->get_name(), b));
 145
 146         for(const Component *c: broken_components)
 147                 for(const string &b: c->get_problems())
 148                         problems.push_back(format("%s/%s: %s", c->get_package().get_name(), c->get_name(), b));
 149
 150         for(const Tool *t: broken_tools)
 151                 for(const string &b: t->get_problems())
 152                         problems.push_back(format("%s: %s", t->get_tag(), b));
 153
 154         return problems;
 155 }
 156
 157 void Builder::load_build_file(const FS::Path &fn, const Config::InputOptions *opts, bool all)
 158 {
 159         IO::BufferedFile in(fn.str());
 160
 161         get_logger().log("files", "Reading %s", fn);
 162
 163         DataFile::Parser parser(in, fn.str());
 164         Loader loader(*this, opts, all);
 165         loader.load(parser);
 166 }
 167
 168 void Builder::save_caches()
 169 {
 170         for(const auto &kvp: package_manager.get_packages())
 171                 kvp.second->save_caches();
 172 }
 173
 174 int Builder::build(unsigned jobs, bool dry_run, bool show_progress)
 175 {
 176         unsigned total = build_graph.count_rebuild_targets();
 177
 178         if(!total)
 179         {
 180                 get_logger().log("summary", "Already up to date");
 181                 return 0;
 182         }
 183         get_logger().log("summary", "Will build %d target%s", total, (total!=1 ? "s" : ""));
 184
 185         vector<Task *> tasks;
 186
 187         unsigned count = 0;
 188
 189         bool fail = false;
 190         bool finish = false;
 191         bool starved = false;
 192
 193         while(!finish)
 194         {
 195                 if(tasks.size()<jobs && !fail && !starved)
 196                 {
 197                         Target *tgt = build_graph.get_buildable_target();
 198                         if(tgt)
 199                         {
 200                                 if(tgt->get_tool())
 201                                 {
 202                                         if(show_progress)
 203                                                 IO::print("\033[K");
 204                                         get_logger().log("tasks", "%-4s  %s", tgt->get_tool()->get_tag(), tgt->get_name());
 205                                 }
 206                                 Task *task = tgt->build();
 207                                 if(task)
 208                                 {
 209                                         get_logger().log("commands", "%s", task->get_command());
 210                                         if(dry_run)
 211                                         {
 212                                                 task->signal_finished.emit(true);
 213                                                 delete task;
 214                                         }
 215                                         else
 216                                         {
 217                                                 task->start();
 218                                                 tasks.push_back(task);
 219                                         }
 220                                 }
 221
 222                                 if(show_progress)
 223                                         IO::print("%d of %d target%s built\033[1G", count, total, (total!=1 ? "s" : ""));
 224                         }
 225                         else if(tasks.empty())
 226                                 finish = true;
 227                         else
 228                                 starved = true;
 229                 }
 230                 else
 231                         Time::sleep(10*Time::msec);
 232
 233                 for(unsigned i=0; i<tasks.size();)
 234                 {
 235                         Task::Status status;
 236                         if(jobs==1 || (tasks.size()==1 && starved))
 237                                 status = tasks[i]->wait();
 238                         else
 239                                 status = tasks[i]->check();
 240
 241                         if(status!=Task::RUNNING)
 242                         {
 243                                 ++count;
 244
 245                                 delete tasks[i];
 246                                 tasks.erase(tasks.begin()+i);
 247                                 if(status==Task::ERROR)
 248                                         fail = true;
 249                                 if(tasks.empty() && fail)
 250                                         finish = true;
 251                                 starved = false;
 252                         }
 253                         else
 254                                 ++i;
 255                 }
 256         }
 257
 258         if(show_progress)
 259                 IO::print("\033[K");
 260         if(fail)
 261                 get_logger().log("summary", "Build failed");
 262         else if(show_progress)
 263                 get_logger().log("summary", "Build complete");
 264
 265         return fail;
 266 }
 267
 268 int Builder::clean(bool all, bool dry_run)
 269 {
 270         // Cleaning doesn't care about ordering, so a simpler method can be used
 271
 272         set<Target *> clean_tgts;
 273         deque<Target *> queue;
 274         queue.push_back(&build_graph.get_goals());
 275
 276         while(!queue.empty())
 277         {
 278                 Target *tgt = queue.front();
 279                 queue.pop_front();
 280
 281                 if(tgt->is_buildable() && (tgt->get_package()==&package_manager.get_main_package() || all))
 282                         clean_tgts.insert(tgt);
 283
 284                 for(Target *t: tgt->get_dependencies())
 285                         if(!clean_tgts.count(t))
 286                                 queue.push_back(t);
 287         }
 288
 289         for(Target *t: clean_tgts)
 290         {
 291                 get_logger().log("tasks", "RM    %s", t->get_name());
 292                 if(!dry_run)
 293                         t->clean();
 294         }
 295
 296         return 0;
 297 }
 298
 299
 300 Builder::Loader::Loader(Builder &b, const Config::InputOptions *o, bool a):
 301         DataFile::ObjectLoader<Builder>(b),
 302         options(o),
 303         conf_all(a)
 304 {
 305         add("architecture", &Loader::architecture);
 306         add("binary_package", &Loader::binpkg);
 307         add("build_type", &Loader::build_type);
 308         add("package", &Loader::package);
 309
 310         if(!obj.top_loader)
 311                 obj.top_loader = this;
 312         else if(!options && obj.top_loader!=this && obj.top_loader->conf_all)
 313                 options = obj.top_loader->options;
 314 }
 315
 316 Builder::Loader::~Loader()
 317 {
 318         if(obj.top_loader==this)
 319                 obj.top_loader = 0;
 320 }
 321
 322 void Builder::Loader::architecture(const string &n)
 323 {
 324         if(obj.current_arch->match_name(n))
 325                 load_sub(*obj.current_arch);
 326 }
 327
 328 void Builder::Loader::binpkg(const string &n)
 329 {
 330         BinaryPackage *pkg = new BinaryPackage(obj, n);
 331         load_sub(*pkg);
 332 }
 333
 334 void Builder::Loader::build_type(const string &n)
 335 {
 336         BuildType btype(n);
 337         load_sub(btype);
 338         auto i = obj.build_types.insert({ n, btype }).first;
 339         if(!obj.build_type)
 340                 obj.build_type = &i->second;
 341 }
 342
 343 void Builder::Loader::package(const string &n)
 344 {
 345         SourcePackage *pkg = new SourcePackage(obj, n, get_source());
 346
 347         load_sub(*pkg, options);
 348
 349         if(obj.build_type)
 350                 pkg->set_build_type(*obj.build_type);
 351 }