source/builder.cpp

   1 #include <set>
   2 #include <msp/core/maputils.h>
   3 #include <msp/datafile/parser.h>
   4 #include <msp/fs/dir.h>
   5 #include <msp/fs/utils.h>
   6 #include <msp/io/buffered.h>
   7 #include <msp/io/file.h>
   8 #include <msp/io/print.h>
   9 #include <msp/strings/format.h>
  10 #include <msp/time/timedelta.h>
  11 #include <msp/time/utils.h>
  12 #include "androidtools.h"
  13 #include "binarypackage.h"
  14 #include "builder.h"
  15 #include "builtintools.h"
  16 #include "clangtools.h"
  17 #include "datatool.h"
  18 #include "gnutools.h"
  19 #include "installedfile.h"
  20 #include "microsofttools.h"
  21 #include "package.h"
  22 #include "sharedlibrary.h"
  23 #include "sourcepackage.h"
  24 #include "task.h"
  25 #include "virtualtarget.h"
  26
  27 using namespace std;
  28 using namespace Msp;
  29
  30 Builder::Builder():
  31         package_manager(*this),
  32         native_arch(*this, string()),
  33         current_arch(0),
  34         build_type(0),
  35         vfs(*this),
  36         build_graph(*this),
  37         logger(&default_logger),
  38         tempdir("temp"),
  39         top_loader(0)
  40 {
  41         set_architecture(string());
  42 }
  43
  44 Builder::~Builder()
  45 {
  46         if(current_arch!=&native_arch)
  47                 delete current_arch;
  48 }
  49
  50 void Builder::set_architecture(const string &name)
  51 {
  52         if(name.empty())
  53         {
  54                 current_arch = &native_arch;
  55                 prefix = FS::get_home_dir()/"local";
  56         }
  57         else
  58         {
  59                 current_arch = new Architecture(*this, name);
  60                 prefix = FS::get_home_dir()/"local"/current_arch->get_name();
  61         }
  62 }
  63
  64 vector<string> Builder::get_build_types() const
  65 {
  66         vector<string> keys;
  67         keys.reserve(build_types.size());
  68         for(const auto &kvp: build_types)
  69                 keys.push_back(kvp.first);
  70         return keys;
  71 }
  72
  73 void Builder::set_build_type(const string &name)
  74 {
  75         build_type = &get_item(build_types, name);
  76 }
  77
  78 void Builder::set_prefix(const FS::Path &p)
  79 {
  80         prefix = p;
  81 }
  82
  83 void Builder::set_temp_directory(const FS::Path &p)
  84 {
  85         tempdir = p;
  86 }
  87
  88 void Builder::add_default_tools()
  89 {
  90         const string &arch_tc = current_arch->get_toolchain();
  91         if(current_arch->get_system()=="android")
  92                 toolchain.add_toolchain(new AndroidTools(*this, *current_arch));
  93         else if(arch_tc=="msvc")
  94                 toolchain.add_toolchain(new MicrosoftTools(*this, *current_arch));
  95         else if(arch_tc=="clang")
  96                 toolchain.add_toolchain(new ClangTools(*this, *current_arch));
  97         else if(arch_tc=="gnu")
  98                 toolchain.add_toolchain(new GnuTools(*this, *current_arch));
  99         toolchain.add_toolchain(new BuiltinTools(*this));
 100         toolchain.add_tool(new DataTool(*this));
 101 }
 102
 103 void Builder::set_logger(const Logger *l)
 104 {
 105         logger = (l ? l : &default_logger);
 106 }
 107
 108 list<string> Builder::collect_problems() const
 109 {
 110         list<string> problems;
 111         set<const Package *> broken_packages;
 112         set<const Component *> broken_components;
 113         set<const Tool *> broken_tools;
 114
 115         for(const auto &kvp: build_graph.get_targets())
 116                 if(kvp.second->is_broken())
 117                 {
 118                         for(const string &p: kvp.second->get_problems())
 119                                 problems.push_back(format("%s: %s", kvp.second->get_name(), p));
 120
 121                         const Package *package = kvp.second->get_package();
 122                         if(package && !package->get_problems().empty())
 123                                 broken_packages.insert(package);
 124
 125                         const Component *component = kvp.second->get_component();
 126                         if(component && !component->get_problems().empty())
 127                                 broken_components.insert(component);
 128
 129                         const Tool *tool = kvp.second->get_tool();
 130                         if(tool && !tool->get_problems().empty())
 131                                 broken_tools.insert(tool);
 132                 }
 133
 134         // TODO Sort components after their packages, and targets last
 135         for(const Package *p: broken_packages)
 136                 for(const string &b: p->get_problems())
 137                         problems.push_back(format("%s: %s", p->get_name(), b));
 138
 139         for(const Component *c: broken_components)
 140                 for(const string &b: c->get_problems())
 141                         problems.push_back(format("%s/%s: %s", c->get_package().get_name(), c->get_name(), b));
 142
 143         for(const Tool *t: broken_tools)
 144                 for(const string &b: t->get_problems())
 145                         problems.push_back(format("%s: %s", t->get_tag(), b));
 146
 147         return problems;
 148 }
 149
 150 void Builder::load_build_file(const FS::Path &fn, const Config::InputOptions *opts, bool all)
 151 {
 152         IO::BufferedFile in(fn.str());
 153
 154         get_logger().log("files", format("Reading %s", fn));
 155
 156         DataFile::Parser parser(in, fn.str());
 157         Loader loader(*this, opts, all);
 158         loader.load(parser);
 159 }
 160
 161 void Builder::save_caches()
 162 {
 163         for(const auto &kvp: package_manager.get_packages())
 164                 kvp.second->save_caches();
 165 }
 166
 167 int Builder::build(unsigned jobs, bool dry_run, bool show_progress)
 168 {
 169         unsigned total = build_graph.count_rebuild_targets();
 170
 171         if(!total)
 172         {
 173                 get_logger().log("summary", "Already up to date");
 174                 return 0;
 175         }
 176         get_logger().log("summary", format("Will build %d target%s", total, (total!=1 ? "s" : "")));
 177
 178         vector<Task *> tasks;
 179
 180         unsigned count = 0;
 181
 182         bool fail = false;
 183         bool finish = false;
 184         bool starved = false;
 185
 186         while(!finish)
 187         {
 188                 if(tasks.size()<jobs && !fail && !starved)
 189                 {
 190                         Target *tgt = build_graph.get_buildable_target();
 191                         if(tgt)
 192                         {
 193                                 if(tgt->get_tool())
 194                                         get_logger().log("tasks", format("%-4s  %s", tgt->get_tool()->get_tag(), tgt->get_name()));
 195                                 Task *task = tgt->build();
 196                                 if(task)
 197                                 {
 198                                         get_logger().log("commands", format("%s", task->get_command()));
 199                                         if(dry_run)
 200                                         {
 201                                                 task->signal_finished.emit(true);
 202                                                 delete task;
 203                                         }
 204                                         else
 205                                         {
 206                                                 task->start();
 207                                                 tasks.push_back(task);
 208                                         }
 209                                 }
 210
 211                                 if(show_progress)
 212                                         IO::print("%d of %d target%s built\033[1G", count, total, (total!=1 ? "s" : ""));
 213                         }
 214                         else if(tasks.empty())
 215                                 finish = true;
 216                         else
 217                                 starved = true;
 218                 }
 219                 else
 220                         Time::sleep(10*Time::msec);
 221
 222                 for(unsigned i=0; i<tasks.size();)
 223                 {
 224                         Task::Status status;
 225                         if(jobs==1 || (tasks.size()==1 && starved))
 226                                 status = tasks[i]->wait();
 227                         else
 228                                 status = tasks[i]->check();
 229
 230                         if(status!=Task::RUNNING)
 231                         {
 232                                 ++count;
 233
 234                                 delete tasks[i];
 235                                 tasks.erase(tasks.begin()+i);
 236                                 if(status==Task::ERROR)
 237                                         fail = true;
 238                                 if(tasks.empty() && fail)
 239                                         finish = true;
 240                                 starved = false;
 241                         }
 242                         else
 243                                 ++i;
 244                 }
 245         }
 246
 247         if(show_progress)
 248                 IO::print("\033[K");
 249         if(fail)
 250                 get_logger().log("summary", "Build failed");
 251         else if(show_progress)
 252                 get_logger().log("summary", "Build complete");
 253
 254         return fail;
 255 }
 256
 257 int Builder::clean(bool all, bool dry_run)
 258 {
 259         // Cleaning doesn't care about ordering, so a simpler method can be used
 260
 261         set<Target *> clean_tgts;
 262         list<Target *> queue;
 263         queue.push_back(&build_graph.get_goals());
 264
 265         while(!queue.empty())
 266         {
 267                 Target *tgt = queue.front();
 268                 queue.pop_front();
 269
 270                 if(tgt->is_buildable() && (tgt->get_package()==&package_manager.get_main_package() || all))
 271                         clean_tgts.insert(tgt);
 272
 273                 for(Target *t: tgt->get_dependencies())
 274                         if(!clean_tgts.count(t))
 275                                 queue.push_back(t);
 276         }
 277
 278         for(Target *t: clean_tgts)
 279         {
 280                 get_logger().log("tasks", format("RM    %s", t->get_name()));
 281                 if(!dry_run)
 282                         t->clean();
 283         }
 284
 285         return 0;
 286 }
 287
 288
 289 Builder::Loader::Loader(Builder &b, const Config::InputOptions *o, bool a):
 290         DataFile::ObjectLoader<Builder>(b),
 291         options(o),
 292         conf_all(a)
 293 {
 294         add("architecture", &Loader::architecture);
 295         add("binary_package", &Loader::binpkg);
 296         add("build_type", &Loader::build_type);
 297         add("package", &Loader::package);
 298
 299         if(!obj.top_loader)
 300                 obj.top_loader = this;
 301         else if(!options && obj.top_loader!=this && obj.top_loader->conf_all)
 302                 options = obj.top_loader->options;
 303 }
 304
 305 Builder::Loader::~Loader()
 306 {
 307         if(obj.top_loader==this)
 308                 obj.top_loader = 0;
 309 }
 310
 311 void Builder::Loader::architecture(const string &n)
 312 {
 313         if(obj.current_arch->match_name(n))
 314                 load_sub(*obj.current_arch);
 315 }
 316
 317 void Builder::Loader::binpkg(const string &n)
 318 {
 319         BinaryPackage *pkg = new BinaryPackage(obj, n);
 320         load_sub(*pkg);
 321 }
 322
 323 void Builder::Loader::build_type(const string &n)
 324 {
 325         BuildType btype(n);
 326         load_sub(btype);
 327         auto i = obj.build_types.insert(BuildTypeMap::value_type(n, btype)).first;
 328         if(!obj.build_type)
 329                 obj.build_type = &i->second;
 330 }
 331
 332 void Builder::Loader::package(const string &n)
 333 {
 334         SourcePackage *pkg = new SourcePackage(obj, n, get_source());
 335
 336         load_sub(*pkg, options);
 337
 338         if(obj.build_type)
 339                 pkg->set_build_type(*obj.build_type);
 340 }