source/builder.cpp

   1 #include <set>
   2 #include <msp/core/maputils.h>
   3 #include <msp/datafile/parser.h>
   4 #include <msp/fs/dir.h>
   5 #include <msp/fs/utils.h>
   6 #include <msp/io/buffered.h>
   7 #include <msp/io/file.h>
   8 #include <msp/io/print.h>
   9 #include <msp/strings/format.h>
  10 #include <msp/time/units.h>
  11 #include <msp/time/utils.h>
  12 #include "androidtools.h"
  13 #include "binarypackage.h"
  14 #include "builder.h"
  15 #include "builtintools.h"
  16 #include "clangtools.h"
  17 #include "datatool.h"
  18 #include "gnutools.h"
  19 #include "installedfile.h"
  20 #include "package.h"
  21 #include "sharedlibrary.h"
  22 #include "sourcepackage.h"
  23 #include "task.h"
  24 #include "virtualtarget.h"
  25
  26 using namespace std;
  27 using namespace Msp;
  28
  29 Builder::Builder():
  30         package_manager(*this),
  31         native_arch(*this, string()),
  32         current_arch(0),
  33         build_type(0),
  34         vfs(*this),
  35         build_graph(*this),
  36         logger(&default_logger),
  37         tempdir("temp"),
  38         top_loader(0)
  39 {
  40         set_architecture(string());
  41 }
  42
  43 Builder::~Builder()
  44 {
  45         if(current_arch!=&native_arch)
  46                 delete current_arch;
  47 }
  48
  49 void Builder::set_architecture(const string &name)
  50 {
  51         if(name.empty())
  52         {
  53                 current_arch = &native_arch;
  54                 prefix = FS::get_home_dir()/"local";
  55         }
  56         else
  57         {
  58                 current_arch = new Architecture(*this, name);
  59                 prefix = FS::get_home_dir()/"local"/current_arch->get_name();
  60         }
  61 }
  62
  63 void Builder::set_build_type(const string &name)
  64 {
  65         build_type = &get_item(build_types, name);
  66 }
  67
  68 void Builder::set_prefix(const FS::Path &p)
  69 {
  70         prefix = p;
  71 }
  72
  73 void Builder::set_temp_directory(const FS::Path &p)
  74 {
  75         tempdir = p;
  76 }
  77
  78 void Builder::add_default_tools()
  79 {
  80         bool use_clang = (current_arch->get_system()=="darwin" || current_arch->get_system()=="freebsd");
  81         if(use_clang && vfs.find_binary("clang"))
  82                 toolchain.add_toolchain(new ClangTools(*this, *current_arch));
  83         if(current_arch->get_system()=="android")
  84                 toolchain.add_toolchain(new AndroidTools(*this, *current_arch));
  85         toolchain.add_toolchain(new GnuTools(*this, *current_arch));
  86         toolchain.add_toolchain(new BuiltinTools(*this));
  87         toolchain.add_tool(new DataTool(*this));
  88 }
  89
  90 void Builder::set_logger(const Logger *l)
  91 {
  92         logger = (l ? l : &default_logger);
  93 }
  94
  95 list<string> Builder::collect_problems() const
  96 {
  97         list<string> problems;
  98         set<const Package *> broken_packages;
  99         set<const Component *> broken_components;
 100         set<const Tool *> broken_tools;
 101
 102         const BuildGraph::TargetMap &targets = build_graph.get_targets();
 103         for(BuildGraph::TargetMap::const_iterator i=targets.begin(); i!=targets.end(); ++i)
 104                 if(i->second->is_broken())
 105                 {
 106                         const list<string> &tgt_problems = i->second->get_problems();
 107                         for(list<string>::const_iterator j=tgt_problems.begin(); j!=tgt_problems.end(); ++j)
 108                                 problems.push_back(format("%s: %s", i->second->get_name(), *j));
 109
 110                         const Package *package = i->second->get_package();
 111                         if(package && !package->get_problems().empty())
 112                                 broken_packages.insert(package);
 113
 114                         const Component *component = i->second->get_component();
 115                         if(component && !component->get_problems().empty())
 116                                 broken_components.insert(component);
 117
 118                         const Tool *tool = i->second->get_tool();
 119                         if(tool && !tool->get_problems().empty())
 120                                 broken_tools.insert(tool);
 121                 }
 122
 123         // TODO Sort components after their packages, and targets last
 124         for(set<const Package *>::const_iterator i=broken_packages.begin(); i!=broken_packages.end(); ++i)
 125         {
 126                 const list<string> &pkg_problems = (*i)->get_problems();
 127                 for(list<string>::const_iterator j=pkg_problems.begin(); j!=pkg_problems.end(); ++j)
 128                         problems.push_back(format("%s: %s", (*i)->get_name(), *j));
 129         }
 130
 131         for(set<const Component *>::const_iterator i=broken_components.begin(); i!=broken_components.end(); ++i)
 132         {
 133                 const list<string> &comp_problems = (*i)->get_problems();
 134                 for(list<string>::const_iterator j=comp_problems.begin(); j!=comp_problems.end(); ++j)
 135                         problems.push_back(format("%s/%s: %s", (*i)->get_package().get_name(), (*i)->get_name(), *j));
 136         }
 137
 138         for(set<const Tool *>::const_iterator i=broken_tools.begin(); i!=broken_tools.end(); ++i)
 139         {
 140                 const list<string> &tool_problems = (*i)->get_problems();
 141                 for(list<string>::const_iterator j=tool_problems.begin(); j!=tool_problems.end(); ++j)
 142                         problems.push_back(format("%s: %s", (*i)->get_tag(), *j));
 143         }
 144
 145         return problems;
 146 }
 147
 148 void Builder::load_build_file(const FS::Path &fn, const Config::InputOptions *opts, bool all)
 149 {
 150         IO::BufferedFile in(fn.str());
 151
 152         get_logger().log("files", format("Reading %s", fn));
 153
 154         DataFile::Parser parser(in, fn.str());
 155         Loader loader(*this, opts, all);
 156         loader.load(parser);
 157 }
 158
 159 int Builder::build(unsigned jobs, bool dry_run, bool show_progress)
 160 {
 161         unsigned total = build_graph.count_rebuild_targets();
 162
 163         if(!total)
 164         {
 165                 get_logger().log("summary", "Already up to date");
 166                 return 0;
 167         }
 168         get_logger().log("summary", format("Will build %d target%s", total, (total!=1 ? "s" : "")));
 169
 170         vector<Task *> tasks;
 171
 172         unsigned count = 0;
 173
 174         bool fail = false;
 175         bool finish = false;
 176         bool starved = false;
 177
 178         while(!finish)
 179         {
 180                 if(tasks.size()<jobs && !fail && !starved)
 181                 {
 182                         Target *tgt = build_graph.get_buildable_target();
 183                         if(tgt)
 184                         {
 185                                 if(tgt->get_tool())
 186                                         get_logger().log("tasks", format("%-4s  %s", tgt->get_tool()->get_tag(), tgt->get_name()));
 187                                 Task *task = tgt->build();
 188                                 if(task)
 189                                 {
 190                                         get_logger().log("commands", format("%s", task->get_command()));
 191                                         if(dry_run)
 192                                         {
 193                                                 task->signal_finished.emit(true);
 194                                                 delete task;
 195                                         }
 196                                         else
 197                                         {
 198                                                 task->start();
 199                                                 tasks.push_back(task);
 200                                         }
 201                                 }
 202
 203                                 if(show_progress)
 204                                         IO::print("%d of %d target%s built\033[1G", count, total, (total!=1 ? "s" : ""));
 205                         }
 206                         else if(tasks.empty())
 207                                 finish = true;
 208                         else
 209                                 starved = true;
 210                 }
 211                 else
 212                         Time::sleep(10*Time::msec);
 213
 214                 for(unsigned i=0; i<tasks.size();)
 215                 {
 216                         Task::Status status;
 217                         if(jobs==1 || (tasks.size()==1 && starved))
 218                                 status = tasks[i]->wait();
 219                         else
 220                                 status = tasks[i]->check();
 221
 222                         if(status!=Task::RUNNING)
 223                         {
 224                                 ++count;
 225
 226                                 delete tasks[i];
 227                                 tasks.erase(tasks.begin()+i);
 228                                 if(status==Task::ERROR)
 229                                         fail = true;
 230                                 if(tasks.empty() && fail)
 231                                         finish = true;
 232                                 starved = false;
 233                         }
 234                         else
 235                                 ++i;
 236                 }
 237         }
 238
 239         if(show_progress)
 240                 IO::print("\033[K");
 241         if(fail)
 242                 get_logger().log("summary", "Build failed");
 243         else if(show_progress)
 244                 get_logger().log("summary", "Build complete");
 245
 246         if(!dry_run)
 247         {
 248                 const PackageManager::PackageMap &packages = package_manager.get_packages();
 249                 for(PackageManager::PackageMap::const_iterator i=packages.begin(); i!=packages.end(); ++i)
 250                         i->second->save_caches();
 251         }
 252
 253         return fail;
 254 }
 255
 256 int Builder::clean(bool all, bool dry_run)
 257 {
 258         // Cleaning doesn't care about ordering, so a simpler method can be used
 259
 260         set<Target *> clean_tgts;
 261         list<Target *> queue;
 262         queue.push_back(&build_graph.get_goals());
 263
 264         while(!queue.empty())
 265         {
 266                 Target *tgt = queue.front();
 267                 queue.pop_front();
 268
 269                 if(tgt->is_buildable() && (tgt->get_package()==&package_manager.get_main_package() || all))
 270                         clean_tgts.insert(tgt);
 271
 272                 const Target::Dependencies &deps = tgt->get_dependencies();
 273                 for(list<Target *>::const_iterator i=deps.begin(); i!=deps.end(); ++i)
 274                         if(!clean_tgts.count(*i))
 275                                 queue.push_back(*i);
 276         }
 277
 278         for(set<Target *>::iterator i=clean_tgts.begin(); i!=clean_tgts.end(); ++i)
 279         {
 280                 get_logger().log("tasks", format("RM    %s", (*i)->get_name()));
 281                 if(!dry_run)
 282                         (*i)->clean();
 283         }
 284
 285         return 0;
 286 }
 287
 288
 289 Builder::Loader::Loader(Builder &b, const Config::InputOptions *o, bool a):
 290         DataFile::ObjectLoader<Builder>(b),
 291         options(o),
 292         conf_all(a)
 293 {
 294         add("architecture", &Loader::architecture);
 295         add("binary_package", &Loader::binpkg);
 296         add("build_type", &Loader::build_type);
 297         add("package", &Loader::package);
 298
 299         if(!obj.top_loader)
 300                 obj.top_loader = this;
 301         else if(!options && obj.top_loader!=this && obj.top_loader->conf_all)
 302                 options = obj.top_loader->options;
 303 }
 304
 305 Builder::Loader::~Loader()
 306 {
 307         if(obj.top_loader==this)
 308                 obj.top_loader = 0;
 309 }
 310
 311 void Builder::Loader::architecture(const string &n)
 312 {
 313         if(obj.current_arch->match_name(n))
 314                 load_sub(*obj.current_arch);
 315 }
 316
 317 void Builder::Loader::binpkg(const string &n)
 318 {
 319         BinaryPackage *pkg = new BinaryPackage(obj, n);
 320         load_sub(*pkg);
 321 }
 322
 323 void Builder::Loader::build_type(const string &n)
 324 {
 325         BuildType btype(n);
 326         load_sub(btype);
 327         BuildTypeMap::iterator i = obj.build_types.insert(BuildTypeMap::value_type(n, btype)).first;
 328         if(!obj.build_type)
 329                 obj.build_type = &i->second;
 330 }
 331
 332 void Builder::Loader::package(const string &n)
 333 {
 334         SourcePackage *pkg = new SourcePackage(obj, n, get_source());
 335
 336         if(options)
 337                 load_sub(*pkg, *options);
 338         else
 339                 load_sub(*pkg);
 340
 341         if(obj.build_type)
 342                 pkg->set_build_type(*obj.build_type);
 343 }