source/builder.cpp

   1 #include <set>
   2 #include <msp/core/maputils.h>
   3 #include <msp/datafile/parser.h>
   4 #include <msp/fs/dir.h>
   5 #include <msp/fs/utils.h>
   6 #include <msp/io/buffered.h>
   7 #include <msp/io/file.h>
   8 #include <msp/io/print.h>
   9 #include <msp/strings/format.h>
  10 #include <msp/time/timedelta.h>
  11 #include <msp/time/utils.h>
  12 #include "androidtools.h"
  13 #include "binarypackage.h"
  14 #include "builder.h"
  15 #include "builtintools.h"
  16 #include "clangtools.h"
  17 #include "datatool.h"
  18 #include "gnutools.h"
  19 #include "installedfile.h"
  20 #include "package.h"
  21 #include "sharedlibrary.h"
  22 #include "sourcepackage.h"
  23 #include "task.h"
  24 #include "virtualtarget.h"
  25
  26 using namespace std;
  27 using namespace Msp;
  28
  29 Builder::Builder():
  30         package_manager(*this),
  31         native_arch(*this, string()),
  32         current_arch(0),
  33         build_type(0),
  34         vfs(*this),
  35         build_graph(*this),
  36         logger(&default_logger),
  37         tempdir("temp"),
  38         top_loader(0)
  39 {
  40         set_architecture(string());
  41 }
  42
  43 Builder::~Builder()
  44 {
  45         if(current_arch!=&native_arch)
  46                 delete current_arch;
  47 }
  48
  49 void Builder::set_architecture(const string &name)
  50 {
  51         if(name.empty())
  52         {
  53                 current_arch = &native_arch;
  54                 prefix = FS::get_home_dir()/"local";
  55         }
  56         else
  57         {
  58                 current_arch = new Architecture(*this, name);
  59                 prefix = FS::get_home_dir()/"local"/current_arch->get_name();
  60         }
  61 }
  62
  63 vector<string> Builder::get_build_types() const
  64 {
  65         vector<string> keys;
  66         keys.reserve(build_types.size());
  67         for(BuildTypeMap::const_iterator i=build_types.begin(); i!=build_types.end(); ++i)
  68                 keys.push_back(i->first);
  69         return keys;
  70 }
  71
  72 void Builder::set_build_type(const string &name)
  73 {
  74         build_type = &get_item(build_types, name);
  75 }
  76
  77 void Builder::set_prefix(const FS::Path &p)
  78 {
  79         prefix = p;
  80 }
  81
  82 void Builder::set_temp_directory(const FS::Path &p)
  83 {
  84         tempdir = p;
  85 }
  86
  87 void Builder::add_default_tools()
  88 {
  89         const string &arch_tc = current_arch->get_toolchain();
  90         if(current_arch->get_system()=="android")
  91                 toolchain.add_toolchain(new AndroidTools(*this, *current_arch));
  92         else if(arch_tc=="clang")
  93                 toolchain.add_toolchain(new ClangTools(*this, *current_arch));
  94         else if(arch_tc=="gnu")
  95                 toolchain.add_toolchain(new GnuTools(*this, *current_arch));
  96         toolchain.add_toolchain(new BuiltinTools(*this));
  97         toolchain.add_tool(new DataTool(*this));
  98 }
  99
 100 void Builder::set_logger(const Logger *l)
 101 {
 102         logger = (l ? l : &default_logger);
 103 }
 104
 105 list<string> Builder::collect_problems() const
 106 {
 107         list<string> problems;
 108         set<const Package *> broken_packages;
 109         set<const Component *> broken_components;
 110         set<const Tool *> broken_tools;
 111
 112         const BuildGraph::TargetMap &targets = build_graph.get_targets();
 113         for(BuildGraph::TargetMap::const_iterator i=targets.begin(); i!=targets.end(); ++i)
 114                 if(i->second->is_broken())
 115                 {
 116                         const list<string> &tgt_problems = i->second->get_problems();
 117                         for(list<string>::const_iterator j=tgt_problems.begin(); j!=tgt_problems.end(); ++j)
 118                                 problems.push_back(format("%s: %s", i->second->get_name(), *j));
 119
 120                         const Package *package = i->second->get_package();
 121                         if(package && !package->get_problems().empty())
 122                                 broken_packages.insert(package);
 123
 124                         const Component *component = i->second->get_component();
 125                         if(component && !component->get_problems().empty())
 126                                 broken_components.insert(component);
 127
 128                         const Tool *tool = i->second->get_tool();
 129                         if(tool && !tool->get_problems().empty())
 130                                 broken_tools.insert(tool);
 131                 }
 132
 133         // TODO Sort components after their packages, and targets last
 134         for(set<const Package *>::const_iterator i=broken_packages.begin(); i!=broken_packages.end(); ++i)
 135         {
 136                 const list<string> &pkg_problems = (*i)->get_problems();
 137                 for(list<string>::const_iterator j=pkg_problems.begin(); j!=pkg_problems.end(); ++j)
 138                         problems.push_back(format("%s: %s", (*i)->get_name(), *j));
 139         }
 140
 141         for(set<const Component *>::const_iterator i=broken_components.begin(); i!=broken_components.end(); ++i)
 142         {
 143                 const list<string> &comp_problems = (*i)->get_problems();
 144                 for(list<string>::const_iterator j=comp_problems.begin(); j!=comp_problems.end(); ++j)
 145                         problems.push_back(format("%s/%s: %s", (*i)->get_package().get_name(), (*i)->get_name(), *j));
 146         }
 147
 148         for(set<const Tool *>::const_iterator i=broken_tools.begin(); i!=broken_tools.end(); ++i)
 149         {
 150                 const list<string> &tool_problems = (*i)->get_problems();
 151                 for(list<string>::const_iterator j=tool_problems.begin(); j!=tool_problems.end(); ++j)
 152                         problems.push_back(format("%s: %s", (*i)->get_tag(), *j));
 153         }
 154
 155         return problems;
 156 }
 157
 158 void Builder::load_build_file(const FS::Path &fn, const Config::InputOptions *opts, bool all)
 159 {
 160         IO::BufferedFile in(fn.str());
 161
 162         get_logger().log("files", format("Reading %s", fn));
 163
 164         DataFile::Parser parser(in, fn.str());
 165         Loader loader(*this, opts, all);
 166         loader.load(parser);
 167 }
 168
 169 int Builder::build(unsigned jobs, bool dry_run, bool show_progress)
 170 {
 171         unsigned total = build_graph.count_rebuild_targets();
 172
 173         if(!total)
 174         {
 175                 get_logger().log("summary", "Already up to date");
 176                 return 0;
 177         }
 178         get_logger().log("summary", format("Will build %d target%s", total, (total!=1 ? "s" : "")));
 179
 180         vector<Task *> tasks;
 181
 182         unsigned count = 0;
 183
 184         bool fail = false;
 185         bool finish = false;
 186         bool starved = false;
 187
 188         while(!finish)
 189         {
 190                 if(tasks.size()<jobs && !fail && !starved)
 191                 {
 192                         Target *tgt = build_graph.get_buildable_target();
 193                         if(tgt)
 194                         {
 195                                 if(tgt->get_tool())
 196                                         get_logger().log("tasks", format("%-4s  %s", tgt->get_tool()->get_tag(), tgt->get_name()));
 197                                 Task *task = tgt->build();
 198                                 if(task)
 199                                 {
 200                                         get_logger().log("commands", format("%s", task->get_command()));
 201                                         if(dry_run)
 202                                         {
 203                                                 task->signal_finished.emit(true);
 204                                                 delete task;
 205                                         }
 206                                         else
 207                                         {
 208                                                 task->start();
 209                                                 tasks.push_back(task);
 210                                         }
 211                                 }
 212
 213                                 if(show_progress)
 214                                         IO::print("%d of %d target%s built\033[1G", count, total, (total!=1 ? "s" : ""));
 215                         }
 216                         else if(tasks.empty())
 217                                 finish = true;
 218                         else
 219                                 starved = true;
 220                 }
 221                 else
 222                         Time::sleep(10*Time::msec);
 223
 224                 for(unsigned i=0; i<tasks.size();)
 225                 {
 226                         Task::Status status;
 227                         if(jobs==1 || (tasks.size()==1 && starved))
 228                                 status = tasks[i]->wait();
 229                         else
 230                                 status = tasks[i]->check();
 231
 232                         if(status!=Task::RUNNING)
 233                         {
 234                                 ++count;
 235
 236                                 delete tasks[i];
 237                                 tasks.erase(tasks.begin()+i);
 238                                 if(status==Task::ERROR)
 239                                         fail = true;
 240                                 if(tasks.empty() && fail)
 241                                         finish = true;
 242                                 starved = false;
 243                         }
 244                         else
 245                                 ++i;
 246                 }
 247         }
 248
 249         if(show_progress)
 250                 IO::print("\033[K");
 251         if(fail)
 252                 get_logger().log("summary", "Build failed");
 253         else if(show_progress)
 254                 get_logger().log("summary", "Build complete");
 255
 256         if(!dry_run)
 257         {
 258                 const PackageManager::PackageMap &packages = package_manager.get_packages();
 259                 for(PackageManager::PackageMap::const_iterator i=packages.begin(); i!=packages.end(); ++i)
 260                         i->second->save_caches();
 261         }
 262
 263         return fail;
 264 }
 265
 266 int Builder::clean(bool all, bool dry_run)
 267 {
 268         // Cleaning doesn't care about ordering, so a simpler method can be used
 269
 270         set<Target *> clean_tgts;
 271         list<Target *> queue;
 272         queue.push_back(&build_graph.get_goals());
 273
 274         while(!queue.empty())
 275         {
 276                 Target *tgt = queue.front();
 277                 queue.pop_front();
 278
 279                 if(tgt->is_buildable() && (tgt->get_package()==&package_manager.get_main_package() || all))
 280                         clean_tgts.insert(tgt);
 281
 282                 const Target::Dependencies &deps = tgt->get_dependencies();
 283                 for(list<Target *>::const_iterator i=deps.begin(); i!=deps.end(); ++i)
 284                         if(!clean_tgts.count(*i))
 285                                 queue.push_back(*i);
 286         }
 287
 288         for(set<Target *>::iterator i=clean_tgts.begin(); i!=clean_tgts.end(); ++i)
 289         {
 290                 get_logger().log("tasks", format("RM    %s", (*i)->get_name()));
 291                 if(!dry_run)
 292                         (*i)->clean();
 293         }
 294
 295         return 0;
 296 }
 297
 298
 299 Builder::Loader::Loader(Builder &b, const Config::InputOptions *o, bool a):
 300         DataFile::ObjectLoader<Builder>(b),
 301         options(o),
 302         conf_all(a)
 303 {
 304         add("architecture", &Loader::architecture);
 305         add("binary_package", &Loader::binpkg);
 306         add("build_type", &Loader::build_type);
 307         add("package", &Loader::package);
 308
 309         if(!obj.top_loader)
 310                 obj.top_loader = this;
 311         else if(!options && obj.top_loader!=this && obj.top_loader->conf_all)
 312                 options = obj.top_loader->options;
 313 }
 314
 315 Builder::Loader::~Loader()
 316 {
 317         if(obj.top_loader==this)
 318                 obj.top_loader = 0;
 319 }
 320
 321 void Builder::Loader::architecture(const string &n)
 322 {
 323         if(obj.current_arch->match_name(n))
 324                 load_sub(*obj.current_arch);
 325 }
 326
 327 void Builder::Loader::binpkg(const string &n)
 328 {
 329         BinaryPackage *pkg = new BinaryPackage(obj, n);
 330         load_sub(*pkg);
 331 }
 332
 333 void Builder::Loader::build_type(const string &n)
 334 {
 335         BuildType btype(n);
 336         load_sub(btype);
 337         BuildTypeMap::iterator i = obj.build_types.insert(BuildTypeMap::value_type(n, btype)).first;
 338         if(!obj.build_type)
 339                 obj.build_type = &i->second;
 340 }
 341
 342 void Builder::Loader::package(const string &n)
 343 {
 344         SourcePackage *pkg = new SourcePackage(obj, n, get_source());
 345
 346         if(options)
 347                 load_sub(*pkg, *options);
 348         else
 349                 load_sub(*pkg);
 350
 351         if(obj.build_type)
 352                 pkg->set_build_type(*obj.build_type);
 353 }