source/builder.cpp

   1 #include <set>
   2 #include <msp/core/maputils.h>
   3 #include <msp/datafile/parser.h>
   4 #include <msp/fs/dir.h>
   5 #include <msp/fs/utils.h>
   6 #include <msp/io/buffered.h>
   7 #include <msp/io/file.h>
   8 #include <msp/io/print.h>
   9 #include <msp/strings/format.h>
  10 #include <msp/time/units.h>
  11 #include <msp/time/utils.h>
  12 #include "binarypackage.h"
  13 #include "builder.h"
  14 #include "builtintools.h"
  15 #include "datatool.h"
  16 #include "gnutools.h"
  17 #include "installedfile.h"
  18 #include "package.h"
  19 #include "sharedlibrary.h"
  20 #include "sourcepackage.h"
  21 #include "task.h"
  22 #include "virtualtarget.h"
  23
  24 using namespace std;
  25 using namespace Msp;
  26
  27 Builder::Builder():
  28         package_manager(*this),
  29         native_arch(*this, string()),
  30         current_arch(0),
  31         build_type(0),
  32         vfs(*this),
  33         build_graph(*this),
  34         logger(&default_logger),
  35         tempdir("temp"),
  36         top_loader(0)
  37 {
  38         set_architecture(string());
  39 }
  40
  41 Builder::~Builder()
  42 {
  43         if(current_arch!=&native_arch)
  44                 delete current_arch;
  45 }
  46
  47 void Builder::set_architecture(const string &name)
  48 {
  49         if(name.empty())
  50         {
  51                 current_arch = &native_arch;
  52                 prefix = FS::get_home_dir()/"local";
  53         }
  54         else
  55         {
  56                 current_arch = new Architecture(*this, name);
  57                 prefix = FS::get_home_dir()/"local"/current_arch->get_name();
  58         }
  59 }
  60
  61 void Builder::set_build_type(const string &name)
  62 {
  63         build_type = &get_item(build_types, name);
  64 }
  65
  66 void Builder::set_prefix(const FS::Path &p)
  67 {
  68         prefix = p;
  69 }
  70
  71 void Builder::set_temp_directory(const FS::Path &p)
  72 {
  73         tempdir = p;
  74 }
  75
  76 void Builder::add_default_tools()
  77 {
  78         toolchain.add_toolchain(new GnuTools(*this, *current_arch));
  79         toolchain.add_toolchain(new BuiltinTools(*this));
  80         toolchain.add_tool(new DataTool(*this));
  81 }
  82
  83 void Builder::set_logger(const Logger *l)
  84 {
  85         logger = (l ? l : &default_logger);
  86 }
  87
  88 list<string> Builder::collect_problems() const
  89 {
  90         list<string> problems;
  91         set<const Package *> broken_packages;
  92         set<const Component *> broken_components;
  93         set<const Tool *> broken_tools;
  94
  95         const BuildGraph::TargetMap &targets = build_graph.get_targets();
  96         for(BuildGraph::TargetMap::const_iterator i=targets.begin(); i!=targets.end(); ++i)
  97                 if(i->second->is_broken())
  98                 {
  99                         const list<string> &tgt_problems = i->second->get_problems();
 100                         for(list<string>::const_iterator j=tgt_problems.begin(); j!=tgt_problems.end(); ++j)
 101                                 problems.push_back(format("%s: %s", i->second->get_name(), *j));
 102
 103                         const Package *package = i->second->get_package();
 104                         if(package && !package->get_problems().empty())
 105                                 broken_packages.insert(package);
 106
 107                         const Component *component = i->second->get_component();
 108                         if(component && !component->get_problems().empty())
 109                                 broken_components.insert(component);
 110
 111                         const Tool *tool = i->second->get_tool();
 112                         if(tool && !tool->get_problems().empty())
 113                                 broken_tools.insert(tool);
 114                 }
 115
 116         // TODO Sort components after their packages, and targets last
 117         for(set<const Package *>::const_iterator i=broken_packages.begin(); i!=broken_packages.end(); ++i)
 118         {
 119                 const list<string> &pkg_problems = (*i)->get_problems();
 120                 for(list<string>::const_iterator j=pkg_problems.begin(); j!=pkg_problems.end(); ++j)
 121                         problems.push_back(format("%s: %s", (*i)->get_name(), *j));
 122         }
 123
 124         for(set<const Component *>::const_iterator i=broken_components.begin(); i!=broken_components.end(); ++i)
 125         {
 126                 const list<string> &comp_problems = (*i)->get_problems();
 127                 for(list<string>::const_iterator j=comp_problems.begin(); j!=comp_problems.end(); ++j)
 128                         problems.push_back(format("%s/%s: %s", (*i)->get_package().get_name(), (*i)->get_name(), *j));
 129         }
 130
 131         for(set<const Tool *>::const_iterator i=broken_tools.begin(); i!=broken_tools.end(); ++i)
 132         {
 133                 const list<string> &tool_problems = (*i)->get_problems();
 134                 for(list<string>::const_iterator j=tool_problems.begin(); j!=tool_problems.end(); ++j)
 135                         problems.push_back(format("%s: %s", (*i)->get_tag(), *j));
 136         }
 137
 138         return problems;
 139 }
 140
 141 void Builder::load_build_file(const FS::Path &fn, const Config::InputOptions *opts, bool all)
 142 {
 143         IO::BufferedFile in(fn.str());
 144
 145         get_logger().log("files", format("Reading %s", fn));
 146
 147         DataFile::Parser parser(in, fn.str());
 148         Loader loader(*this, opts, all);
 149         loader.load(parser);
 150 }
 151
 152 int Builder::build(unsigned jobs, bool dry_run, bool show_progress)
 153 {
 154         unsigned total = build_graph.count_rebuild_targets();
 155
 156         if(!total)
 157         {
 158                 get_logger().log("summary", "Already up to date");
 159                 return 0;
 160         }
 161         get_logger().log("summary", format("Will build %d target%s", total, (total!=1 ? "s" : "")));
 162
 163         vector<Task *> tasks;
 164
 165         unsigned count = 0;
 166
 167         bool fail = false;
 168         bool finish = false;
 169         bool starved = false;
 170
 171         while(!finish)
 172         {
 173                 if(tasks.size()<jobs && !fail && !starved)
 174                 {
 175                         Target *tgt = build_graph.get_buildable_target();
 176                         if(tgt)
 177                         {
 178                                 if(tgt->get_tool())
 179                                         get_logger().log("tasks", format("%-4s  %s", tgt->get_tool()->get_tag(), tgt->get_name()));
 180                                 Task *task = tgt->build();
 181                                 if(task)
 182                                 {
 183                                         get_logger().log("commands", format("%s", task->get_command()));
 184                                         if(dry_run)
 185                                         {
 186                                                 task->signal_finished.emit(true);
 187                                                 delete task;
 188                                         }
 189                                         else
 190                                         {
 191                                                 task->start();
 192                                                 tasks.push_back(task);
 193                                         }
 194                                 }
 195
 196                                 if(show_progress)
 197                                         IO::print("%d of %d target%s built\033[1G", count, total, (total!=1 ? "s" : ""));
 198                         }
 199                         else if(tasks.empty())
 200                                 finish = true;
 201                         else
 202                                 starved = true;
 203                 }
 204                 else
 205                         Time::sleep(10*Time::msec);
 206
 207                 for(unsigned i=0; i<tasks.size();)
 208                 {
 209                         Task::Status status;
 210                         if(jobs==1 || (tasks.size()==1 && starved))
 211                                 status = tasks[i]->wait();
 212                         else
 213                                 status = tasks[i]->check();
 214
 215                         if(status!=Task::RUNNING)
 216                         {
 217                                 ++count;
 218
 219                                 delete tasks[i];
 220                                 tasks.erase(tasks.begin()+i);
 221                                 if(status==Task::ERROR)
 222                                         fail = true;
 223                                 if(tasks.empty() && fail)
 224                                         finish = true;
 225                                 starved = false;
 226                         }
 227                         else
 228                                 ++i;
 229                 }
 230         }
 231
 232         if(show_progress)
 233                 IO::print("\033[K");
 234         if(fail)
 235                 get_logger().log("summary", "Build failed");
 236         else if(show_progress)
 237                 get_logger().log("summary", "Build complete");
 238
 239         if(!dry_run)
 240         {
 241                 const PackageManager::PackageMap &packages = package_manager.get_packages();
 242                 for(PackageManager::PackageMap::const_iterator i=packages.begin(); i!=packages.end(); ++i)
 243                         i->second->save_caches();
 244         }
 245
 246         return fail;
 247 }
 248
 249 int Builder::clean(bool all, bool dry_run)
 250 {
 251         // Cleaning doesn't care about ordering, so a simpler method can be used
 252
 253         set<Target *> clean_tgts;
 254         list<Target *> queue;
 255         queue.push_back(build_graph.get_target("cmdline"));
 256
 257         while(!queue.empty())
 258         {
 259                 Target *tgt = queue.front();
 260                 queue.erase(queue.begin());
 261
 262                 if(tgt->is_buildable() && (tgt->get_package()==&package_manager.get_main_package() || all))
 263                         clean_tgts.insert(tgt);
 264
 265                 const Target::Dependencies &deps = tgt->get_dependencies();
 266                 for(list<Target *>::const_iterator i=deps.begin(); i!=deps.end(); ++i)
 267                         if(!clean_tgts.count(*i))
 268                                 queue.push_back(*i);
 269         }
 270
 271         for(set<Target *>::iterator i=clean_tgts.begin(); i!=clean_tgts.end(); ++i)
 272         {
 273                 get_logger().log("tasks", format("RM    %s", (*i)->get_name()));
 274                 if(!dry_run)
 275                         (*i)->clean();
 276         }
 277
 278         return 0;
 279 }
 280
 281
 282 Builder::Loader::Loader(Builder &b, const Config::InputOptions *o, bool a):
 283         DataFile::ObjectLoader<Builder>(b),
 284         options(o),
 285         conf_all(a)
 286 {
 287         add("architecture", &Loader::architecture);
 288         add("binary_package", &Loader::binpkg);
 289         add("build_type", &Loader::build_type);
 290         add("package", &Loader::package);
 291
 292         if(!obj.top_loader)
 293                 obj.top_loader = this;
 294         else if(!options && obj.top_loader!=this && obj.top_loader->conf_all)
 295                 options = obj.top_loader->options;
 296 }
 297
 298 Builder::Loader::~Loader()
 299 {
 300         if(obj.top_loader==this)
 301                 obj.top_loader = 0;
 302 }
 303
 304 void Builder::Loader::architecture(const string &n)
 305 {
 306         if(obj.current_arch->match_name(n))
 307                 load_sub(*obj.current_arch);
 308 }
 309
 310 void Builder::Loader::binpkg(const string &n)
 311 {
 312         BinaryPackage *pkg = new BinaryPackage(obj, n);
 313         load_sub(*pkg);
 314 }
 315
 316 void Builder::Loader::build_type(const string &n)
 317 {
 318         BuildType btype(n);
 319         load_sub(btype);
 320         BuildTypeMap::iterator i = obj.build_types.insert(BuildTypeMap::value_type(n, btype)).first;
 321         if(!obj.build_type)
 322                 obj.build_type = &i->second;
 323 }
 324
 325 void Builder::Loader::package(const string &n)
 326 {
 327         SourcePackage *pkg = new SourcePackage(obj, n, get_source());
 328
 329         if(options)
 330                 load_sub(*pkg, *options);
 331         else
 332                 load_sub(*pkg);
 333
 334         if(obj.build_type)
 335                 pkg->set_build_type(*obj.build_type);
 336 }