]> git.tdb.fi Git - builder.git/blob - source/builder.cpp
c271c2bb56cf81b211efd8013e4ade5de0e92b59
[builder.git] / source / builder.cpp
1 #include <fstream>
2 #include <iostream>
3 #include <msp/progress.h>
4 #include <msp/strconv.h>
5 #include <msp/strutils.h>
6 #include <msp/core/error.h>
7 #include <msp/getopt++/getopt++.h>
8 #include <msp/parser/parser.h>
9 #include <msp/path/utils.h>
10 #include <msp/time/units.h>
11 #include <msp/time/utils.h>
12 #include "action.h"
13 #include "analyzer.h"
14 #include "builder.h"
15 #include "executable.h"
16 #include "header.h"
17 #include "install.h"
18 #include "misc.h"
19 #include "objectfile.h"
20 #include "package.h"
21 #include "sharedlibrary.h"
22 #include "staticlibrary.h"
23 #include "systemlibrary.h"
24 #include "virtualtarget.h"
25
26 using namespace std;
27 using namespace Msp;
28
29 Builder::Builder(int argc, char **argv):
30         analyzer(0),
31         do_build(true),
32         verbose(1),
33         chrome(false),
34         build_file("Build"),
35         jobs(1)
36 {
37         GetOpt getopt;
38         getopt.add_option(GetOpt::Option('a', "analyze", GetOpt::REQUIRED));
39         getopt.add_option(GetOpt::Option('b', "build", GetOpt::NONE));
40         getopt.add_option(GetOpt::Option('c', "clean", GetOpt::NONE));
41         getopt.add_option(GetOpt::Option('f', "file", GetOpt::REQUIRED, "Build"));
42         getopt.add_option(GetOpt::Option('h', "help", GetOpt::NONE));
43         getopt.add_option(GetOpt::Option('j', "jobs", GetOpt::REQUIRED, "1"));
44         getopt.add_option(GetOpt::Option('n', "dry-run", GetOpt::NONE));
45         getopt.add_option(GetOpt::Option('v', "verbose", GetOpt::NONE));
46         getopt.add_option(GetOpt::Option('A', "conf-all", GetOpt::NONE));
47         getopt.add_option(GetOpt::Option('B', "build-all", GetOpt::NONE));
48         getopt.add_option(GetOpt::Option('C', "chdir", GetOpt::REQUIRED));
49         getopt.add_option(GetOpt::Option('W', "what-if", GetOpt::REQUIRED));
50         getopt.add_option(GetOpt::Option("chrome", GetOpt::NONE));
51         getopt.add_option(GetOpt::Option("full-paths", GetOpt::NONE));
52         getopt.add_option(GetOpt::Option("max-depth", GetOpt::REQUIRED));
53         int index=getopt(argc, argv);
54
55         verbose+=getopt['v'].count();
56
57         if(getopt['a'])
58         {
59                 analyzer=new Analyzer(*this);
60
61                 string mode=getopt['a'].arg();
62                 if(mode=="deps")
63                         analyzer->set_mode(Analyzer::DEPS);
64                 else if(mode=="alldeps")
65                         analyzer->set_mode(Analyzer::ALLDEPS);
66                 else if(mode=="rebuild")
67                         analyzer->set_mode(Analyzer::REBUILD);
68                 else if(mode=="rdeps")
69                         analyzer->set_mode(Analyzer::RDEPS);
70                 else
71                         throw UsageError("Invalid analysis mode");
72
73                 if(getopt["max-depth"])
74                         analyzer->set_max_depth(strtol(getopt["max-depth"].arg()));
75                 analyzer->set_full_paths(getopt["full-paths"]);
76
77                 if(!getopt['b'])
78                         do_build=false;
79         }
80
81         dry_run=getopt['n'];
82         jobs=max(strtol(getopt['j'].arg()), 1L);
83         chrome=getopt["chrome"];
84         conf_all=getopt['A'];
85         build_file=getopt['f'].arg();
86         build_all=getopt['B'];
87         help=getopt['h'];
88
89         if(getopt['C'])
90                 chdir(getopt['C'].arg().c_str());
91
92         for(int i=index; i<argc; ++i)
93         {
94                 string v(argv[i]);
95                 unsigned equal=v.find('=');
96                 if(equal!=string::npos)
97                         cmdline_options.insert(RawOptionMap::value_type(v.substr(0, equal), v.substr(equal+1)));
98                 else
99                         cmdline_targets.push_back(argv[i]);
100         }
101
102         if(cmdline_targets.empty())
103                 cmdline_targets.push_back("default");
104
105         if(getopt['W'])
106                 what_if.push_back(getopt['W'].arg());
107
108         cwd=Path::getcwd();
109 }
110
111 /**
112 Gets a package with the specified name, possibly creating it.
113
114 @param   n  Package name
115
116 @return  Pointer to the package, or 0 if the package could not be located
117 */
118 Package *Builder::get_package(const string &n)
119 {
120         PackageMap::iterator i=packages.find(n);
121         if(i!=packages.end())
122                 return i->second;
123
124         // Try to get source directory with pkgconfig
125         list<string> argv;
126         argv.push_back("pkg-config");
127         argv.push_back("--variable=source");
128         argv.push_back(n);
129         string srcdir=strip(run_command(argv));
130         
131         PathList dirs;
132         if(!srcdir.empty())
133                 dirs.push_back(srcdir);
134
135         // Make some other guesses about the source directory
136         string dirname=n;
137         if(!dirname.compare(0, 3, "msp"))
138                 dirname.erase(0, 3);
139         dirs.push_back(cwd/dirname);
140         dirs.push_back(cwd/".."/dirname);
141
142         // Go through the candidate directories and look for a Build file
143         for(PathList::iterator j=dirs.begin(); j!=dirs.end(); ++j)
144                 if(!load_build_file(*j/"Build"))
145                 {
146                         i=packages.find(n);
147                         if(i!=packages.end())
148                                 return i->second;
149                         break;
150                 }
151
152         // Package source not found - create a binary package
153         Package *pkg=Package::create(*this, n);
154         packages.insert(PackageMap::value_type(n, pkg));
155         if(pkg)
156                 new_pkgs.push_back(pkg);
157
158         return pkg;
159 }
160
161 /**
162 Returns the target with the given name, or 0 if no such target exists.
163 */
164 Target *Builder::get_target(const string &n)
165 {
166         TargetMap::iterator i=targets.find(n);
167         if(i!=targets.end())
168                 return i->second;
169         return 0;
170 }
171
172 /**
173 Tries to locate a header included from a given location and with a given include
174 path.  Considers known targets as well as existing files.  If a matching target
175 is not found but a file exists, a new SystemHeader target will be created and
176 returned.
177 */
178 Target *Builder::get_header(const string &include, const string &from, const list<string> &path)
179 {
180         string hash(8, 0);
181         update_hash(hash, from);
182         for(list<string>::const_iterator i=path.begin(); i!=path.end(); ++i)
183                 update_hash(hash, *i);
184
185         string id=hash+include;
186         TargetMap::iterator i=includes.find(id);
187         if(i!=includes.end())
188                 return i->second;
189
190         string fn=include.substr(1);
191         Target *tgt=0;
192         if(include[0]=='"' && (tgt=check_header(Path::Path(from)/fn)))
193                 return tgt;
194         if((tgt=check_header(Path::Path("/usr/include")/fn)))
195                 return tgt;
196         //XXX Determine the C++ header location dynamically
197         if((tgt=check_header(Path::Path("/usr/include/c++/4.1.2")/fn)))
198                 return tgt;
199         for(list<string>::const_iterator j=path.begin(); j!=path.end(); ++j)
200                 if((tgt=check_header(cwd/ *j/fn)))
201                         return tgt;
202         
203         return 0;
204 }
205
206 /**
207 Tries to locate a library with the given library path.  Considers known targets
208 as well as existing files.  If a matching target is not found but a file exists,
209 a new SystemLibrary target will be created and returned.
210 */
211 Target *Builder::get_library(const string &lib, const list<string> &path)
212 {
213         string hash(8, 0);
214         for(list<string>::const_iterator i=path.begin(); i!=path.end(); ++i)
215                 update_hash(hash, *i);
216
217         string id=hash+lib;
218         TargetMap::iterator i=libraries.find(id);
219         if(i!=libraries.end())
220                 return i->second;
221
222         string basename="lib"+lib+".so";
223         for(list<string>::const_iterator j=path.begin(); j!=path.end(); ++j)
224         {
225                 string full=(cwd/ *j/basename).str();
226                 Target *tgt=get_target(full);
227                 if(tgt) return tgt;
228                 
229                 if(Path::exists(full))
230                 {
231                         add_target(tgt=new SystemLibrary(*this, full));
232                         return tgt;
233                 }
234         }
235
236         return 0;
237 }
238
239 int Builder::main()
240 {
241         if(load_build_file(cwd/build_file))
242         {
243                 cerr<<"No build info here.\n";
244                 return 1;
245         }
246
247         default_pkg=packages.begin()->second;
248
249         while(!new_pkgs.empty())
250         {
251                 Package *pkg=new_pkgs.front();
252                 if(pkg==default_pkg || conf_all)
253                         pkg->process_options(cmdline_options);
254                 new_pkgs.erase(new_pkgs.begin());
255                 pkg->resolve_refs();
256         }
257
258         if(help)
259         {
260                 usage("builder", false);
261                 cout<<'\n';
262                 package_help();
263                 return 0;
264         }
265
266         StringList missing;
267         for(PackageMap::iterator i=packages.begin(); i!=packages.end(); ++i)
268                 if(!i->second)
269                         missing.push_back(i->first);
270
271         if(!missing.empty())
272         {
273                 missing.sort();
274                 cerr<<"The following packages were not found on the system:\n";
275                 for(list<string>::iterator i=missing.begin(); i!=missing.end(); ++i)
276                         cerr<<"  "<<*i<<'\n';
277                 cerr<<"Please install them and try again.\n";
278                 return 1;
279         }
280
281         default_pkg->create_build_info();
282
283         if(create_targets())
284                 return 1;
285
286         cout<<packages.size()<<" packages, "<<targets.size()<<" targets\n";
287         if(verbose>=2)
288         {
289                 for(PackageMap::iterator i=packages.begin(); i!=packages.end(); ++i)
290                 {
291                         cout<<' '<<i->second->get_name();
292                         if(i->second->get_buildable())
293                                 cout<<'*';
294                         unsigned count=0;
295                         unsigned ood_count=0;
296                         for(TargetMap::iterator j=targets.begin(); j!=targets.end(); ++j)
297                                 if(j->second->get_package()==i->second)
298                                 {
299                                         ++count;
300                                         if(j->second->get_rebuild())
301                                                 ++ood_count;
302                                 }
303                         if(count)
304                         {
305                                 cout<<" ("<<count<<" targets";
306                                 if(ood_count)
307                                         cout<<", "<<ood_count<<" out-of-date";
308                                 cout<<')';
309                         }
310                         cout<<'\n';
311                 }
312         }
313
314         if(analyzer)
315                 analyzer->analyze();
316
317         if(do_build)
318                 build();
319
320         return exit_code;
321 }
322
323 Builder::~Builder()
324 {
325         for(PackageMap::iterator i=packages.begin(); i!=packages.end(); ++i)
326                 delete i->second;
327         for(TargetMap::iterator i=targets.begin(); i!=targets.end(); ++i)
328                 delete i->second;
329         delete analyzer;
330 }
331
332 void Builder::usage(const char *argv0, bool brief)
333 {
334         if(brief)
335                 cerr<<"Usage: "<<argv0<<" [-a|--analyze MODE] [-b|--build] [-c|--clean] [-f|--file FILE] [-h|--help] [-j|--jobs NUM] [-n||--dry-run] [-v|--verbose] [-A|--conf-all] [-B|--build-all] [-C|--chdir DIRECTORY] [-W|--what-if FILE] [--chrome] [--full-paths] [--max-depth NUM] [<target> ...]";
336         else
337         {
338                 cerr<<
339                         "Usage: "<<argv0<<" [options] [<target> ...]\n"
340                         "\n"
341                         "Options:\n"
342                         "  -a, --analyze MODE  Perform analysis.  MODE can be deps, alldeps or rebuild.\n"
343                         "  -b, --build         Perform build even if doing analysis.\n"
344                         "  -c, --clean         Clean buildable targets.\n"
345                         "  -f, --file FILE     Read info from FILE instead of Build.\n"
346                         "  -h, --help          Print this message.\n"
347                         "  -j, --jobs NUM      Run NUM commands at once, whenever possible.\n"
348                         "  -n, --dry-run       Don't actually do anything, only show what would be done.\n"
349                         "  -v, --verbose       Print more information about what's going on.\n"
350                         "  -A, --conf-all      Apply configuration to all packages.\n"
351                         "  -B, --build-all     Build all targets unconditionally.\n"
352                         "  -C, --chdir DIR     Change to DIR before doing anything else.\n"
353                         "  -W, --what-if FILE  Pretend that FILE has changed.\n"
354                         "  --chrome            Use extra chrome to print status.\n"
355                         "  --full-paths        Output full paths in analysis.\n"
356                         "  --max-depth NUM     Maximum depth to show in analysis.\n";
357         }
358 }
359
360 /**
361 Loads the given build file.
362
363 @param   fn  Path to the file
364
365 @return  0 on success, -1 if the file could not be opened
366 */
367 int Builder::load_build_file(const Path::Path &fn)
368 {
369         ifstream in(fn.str().c_str());
370         if(!in)
371                 return -1;
372
373         Parser::Parser parser(in, fn.str());
374         Loader loader(*this, fn.subpath(0, fn.size()-1));
375         loader.load(parser);
376
377         return 0;
378 }
379
380 /**
381 Creates targets for all packages and prepares them for building.
382
383 @return  0 if everything went ok, -1 if something bad happened and a build
384          shouldn't be attempted
385 */
386 int Builder::create_targets()
387 {
388         Target *world=new VirtualTarget(*this, "world");
389         add_target(world);
390
391         Target *def_tgt=new VirtualTarget(*this, "default");
392         add_target(def_tgt);
393         world->add_depend(def_tgt);
394
395         Target *install=new VirtualTarget(*this, "install");
396         add_target(install);
397         world->add_depend(install);
398
399         for(PackageMap::iterator i=packages.begin(); i!=packages.end(); ++i)
400         {
401                 if(!i->second)
402                         continue;
403                 if(!i->second->get_buildable())
404                         continue;
405
406                 Path::Path inst_base;
407                 if(i->second->get_config().is_option("prefix"))
408                         inst_base=i->second->get_config().get_option("prefix").value;
409
410                 const ComponentList &components=i->second->get_components();
411                 for(ComponentList::const_iterator j=components.begin(); j!=components.end(); ++j)
412                 {
413                         // Collect all files belonging to the component
414                         PathList files;
415                         const PathList &sources=j->get_sources();
416                         for(PathList::const_iterator k=sources.begin(); k!=sources.end(); ++k)
417                         {
418                                 struct stat st;
419                                 stat(*k, st);
420                                 if(S_ISDIR(st.st_mode))
421                                 {
422                                         list<string> sfiles=list_files(*k);
423                                         for(list<string>::iterator l=sfiles.begin(); l!=sfiles.end(); ++l)
424                                                 files.push_back(*k / *l);
425                                 }
426                                 else
427                                         files.push_back(*k);
428                         }
429
430                         bool build_exe=j->get_type()!=Component::HEADERS;
431                         
432                         list<ObjectFile *> objs;
433                         for(PathList::iterator k=files.begin(); k!=files.end(); ++k)
434                         {
435                                 string basename=(*k)[-1];
436                                 string ext=Path::splitext(basename).ext;
437                                 if((ext==".cpp" || ext==".c") && build_exe)
438                                 {
439                                         SourceFile *src=new SourceFile(*this, &*j, k->str());
440                                         add_target(src);
441
442                                         // Compile sources
443                                         ObjectFile *obj=new ObjectFile(*this, *j, *src);
444                                         add_target(obj);
445                                         objs.push_back(obj);
446                                 }
447                                 else if(ext==".h")
448                                 {
449                                         Target *hdr=get_target(k->str());
450                                         if(!hdr)
451                                         {
452                                                 hdr=new Header(*this, &*j, k->str());
453                                                 add_target(hdr);
454                                         }
455
456                                         // Install headers if requested
457                                         if(!j->get_install_headers().empty())
458                                         {
459                                                 Path::Path inst_path=inst_base/"include"/j->get_install_headers()/basename;
460                                                 Install *inst=new Install(*this, *i->second, *hdr, inst_path.str());
461                                                 add_target(inst);
462                                                 install->add_depend(inst);
463                                         }
464                                 }
465                         }
466
467                         if(build_exe)
468                         {
469                                 Executable    *exe=0;
470                                 StaticLibrary *slib=0;
471                                 if(j->get_type()==Component::LIBRARY)
472                                 {
473                                         exe=new SharedLibrary(*this, *j, objs);
474                                         slib=new StaticLibrary(*this, *j, objs);
475                                         add_target(slib);
476                                 }
477                                 else
478                                         exe=new Executable(*this, *j, objs);
479                                 
480                                 add_target(exe);
481                                 if(i->second==default_pkg)
482                                 {
483                                         def_tgt->add_depend(exe);
484                                         if(slib) def_tgt->add_depend(slib);
485                                 }
486                                 else
487                                 {
488                                         world->add_depend(exe);
489                                         if(slib) world->add_depend(slib);
490                                 }
491
492                                 if(j->get_install())
493                                 {
494                                         string inst_dir;
495                                         if(j->get_type()==Component::PROGRAM)
496                                                 inst_dir="bin";
497                                         else if(j->get_type()==Component::LIBRARY)
498                                                 inst_dir="lib";
499                                         if(!inst_dir.empty())
500                                         {
501                                                 Install *inst=new Install(*this, *i->second, *exe, (inst_base/inst_dir/Path::basename(exe->get_name())).str());
502                                                 add_target(inst);
503                                                 install->add_depend(inst);
504
505                                                 if(slib)
506                                                 {
507                                                         inst=new Install(*this, *i->second, *slib, (inst_base/inst_dir/Path::basename(slib->get_name())).str());
508                                                         add_target(inst);
509                                                         install->add_depend(inst);
510                                                 }
511                                         }
512                                 }
513                         }
514                 }
515         }
516
517         // Find dependencies until no new targets are created
518         while(!new_tgts.empty())
519         {
520                 Target *tgt=new_tgts.front();
521                 new_tgts.erase(new_tgts.begin());
522                 tgt->find_depends();
523                 if(!tgt->get_depends_ready())
524                         new_tgts.push_back(tgt);
525         }
526
527         // Apply what-ifs
528         for(StringList::iterator i=what_if.begin(); i!=what_if.end(); ++i)
529         {
530                 Target *tgt=get_target((cwd/ *i).str());
531                 if(!tgt)
532                 {
533                         cerr<<"Unknown what-if target "<<*i<<'\n';
534                         return -1;
535                 }
536                 tgt->touch();
537         }
538
539         // Make the cmdline target depend on all targets mentioned on the command line
540         Target *cmdline=new VirtualTarget(*this, "cmdline");
541         add_target(cmdline);
542         world->add_depend(cmdline);
543         for(list<string>::iterator i=cmdline_targets.begin(); i!=cmdline_targets.end(); ++i)
544         {
545                 Target *tgt=get_target(*i);
546                 if(!tgt)
547                         tgt=get_target((cwd/ *i).str());
548                 if(!tgt)
549                 {
550                         cerr<<"I don't know anything about "<<*i<<'\n';
551                         return -1;
552                 }
553                 cmdline->add_depend(tgt);
554         }
555
556         world->prepare();
557
558         return 0;
559 }
560
561 /**
562 Check if a header exists, either as a target or a file.  Either an existing
563 target or a new SystemHeader target will be returned.
564 */
565 Target *Builder::check_header(const Msp::Path::Path &fn)
566 {
567         Target *tgt=get_target(fn.str());
568         if(tgt) return tgt;
569
570         if(Path::exists(fn))
571         {
572                 add_target(tgt=new SystemHeader(*this, fn.str()));
573                 return tgt;
574         }
575         return 0;
576 }
577
578 /**
579 Adds a target to both the target map and the new target queue.
580 */
581 void Builder::add_target(Target *t)
582 {
583         targets.insert(TargetMap::value_type(t->get_name(), t));
584         new_tgts.push_back(t);
585 }
586
587 /**
588 Updates a hash with a string.  This is used from get_header and get_library.
589 */
590 void Builder::update_hash(string &hash, const string &value)
591 {
592         for(unsigned i=0; i<value.size(); ++i)
593                 hash[i%hash.size()]^=value[i];
594 }
595
596 /**
597 This function supervises the build process, starting new actions when slots
598 become available.
599 */
600 int Builder::build()
601 {
602         Target *cmdline=get_target("cmdline");
603
604         unsigned total=cmdline->count_rebuild();
605         if(!total)
606         {
607                 cout<<"Already up to date\n";
608                 return 0;
609         }
610         cout<<"Will build "<<total<<" target(s)\n";
611
612         vector<Action *> actions;
613
614         if(chrome)
615                 cout<<"0 targets built\n";
616         unsigned count=0;
617
618         bool fail=false;
619         bool finish=false;
620
621         while(!finish)
622         {
623                 if(actions.size()<jobs && !fail)
624                 {
625                         Target *tgt=cmdline->get_buildable_target();
626                         if(tgt)
627                         {
628                                 Action *action=tgt->build();
629                                 if(action)
630                                         actions.push_back(action);
631                         }
632                         else if(actions.empty())
633                                 finish=true;
634                 }
635                 else
636                         Time::sleep(10*Time::msec);
637
638                 for(unsigned i=0; i<actions.size();)
639                 {
640                         int status=actions[i]->check();
641                         if(status>=0)
642                         {
643                                 ++count;
644                                 if(chrome)
645                                 {
646                                         cout<<"\e["<<actions.size()+1<<'A';
647                                         cout<<count<<" targets built\n";
648                                         if(i)
649                                                 cout<<"\e["<<i<<"B";
650                                         cout<<"\e[M";
651                                         if(i<actions.size()-1)
652                                                 cout<<"\e["<<actions.size()-i-1<<"B";
653                                         cout.flush();
654                                 }
655                                 delete actions[i];
656                                 actions.erase(actions.begin()+i);
657                                 if(status>0)
658                                         fail=true;
659                                 if(actions.empty() && fail)
660                                         finish=true;
661                         }
662                         else
663                                 ++i;
664                 }
665         }
666
667         return fail?-1:0;
668 }
669
670 /**
671 Prints out information about the default package.
672 */
673 void Builder::package_help()
674 {
675         const Config &config=default_pkg->get_config();
676         const Config::OptionMap &options=config.get_options();
677
678         cout<<"Required packages:\n  ";
679         const PkgRefList &requires=default_pkg->get_requires();
680         for(PkgRefList::const_iterator i=requires.begin(); i!=requires.end(); ++i)
681         {
682                 if(i!=requires.begin())
683                         cout<<", ";
684                 cout<<i->get_name();
685         }
686         cout<<"\n\n";
687         cout<<"Package configuration:\n";
688         for(Config::OptionMap::const_iterator i=options.begin(); i!=options.end(); ++i)
689         {
690                 const Config::Option &opt=i->second;
691                 cout<<"  "<<opt.name<<": "<<opt.descr<<" ("<<opt.value<<") ["<<opt.defv<<"]\n";
692         }
693 }
694
695 Application::RegApp<Builder> Builder::reg;
696
697 Builder::Loader::Loader(Builder &b, const Path::Path &s):
698         bld(b),
699         src(s)
700 {
701         add("package", &Loader::package);
702 }
703
704 void Builder::Loader::package(const string &n)
705 {
706         Package *pkg=new Package(bld, n, src);
707         load_sub(*pkg);
708         bld.packages.insert(PackageMap::value_type(n, pkg));
709         bld.new_pkgs.push_back(pkg);
710 }
711