X-Git-Url: http://git.tdb.fi/?a=blobdiff_plain;f=source%2Fbuilder.cpp;h=e453a84eb73e178c4633799a59afc34810568933;hb=69cd5cc73f466181420711fb907979de58687a4c;hp=11f2b1a0e32a307ab899cd10352e8607a920e7b3;hpb=0f12a599f28748cef9c09dfb7a3b018b29ade6df;p=builder.git diff --git a/source/builder.cpp b/source/builder.cpp index 11f2b1a..e453a84 100644 --- a/source/builder.cpp +++ b/source/builder.cpp @@ -1,580 +1,343 @@ -#include -#include -#include -#include -#include -#include -#include -#include -#include +#include +#include +#include +#include +#include +#include +#include +#include +#include #include #include -#include "action.h" -#include "analyzer.h" +#include "androidtools.h" +#include "binarypackage.h" #include "builder.h" -#include "executable.h" -#include "header.h" -#include "install.h" -#include "misc.h" -#include "objectfile.h" +#include "builtintools.h" +#include "clangtools.h" +#include "datatool.h" +#include "gnutools.h" +#include "installedfile.h" #include "package.h" -#include "systemlibrary.h" +#include "sharedlibrary.h" +#include "sourcepackage.h" +#include "task.h" #include "virtualtarget.h" using namespace std; using namespace Msp; -Builder::Builder(int argc, char **argv): - verbose(1), - cwd(Path::getcwd()), - build_file("Build"), - do_build(true), - analyzer(0), - jobs(1), - chrome(false) +Builder::Builder(): + package_manager(*this), + native_arch(*this, string()), + current_arch(0), + build_type(0), + vfs(*this), + build_graph(*this), + logger(&default_logger), + tempdir("temp"), + top_loader(0) { - GetOpt getopt; - getopt.add_option(GetOpt::Option('v', "verbose", GetOpt::NONE)); - getopt.add_option(GetOpt::Option('a', "analyze", GetOpt::REQUIRED)); - getopt.add_option(GetOpt::Option('b', "build", GetOpt::NONE)); - getopt.add_option(GetOpt::Option("max-depth", GetOpt::REQUIRED)); - getopt.add_option(GetOpt::Option('n', "dry-run", GetOpt::NONE)); - getopt.add_option(GetOpt::Option('W', "what-if", GetOpt::REQUIRED)); - getopt.add_option(GetOpt::Option('B', "build-all", GetOpt::NONE)); - getopt.add_option(GetOpt::Option('C', "chdir", GetOpt::REQUIRED)); - getopt.add_option(GetOpt::Option('j', "jobs", GetOpt::REQUIRED, "1")); - getopt.add_option(GetOpt::Option('h', "help", GetOpt::NONE)); - getopt.add_option(GetOpt::Option('c', "clean", GetOpt::NONE)); - getopt.add_option(GetOpt::Option('f', "file", GetOpt::REQUIRED, "Build")); - getopt.add_option(GetOpt::Option("chrome", GetOpt::NONE)); - getopt.add_option(GetOpt::Option("full-paths", GetOpt::NONE)); - getopt.add_option(GetOpt::Option('A', "conf-all", GetOpt::NONE)); - int index=getopt(argc, argv); - - verbose+=getopt['v'].count(); - - if(getopt['a']) - { - analyzer=new Analyzer(*this); - - string mode=getopt['a'].arg(); - if(mode=="deps") - analyzer->set_mode(Analyzer::DEPS); - else if(mode=="alldeps") - analyzer->set_mode(Analyzer::ALLDEPS); - else if(mode=="rebuild") - analyzer->set_mode(Analyzer::REBUILD); - else if(mode=="rdeps") - analyzer->set_mode(Analyzer::RDEPS); - else - throw UsageError("Invalid analysis mode"); + set_architecture(string()); +} - if(getopt["max-depth"]) - analyzer->set_max_depth(strtol(getopt["max-depth"].arg())); - analyzer->set_full_paths(getopt["full-paths"]); +Builder::~Builder() +{ + if(current_arch!=&native_arch) + delete current_arch; +} - if(!getopt['b']) - do_build=false; +void Builder::set_architecture(const string &name) +{ + if(name.empty()) + { + current_arch = &native_arch; + prefix = FS::get_home_dir()/"local"; } - - dry_run=getopt['n']; - - jobs=max(strtol(getopt['j'].arg()), 1L); - - chrome=getopt["chrome"]; - - conf_all=getopt['A']; - - if(getopt['C']) - chdir(getopt['C'].arg().c_str()); - - build_file=getopt['f'].arg(); - - for(int i=index; iget_name(); } - - if(cmdline_targets.empty()) - cmdline_targets.push_back("default"); - - if(getopt['W']) - what_if.push_back(getopt['W'].arg()); } -/** -Gets a package with the specified name, possibly creating it. - -@param n Package name - -@return Pointer to the package, or 0 if the package could not be located -*/ -Package *Builder::get_package(const string &n) +void Builder::set_build_type(const string &name) { - PackageMap::iterator i=packages.find(n); - if(i!=packages.end()) - return i->second; - - // Try to get source directory with pkgconfig - list argv; - argv.push_back("pkg-config"); - argv.push_back("--variable=source"); - argv.push_back(n); - string srcdir=strip(run_command(argv)); - - PathList dirs; - if(!srcdir.empty()) - dirs.push_back(srcdir); - - // Make some other guesses about the source directory - string dirname=n; - if(!dirname.compare(0, 3, "msp")) - dirname.erase(0, 3); - dirs.push_back(cwd/dirname); - dirs.push_back(cwd/".."/dirname); - - // Go through the candidate directories and look for a Build file - for(PathList::iterator j=dirs.begin(); j!=dirs.end(); ++j) - if(!load_build_file(*j/"Build")) - { - i=packages.find(n); - if(i!=packages.end()) - return i->second; - break; - } - - // Package source not found - create a binary package - Package *pkg=Package::create(*this, n); - packages.insert(PackageMap::value_type(n, pkg)); - if(pkg) - new_pkgs.push_back(pkg); - - return pkg; + build_type = &get_item(build_types, name); } -/** -Returns the target with the given name, or 0 if no such target exists. -*/ -Target *Builder::get_target(const string &n) +void Builder::set_prefix(const FS::Path &p) { - TargetMap::iterator i=targets.find(n); - if(i!=targets.end()) - return i->second; - return 0; + prefix = p; } -/** -Tries to locate a header included from a given location and with a given include -path. Considers known targets as well as existing files. -*/ -Target *Builder::get_header(const string &include, const string &from, const list &path) +void Builder::set_temp_directory(const FS::Path &p) { - string hash(8, 0); - update_hash(hash, from); - for(list::const_iterator i=path.begin(); i!=path.end(); ++i) - update_hash(hash, *i); - - string id=hash+include; - TargetMap::iterator i=includes.find(id); - if(i!=includes.end()) - return i->second; - - string fn=include.substr(1); - Target *tgt=0; - if(include[0]=='"' && (tgt=check_header(Path::Path(from)/fn))) - return tgt; - if((tgt=check_header(Path::Path("/usr/include")/fn))) - return tgt; - //XXX Determine the C++ header location dynamically - if((tgt=check_header(Path::Path("/usr/include/c++/4.1.2")/fn))) - return tgt; - for(list::const_iterator j=path.begin(); j!=path.end(); ++j) - if((tgt=check_header(Path::Path(*j)/fn))) - return tgt; - - return 0; + tempdir = p; } -Target *Builder::get_library(const string &lib, const list &path) +void Builder::add_default_tools() { - string hash(8, 0); - for(list::const_iterator i=path.begin(); i!=path.end(); ++i) - update_hash(hash, *i); + bool use_clang = (current_arch->get_system()=="darwin" || current_arch->get_system()=="freebsd"); + if(use_clang && vfs.find_binary("clang")) + toolchain.add_toolchain(new ClangTools(*this, *current_arch)); + if(current_arch->get_system()=="android") + toolchain.add_toolchain(new AndroidTools(*this, *current_arch)); + toolchain.add_toolchain(new GnuTools(*this, *current_arch)); + toolchain.add_toolchain(new BuiltinTools(*this)); + toolchain.add_tool(new DataTool(*this)); +} - string id=hash+lib; - TargetMap::iterator i=libraries.find(id); - if(i!=libraries.end()) - return i->second; +void Builder::set_logger(const Logger *l) +{ + logger = (l ? l : &default_logger); +} - string basename="lib"+lib+".so"; - for(list::const_iterator j=path.begin(); j!=path.end(); ++j) - { - string full=(Path::Path(*j)/basename).str(); - Target *tgt=get_target(full); - if(tgt) return tgt; - - if(Path::exists(full)) +list Builder::collect_problems() const +{ + list problems; + set broken_packages; + set broken_components; + set broken_tools; + + const BuildGraph::TargetMap &targets = build_graph.get_targets(); + for(BuildGraph::TargetMap::const_iterator i=targets.begin(); i!=targets.end(); ++i) + if(i->second->is_broken()) { - add_target(tgt=new SystemLibrary(*this, full)); - return tgt; - } - } + const list &tgt_problems = i->second->get_problems(); + for(list::const_iterator j=tgt_problems.begin(); j!=tgt_problems.end(); ++j) + problems.push_back(format("%s: %s", i->second->get_name(), *j)); - return 0; -} + const Package *package = i->second->get_package(); + if(package && !package->get_problems().empty()) + broken_packages.insert(package); -int Builder::main() -{ - if(load_build_file(cwd/build_file)) - { - cerr<<"No build info here.\n"; - return 1; - } + const Component *component = i->second->get_component(); + if(component && !component->get_problems().empty()) + broken_components.insert(component); - default_pkg=packages.begin()->second; + const Tool *tool = i->second->get_tool(); + if(tool && !tool->get_problems().empty()) + broken_tools.insert(tool); + } - while(!new_pkgs.empty()) + // TODO Sort components after their packages, and targets last + for(set::const_iterator i=broken_packages.begin(); i!=broken_packages.end(); ++i) { - Package *pkg=new_pkgs.front(); - if(pkg==default_pkg || conf_all) - pkg->process_options(cmdline_options); - new_pkgs.erase(new_pkgs.begin()); - pkg->resolve_refs(); + const list &pkg_problems = (*i)->get_problems(); + for(list::const_iterator j=pkg_problems.begin(); j!=pkg_problems.end(); ++j) + problems.push_back(format("%s: %s", (*i)->get_name(), *j)); } - std::list missing; - for(PackageMap::iterator i=packages.begin(); i!=packages.end(); ++i) - if(!i->second) - missing.push_back(i->first); - - if(!missing.empty()) + for(set::const_iterator i=broken_components.begin(); i!=broken_components.end(); ++i) { - missing.sort(); - cerr<<"The following packages were not found on the system:\n"; - for(list::iterator i=missing.begin(); i!=missing.end(); ++i) - cerr<<" "<<*i<<'\n'; - cerr<<"Please install them and try again.\n"; - return 1; + const list &comp_problems = (*i)->get_problems(); + for(list::const_iterator j=comp_problems.begin(); j!=comp_problems.end(); ++j) + problems.push_back(format("%s/%s: %s", (*i)->get_package().get_name(), (*i)->get_name(), *j)); } - default_pkg->create_build_info(); - - if(create_targets()) - return 1; - - cout<=2) + for(set::const_iterator i=broken_tools.begin(); i!=broken_tools.end(); ++i) { - for(PackageMap::iterator i=packages.begin(); i!=packages.end(); ++i) - { - cout<<' '<second->get_name(); - if(i->second->get_buildable()) - cout<<'*'; - unsigned count=0; - for(TargetMap::iterator j=targets.begin(); j!=targets.end(); ++j) - if(j->second->get_package()==i->second) - ++count; - cout<<" ("< &tool_problems = (*i)->get_problems(); + for(list::const_iterator j=tool_problems.begin(); j!=tool_problems.end(); ++j) + problems.push_back(format("%s: %s", (*i)->get_tag(), *j)); } - if(analyzer) - analyzer->analyze(); - - if(do_build) - build(); - - return exit_code; + return problems; } -Builder::~Builder() +void Builder::load_build_file(const FS::Path &fn, const Config::InputOptions *opts, bool all) { - for(PackageMap::iterator i=packages.begin(); i!=packages.end(); ++i) - delete i->second; - for(TargetMap::iterator i=targets.begin(); i!=targets.end(); ++i) - delete i->second; - delete analyzer; -} + IO::BufferedFile in(fn.str()); -int Builder::load_build_file(const Path::Path &fn) -{ - ifstream in(fn.str().c_str()); - if(!in) - return -1; + get_logger().log("files", format("Reading %s", fn)); - Parser::Parser parser(in, fn.str()); - Loader loader(*this, fn.subpath(0, fn.size()-1)); + DataFile::Parser parser(in, fn.str()); + Loader loader(*this, opts, all); loader.load(parser); - - return 0; } -int Builder::create_targets() +int Builder::build(unsigned jobs, bool dry_run, bool show_progress) { - Target *world=new VirtualTarget(*this, "world"); - add_target(world); + unsigned total = build_graph.count_rebuild_targets(); - Target *def_tgt=new VirtualTarget(*this, "default"); - add_target(def_tgt); - world->add_depend(def_tgt); + if(!total) + { + get_logger().log("summary", "Already up to date"); + return 0; + } + get_logger().log("summary", format("Will build %d target%s", total, (total!=1 ? "s" : ""))); - Target *install=new VirtualTarget(*this, "install"); - add_target(install); - world->add_depend(install); + vector tasks; - for(PackageMap::iterator i=packages.begin(); i!=packages.end(); ++i) - { - if(!i->second) - continue; - if(!i->second->get_buildable()) - continue; + unsigned count = 0; - Path::Path inst_base; - if(i->second->get_config().is_option("prefix")) - inst_base=i->second->get_config().get_option("prefix").value; + bool fail = false; + bool finish = false; + bool starved = false; - const ComponentList &components=i->second->get_components(); - for(ComponentList::const_iterator j=components.begin(); j!=components.end(); ++j) + while(!finish) + { + if(tasks.size()get_sources(); - for(PathList::const_iterator k=sources.begin(); k!=sources.end(); ++k) - { - struct stat st; - stat(*k, st); - if(S_ISDIR(st.st_mode)) - { - list sfiles=list_files(*k); - for(list::iterator l=sfiles.begin(); l!=sfiles.end(); ++l) - files.push_back(*k / *l); - } - else - files.push_back(*k); - } - - bool build_exe=j->get_type()!=Component::HEADERS; - - list objs; - for(PathList::iterator k=files.begin(); k!=files.end(); ++k) + Target *tgt = build_graph.get_buildable_target(); + if(tgt) { - string basename=(*k)[-1]; - string ext=Path::splitext(basename).ext; - if(ext==".cpp" || ext==".c") - { - if(build_exe) - { - SourceFile *src=new SourceFile(*this, &*j, k->str()); - add_target(src); - - ObjectFile *obj=new ObjectFile(*this, *j, *src); - add_target(obj); - objs.push_back(obj); - } - } - else if(ext==".h") + if(tgt->get_tool()) + get_logger().log("tasks", format("%-4s %s", tgt->get_tool()->get_tag(), tgt->get_name())); + Task *task = tgt->build(); + if(task) { - Target *hdr=get_target(k->str()); - if(!hdr) + get_logger().log("commands", format("%s", task->get_command())); + if(dry_run) { - hdr=new Header(*this, &*j, k->str()); - add_target(hdr); + task->signal_finished.emit(true); + delete task; } - if(!j->get_install_headers().empty()) + else { - Path::Path inst_path=inst_base/"include"/j->get_install_headers()/basename; - Install *inst=new Install(*this, *i->second, *hdr, inst_path.str()); - add_target(inst); - install->add_depend(inst); + task->start(); + tasks.push_back(task); } } + + if(show_progress) + IO::print("%d of %d target%s built\033[1G", count, total, (total!=1 ? "s" : "")); } + else if(tasks.empty()) + finish = true; + else + starved = true; + } + else + Time::sleep(10*Time::msec); - if(build_exe) + for(unsigned i=0; iwait(); + else + status = tasks[i]->check(); + + if(status!=Task::RUNNING) { - Executable *exe=new Executable(*this, *j, objs); - add_target(exe); - if(i->second==default_pkg) - def_tgt->add_depend(exe); - else - world->add_depend(exe); - - if(j->get_install()) - { - string inst_dir; - if(j->get_type()==Component::PROGRAM) - inst_dir="bin"; - else if(j->get_type()==Component::LIBRARY) - inst_dir="lib"; - if(!inst_dir.empty()) - { - Install *inst=new Install(*this, *i->second, *exe, (inst_base/inst_dir/Path::basename(exe->get_name())).str()); - add_target(inst); - install->add_depend(inst); - } - } + ++count; + + delete tasks[i]; + tasks.erase(tasks.begin()+i); + if(status==Task::ERROR) + fail = true; + if(tasks.empty() && fail) + finish = true; + starved = false; } + else + ++i; } } - while(!new_tgts.empty()) + if(show_progress) + IO::print("\033[K"); + if(fail) + get_logger().log("summary", "Build failed"); + else if(show_progress) + get_logger().log("summary", "Build complete"); + + if(!dry_run) { - Target *tgt=new_tgts.front(); - new_tgts.erase(new_tgts.begin()); - tgt->find_depends(); - if(!tgt->get_depends_ready()) - new_tgts.push_back(tgt); + const PackageManager::PackageMap &packages = package_manager.get_packages(); + for(PackageManager::PackageMap::const_iterator i=packages.begin(); i!=packages.end(); ++i) + i->second->save_caches(); } - for(list::iterator i=what_if.begin(); i!=what_if.end(); ++i) + return fail; +} + +int Builder::clean(bool all, bool dry_run) +{ + // Cleaning doesn't care about ordering, so a simpler method can be used + + set clean_tgts; + list queue; + queue.push_back(&build_graph.get_goals()); + + while(!queue.empty()) { - Target *tgt=get_target((cwd/ *i).str()); - if(!tgt) - { - cerr<<"Unknown what-if target "<<*i<<'\n'; - return -1; - } - tgt->touch(); + Target *tgt = queue.front(); + queue.pop_front(); + + if(tgt->is_buildable() && (tgt->get_package()==&package_manager.get_main_package() || all)) + clean_tgts.insert(tgt); + + const Target::Dependencies &deps = tgt->get_dependencies(); + for(list::const_iterator i=deps.begin(); i!=deps.end(); ++i) + if(!clean_tgts.count(*i)) + queue.push_back(*i); } - Target *cmdline=new VirtualTarget(*this, "cmdline"); - add_target(cmdline); - world->add_depend(cmdline); - for(list::iterator i=cmdline_targets.begin(); i!=cmdline_targets.end(); ++i) + for(set::iterator i=clean_tgts.begin(); i!=clean_tgts.end(); ++i) { - Target *tgt=get_target(*i); - if(!tgt) - tgt=get_target((cwd/ *i).str()); - if(!tgt) - { - cerr<<"I don't know anything about "<<*i<<'\n'; - return -1; - } - cmdline->add_depend(tgt); + get_logger().log("tasks", format("RM %s", (*i)->get_name())); + if(!dry_run) + (*i)->clean(); } - world->prepare(); - return 0; } -Target *Builder::check_header(const Msp::Path::Path &fn) + +Builder::Loader::Loader(Builder &b, const Config::InputOptions *o, bool a): + DataFile::ObjectLoader(b), + options(o), + conf_all(a) { - Target *tgt=get_target(fn.str()); - if(tgt) return tgt; + add("architecture", &Loader::architecture); + add("binary_package", &Loader::binpkg); + add("build_type", &Loader::build_type); + add("package", &Loader::package); - if(Path::exists(fn)) - { - add_target(tgt=new SystemHeader(*this, fn.str())); - return tgt; - } - return 0; + if(!obj.top_loader) + obj.top_loader = this; + else if(!options && obj.top_loader!=this && obj.top_loader->conf_all) + options = obj.top_loader->options; } -void Builder::add_target(Target *t) +Builder::Loader::~Loader() { - targets.insert(TargetMap::value_type(t->get_name(), t)); - new_tgts.push_back(t); + if(obj.top_loader==this) + obj.top_loader = 0; } -void Builder::update_hash(string &hash, const string &value) +void Builder::Loader::architecture(const string &n) { - for(unsigned i=0; imatch_name(n)) + load_sub(*obj.current_arch); } -/** -This function supervises the build process, starting new actions when slots -become available. -*/ -int Builder::build() +void Builder::Loader::binpkg(const string &n) { - Target *cmdline=get_target("cmdline"); - - unsigned total=cmdline->count_rebuild(); - if(!total) - { - cout<<"Already up to date\n"; - return 0; - } - cout<<"Will build "< actions; - - if(chrome) - cout<<"0 targets built\n"; - unsigned count=0; - - bool fail=false; - bool finish=false; - - while(!finish) - { - if(actions.size()get_buildable_target(); - if(tgt) - { - Action *action=tgt->build(); - if(action) - actions.push_back(action); - } - else if(actions.empty()) - finish=true; - } - else - Time::sleep(10*Time::msec); - - for(unsigned i=0; icheck(); - if(status>=0) - { - ++count; - if(chrome) - { - cout<<"\e["<0) - fail=true; - if(actions.empty() && fail) - finish=true; - } - else - ++i; - } - } - - return fail?-1:0; + BinaryPackage *pkg = new BinaryPackage(obj, n); + load_sub(*pkg); } -Application::RegApp Builder::reg; - -Builder::Loader::Loader(Builder &b, const Path::Path &s): - bld(b), - src(s) +void Builder::Loader::build_type(const string &n) { - add("package", &Loader::package); + BuildType btype(n); + load_sub(btype); + BuildTypeMap::iterator i = obj.build_types.insert(BuildTypeMap::value_type(n, btype)).first; + if(!obj.build_type) + obj.build_type = &i->second; } void Builder::Loader::package(const string &n) { - Package *pkg=new Package(bld, n, src); - load_sub(*pkg); - bld.packages.insert(PackageMap::value_type(n, pkg)); - bld.new_pkgs.push_back(pkg); -} + SourcePackage *pkg = new SourcePackage(obj, n, get_source()); + if(options) + load_sub(*pkg, *options); + else + load_sub(*pkg); + + if(obj.build_type) + pkg->set_build_type(*obj.build_type); +}