X-Git-Url: http://git.tdb.fi/?a=blobdiff_plain;f=source%2Fbuilder.cpp;h=6794842c50d2c872e8ac36e560d2d2dc58fd624c;hb=4ba3ef9601166127d0782b3e6a3352b71db7ccc9;hp=3f93a672cfe46e8ef0de8c7e9c8ca5da8279ee3c;hpb=4dc31cca056ea293d320928f61fef0558089d32d;p=builder.git diff --git a/source/builder.cpp b/source/builder.cpp index 3f93a67..6794842 100644 --- a/source/builder.cpp +++ b/source/builder.cpp @@ -1,222 +1,657 @@ -#include -#include -#include -#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include "analyzer.h" +#include "binarypackage.h" #include "builder.h" -#include "executable.h" -#include "header.h" -#include "misc.h" -#include "objectfile.h" +#include "copy.h" +#include "gnuarchiver.h" +#include "gnuccompiler.h" +#include "gnucxxcompiler.h" +#include "gnulinker.h" +#include "installedfile.h" +#include "mingwdlltool.h" #include "package.h" +#include "pkgconfiggenerator.h" +#include "sharedlibrary.h" +#include "sourcepackage.h" +#include "tar.h" +#include "task.h" #include "virtualtarget.h" using namespace std; using namespace Msp; Builder::Builder(int argc, char **argv): - verbose(1), - cwd(Path::getcwd()) + package_manager(*this), + main_pkg(0), + native_arch(*this, string()), + build_type(0), + vfs(*this), + analyzer(0), + build(false), + clean(0), + dry_run(false), + help(false), + show_progress(false), + build_file("Build"), + jobs(1), + conf_all(false), + conf_only(false), + build_all(false), + create_makefile(false), + tempdir("temp") { - for(int i=1; i log_channels; + string build_type_name; + + GetOpt getopt; + getopt.add_option('a', "analyze", analyze_mode, GetOpt::REQUIRED_ARG).set_help("Perform dependency analysis.", "MODE"); + getopt.add_option('b', "build", build, GetOpt::NO_ARG).set_help("Perform build even if also doing something else."); + getopt.add_option('c', "clean", clean, GetOpt::NO_ARG).set_help("Clean buildable targets."); + getopt.add_option('f', "file", build_file, GetOpt::REQUIRED_ARG).set_help("Read build instructions from FILE.", "FILE"); + getopt.add_option('h', "help", help, GetOpt::NO_ARG).set_help("Print this message."); + getopt.add_option('j', "jobs", jobs, GetOpt::REQUIRED_ARG).set_help("Run up to NUM tasks in parallel.", "NUM"); + getopt.add_option('l', "log", log_channels, GetOpt::REQUIRED_ARG).set_help("Enable listed log channels.", "LIST"); + getopt.add_option('n', "dry-run", dry_run, GetOpt::NO_ARG).set_help("Show what would be done without actually doing it."); + getopt.add_option('s', "silent", silent, GetOpt::NO_ARG).set_help("Don't print any messages other than errors."); + getopt.add_option('t', "build-type", build_type_name, GetOpt::REQUIRED_ARG).set_help("Set build type.", "TYPE"); + getopt.add_option('v', "verbose", verbose, GetOpt::NO_ARG).set_help("Print more information about what's going on."); + getopt.add_option('x', "no-externals", no_externals, GetOpt::NO_ARG).set_help("Do not load external source packages."); + getopt.add_option('A', "conf-all", conf_all, GetOpt::NO_ARG).set_help("Apply configuration to all packages."); + getopt.add_option('B', "build-all", build_all, GetOpt::NO_ARG).set_help("Build all targets unconditionally."); + getopt.add_option('C', "chdir", work_dir, GetOpt::REQUIRED_ARG).set_help("Change to DIR before doing anything else.", "DIR"); + getopt.add_option('P', "progress", show_progress, GetOpt::NO_ARG).set_help("Display progress while building."); + getopt.add_option('W', "what-if", what_if, GetOpt::REQUIRED_ARG).set_help("Pretend that FILE has changed.", "FILE"); + getopt.add_option( "arch", arch, GetOpt::REQUIRED_ARG).set_help("Build for architecture ARCH.", "ARCH"); + getopt.add_option( "conf-only", conf_only, GetOpt::NO_ARG).set_help("Stop after configuring packages."); + getopt.add_option( "full-paths", full_paths, GetOpt::NO_ARG).set_help("Output full paths in analysis."); + getopt.add_option( "max-depth", max_depth, GetOpt::REQUIRED_ARG).set_help("Show up to NUM levels in analysis.", "NUM"); + getopt.add_option( "prefix", prfx, GetOpt::REQUIRED_ARG).set_help("Install things to DIR.", "DIR"); + getopt.add_option( "tempdir", temp_str, GetOpt::REQUIRED_ARG).set_help("Store temporary files in DIR.", "DIR"); + usagemsg = getopt.generate_usage(argv[0])+" [ ...]"; + helpmsg = getopt.generate_help(); + getopt(argc, argv); + + if(silent) + --verbose; + if(verbose>=1) + { + logger.enable_channel("summary"); + logger.enable_channel("tasks"); + } + if(verbose>=2) + { + logger.enable_channel("environment"); + logger.enable_channel("packages"); + logger.enable_channel("commands"); + } + if(verbose>=3) + { + logger.enable_channel("files"); + logger.enable_channel("auxcommands"); + } + for(list::const_iterator i=log_channels.begin(); i!=log_channels.end(); ++i) + { + vector parts = split(*i, ','); + for(vector::const_iterator j=parts.begin(); j!=parts.end(); ++j) + logger.enable_channel(*j); + } + + if(!analyze_mode.empty()) + { + analyzer = new Analyzer(*this); + + if(analyze_mode=="deps") + analyzer->set_mode(Analyzer::DEPS); + else if(analyze_mode=="alldeps") + analyzer->set_mode(Analyzer::ALLDEPS); + else if(analyze_mode=="rebuild") + analyzer->set_mode(Analyzer::REBUILD); + else if(analyze_mode=="rdeps") + analyzer->set_mode(Analyzer::RDEPS); + else + throw usage_error("Invalid analyze mode"); + + analyzer->set_max_depth(max_depth); + analyzer->set_full_paths(full_paths); + } + else if(!clean && !create_makefile) + build = true; + + const vector &args = getopt.get_args(); + for(vector::const_iterator i=args.begin(); i!=args.end(); ++i) + { + string::size_type equal = i->find('='); + if(equal!=string::npos) + cmdline_options.insert(Config::InputOptions::value_type(i->substr(0, equal), i->substr(equal+1))); + else + cmdline_targets.push_back(*i); + } + if(cmdline_targets.empty()) cmdline_targets.push_back("default"); + + if(!work_dir.empty()) + FS::chdir(work_dir); + + cwd = FS::getcwd(); + + package_manager.append_package_path(cwd); + package_manager.append_package_path(cwd/".."); + package_manager.append_binary_package_path(FS::get_sys_data_dir(argv[0], "builder")); + + package_manager.set_no_externals(no_externals); + + if(arch.empty()) + current_arch = &native_arch; + else + current_arch = new Architecture(*this, arch); + + list start_files; + start_files.push_back(FS::get_sys_data_dir(argv[0], "builder")/"builderrc"); + start_files.push_back(FS::get_user_data_dir("builder")/"rc"); + for(list::const_iterator i=start_files.begin(); i!=start_files.end(); ++i) + if(FS::exists(*i)) + load_build_file(*i); + + if(prfx.empty()) + { + if(current_arch->is_native()) + prefix = (FS::get_home_dir()/"local").str(); + else + prefix = (FS::get_home_dir()/"local"/current_arch->get_name()).str(); + } + else + prefix = cwd/prfx; + + if(!temp_str.empty()) + tempdir = temp_str; + + if(!build_type_name.empty()) + { + BuildTypeMap::iterator i = build_types.find(build_type_name); + if(i==build_types.end()) + throw usage_error("Unknown build type"); + build_type = &i->second; + } + + toolchain.add_tool(new GnuCCompiler(*this, *current_arch)); + toolchain.add_tool(new GnuCxxCompiler(*this, *current_arch)); + toolchain.add_tool(new GnuLinker(*this, *current_arch)); + toolchain.add_tool(new GnuArchiver(*this, *current_arch)); + toolchain.add_tool(new Copy(*this)); + toolchain.add_tool(new Tar(*this)); + toolchain.add_tool(new PkgConfigGenerator(*this)); + if(current_arch->get_system()=="windows") + toolchain.add_tool(new MingwDllTool(*this, *current_arch)); } -Package *Builder::get_package(const string &n) +Builder::~Builder() { - PackageMap::iterator i=packages.find(n); - if(i!=packages.end()) - return i->second; + for(TargetMap::iterator i=targets.begin(); i!=targets.end(); ++i) + delete i->second; + delete analyzer; +} + +int Builder::main() +{ + FS::Path main_file = cwd/build_file; + if(!FS::exists(main_file)) + { + if(help) + { + usage(0, "builder", false); + return 0; + } + else + { + IO::print(IO::cerr, "The file %s does not exist.\n", main_file); + return 1; + } + } - list argv; - argv.push_back("pkg-config"); - argv.push_back("--variable=source"); - argv.push_back(n); - Path::Path srcdir=run_command(argv); - if(srcdir.empty()) - { - string dirname=n; - if(dirname.compare(0, 3, "msp")) - dirname.erase(0, 3); - if(Path::exists(cwd/dirname)) - srcdir=cwd/dirname; - else if(Path::exists(cwd/".."/dirname)) - srcdir=cwd/".."/dirname; + load_build_file(main_file); + + if(help) + { + usage(0, "builder", false); + IO::print("\n"); + package_help(); + return 0; } + + if(!prepare_build()) + return 1; + + if(conf_only) + return 0; + + logger.log("environment", format("Building on %s, for %s%s", native_arch.get_name(), + current_arch->get_name(), (current_arch->is_native() ? " (native)" : ""))); + logger.log("environment", format("Prefix is %s", prefix)); + if(tempdir.is_absolute()) + logger.log("environment", format("Temporary directory is %s", tempdir)); else - srcdir=strip(srcdir.str()); - - if(!srcdir.empty()) - load_build_file(srcdir/"Build"); + logger.log("environment", format("Using per-package temporary directory %s", tempdir)); + if(build_type) + logger.log("environment", format("Build type is %s", build_type->get_name())); - return 0; + const PackageManager::PackageMap &packages = package_manager.get_packages(); + list package_details; + for(PackageManager::PackageMap::const_iterator i=packages.begin(); i!=packages.end(); ++i) + { + if(!i->second->is_prepared()) + continue; + + string line = i->second->get_name(); + if(dynamic_cast(i->second)) + { + line += '*'; + + unsigned count = 0; + unsigned to_be_built = 0; + for(TargetMap::iterator j=targets.begin(); j!=targets.end(); ++j) + if(j->second->get_package()==i->second) + { + ++count; + if(j->second->needs_rebuild()) + ++to_be_built; + } + if(count) + { + line += format(" (%d targets", count); + if(to_be_built) + line += format(", %d to be built", to_be_built); + line += ')'; + } + } + + package_details.push_back(line); + } + + logger.log("summary", format("%d active packages, %d targets", package_details.size(), targets.size())); + for(list::const_iterator i=package_details.begin(); i!=package_details.end(); ++i) + logger.log("packages", *i); + + if(analyzer) + analyzer->analyze(); + + if(!problems.empty()) + { + IO::print(IO::cerr, "The following problems were detected:\n"); + for(ProblemList::iterator i=problems.begin(); i!=problems.end(); ++i) + IO::print(IO::cerr, " %s: %s\n", i->package, i->descr); + if(!analyzer) + IO::print(IO::cerr, "Please fix them and try again.\n"); + return 1; + } + + if(clean) + exit_code = do_clean(); + if(build) + exit_code = do_build(); + + return exit_code; } -Target *Builder::get_target(const string &n) +Target *Builder::get_target(const string &n) const { - TargetMap::iterator i=targets.find(n); + TargetMap::const_iterator i = targets.find(n); if(i!=targets.end()) return i->second; return 0; } -Target *Builder::get_header(const string &include, const string &from, const list &path) +void Builder::problem(const string &p, const string &d) { - string id=from+":"+include; - TargetMap::iterator i=includes.find(id); - if(i!=includes.end()) - return i->second; + problems.push_back(Problem(p, d)); +} - string fn=include.substr(1); - Target *tgt; - if(include[0]=='"' && (tgt=check_header(Path::Path(from)/fn))) - return tgt; - if((tgt=check_header(Path::Path("/usr/include")/fn))) - return tgt; - if((tgt=check_header(Path::Path("/usr/include/c++/4.1.2")/fn))) - return tgt; - for(list::const_iterator j=path.begin(); j!=path.end(); ++j) - if((tgt=check_header(Path::Path(*j)/fn))) - return tgt; - - return 0; +void Builder::add_target(Target *t) +{ + targets.insert(TargetMap::value_type(t->get_name(), t)); } -int Builder::main() +void Builder::add_primary_target(Target &t) { - if(load_build_file("Build")) + get_target("world")->add_dependency(t); + + if(t.get_package()==main_pkg && t.get_component() && t.get_component()->is_default()) + get_target("default")->add_dependency(t); +} + +void Builder::add_installed_target(Target &t) +{ + Target *inst_tgt = 0; + if(const Tool *tool = t.get_tool()) + inst_tgt = tool->create_install(t); + if(!inst_tgt) + inst_tgt = toolchain.get_tool("CP").create_target(t); + get_target("install")->add_dependency(*inst_tgt); +} + +void Builder::usage(const char *reason, const char *argv0, bool brief) +{ + if(reason) + IO::print(IO::cerr, "%s\n", reason); + + if(brief) + IO::print(IO::cerr, "Usage: %s\n", usagemsg); + else { - cerr<<"No build info here.\n"; - return 1; + IO::print(IO::cerr, "Builder 1.0\n\n"); + IO::print(IO::cerr, "Usage: %s [options] [ ...]\n\n", argv0); + IO::print(IO::cerr, "Options:\n"); + IO::print(IO::cerr, helpmsg); } +} + +void Builder::load_build_file(const FS::Path &fn) +{ + IO::BufferedFile in(fn.str()); - default_pkg=packages.begin()->second; + logger.log("files", format("Reading %s", fn)); - while(!new_pkgs.empty()) + DataFile::Parser parser(in, fn.str()); + Loader loader(*this); + loader.load(parser); +} + +bool Builder::prepare_build() +{ + Target *world = new VirtualTarget(*this, "world"); + + Target *def_tgt = new VirtualTarget(*this, "default"); + world->add_dependency(*def_tgt); + + Target *install = new VirtualTarget(*this, "install"); + world->add_dependency(*install); + + Target *tarballs = new VirtualTarget(*this, "tarballs"); + world->add_dependency(*tarballs); + + main_pkg->prepare(); + + // Make the cmdline target depend on all targets mentioned on the command line + Target *cmdline = new VirtualTarget(*this, "cmdline"); + for(NameList::iterator i=cmdline_targets.begin(); i!=cmdline_targets.end(); ++i) { - Package *pkg=new_pkgs.front(); - new_pkgs.erase(new_pkgs.begin()); - pkg->resolve_refs(); + Target *tgt = get_target(*i); + if(!tgt) + tgt = vfs.get_target(*i); + if(!tgt) + tgt = vfs.get_target(cwd/ *i); + if(!tgt) + { + IO::print("I don't know anything about %s\n", *i); + return false; + } + + cmdline->add_dependency(*tgt); } - cout<<"Active packages:"; - for(PackageMap::iterator i=packages.begin(); i!=packages.end(); ++i) + cmdline->prepare(); + + // Apply what-ifs + for(NameList::iterator i=what_if.begin(); i!=what_if.end(); ++i) { - cout<<' '<second->get_name(); - if(i->second->get_buildable()) - cout<<'*'; + FileTarget *tgt = vfs.get_target(cwd/ *i); + if(!tgt) + { + IO::print(IO::cerr, "Unknown what-if target %s\n", *i); + return false; + } + tgt->touch(); } - cout<<'\n'; - - create_targets(); - for(TargetMap::iterator i=targets.begin(); i!=targets.end(); ++i) - cout<second->get_name()<<' '<second->get_type()<<'\n'; + if(build_all) + { + for(TargetMap::iterator i=targets.begin(); i!=targets.end(); ++i) + if(i->second->is_buildable() && !i->second->needs_rebuild()) + i->second->force_rebuild(); + } - cout<<"Active targets: "<second->save_caches(); + } - return 0; + return true; } -int Builder::load_build_file(const Path::Path &fn) +int Builder::do_build() { - ifstream in(fn.str().c_str()); - if(!in) - return -1; + Target *cmdline = get_target("cmdline"); - Parser::Parser parser(in, fn.str()); - Loader loader(*this, cwd/fn.subpath(0, fn.size()-1)); - loader.load(parser); + unsigned total = 0; + for(map::const_iterator i=targets.begin(); i!=targets.end(); ++i) + if(i->second->is_buildable() && i->second->needs_rebuild()) + ++total; - return 0; -} + if(!total) + { + logger.log("summary", "Already up to date"); + return 0; + } + logger.log("summary", format("Will build %d target%s", total, (total!=1 ? "s" : ""))); -void Builder::create_targets() -{ - Target *world=new VirtualTarget(*this, "world"); - add_target(world); - Target *def_tgt=new VirtualTarget(*this, "default"); - add_target(def_tgt); + vector tasks; + + unsigned count = 0; - for(PackageMap::iterator i=packages.begin(); i!=packages.end(); ++i) + bool fail = false; + bool finish = false; + bool starved = false; + + while(!finish) { - cout<second->get_source()<<'\n'; - const ComponentList &components=i->second->get_components(); - for(ComponentList::const_iterator j=components.begin(); j!=components.end(); ++j) + if(tasks.size()second->get_source()/j->get_source(); - cout< files=list_files(base); - - list objs; - for(list::iterator k=files.begin(); k!=files.end(); ++k) + Target *tgt = cmdline->get_buildable_target(); + if(tgt) { - Path::Path fn=base/ *k; - //cout<<*k<<'\n'; - string ext=Path::splitext(*k).ext; - if(ext==".cpp" || ext==".c") + if(tgt->get_tool()) + logger.log("tasks", format("%-4s %s", tgt->get_tool()->get_tag(), tgt->get_name())); + Task *task = tgt->build(); + if(task) { - SourceFile *src=new SourceFile(*this, &*j, fn.str()); - add_target(src); - - ObjectFile *obj=new ObjectFile(*this, *j, *src); - add_target(obj); - objs.push_back(obj); + logger.log("commands", format("%s", task->get_command())); + if(dry_run) + { + task->signal_finished.emit(true); + delete task; + } + else + { + task->start(); + tasks.push_back(task); + } } - else if(ext==".h") - add_target(new Header(*this, &*j, fn.str())); + + if(show_progress) + IO::print("%d of %d target%s built\033[1G", count, total, (total!=1 ? "s" : "")); } + else if(tasks.empty()) + finish = true; + else + starved = true; + } + else + Time::sleep(10*Time::msec); + + for(unsigned i=0; iwait(); + else + status = tasks[i]->check(); + + if(status!=Task::RUNNING) + { + ++count; - Executable *exe=new Executable(*this, *j, objs); - add_target(exe); - if(i->second==default_pkg) - def_tgt->add_depend(exe); + delete tasks[i]; + tasks.erase(tasks.begin()+i); + if(status==Task::ERROR) + fail = true; + if(tasks.empty() && fail) + finish = true; + starved = false; + } + else + ++i; } } - while(!new_tgts.empty()) + if(show_progress) + IO::print("\033[K"); + if(fail) + logger.log("summary", "Build failed"); + else if(show_progress) + logger.log("summary", "Build complete"); + + if(!dry_run) { - Target *tgt=new_tgts.front(); - new_tgts.erase(new_tgts.begin()); - tgt->find_depends(); + const PackageManager::PackageMap &packages = package_manager.get_packages(); + for(PackageManager::PackageMap::const_iterator i=packages.begin(); i!=packages.end(); ++i) + i->second->save_caches(); } + + return fail; } -Target *Builder::check_header(const Msp::Path::Path &fn) +int Builder::do_clean() { - Target *tgt=get_target(fn.str()); - if(tgt) return tgt; + // Cleaning doesn't care about ordering, so a simpler method can be used + + set clean_tgts; + list queue; + queue.push_back(get_target("cmdline")); - if(Path::exists(fn)) + while(!queue.empty()) { - add_target(tgt=new SystemHeader(*this, fn.str())); - return tgt; + Target *tgt = queue.front(); + queue.erase(queue.begin()); + + if(tgt->is_buildable() && (tgt->get_package()==main_pkg || clean>=2)) + clean_tgts.insert(tgt); + + const Target::Dependencies &deps = tgt->get_dependencies(); + for(list::const_iterator i=deps.begin(); i!=deps.end(); ++i) + if(!clean_tgts.count(*i)) + queue.push_back(*i); + } + + for(set::iterator i=clean_tgts.begin(); i!=clean_tgts.end(); ++i) + { + logger.log("tasks", format("RM %s", (*i)->get_name())); + if(!dry_run) + (*i)->clean(); } + return 0; } -void Builder::add_target(Target *t) +void Builder::package_help() { - targets.insert(TargetMap::value_type(t->get_name(), t)); - new_tgts.push_back(t); + const Config &config = main_pkg->get_config(); + const Config::OptionMap &options = config.get_options(); + + IO::print("Required packages:\n "); + const Package::Requirements &requires = main_pkg->get_required_packages(); + for(Package::Requirements::const_iterator i=requires.begin(); i!=requires.end(); ++i) + { + if(i!=requires.begin()) + IO::print(", "); + IO::print((*i)->get_name()); + } + IO::print("\n\nPackage configuration:\n"); + for(Config::OptionMap::const_iterator i=options.begin(); i!=options.end(); ++i) + { + const Config::Option &opt = i->second; + IO::print(" %s: %s (%s)", opt.name, opt.description, opt.value); + if(opt.value!=opt.default_value) + IO::print(" [%s]", opt.default_value); + IO::print("\n"); + } } -Application::RegApp Builder::reg; +string Builder::usagemsg; +string Builder::helpmsg; -Builder::Loader::Loader(Builder &b, const Path::Path &s): - bld(b), - src(s) + +Builder::Loader::Loader(Builder &b): + DataFile::ObjectLoader(b) { + add("architecture", &Loader::architecture); + add("binary_package", &Loader::binpkg); + add("build_type", &Loader::build_type); + add("profile", &Loader::profile); add("package", &Loader::package); } -void Builder::Loader::package(const string &n) +void Builder::Loader::architecture(const string &n) { - Package *pkg=new Package(bld, n, src); + if(obj.current_arch->match_name(n)) + load_sub(*obj.current_arch); +} + +void Builder::Loader::binpkg(const string &n) +{ + BinaryPackage *pkg = new BinaryPackage(obj, n); load_sub(*pkg); - bld.packages.insert(PackageMap::value_type(n, pkg)); - bld.new_pkgs.push_back(pkg); - //cout<<"loaded "<get_name()<<'\n'; } +void Builder::Loader::build_type(const string &n) +{ + BuildType btype(n); + load_sub(btype); + BuildTypeMap::iterator i = obj.build_types.insert(BuildTypeMap::value_type(n, btype)).first; + if(!obj.build_type) + obj.build_type = &i->second; +} + +void Builder::Loader::profile(const string &) +{ + IO::print("Profiles are deprecated\n"); +} + +void Builder::Loader::package(const string &n) +{ + SourcePackage *pkg = new SourcePackage(obj, n, get_source()); + if(!obj.main_pkg) + obj.main_pkg = pkg; + + if(obj.conf_all || pkg==obj.main_pkg) + load_sub(*pkg, obj.cmdline_options); + else + load_sub(*pkg); + + if(obj.build_type) + pkg->set_build_type(*obj.build_type); +}