From: Mikko Rasa <tdb@tdb.fi>
Date: Sun, 21 Feb 2021 14:23:07 +0000 (+0200)
Subject: Redesign the way shader programs are loaded
X-Git-Url: http://git.tdb.fi/?p=libs%2Fgl.git;a=commitdiff_plain;h=dccad64b2ec82249d850b9745614042171499972

Redesign the way shader programs are loaded

Vulkan uses SPIR-V modules which can be specialized so there has to be a
way to keep the module around.  A module can also contain multiple shader
stages.  The abstraction for individual stages with the Shader class is
outdated and has been replaced with Module.

GLSL code is stored in the Module in combined form and split into stages
when loaded to a Program.
---

diff --git a/blender/io_mspgl/export_material.py b/blender/io_mspgl/export_material.py
index 1e5ebe62..41ed287b 100644
--- a/blender/io_mspgl/export_material.py
+++ b/blender/io_mspgl/export_material.py
@@ -11,7 +11,10 @@ def create_technique_resource(material, resources):
 		st.sub.append(tech_res.create_embed_statement("material", mat_res))
 
 	if material.render_mode=='CUSTOM':
-		st.sub.append(Statement("shader", material.shader))
+		shader = material.shader
+		if shader.endswith(".glsl"):
+			shader += ".shader"
+		st.sub.append(Statement("shader", shader))
 
 	tech_res.statements.append(st)
 
diff --git a/source/core/module.cpp b/source/core/module.cpp
new file mode 100644
index 00000000..406b4739
--- /dev/null
+++ b/source/core/module.cpp
@@ -0,0 +1,37 @@
+#include "module.h"
+#include "resources.h"
+#include "glsl/compiler.h"
+
+using namespace std;
+
+namespace Msp {
+namespace GL {
+
+Module::Module()
+{ }
+
+void Module::set_source(const string &src)
+{
+	SL::Compiler compiler;
+	compiler.set_source(src);
+	compiler.compile(SL::Compiler::MODULE);
+	prepared_source = compiler.get_combined_glsl();
+	source_map = compiler.get_source_map();
+}
+
+void Module::load_source(IO::Base &io, Resources *res, const string &name)
+{
+	SL::Compiler compiler;
+	compiler.load_source(io, res, name);
+	compiler.compile(SL::Compiler::MODULE);
+	prepared_source = compiler.get_combined_glsl();
+	source_map = compiler.get_source_map();
+}
+
+void Module::load_source(IO::Base &io, const string &name)
+{
+	load_source(io, 0, name);
+}
+
+} // namespace GL
+} // namespace Msp
diff --git a/source/core/module.h b/source/core/module.h
new file mode 100644
index 00000000..b1c0f8ab
--- /dev/null
+++ b/source/core/module.h
@@ -0,0 +1,33 @@
+#ifndef MSP_GL_MODULE_H_
+#define MSP_GL_MODULE_H_
+
+#include <string>
+#include <msp/io/base.h>
+#include "glsl/sourcemap.h"
+
+namespace Msp {
+namespace GL {
+
+class Resources;
+
+class Module
+{
+private:
+	std::string prepared_source;
+	SL::SourceMap source_map;
+
+public:
+	Module();
+
+	void set_source(const std::string &);
+	void load_source(IO::Base &, Resources *, const std::string &);
+	void load_source(IO::Base &, const std::string &);
+
+	const std::string &get_prepared_source() const { return prepared_source; }
+	const SL::SourceMap &get_source_map() const { return source_map; }
+};
+
+} // namespace GL
+} // namespace Msp
+
+#endif
diff --git a/source/core/program.cpp b/source/core/program.cpp
index 4cdbf97c..2a34feb3 100644
--- a/source/core/program.cpp
+++ b/source/core/program.cpp
@@ -3,6 +3,8 @@
 #include <set>
 #include <msp/core/hash.h>
 #include <msp/core/maputils.h>
+#include <msp/gl/extensions/arb_fragment_shader.h>
+#include <msp/gl/extensions/arb_geometry_shader4.h>
 #include <msp/gl/extensions/arb_shader_objects.h>
 #include <msp/gl/extensions/arb_uniform_buffer_object.h>
 #include <msp/gl/extensions/arb_vertex_shader.h>
@@ -13,6 +15,7 @@
 #include "buffer.h"
 #include "error.h"
 #include "misc.h"
+#include "module.h"
 #include "program.h"
 #include "resources.h"
 #include "shader.h"
@@ -32,27 +35,23 @@ Program::Program(const std::string &source)
 {
 	init();
 
-	SL::Compiler compiler;
-	if(source.find(';')==string::npos && source.size()>5 && !source.compare(source.size()-5, 5, ".glsl"))
-	{
-		if(RefPtr<IO::Seekable> io = Resources::get_builtins().open(source))
-			compiler.load_source(*io, source);
-		else
-			throw IO::file_not_found(source);
-	}
-	else
-		compiler.set_source(source);
-	compiler.compile();
-	compiler.add_shaders(*this);
+	Module mod;
+	mod.set_source(source);
+	add_stages(mod);
+
 	link();
+	module = 0;
 }
 
 Program::Program(const string &vert, const string &frag)
 {
 	init();
 
+#pragma GCC diagnostic push
+#pragma GCC diagnostic ignored "-Wdeprecated-declarations"
 	attach_shader_owned(new VertexShader(vert));
 	attach_shader_owned(new FragmentShader(frag));
+#pragma GCC diagnostic pop
 	link();
 }
 
@@ -60,41 +59,97 @@ void Program::init()
 {
 	static Require _req(ARB_shader_objects);
 
-	linked = false;
 	id = glCreateProgram();
+	module = 0;
+	linked = false;
 }
 
 Program::~Program()
 {
-	for(ShaderList::iterator i=owned_data.begin(); i!=owned_data.end(); ++i)
-		delete *i;
+	for(vector<unsigned>::iterator i=shader_ids.begin(); i!=shader_ids.end(); ++i)
+		glDeleteShader(*i);
 	glDeleteProgram(id);
 }
 
-void Program::attach_shader(Shader &shader)
+void Program::add_stages(const Module &mod)
 {
-	if(find(shaders.begin(), shaders.end(), &shader)==shaders.end())
+	module = &mod;
+
+	SL::Compiler compiler;
+	compiler.set_source(module->get_prepared_source(), "<module>");
+	compiler.compile(SL::Compiler::PROGRAM);
+
+	vector<SL::Stage::Type> stages = compiler.get_stages();
+	for(vector<SL::Stage::Type>::const_iterator i=stages.begin(); i!=stages.end(); ++i)
 	{
-		glAttachShader(id, shader.get_id());
-		shaders.push_back(&shader);
+		GLenum type;
+		switch(*i)
+		{
+		case SL::Stage::VERTEX:
+			{ static Require _req(ARB_vertex_shader); }
+			type = GL_VERTEX_SHADER;
+			break;
+		case SL::Stage::GEOMETRY:
+			{ static Require _req(ARB_geometry_shader4); }
+			type = GL_GEOMETRY_SHADER;
+			break;
+		case SL::Stage::FRAGMENT:
+			{ static Require _req(ARB_fragment_shader); }
+			type = GL_FRAGMENT_SHADER;
+			break;
+		default:
+			throw invalid_operation("Program::add_stages");
+		}
+
+		unsigned stage_id = glCreateShader(type);
+		shader_ids.push_back(stage_id);
+		glAttachShader(id, stage_id);
+
+		string stage_src = compiler.get_stage_glsl(*i);
+		const char *src_ptr = stage_src.data();
+		int src_len = stage_src.size();
+		glShaderSource(stage_id, 1, &src_ptr, &src_len);
+
+		if(*i==SL::Stage::VERTEX)
+		{
+			const map<string, unsigned> &attribs = compiler.get_vertex_attributes();
+			for(map<string, unsigned>::const_iterator j=attribs.begin(); j!=attribs.end(); ++j)
+				glBindAttribLocation(id, j->second, j->first.c_str());
+		}
+
+		if(*i==SL::Stage::FRAGMENT && EXT_gpu_shader4)
+		{
+			const map<string, unsigned> &frag_outs = compiler.get_fragment_outputs();
+			for(map<string, unsigned>::const_iterator j=frag_outs.begin(); j!=frag_outs.end(); ++j)
+				glBindFragDataLocation(id, j->second, j->first.c_str());
+		}
 	}
 }
 
+#pragma GCC diagnostic push
+#pragma GCC diagnostic ignored "-Wdeprecated-declarations"
+void Program::attach_shader(Shader &shader)
+{
+	unsigned shader_id = shader.steal_id();
+	if(!shader_id)
+		throw invalid_argument("Program::attach_shader");
+	shader_ids.push_back(shader_id);
+}
+
 void Program::attach_shader_owned(Shader *shader)
 {
 	attach_shader(*shader);
-	if(find(owned_data.begin(), owned_data.end(), shader)==owned_data.end())
-		owned_data.push_back(shader);
+	delete shader;
 }
 
-void Program::detach_shader(Shader &shader)
+void Program::detach_shader(Shader &)
 {
-	ShaderList::iterator i = remove(shaders.begin(), shaders.end(), &shader);
-	if(i!=shaders.end())
-	{
-		shaders.erase(i, shaders.end());
-		glDetachShader(id, shader.get_id());
-	}
+}
+
+const vector<Shader *> &Program::get_attached_shaders() const
+{
+	static vector<Shader *> dummy;
+	return dummy;
 }
 
 void Program::bind_attribute(unsigned index, const string &name)
@@ -113,22 +168,43 @@ void Program::bind_fragment_data(unsigned index, const string &name)
 	static Require _req(EXT_gpu_shader4);
 	glBindFragDataLocation(id, index, name.c_str());
 }
+#pragma GCC diagnostic pop
 
 void Program::link()
 {
-	for(ShaderList::iterator i=shaders.begin(); i!=shaders.end(); ++i)
-		if(!(*i)->is_compiled())
-			(*i)->compile();
+	for(vector<unsigned>::const_iterator i=shader_ids.begin(); i!=shader_ids.end(); ++i)
+	{
+		glCompileShader(*i);
+		bool compiled = get_shader_i(*i, GL_COMPILE_STATUS);
+
+		GLsizei info_log_len = get_shader_i(*i, GL_INFO_LOG_LENGTH);
+		string info_log(info_log_len+1, 0);
+		glGetShaderInfoLog(*i, info_log_len+1, &info_log_len, &info_log[0]);
+		info_log.erase(info_log_len);
+		info_log = module->get_source_map().translate_errors(info_log);
+
+		if(!compiled)
+			throw compile_error(info_log);
+#ifdef DEBUG
+		if(!info_log.empty())
+			IO::print("Shader compile info log:\n%s", info_log);
+#endif
+	}
 
 	uniforms.clear();
 
 	glLinkProgram(id);
 	linked = get_program_i(id, GL_LINK_STATUS);
-	if(!linked)
-		throw compile_error(get_info_log());
 
+	GLsizei info_log_len = get_program_i(id, GL_INFO_LOG_LENGTH);
+	string info_log(info_log_len+1, 0);
+	glGetProgramInfoLog(id, info_log_len+1, &info_log_len, &info_log[0]);
+	info_log.erase(info_log_len);
+	info_log = module->get_source_map().translate_errors(info_log);
+
+	if(!linked)
+		throw compile_error(info_log);
 #ifdef DEBUG
-	std::string info_log = get_info_log();
 	if(!info_log.empty())
 		IO::print("Program link info log:\n%s", info_log);
 #endif
@@ -384,9 +460,12 @@ void Program::unbind()
 }
 
 
-Program::Loader::Loader(Program &p):
-	DataFile::ObjectLoader<Program>(p)
+Program::Loader::Loader(Program &p, Collection &c):
+	DataFile::CollectionObjectLoader<Program>(p, &c)
 {
+	add("module",          &Loader::module);
+
+	// Deprecated
 	add("attribute",       &Loader::attribute);
 	add("fragment_shader", &Loader::fragment_shader);
 	add("geometry_shader", &Loader::geometry_shader);
@@ -398,6 +477,13 @@ void Program::Loader::finish()
 	obj.link();
 }
 
+void Program::Loader::module(const string &n)
+{
+	obj.add_stages(get_collection().get<Module>(n));
+}
+
+#pragma GCC diagnostic push
+#pragma GCC diagnostic ignored "-Wdeprecated-declarations"
 void Program::Loader::attribute(unsigned i, const string &n)
 {
 	obj.bind_attribute(i, n);
@@ -417,6 +503,7 @@ void Program::Loader::vertex_shader(const string &src)
 {
 	obj.attach_shader_owned(new VertexShader(src));
 }
+#pragma GCC diagnostic pop
 
 } // namespace GL
 } // namespace Msp
diff --git a/source/core/program.h b/source/core/program.h
index fc60db46..e86ee059 100644
--- a/source/core/program.h
+++ b/source/core/program.h
@@ -11,6 +11,7 @@
 namespace Msp {
 namespace GL {
 
+class Module;
 class Shader;
 
 /**
@@ -20,10 +21,10 @@ generated with a set of standard features.
 class Program: public Bindable<Program>
 {
 public:
-	class Loader: public DataFile::ObjectLoader<Program>
+	class Loader: public DataFile::CollectionObjectLoader<Program>
 	{
 	public:
-		Loader(Program &);
+		Loader(Program &, Collection &);
 
 	private:
 		virtual void finish();
@@ -31,6 +32,7 @@ public:
 		void attribute(unsigned, const std::string &);
 		void fragment_shader(const std::string &);
 		void geometry_shader(const std::string &);
+		void module(const std::string &);
 		void vertex_shader(const std::string &);
 	};
 
@@ -65,15 +67,14 @@ public:
 		GLenum type;
 	};
 
-	typedef std::vector<Shader *> ShaderList;
 	typedef std::map<std::string, UniformInfo> UniformMap;
 	typedef std::map<std::string, UniformBlockInfo> UniformBlockMap;
 	typedef std::map<std::string, AttributeInfo> AttributeMap;
 
 private:
 	unsigned id;
-	ShaderList shaders;
-	ShaderList owned_data;
+	std::vector<unsigned> shader_ids;
+	const Module *module;
 	bool linked;
 	UniformBlockMap uniform_blocks;
 	UniformMap uniforms;
@@ -88,21 +89,23 @@ public:
 	Program(const std::string &);
 
 	/// Constructs a Program from vertex and fragment shader source code.
-	Program(const std::string &, const std::string &);
+	DEPRECATED Program(const std::string &, const std::string &);
 
 private:
 	void init();
 public:
 	virtual ~Program();
 
-	void attach_shader(Shader &shader);
-	void attach_shader_owned(Shader *shader);
-	void detach_shader(Shader &shader);
-	const ShaderList &get_attached_shaders() const { return shaders; }
+	void add_stages(const Module &);
 
-	void bind_attribute(unsigned, const std::string &);
-	void bind_attribute(VertexComponent, const std::string &);
-	void bind_fragment_data(unsigned, const std::string &);
+	DEPRECATED void attach_shader(Shader &shader);
+	DEPRECATED void attach_shader_owned(Shader *shader);
+	DEPRECATED void detach_shader(Shader &shader);
+	DEPRECATED const std::vector<Shader *> &get_attached_shaders() const;
+
+	DEPRECATED void bind_attribute(unsigned, const std::string &);
+	DEPRECATED void bind_attribute(VertexComponent, const std::string &);
+	DEPRECATED void bind_fragment_data(unsigned, const std::string &);
 
 	void link();
 private:
@@ -114,7 +117,7 @@ private:
 	static bool uniform_location_compare(const UniformInfo *, const UniformInfo *);
 public:
 	bool is_linked() const { return linked; }
-	std::string get_info_log() const;
+	DEPRECATED std::string get_info_log() const;
 
 	LayoutHash get_uniform_layout_hash() const { return uniform_layout_hash; }
 	const UniformBlockMap &get_uniform_blocks() const { return uniform_blocks; }
diff --git a/source/core/shader.cpp b/source/core/shader.cpp
index 02dcc0dc..9950e497 100644
--- a/source/core/shader.cpp
+++ b/source/core/shader.cpp
@@ -42,7 +42,8 @@ void Shader::init(GLenum t)
 
 Shader::~Shader()
 {
-	glDeleteShader(id);
+	if(id)
+		glDeleteShader(id);
 }
 
 void Shader::source(unsigned count, const char **str, const int *len)
@@ -74,6 +75,13 @@ void Shader::compile()
 #endif
 }
 
+unsigned Shader::steal_id()
+{
+	unsigned result = id;
+	id = 0;
+	return result;
+}
+
 string Shader::get_info_log() const
 {
 	GLsizei len = get_shader_i(id, GL_INFO_LOG_LENGTH);
diff --git a/source/core/shader.h b/source/core/shader.h
index bab11a7e..e02aa84a 100644
--- a/source/core/shader.h
+++ b/source/core/shader.h
@@ -1,6 +1,7 @@
 #ifndef MSP_GL_SHADER_H_
 #define MSP_GL_SHADER_H_
 
+#include <msp/core/attributes.h>
 #include <string>
 #include "gl.h"
 
@@ -32,9 +33,10 @@ public:
 	void source(const char *str, int len);
 	void compile();
 	unsigned get_id() const { return id; }
+	unsigned steal_id();
 	bool is_compiled() const { return compiled; }
 	std::string get_info_log() const;
-};
+} DEPRECATED;
 
 
 class VertexShader: public Shader
diff --git a/source/effects/ambientocclusion.cpp b/source/effects/ambientocclusion.cpp
index e5de04b4..bae5e780 100644
--- a/source/effects/ambientocclusion.cpp
+++ b/source/effects/ambientocclusion.cpp
@@ -14,8 +14,8 @@ namespace GL {
 
 AmbientOcclusion::AmbientOcclusion(Resources &resources, unsigned w, unsigned h, float):
 	occlude_target(w, h, (RENDER_COLOR,R8)),
-	occlude_shader(resources.get<Program>("_ambientocclusion_occlude.glsl")),
-	combine_shader(resources.get<Program>("_ambientocclusion_combine.glsl")),
+	occlude_shader(resources.get<Program>("_ambientocclusion_occlude.glsl.shader")),
+	combine_shader(resources.get<Program>("_ambientocclusion_combine.glsl.shader")),
 	quad(resources.get<Mesh>("_fullscreen_quad.mesh")),
 	linear_sampler(resources.get<Sampler>("_linear_clamp.samp")),
 	nearest_sampler(resources.get<Sampler>("_nearest_clamp.samp"))
diff --git a/source/effects/bloom.cpp b/source/effects/bloom.cpp
index 1204c81a..1e39c686 100644
--- a/source/effects/bloom.cpp
+++ b/source/effects/bloom.cpp
@@ -15,8 +15,8 @@ namespace Msp {
 namespace GL {
 
 Bloom::Bloom(Resources &resources, unsigned w, unsigned h):
-	blur_shader(resources.get<Program>("_bloom_blur.glsl")),
-	combine_shader(resources.get<Program>("_bloom_combine.glsl")),
+	blur_shader(resources.get<Program>("_bloom_blur.glsl.shader")),
+	combine_shader(resources.get<Program>("_bloom_combine.glsl.shader")),
 	quad(resources.get<Mesh>("_fullscreen_quad.mesh")),
 	nearest_sampler(resources.get<Sampler>("_nearest_clamp.samp")),
 	linear_sampler(resources.get<Sampler>("_linear_clamp.samp"))
diff --git a/source/effects/colorcurve.cpp b/source/effects/colorcurve.cpp
index 697c7257..7b8c63b0 100644
--- a/source/effects/colorcurve.cpp
+++ b/source/effects/colorcurve.cpp
@@ -13,7 +13,7 @@ namespace Msp {
 namespace GL {
 
 ColorCurve::ColorCurve(Resources &resources):
-	shprog(resources.get<Program>("_colorcurve.glsl")),
+	shprog(resources.get<Program>("_colorcurve.glsl.shader")),
 	quad(resources.get<Mesh>("_fullscreen_quad.mesh")),
 	linear_sampler(resources.get<Sampler>("_linear_clamp.samp")),
 	nearest_sampler(resources.get<Sampler>("_nearest_clamp.samp"))
diff --git a/source/glsl/compiler.cpp b/source/glsl/compiler.cpp
index db9017cf..89ffd7f3 100644
--- a/source/glsl/compiler.cpp
+++ b/source/glsl/compiler.cpp
@@ -56,10 +56,10 @@ void Compiler::load_source(IO::Base &io, const string &src_name)
 	load_source(io, 0, src_name);
 }
 
-void Compiler::compile()
+void Compiler::compile(Mode mode)
 {
 	for(list<Stage>::iterator i=module->stages.begin(); i!=module->stages.end(); ++i)
-		generate(*i);
+		generate(*i, mode);
 	for(list<Stage>::iterator i=module->stages.begin(); i!=module->stages.end(); )
 	{
 		if(optimize(*i))
@@ -68,43 +68,62 @@ void Compiler::compile()
 			++i;
 	}
 	for(list<Stage>::iterator i=module->stages.begin(); i!=module->stages.end(); ++i)
-		finalize(*i);
+		finalize(*i, mode);
 }
 
-void Compiler::add_shaders(Program &program)
+string Compiler::get_combined_glsl() const
 {
-	if(!module)
-		throw invalid_operation("Compiler::add_shaders");
+	string glsl;
 
-	try
-	{
-		for(list<Stage>::iterator i=module->stages.begin(); i!=module->stages.end(); ++i)
-		{
-			string stage_src = Formatter().apply(*i);
-
-			if(i->type==Stage::VERTEX)
-			{
-				program.attach_shader_owned(new VertexShader(stage_src));
-				for(map<string, unsigned>::iterator j=i->locations.begin(); j!=i->locations.end(); ++j)
-					program.bind_attribute(j->second, j->first);
-			}
-			else if(i->type==Stage::GEOMETRY)
-				program.attach_shader_owned(new GeometryShader(stage_src));
-			else if(i->type==Stage::FRAGMENT)
-			{
-				program.attach_shader_owned(new FragmentShader(stage_src));
-				if(EXT_gpu_shader4)
-				{
-					for(map<string, unsigned>::iterator j=i->locations.begin(); j!=i->locations.end(); ++j)
-						program.bind_fragment_data(j->second, j->first);
-				}
-			}
-		}
-	}
-	catch(const compile_error &e)
+	unsigned source_count = module->source_map.get_count();
+	for(unsigned i=1; i<source_count; ++i)
+		glsl += format("#pragma MSP source(%d, \"%s\")\n", i, module->source_map.get_name(i));
+	for(list<Stage>::iterator i=module->stages.begin(); i!=module->stages.end(); ++i)
 	{
-		throw compile_error(module->source_map.translate_errors(e.what()));
+		glsl += format("#pragma MSP stage(%s)\n", Stage::get_stage_name(i->type));
+		glsl += Formatter().apply(*i, MODULE);
+		glsl += '\n';
 	}
+
+	return glsl;
+}
+
+vector<Stage::Type> Compiler::get_stages() const
+{
+	vector<Stage::Type> stage_types;
+	stage_types.reserve(module->stages.size());
+	for(list<Stage>::const_iterator i=module->stages.begin(); i!=module->stages.end(); ++i)
+		stage_types.push_back(i->type);
+	return stage_types;
+}
+
+string Compiler::get_stage_glsl(Stage::Type stage_type) const
+{
+	for(list<Stage>::iterator i=module->stages.begin(); i!=module->stages.end(); ++i)
+		if(i->type==stage_type)
+			return Formatter().apply(*i, PROGRAM);
+	throw key_error(Stage::get_stage_name(stage_type));
+}
+
+const map<string, unsigned> &Compiler::get_vertex_attributes() const
+{
+	for(list<Stage>::const_iterator i=module->stages.begin(); i!=module->stages.end(); ++i)
+		if(i->type==Stage::VERTEX)
+			return i->locations;
+	throw invalid_operation("Compiler::get_vertex_attributes");
+}
+
+const map<string, unsigned> &Compiler::get_fragment_outputs() const
+{
+	for(list<Stage>::const_iterator i=module->stages.begin(); i!=module->stages.end(); ++i)
+		if(i->type==Stage::FRAGMENT)
+			return i->locations;
+	throw invalid_operation("Compiler::get_fragment_outputs");
+}
+
+const SourceMap &Compiler::get_source_map() const
+{
+	return module->source_map;
 }
 
 void Compiler::append_module(Module &mod, DataFile::Collection *res)
@@ -164,7 +183,7 @@ void Compiler::import(DataFile::Collection *resources, const string &name)
 	append_module(import_parser.parse(*io, fn, module->source_map.get_count()), resources);
 }
 
-void Compiler::generate(Stage &stage)
+void Compiler::generate(Stage &stage, Mode mode)
 {
 	if(module->shared.required_version>stage.required_version)
 		stage.required_version = module->shared.required_version;
@@ -178,7 +197,8 @@ void Compiler::generate(Stage &stage)
 	VariableResolver().apply(stage);
 	DeclarationReorderer().apply(stage);
 	FunctionResolver().apply(stage);
-	LegacyConverter().apply(stage);
+	if(mode==PROGRAM)
+		LegacyConverter().apply(stage);
 }
 
 bool Compiler::optimize(Stage &stage)
@@ -195,11 +215,11 @@ bool Compiler::optimize(Stage &stage)
 	return result;
 }
 
-void Compiler::finalize(Stage &stage)
+void Compiler::finalize(Stage &stage, Mode mode)
 {
-	if(get_gl_api()==OPENGL_ES2)
+	if(get_gl_api()==OPENGL_ES2 && mode==PROGRAM)
 		DefaultPrecisionGenerator().apply(stage);
-	else
+	else if(mode==MODULE)
 		PrecisionRemover().apply(stage);
 }
 
diff --git a/source/glsl/compiler.h b/source/glsl/compiler.h
index 8e839906..0ecec70d 100644
--- a/source/glsl/compiler.h
+++ b/source/glsl/compiler.h
@@ -3,7 +3,6 @@
 
 #include <vector>
 #include "parser.h"
-#include "program.h"
 #include "syntax.h"
 
 namespace Msp {
@@ -12,6 +11,13 @@ namespace SL {
 
 class Compiler
 {
+public:
+	enum Mode
+	{
+		MODULE,
+		PROGRAM
+	};
+
 private:
 	Module *module;
 	std::vector<std::string> imported_names;
@@ -26,16 +32,22 @@ public:
 	void set_source(const std::string &, const std::string & = "<string>");
 	void load_source(IO::Base &, DataFile::Collection * = 0, const std::string & = "<file>");
 	void load_source(IO::Base &, const std::string &);
-	void compile();
-	void add_shaders(Program &);
-
+	void compile(Mode);
+
+	std::string get_combined_glsl() const;
+	std::vector<Stage::Type> get_stages() const;
+	std::string get_stage_glsl(Stage::Type) const;
+	const std::map<std::string, unsigned> &get_vertex_attributes() const;
+	const std::map<std::string, unsigned> &get_fragment_outputs() const;
+	const SourceMap &get_source_map() const;
 private:
+
 	void append_module(Module &, DataFile::Collection *);
 	void append_stage(Stage &);
 	void import(DataFile::Collection *, const std::string &);
-	void generate(Stage &);
+	void generate(Stage &, Mode);
 	bool optimize(Stage &);
-	void finalize(Stage &);
+	void finalize(Stage &, Mode);
 	static void inject_block(Block &, const Block &);
 };
 
diff --git a/source/glsl/output.cpp b/source/glsl/output.cpp
index f97b8cdb..9d4383e4 100644
--- a/source/glsl/output.cpp
+++ b/source/glsl/output.cpp
@@ -10,14 +10,16 @@ namespace SL {
 
 Formatter::Formatter():
 	stage(0),
+	mode(Compiler::PROGRAM),
 	source_index(0),
 	source_line(1),
 	indent(0),
 	parameter_list(false)
 { }
 
-const string &Formatter::apply(Stage &s)
+const string &Formatter::apply(Stage &s, Compiler::Mode m)
 {
+	mode = m;
 	stage = &s;
 
 	GLApi api = get_gl_api();
@@ -65,7 +67,7 @@ void Formatter::set_source(unsigned index, unsigned line)
 		else
 		{
 			unsigned l = line;
-			if(stage && stage->required_version<Version(3, 30))
+			if(mode==Compiler::PROGRAM && stage && stage->required_version<Version(3, 30))
 				--l;
 			formatted += format("#line %d %d\n", l, index);
 		}
@@ -214,7 +216,7 @@ void Formatter::visit(VariableDeclaration &var)
 	if(!var.interface.empty() && var.interface!=block_interface)
 	{
 		string interface = var.interface;
-		if(stage && stage->required_version<Version(1, 30))
+		if(mode==Compiler::PROGRAM && stage && stage->required_version<Version(1, 30))
 		{
 			if(stage->type==Stage::VERTEX && var.interface=="in")
 				interface = "attribute";
diff --git a/source/glsl/output.h b/source/glsl/output.h
index 5f741909..aec63721 100644
--- a/source/glsl/output.h
+++ b/source/glsl/output.h
@@ -1,6 +1,7 @@
 #ifndef MSP_GL_SL_OUTPUT_H_
 #define MSP_GL_SL_OUTPUT_H_
 
+#include "compiler.h"
 #include "syntax.h"
 #include "visitor.h"
 
@@ -12,6 +13,7 @@ class Formatter: private TraversingVisitor
 {
 private:
 	Stage *stage;
+	Compiler::Mode mode;
 	std::string formatted;
 	unsigned source_index;
 	unsigned source_line;
@@ -22,7 +24,7 @@ private:
 public:
 	Formatter();
 
-	const std::string &apply(Stage &);
+	const std::string &apply(Stage &, Compiler::Mode);
 	const std::string &apply(Node &n) { n.visit(*this); return formatted; }
 
 private:
diff --git a/source/materials/renderpass.cpp b/source/materials/renderpass.cpp
index 835e78a6..561eab6a 100644
--- a/source/materials/renderpass.cpp
+++ b/source/materials/renderpass.cpp
@@ -1,4 +1,5 @@
 #include <msp/datafile/collection.h>
+#include <msp/io/print.h>
 #include <msp/strings/format.h>
 #include "error.h"
 #include "material.h"
@@ -175,6 +176,17 @@ void RenderPass::Loader::init()
 	add("uniform_slot", &Loader::uniform_slot2);
 }
 
+// Temporary compatibility feature
+string RenderPass::Loader::get_shader_name(const string &n)
+{
+	if(n.size()>=5 && !n.compare(n.size()-5, 5, ".glsl"))
+	{
+		IO::print(IO::cerr, "Warning: Loading module '%s' as shader is deprecated\n", n);
+		return n+".shader";
+	}
+	return n;
+}
+
 void RenderPass::Loader::material_inline()
 {
 	Material::GenericLoader ldr(coll);
@@ -192,7 +204,7 @@ void RenderPass::Loader::material(const string &name)
 
 void RenderPass::Loader::shader(const string &n)
 {
-	obj.shprog = &get_collection().get<Program>(n);
+	obj.shprog = &get_collection().get<Program>(get_shader_name(n));
 	obj.shprog.keep();
 	obj.shprog_from_material = false;
 	if(obj.shdata)
diff --git a/source/materials/renderpass.h b/source/materials/renderpass.h
index 9e704376..e01b872d 100644
--- a/source/materials/renderpass.h
+++ b/source/materials/renderpass.h
@@ -31,6 +31,8 @@ public:
 	private:
 		void init();
 
+		static std::string get_shader_name(const std::string &);
+
 		void material_inline();
 		void material(const std::string &);
 		void shader(const std::string &);
diff --git a/source/render/occludedscene.cpp b/source/render/occludedscene.cpp
index bfa9fed7..b3346385 100644
--- a/source/render/occludedscene.cpp
+++ b/source/render/occludedscene.cpp
@@ -14,7 +14,7 @@ namespace GL {
 
 OccludedScene::OccludedScene(Resources &resources):
 	bounding_mesh(resources.get<Mesh>("_occluder.mesh")),
-	bounding_shader(resources.get<Program>("_occluder.glsl")),
+	bounding_shader(resources.get<Program>("_occluder.glsl.shader")),
 	occluder_min_size(0.25f),
 	cache_dirty(false)
 {
diff --git a/source/resources/resources.cpp b/source/resources/resources.cpp
index dc034746..ef8b9325 100644
--- a/source/resources/resources.cpp
+++ b/source/resources/resources.cpp
@@ -8,6 +8,7 @@
 #include "lighting.h"
 #include "material.h"
 #include "mesh.h"
+#include "module.h"
 #include "object.h"
 #include "pipelinetemplate.h"
 #include "pose.h"
@@ -44,10 +45,11 @@ Resources::Resources():
 	add_type<Lighting>().suffix(".lightn").keyword("lighting");
 	add_type<Material>().suffix(".mat").creator(&Resources::create_material);
 	add_type<Mesh>().keyword("mesh").creator(&Resources::create_mesh);
+	add_type<Module>().suffix(".glsl").creator(&Resources::create_module);
 	add_type<Object>().keyword("object");
 	add_type<PipelineTemplate>().suffix(".pipe").keyword("pipeline");
 	add_type<Pose>().keyword("pose");
-	add_type<Program>().keyword("shader").suffix(".glsl").creator(&Resources::create_program);
+	add_type<Program>().keyword("shader").creator(&Resources::create_program);
 	add_type<Sampler>().suffix(".samp").keyword("sampler");
 	add_type<Technique>().suffix(".tech").keyword("technique");
 	add_type<Texture1D>().base<Texture>().suffix(".tex1d").keyword("texture1d");
@@ -166,21 +168,30 @@ Texture2D *Resources::create_texture2d(const string &name)
 	return 0;
 }
 
+Module *Resources::create_module(const string &name)
+{
+	if(RefPtr<IO::Seekable> io = open_raw(name))
+	{
+		RefPtr<Module> module = new Module;
+		module->load_source(*io, this, name);
+		return module.release();
+	}
+
+	return 0;
+}
+
 Program *Resources::create_program(const string &name)
 {
 	string ext = FS::extpart(name);
-	if(ext==".shader")
-		return 0;
-
-	if(RefPtr<IO::Seekable> io = open_raw(name))
+	string base = FS::basepart(name);
+	string ext2 = FS::extpart(base);
+	if(ext==".shader" && ext2==".glsl")
 	{
-		SL::Compiler compiler;
-		compiler.load_source(*io, this, name);
-		compiler.compile();
-		RefPtr<Program> program = new Program;
-		compiler.add_shaders(*program);
-		program->link();
-		return program.release();
+		Module &module = get<Module>(base);
+		RefPtr<Program> shprog = new Program;
+		shprog->add_stages(module);
+		shprog->link();
+		return shprog.release();
 	}
 
 	return 0;
diff --git a/source/resources/resources.h b/source/resources/resources.h
index 6f04a260..3036c2f4 100644
--- a/source/resources/resources.h
+++ b/source/resources/resources.h
@@ -9,6 +9,7 @@ namespace GL {
 
 class Material;
 class Mesh;
+class Module;
 class Program;
 class ResourceManager;
 class Texture2D;
@@ -45,6 +46,7 @@ protected:
 	Material *create_material(const std::string &);
 	Mesh *create_mesh(const std::string &);
 	Texture2D *create_texture2d(const std::string &);
+	Module *create_module(const std::string &);
 	Program *create_program(const std::string &);
 };