From 5687bb2b8cd7715b5a5c5bb1d75f42992a23a56c Mon Sep 17 00:00:00 2001
From: Mikko Rasa <tdb@tdb.fi>
Date: Sun, 19 Dec 2010 13:22:54 +0000
Subject: [PATCH] Improve the ambient occlusion algorithm Allow changing the
 depth ratio Use correct winding for the triangle strip Set pipeline texture
 wrapping to CLAMP_TO_EDGE

---
 source/ambientocclusion.cpp | 47 ++++++++++++++++++++-----------------
 source/ambientocclusion.h   |  2 +-
 source/pipeline.cpp         |  2 ++
 3 files changed, 29 insertions(+), 22 deletions(-)

diff --git a/source/ambientocclusion.cpp b/source/ambientocclusion.cpp
index a3d8894c..2775e854 100644
--- a/source/ambientocclusion.cpp
+++ b/source/ambientocclusion.cpp
@@ -31,19 +31,21 @@ const char occlude_fs[] =
 	"void main()\n"
 	"{\n"
 	"	mat2 transform = mat2(texture2D(rotate, texcoord*screen_size/4.0)*2.0-1.0)\n"
- 	"		*mat2(screen_size.y/screen_size.x, 0.0, 0.0, 1.0)*0.02;\n"
-	"	float depth_avg = 0.0;\n"
+ 	"		*mat2(screen_size.y/screen_size.x, 0.0, 0.0, 1.0)*3.0/screen_size.y;\n"
+	"	float sample = depth_ratio.x/(texture2D(depth, texcoord).r-depth_ratio.y);\n"
+	"	float sum = 0.0;\n"
 	"	for(int i=0; i<=3; ++i)\n"
 	"		for(int j=0; j<=3; ++j)\n"
 	"		{\n"
 	"			vec2 offs = transform*vec2(float(i)-1.5, float(j)-1.5);\n"
-	"			depth_avg += depth_ratio.x/(texture2D(depth, texcoord+offs).r-depth_ratio.y);\n"
+	"			float dxy = length(offs)*-sample;\n"
+	"			float dz = depth_ratio.x/(texture2D(depth, texcoord+offs).r-depth_ratio.y)-sample;\n"
+	"			if(abs(dz)<3*dxy)\n"
+	"				sum += atan(dz/dxy)/1.570796;\n"
+	"			else if(dz<0)\n"
+	"				sum -= 0.8;\n"
 	"		}\n"
-	"	depth_avg /= 16;\n"
-	"	float sample = depth_ratio.x/(texture2D(depth, texcoord).r-depth_ratio.y);\n"
-	"	float diff = sample-depth_avg;\n"
-	"	float shade = min(diff*darkness, 0.0)+1.0;\n"
-	"	gl_FragColor = vec4(shade, 0.0, 0.0, 1.0);\n"
+	"	gl_FragColor = vec4(min(1.0-sum*darkness/16.0, 1.0), 0.0, 0.0, 1.0);\n"
 	"}\n";
 
 const char combine_fs[] =
@@ -52,7 +54,6 @@ const char combine_fs[] =
 	"uniform sampler2D occlusion;\n"
 	"uniform vec2 screen_size;\n"
 	"uniform vec2 depth_ratio;\n"
-	"uniform float edge_threshold;\n"
 	"varying vec2 texcoord;\n"
 	"void main()\n"
 	"{\n"
@@ -63,8 +64,9 @@ const char combine_fs[] =
 	"		for(int j=0; j<=3; ++j)\n"
 	"		{\n"
 	"			vec2 offs = vec2(float(i)-1.5, float(j)-1.5)/screen_size;\n"
-	"			float depth = depth_ratio.x/(texture2D(depth, texcoord+offs).r-depth_ratio.y);\n"
-	"			if(abs(depth-sample)<edge_threshold)\n"
+	"			float dxy = length(offs)*-sample;\n"
+	"			float dz = depth_ratio.x/(texture2D(depth, texcoord+offs).r-depth_ratio.y)-sample;\n"
+	"			if(abs(dz)<3*dxy)\n"
 	"			{\n"
 	"				sum += texture2D(occlusion, texcoord+offs).r;\n"
 	"				count += 1.0;\n"
@@ -108,34 +110,37 @@ AmbientOcclusion::AmbientOcclusion(unsigned w, unsigned h, float depth_ratio):
 
 	occlude_texturing.attach(1, rotate_lookup);
 
-	depth_ratio = 1/depth_ratio;
-
 	occlude_shdata.uniform(occlude_shader.get_uniform_location("depth"), 0);
 	occlude_shdata.uniform(occlude_shader.get_uniform_location("rotate"), 1);
 	occlude_shdata.uniform(occlude_shader.get_uniform_location("screen_size"), w, h);
-	occlude_shdata.uniform(occlude_shader.get_uniform_location("depth_ratio"), depth_ratio, 1+depth_ratio);
 
 	combine_shdata.uniform(combine_shader.get_uniform_location("color"), 1);
 	combine_shdata.uniform(combine_shader.get_uniform_location("depth"), 0);
 	combine_shdata.uniform(combine_shader.get_uniform_location("occlusion"), 2);
 	combine_shdata.uniform(combine_shader.get_uniform_location("screen_size"), w, h);
-	combine_shdata.uniform(combine_shader.get_uniform_location("depth_ratio"), depth_ratio, 1+depth_ratio);
-	combine_shdata.uniform(combine_shader.get_uniform_location("edge_threshold"), 0.05f);
 
-	set_darkness(15);
+	set_depth_ratio(depth_ratio);
+	set_darkness(1.5);
 
 	MeshBuilder bld(quad);
 	bld.begin(TRIANGLE_STRIP);
-	bld.vertex(-1, -1);
 	bld.vertex(-1, 1);
-	bld.vertex(1, -1);
+	bld.vertex(-1, -1);
 	bld.vertex(1, 1);
+	bld.vertex(1, -1);
 	bld.end();
 }
 
-void AmbientOcclusion::set_darkness(float d)
+void AmbientOcclusion::set_depth_ratio(float depth_ratio)
+{
+	depth_ratio = 1/depth_ratio;
+
+	occlude_shdata.uniform(occlude_shader.get_uniform_location("depth_ratio"), depth_ratio, 1+depth_ratio);
+	combine_shdata.uniform(combine_shader.get_uniform_location("depth_ratio"), depth_ratio, 1+depth_ratio);
+}
+
+void AmbientOcclusion::set_darkness(float darkness)
 {
-	darkness = d;
 	occlude_shdata.uniform(occlude_shader.get_uniform_location("darkness"), darkness);
 }
 
diff --git a/source/ambientocclusion.h b/source/ambientocclusion.h
index 1866baee..4affd049 100644
--- a/source/ambientocclusion.h
+++ b/source/ambientocclusion.h
@@ -37,11 +37,11 @@ private:
 	Program combine_shader;
 	ProgramData combine_shdata;
 	Mesh quad;
-	float darkness;
 
 public:
 	AmbientOcclusion(unsigned, unsigned, float);
 
+	void set_depth_ratio(float);
 	void set_darkness(float);
 
 	virtual void render(const Texture2D &, const Texture2D &);
diff --git a/source/pipeline.cpp b/source/pipeline.cpp
index 63bdc4e8..5294ffbd 100644
--- a/source/pipeline.cpp
+++ b/source/pipeline.cpp
@@ -120,12 +120,14 @@ void Pipeline::add_postprocessor(PostProcessor &pp)
 		color_buf = new Texture2D;
 		color_buf->set_min_filter(NEAREST);
 		color_buf->set_mag_filter(NEAREST);
+		color_buf->set_wrap(CLAMP_TO_EDGE);
 		color_buf->storage((hdr ? RGB16F : RGB), width, height);
 		fbo->attach(COLOR_ATTACHMENT0, *color_buf, 0);
 
 		depth_buf = new Texture2D;
 		depth_buf->set_min_filter(NEAREST);
 		depth_buf->set_mag_filter(NEAREST);
+		depth_buf->set_wrap(CLAMP_TO_EDGE);
 		depth_buf->storage(DEPTH_COMPONENT, width, height);
 		fbo->attach(DEPTH_ATTACHMENT, *depth_buf, 0);
 	}
-- 
2.43.0