Improve ambient occlusion edge detection

The depth thresholds are now based on the distance of the sample from the
center, which works better in the presence of sloped flat surfaces.

diff --git a/shaderlib/ambientocclusion_combine.glsl b/shaderlib/ambientocclusion_combine.glsl
index 1e8bd821d48d1fa4b2ad749dd19bd4ad0b01d6ec..05aea60a6c811d3f802f6a2df53db08a99597d6d 100644 (file)
--- a/shaderlib/ambientocclusion_combine.glsl
+++ b/shaderlib/ambientocclusion_combine.glsl
@@ -5,17 +5,19 @@ import ambientocclusion;
 void main()
 {
        vec3 center = unproject(vec3(vertex.xy, texture(depth, texcoord).r));
-       float min_depth = project(vec3(center.xy, center.z+occlusion_radius*0.1)).z;
-       float max_depth = project(vec3(center.xy, center.z-occlusion_radius*0.1)).z;
        vec2 tex_scale = 1.0/vec2(textureSize(occlusion, 0));
        float sum = 0.0;
        float count = 0.0;
        for(int i=0; i<4; ++i)
                for(int j=0; j<4; ++j)
                {
-                       vec2 offset = vec2(float(i)-1.5, float(j)-1.5)*tex_scale;
-                       float occ = texture(occlusion, texcoord+offset).r;
-                       float sample = texture(depth, texcoord+offset).r;
+                       vec2 offset = vec2(float(i), float(j))-1.5;
+                       vec2 sample_coord = texcoord+offset*tex_scale;
+                       float occ = texture(occlusion, sample_coord).r;
+                       float sample = texture(depth, sample_coord).r;
+                       float z_range = occlusion_radius*length(offset)*0.1;
+                       float min_depth = project(vec3(center.xy, center.z+z_range)).z;
+                       float max_depth = project(vec3(center.xy, center.z-z_range)).z;
                        if(sample>=min_depth && sample<=max_depth)
                        {
                                sum += occ;