]> git.tdb.fi Git - geometrycompositor.git/commitdiff
Mask out transparent parts of shaped windows
authorMikko Rasa <tdb@tdb.fi>
Sun, 13 Dec 2015 15:46:03 +0000 (17:46 +0200)
committerMikko Rasa <tdb@tdb.fi>
Sun, 13 Dec 2015 15:53:45 +0000 (17:53 +0200)
source/main.c

index afcd7646b9ecdd38df1d7a557d85f8a366a64d82..970774d2eac52a7ff5350891f8442619287eb265 100644 (file)
@@ -26,6 +26,8 @@ typedef struct CompositedWindow
        Pixmap pixmap;
        GLXPixmap glx_pixmap;
        unsigned texture;
+       unsigned mask_texture;
+       int use_mask;
 } CompositedWindow;
 
 typedef struct CompositedMonitor
@@ -58,9 +60,11 @@ typedef struct CompositedScreen
        GLXFBConfig fbconfig;
        GLXWindow glx_window;
        GLXContext glx_context;
-       unsigned shaders[2];
+       unsigned shaders[3];
        unsigned program;
-       unsigned geometry_loc;
+       int geometry_loc;
+       unsigned masked_program;
+       int masked_geometry_loc;
        unsigned window_vertex_buffer;
        unsigned window_vertex_array;
        unsigned framebuffer;
@@ -79,6 +83,7 @@ typedef struct Compositor
        CompositedScreen *screens;
        unsigned nscreens;
        int damage_event;
+       int shape_event;
        PFNGLXCREATECONTEXTATTRIBSARBPROC glXCreateContextAttribsARB;
        PFNGLXBINDTEXIMAGEEXTPROC glXBindTexImageEXT;
        PFNGLXRELEASETEXIMAGEEXTPROC glXReleaseTexImageEXT;
@@ -87,7 +92,7 @@ typedef struct Compositor
        int dirty;
 } Compositor;
 
-static const char *vshader =
+static const char *vshader_src =
        "#version 150\n"
        "uniform vec4 geometry;\n"
        "in vec2 vertex;\n"
@@ -99,7 +104,7 @@ static const char *vshader =
        "  texcoord = texture_coord;\n"
        "}\n";
 
-static const char *fshader =
+static const char *fshader_src =
        "#version 150\n"
        "uniform sampler2D image;\n"
        "in vec2 texcoord;\n"
@@ -109,6 +114,19 @@ static const char *fshader =
        "       frag_color = texture(image, texcoord);\n"
        "}\n";
 
+static const char *masked_fshader_src =
+       "#version 150\n"
+       "uniform sampler2D image;\n"
+       "uniform sampler2D mask;\n"
+       "in vec2 texcoord;\n"
+       "out vec4 frag_color;\n"
+       "void main()\n"
+       "{\n"
+       "  if(texture(mask, texcoord).r==0.0)\n"
+       "    discard;\n"
+       "  frag_color = texture(image, texcoord);\n"
+       "}\n";
+
 static const float window_vertices[] =
 {
        /* vertex    texcoord */
@@ -211,17 +229,16 @@ unsigned compile_shader(GLenum type, const char *source)
        return shader;
 }
 
-unsigned link_program(unsigned *shaders, unsigned nshaders)
+unsigned link_program(unsigned vshader, unsigned fshader)
 {
        unsigned program;
-       unsigned i;
        int status;
        char info_log[1024];
        GLsizei length;
 
        program = glCreateProgram();
-       for(i=0; i<nshaders; ++i)
-               glAttachShader(program, shaders[i]);
+       glAttachShader(program, vshader);
+       glAttachShader(program, fshader);
        glBindAttribLocation(program, 0, "vertex");
        glBindAttribLocation(program, 1, "texture_coord");
        glBindFragDataLocation(program, 0, "frag_color");
@@ -255,19 +272,33 @@ unsigned create_2d_texture()
 int create_gl_resources(Compositor *compositor, CompositedScreen *screen)
 {
        unsigned stride;
+       int loc;
 
        use_gl(compositor, screen);
 
-       screen->shaders[0] = compile_shader(GL_VERTEX_SHADER, vshader);
-       screen->shaders[1] = compile_shader(GL_FRAGMENT_SHADER, fshader);
-       if(!screen->shaders[0] || !screen->shaders[1])
+       screen->shaders[0] = compile_shader(GL_VERTEX_SHADER, vshader_src);
+       screen->shaders[1] = compile_shader(GL_FRAGMENT_SHADER, fshader_src);
+       screen->shaders[2] = compile_shader(GL_FRAGMENT_SHADER, masked_fshader_src);
+       if(!screen->shaders[0] || !screen->shaders[1] || !screen->shaders[2])
                return 0;
 
-       screen->program = link_program(screen->shaders, 2);
+       screen->program = link_program(screen->shaders[0], screen->shaders[1]);
        if(!screen->program)
                return 0;
 
+       screen->masked_program = link_program(screen->shaders[0], screen->shaders[2]);
+       if(!screen->masked_program)
+               return 0;
+
        screen->geometry_loc = glGetUniformLocation(screen->program, "geometry");
+       screen->masked_geometry_loc = glGetUniformLocation(screen->masked_program, "geometry");
+
+       loc = glGetUniformLocation(screen->masked_program, "mask");
+       if(loc>=0)
+       {
+               glUseProgram(screen->masked_program);
+               glUniform1i(loc, 1);
+       }
 
        glGenBuffers(1, &screen->window_vertex_buffer);
        glBindBuffer(GL_ARRAY_BUFFER, screen->window_vertex_buffer);
@@ -348,6 +379,75 @@ void create_window_pixmap(Compositor *compositor, CompositedScreen *screen, Comp
        window->glx_pixmap = glXCreatePixmap(compositor->display, screen->fbconfig, window->pixmap, attribs);
 }
 
+void update_window_mask(Compositor *compositor, CompositedWindow *window)
+{
+       Bool bounding_shaped;
+       Bool clip_shaped;
+       int xi, yi;
+       XRectangle *rects;
+       int rect_count;
+       int rect_order;
+       unsigned width;
+       unsigned height;
+       unsigned char *data;
+       int i;
+       unsigned y;
+
+       XShapeQueryExtents(compositor->display, window->window, &bounding_shaped, &xi, &yi, &width, &height, &clip_shaped, &xi, &yi, &width, &height);
+       window->use_mask = bounding_shaped;
+       if(!window->use_mask)
+               return;
+
+       rects = XShapeGetRectangles(compositor->display, window->window, ShapeBounding, &rect_count, &rect_order);
+
+       width = window->width+2*window->border;
+       height = window->height+2*window->border;
+       data = (unsigned char *)malloc(width*height);
+       memset(data, 0, width*height);
+       for(i=0; i<rect_count; ++i)
+       {
+               rects[i].x += window->border;
+               rects[i].y += window->border;
+               if(rects[i].x>=(int)width || rects[i].y>=(int)height)
+                       continue;
+
+               if(rects[i].x<0)
+               {
+                       if(-rects[i].x>rects[i].width)
+                               continue;
+                       rects[i].width += rects[i].x;
+                       rects[i].x = 0;
+               }
+
+               if(rects[i].y<0)
+               {
+                       if(-rects[i].y>rects[i].height)
+                               continue;
+                       rects[i].height += rects[i].y;
+                       rects[i].y = 0;
+               }
+
+               if(rects[i].x+rects[i].width>(int)width)
+                       rects[i].width = width-rects[i].x;
+               if(rects[i].y+rects[i].height>(int)height)
+                       rects[i].height = height-rects[i].y;
+
+               for(y=0; y<rects[i].height; ++y)
+               {
+                       unsigned char *row = data+(rects[i].y+y)*width+rects[i].x;
+                       memset(row, 255, rects[i].width);
+               }
+       }
+
+       XFree(rects);
+
+       glBindTexture(GL_TEXTURE_2D, window->mask_texture);
+       glPixelStorei(GL_UNPACK_ALIGNMENT, 1);
+       glTexImage2D(GL_TEXTURE_2D, 0, GL_R8, width, height, 0, GL_RED, GL_UNSIGNED_BYTE, data);
+
+       free(data);
+}
+
 CompositedWindow *add_window(Compositor *compositor, CompositedScreen *screen, Window w)
 {
        CompositedWindow *window;
@@ -392,6 +492,12 @@ CompositedWindow *add_window(Compositor *compositor, CompositedScreen *screen, W
                create_window_pixmap(compositor, screen, window);
 
        window->texture = create_2d_texture();
+       window->mask_texture = create_2d_texture();
+
+       if(window->map_state==IsViewable)
+               update_window_mask(compositor, window);
+
+       XShapeSelectInput(compositor->display, window->window, ShapeNotifyMask);
 
        return window;
 }
@@ -786,7 +892,7 @@ int initialize_compositor(Compositor *compositor)
        else if(major_ver<1)
                return with_error("XDamage 1.0 or later is required");
 
-       if(!XShapeQueryExtension(compositor->display, &event_base, &error_base))
+       if(!XShapeQueryExtension(compositor->display, &compositor->shape_event, &error_base))
                return with_error("XShape is required but was not found");
        else if(!XShapeQueryVersion(compositor->display, &major_ver, &minor_ver))
                return with_error("Cannot determine XShape version");
@@ -827,6 +933,7 @@ void shutdown_screen(Compositor *compositor, CompositedScreen *screen)
        for(i=0; i<screen->nwindows; ++i)
        {
                glDeleteTextures(1, &screen->windows[i].texture);
+               glDeleteTextures(1, &screen->windows[i].mask_texture);
                if(screen->windows[i].pixmap)
                {
                        glXDestroyPixmap(compositor->display, screen->windows[i].glx_pixmap);
@@ -851,8 +958,9 @@ void shutdown_screen(Compositor *compositor, CompositedScreen *screen)
        glDeleteFramebuffers(1, &screen->framebuffer);
        glDeleteTextures(1, &screen->fb_texture);
        glDeleteProgram(screen->program);
-       glDeleteShader(screen->shaders[0]);
-       glDeleteShader(screen->shaders[1]);
+       glDeleteProgram(screen->masked_program);
+       for(i=0; i<3; ++i)
+               glDeleteShader(screen->shaders[i]);
 
        glXMakeContextCurrent(compositor->display, 0, 0, NULL);
        glXDestroyContext(compositor->display, screen->glx_context);
@@ -915,6 +1023,7 @@ void process_map_event(Compositor *compositor, XMapEvent *event)
 
        window->map_state = IsViewable;
        create_window_pixmap(compositor, screen, window);
+       update_window_mask(compositor, window);
 
        mark_dirty(compositor, screen);
 }
@@ -980,6 +1089,7 @@ void process_configure_event(Compositor *compositor, XConfigureEvent *event)
                window->height = event->height;
                window->border = event->border_width;
                create_window_pixmap(compositor, screen, window);
+               update_window_mask(compositor, window);
        }
        reorder_window(screen, window, event->above);
 
@@ -1011,6 +1121,22 @@ void process_damage_event(Compositor *compositor, XDamageNotifyEvent *event)
                mark_dirty(compositor, screen);
 }
 
+void process_shape_event(Compositor *compositor, XShapeEvent *event)
+{
+       CompositedScreen *screen;
+       CompositedWindow *window;
+
+       if(event->kind!=ShapeBounding)
+               return;
+
+       window = find_window_global(compositor, event->window, &screen);
+       if(window && window->map_state==IsViewable)
+       {
+               update_window_mask(compositor, window);
+               mark_dirty(compositor, screen);
+       }
+}
+
 int process_event(Compositor *compositor)
 {
        XEvent event;
@@ -1048,6 +1174,8 @@ int process_event(Compositor *compositor)
        default:
                if(event.type==compositor->damage_event+XDamageNotify)
                        process_damage_event(compositor, (XDamageNotifyEvent *)&event);
+               else if(event.type==compositor->shape_event+ShapeNotify)
+                       process_shape_event(compositor, (XShapeEvent *)&event);
                else
                        printf("Event %d\n", event.type);
        }
@@ -1058,6 +1186,7 @@ int process_event(Compositor *compositor)
 void refresh_screen(Compositor *compositor, CompositedScreen *screen)
 {
        unsigned i;
+       int use_mask;
 
        use_gl(compositor, screen);
 
@@ -1066,7 +1195,7 @@ void refresh_screen(Compositor *compositor, CompositedScreen *screen)
        glClearColor(0.5f, 0.5f, 0.5f, 0.0f);
        glClear(GL_COLOR_BUFFER_BIT);
 
-       glUseProgram(screen->program);
+       use_mask = -1;
        glBindVertexArray(screen->window_vertex_array);
        for(i=0; i<screen->nwindows; ++i)
        {
@@ -1074,11 +1203,24 @@ void refresh_screen(Compositor *compositor, CompositedScreen *screen)
                if(window->map_state!=IsViewable)
                        continue;
 
+               if(window->use_mask!=use_mask)
+               {
+                       use_mask = window->use_mask;
+                       glUseProgram(use_mask ? screen->masked_program : screen->program);
+               }
+
+               if(window->use_mask)
+               {
+                       glActiveTexture(GL_TEXTURE1);
+                       glBindTexture(GL_TEXTURE_2D, window->mask_texture);
+                       glActiveTexture(GL_TEXTURE0);
+               }
+
                XDamageSubtract(compositor->display, window->damage, None, None);
 
                glBindTexture(GL_TEXTURE_2D, window->texture);
                compositor->glXBindTexImageEXT(compositor->display, window->glx_pixmap, GLX_FRONT_LEFT_EXT, NULL);
-               glUniform4f(screen->geometry_loc,
+               glUniform4f((use_mask ? screen->masked_geometry_loc : screen->geometry_loc),
                        (float)window->x/screen->width, 1.0f-(float)(window->y+window->height)/screen->height,
                        (float)(window->width+2*window->border)/screen->width, (float)(window->height+2*window->border)/screen->height);
                glDrawArrays(GL_TRIANGLE_STRIP, 0, 4);
@@ -1091,6 +1233,8 @@ void refresh_screen(Compositor *compositor, CompositedScreen *screen)
        glBindTexture(GL_TEXTURE_2D, screen->fb_texture);
        glEnable(GL_PRIMITIVE_RESTART);
        glPrimitiveRestartIndex(0xFFFF);
+       if(use_mask)
+               glUseProgram(screen->program);
 
        for(i=0; i<screen->nmonitors; ++i)
        {