Updated 2011-07-04 17:28:47 by RLE

To create an application toplevel without the decorations inserted by a window manager, and at the same time with a non-rectangular shape, we have few options. One way is to set override redirect, and the window manager will not touch your window, then use the nice Shape extension by Donal Fellows to have the fancy window form. But this unfortunately creates a problem; you will have to do a global grab to have the keys sent to your application's widgets, and this is cumbersome and dangerous. Another problem is that you are on your own in iconifying the application, without any window manager help.

Another way is to create a separate window, set its shape and populate its wm hints to inform the window manager you don't want decorations. Here is a minimalist implementation of this, with wm hints for the most common desktops (kde, gnome, and motif/cde), which receives a shape from a xpm image file. This is specific for linux and other unices (X Window System). It creates a container window and uses it with the toplevel '-use' switch to make the application's toplevel embedded.

NEW and better - See below for a really minimal implementation, by swallowing the "frame window" (our window reparented by the window manager), which solves the problems with focus and works under fvwm, kde and gnome.

Rildo Pragana
 /*
    Tk window manager miscellaneous extensions

    (C) Copyright 2005, Rildo Pragana <[email protected]>

    The authors hereby grant permission to use, copy, modify, distribute,
    and license this software and its documentation for any purpose, provided
    that existing copyright notices are retained in all copies and that this
    notice is included verbatim in any distributions. No written agreement,
    license, or royalty fee is required for any of the authorized uses.
    Modifications to this software may be copyrighted by their authors
    and need not follow the licensing terms described here, provided that
    the new terms are clearly indicated on the first page of each file where
    they apply.

    IN NO EVENT SHALL THE AUTHORS OR DISTRIBUTORS BE LIABLE TO ANY PARTY
    FOR DIRECT, INDIRECT, SPECIAL, INCIDENTAL, OR CONSEQUENTIAL DAMAGES
    ARISING OUT OF THE USE OF THIS SOFTWARE, ITS DOCUMENTATION, OR ANY
    DERIVATIVES THEREOF, EVEN IF THE AUTHORS HAVE BEEN ADVISED OF THE
    POSSIBILITY OF SUCH DAMAGE.
    THE AUTHORS AND DISTRIBUTORS SPECIFICALLY DISCLAIM ANY WARRANTIES,
    INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY,
    FITNESS FOR A PARTICULAR PURPOSE, AND NON-INFRINGEMENT.  THIS SOFTWARE
    IS PROVIDED ON AN "AS IS" BASIS, AND THE AUTHORS AND DISTRIBUTORS HAVE
    NO OBLIGATION TO PROVIDE MAINTENANCE, SUPPORT, UPDATES, ENHANCEMENTS, OR
    MODIFICATIONS.

    GOVERNMENT USE: If you are acquiring this software on behalf of the
    U.S. government, the Government shall have only "Restricted Rights"
    in the software and related documentation as defined in the Federal
    Acquisition Regulations (FARs) in Clause 52.227.19 (c) (2).  If you
    are acquiring the software on behalf of the Department of Defense, the
    software shall be classified as "Commercial Computer Software" and the
    Government shall have only "Restricted Rights" as defined in Clause
    252.227-7013 (c) (1) of DFARs.  Notwithstanding the foregoing, the
    authors grant the U.S. Government and others acting in its behalf
    permission to use and distribute the software in accordance with the
    terms specified in this license.
 */

 #include <unistd.h>
 #include <X11/X.h>
 #include <X11/Xlib.h>
 #include <X11/Xatom.h>
 #include <X11/Xutil.h>
 #include <X11/Xmd.h>
 #include <X11/extensions/shape.h>
 #include <X11/cursorfont.h>
 #include <X11/xpm.h>

 #include <tcl.h>
 #include <tk.h>

 static Display *dpy;
 static Window rootw;
 static int screen_number;
 static Screen *screen;
 static unsigned long rmask;
 static Atom wm_activewin;

 /* MWM decorations values */
 #define MWM_DECOR_NONE          0
 #define MWM_DECOR_ALL           (1L << 0)
 #define MWM_DECOR_BORDER        (1L << 1)
 #define MWM_DECOR_RESIZEH       (1L << 2)
 #define MWM_DECOR_TITLE         (1L << 3)
 #define MWM_DECOR_MENU          (1L << 4)
 #define MWM_DECOR_MINIMIZE      (1L << 5)
 #define MWM_DECOR_MAXIMIZE      (1L << 6)

 /* KDE decoration values */
 enum {
  KDE_noDecoration = 0,
  KDE_normalDecoration = 1,
  KDE_tinyDecoration = 2,
  KDE_noFocus = 256,
  KDE_standaloneMenuBar = 512,
  KDE_desktopIcon = 1024 ,
  KDE_staysOnTop = 2048
 };

 void wm_nodecorations(Window window) {
    Atom WM_HINTS;
    int set;

    WM_HINTS = XInternAtom(dpy, "_MOTIF_WM_HINTS", True);
    if ( WM_HINTS != None ) {
        #define MWM_HINTS_DECORATIONS   (1L << 1)
        struct {
          unsigned long flags;
          unsigned long functions;
          unsigned long decorations;
                   long input_mode;
          unsigned long status;
        } MWMHints = { MWM_HINTS_DECORATIONS, 0,
            MWM_DECOR_NONE, 0, 0 };
        XChangeProperty(dpy, window, WM_HINTS, WM_HINTS, 32,
                        PropModeReplace, (unsigned char *)&MWMHints,
                        sizeof(MWMHints)/4);
    }
    WM_HINTS = XInternAtom(dpy, "KWM_WIN_DECORATION", True);
    if ( WM_HINTS != None ) {
        long KWMHints = KDE_tinyDecoration;
        XChangeProperty(dpy, window, WM_HINTS, WM_HINTS, 32,
                        PropModeReplace, (unsigned char *)&KWMHints,
                        sizeof(KWMHints)/4);
    }

    WM_HINTS = XInternAtom(dpy, "_WIN_HINTS", True);
    if ( WM_HINTS != None ) {
        long GNOMEHints = 0;
        XChangeProperty(dpy, window, WM_HINTS, WM_HINTS, 32,
                        PropModeReplace, (unsigned char *)&GNOMEHints,
                        sizeof(GNOMEHints)/4);
    }
    WM_HINTS = XInternAtom(dpy, "_NET_WM_WINDOW_TYPE", True);
    if ( WM_HINTS != None ) {
        Atom NET_WMHints[2];
        NET_WMHints[0] = XInternAtom(dpy,
            "_KDE_NET_WM_WINDOW_TYPE_OVERRIDE", True);
        NET_WMHints[1] = XInternAtom(dpy, "_NET_WM_WINDOW_TYPE_NORMAL", True);
        XChangeProperty(dpy, window,
                        WM_HINTS, XA_ATOM, 32, PropModeReplace,
                        (unsigned char *)&NET_WMHints, 2);
    }
    XSetTransientForHint(dpy, window, rootw);
    XUnmapWindow(dpy, window);
    XMapWindow(dpy, window);
 }

 int shaped_container (ClientData client_data, Tcl_Interp *interp, int objc,
        Tcl_Obj * CONST objv[]) {
    Window w;
    XSizeHints  size_hints;
    XWMHints    wm_hints;
    XClassHint  class_hints;
    Pixmap pixmap, mask;
    XEvent ev;

    size_hints.flags = PSize | PMinSize | PMaxSize;
    size_hints.width = size_hints.min_width =
        size_hints.max_width = 800;
    size_hints.height = size_hints.min_height =
        size_hints.max_height = 600;
    w = XCreateSimpleWindow(dpy, rootw,
        0, 0, 800, 600, 2, 0, 0);

    wm_hints.initial_state = NormalState;
    wm_hints.input = True;
    wm_hints.flags = StateHint | InputHint;
    class_hints.res_name = "teste";
    class_hints.res_class = "Teste";
    XmbSetWMProperties(dpy, w, "Teste", "Teste",
               (void *)0, 0, &size_hints, &wm_hints, &class_hints);
    XMapWindow(dpy, w);
    wm_nodecorations(w);
    Tcl_SetObjResult(interp, Tcl_NewIntObj(w));
    return TCL_OK;
 }

 int setXwinshape(ClientData client_data, Tcl_Interp *interp, int objc,
        Tcl_Obj * CONST objv[]) {
    Window w;
    Pixmap pixmap, mask;
    char *bmap;
    XEvent ev;
    int i;

    if (objc != 3) {
        Tcl_WrongNumArgs(interp,1,objv,"winId bitmap");
        return TCL_ERROR;
    }
    if (XShapeQueryExtension(dpy, &i, &i) == False) {
        return TCL_ERROR;
    }
    Tcl_GetLongFromObj(interp,objv[1],(long *)&w);
    bmap = Tcl_GetStringFromObj(objv[2], NULL);
    mask = Tk_GetBitmap(interp, tkwin, Tk_GetUid(bmap));
    if (mask == None) {
        return TCL_ERROR;
    }

    XShapeCombineMask(dpy, w, ShapeBounding, 0, 0, mask, ShapeSet);
    XSync(dpy, False);
    return TCL_OK;
 }

 int errorHandler(Display *dpylay, XErrorEvent *err) {
    /* ignore all errors */
 }

 int getXWinGeometry(ClientData client_data, Tcl_Interp *interp, int objc,
    Tcl_Obj * CONST objv[]) {
        /* returns the width and height of the specified window */
    Window w;
    XWindowAttributes wa;
    int rx,ry;
    Window junkwin;
    int win_width, win_height;
    int win_x, win_y;
    char loc[12];
    if (objc != 2) {
        Tcl_WrongNumArgs(interp,1,objv,"windowId");
        return TCL_ERROR;
    }
    Tcl_GetLongFromObj(interp,objv[1],(long *)&w);
    XGetWindowAttributes(dpy,w,&wa);
    XTranslateCoordinates (dpy, w, wa.root,
                -wa.border_width, -wa.border_width,
                &rx, &ry, &junkwin);

    sprintf(loc,"%d %d %d %d",wa.width,wa.height, rx, ry);
    Tcl_SetObjResult(interp, Tcl_NewStringObj(loc,-1));
    XSync(dpy,False);
    return TCL_OK;
 }

 int moveXWin(ClientData client_data, Tcl_Interp *interp, int objc,
        Tcl_Obj * CONST objv[]) {
    Window w;
    int win_x, win_y;
    if (objc != 4) {
        Tcl_WrongNumArgs(interp,1,objv,"windowId posX posY");
        return TCL_ERROR;
    }
    Tcl_GetLongFromObj(interp,objv[1],(long *)&w);
    Tcl_GetIntFromObj(interp,objv[2],(int *)&win_x);
    Tcl_GetIntFromObj(interp,objv[3],(int *)&win_y);
    XMoveWindow(dpy,w, win_x, win_y);
    XSync(dpy,False);
    return TCL_OK;

 }

 int focusXwin(ClientData client_data, Tcl_Interp *interp, int objc,
    Tcl_Obj * CONST objv[]) {
    Window w;
    char err[80];
    int i=0;
    XEvent ev;
    unsigned long data[5];
    long mask = SubstructureRedirectMask | SubstructureNotifyMask;

    if (objc != 2) {
        Tcl_WrongNumArgs(interp,1,objv,"windowId");
        return TCL_ERROR;
    }

    Tcl_GetLongFromObj(interp,objv[1],(long *)&w);
    ev.xclient.type = ClientMessage;
    ev.xclient.serial = 0;
    ev.xclient.send_event = True;
    ev.xclient.message_type = wm_activewin;
    ev.xclient.window = w;
    ev.xclient.format = 32;
    ev.xclient.data.l[0] = 2;
    ev.xclient.data.l[1] = CurrentTime;
    ev.xclient.data.l[2] = 0;
    ev.xclient.data.l[3] = 0;
    ev.xclient.data.l[4] = 0;
    XSendEvent(dpy, rootw, False, mask, &ev);
    //XSetInputFocus(dpy,w,None,CurrentTime);
    XSync(dpy, False);
    return TCL_OK;
 }

 int iconifyXwin(ClientData client_data, Tcl_Interp *interp, int objc,
    Tcl_Obj * CONST objv[]) {
    Window w;
    char err[80];
    int i=0;

    if (objc != 2) {
        Tcl_WrongNumArgs(interp,1,objv,"windowId");
        return TCL_ERROR;
    }    Tcl_GetLongFromObj(interp,objv[1],(long *)&w);
    XIconifyWindow(dpy,w,screen_number);
    XFlush(dpy);
    return TCL_OK;
 }

 int deiconifyXwin(ClientData client_data, Tcl_Interp *interp, int objc,
    Tcl_Obj * CONST objv[]) {
    Window w;
    char err[80];
    int i=0;

    if (objc != 2) {
        Tcl_WrongNumArgs(interp,1,objv,"windowId");
        return TCL_ERROR;
    }

    Tcl_GetLongFromObj(interp,objv[1],(long *)&w);
    XMapWindow(dpy,w);
    XFlush(dpy);
    return TCL_OK;
 }

 int initXwmhandler( ClientData client_data, Tcl_Interp *interp, int objc,
        Tcl_Obj * CONST objv[]) {
    Tk_Window tkwin;

    tkwin = Tk_MainWindow(interp);
    dpy = Tk_Display(tkwin);
    screen_number = Tk_ScreenNumber(tkwin);
    screen = ScreenOfDisplay(dpy,screen_number);
    rootw = RootWindow(dpy,screen_number);
    wm_activewin = XInternAtom(dpy,"_NET_ACTIVE_WINDOW",False);
    return TCL_OK;
 }

 int Wmx_Init(Tcl_Interp *interp) {

    XSetErrorHandler(errorHandler);

    Tcl_CreateObjCommand(interp,"shaped_container", shaped_container,
        (ClientData)NULL,(Tcl_CmdDeleteProc *)NULL);
    Tcl_CreateObjCommand(interp,"setXwinshape", setXwinshape,
        (ClientData)NULL,(Tcl_CmdDeleteProc *)NULL);
    Tcl_CreateObjCommand(interp,"focusXwin", focusXwin,
        (ClientData)NULL,(Tcl_CmdDeleteProc *)NULL);
    Tcl_CreateObjCommand(interp,"deiconifyXwin", deiconifyXwin,
        (ClientData)NULL,(Tcl_CmdDeleteProc *)NULL);
    Tcl_CreateObjCommand(interp,"iconifyXwin", iconifyXwin,
        (ClientData)NULL,(Tcl_CmdDeleteProc *)NULL);
    Tcl_CreateObjCommand(interp,"moveXwin", moveXWin,
        (ClientData)NULL,(Tcl_CmdDeleteProc *)NULL);
    Tcl_CreateObjCommand(interp,"getXwingeometry", getXWinGeometry,
        (ClientData)NULL,(Tcl_CmdDeleteProc *)NULL);
    Tcl_PkgProvide(interp, "wmx", "0.1");

    initXwmhandler(0,interp,0,NULL);

    return TCL_OK;
 }

To compile this, use the following makefile, after adjusting it to fit your system's paths:
 INCLUDES = -I/usr/X11R6/include -I/usr/local/include
 LIBS = -lX11 -lXext -ltcl8.4
 LIBPATH =  -L/usr/local/lib -L/usr/X11R6/lib -L/usr/lib
 SHLIB_CFLAGS = -fPIC
 SHLIB_LD = ld -G -z text
 CC = gcc
 CFLAGS=$(INCLUDES) -O -g

 wmx.so: wmx.c
    $(CC) -c $(CFLAGS) ${SHLIB_CFLAGS} wmx.c
    ${SHLIB_LD}  $(LIBPATH) $(LIBS) -o wmx.so wmx.o

A few other commands were added to give us more control on the container window, as tk don't have ways to move or focus it. Follows a simple example of its usage. To move the window, click with the left mouse button anywhere and drag the window. Notice that kde, gnome, etc, will show the window (named "teste" here) and allow you to set focus on it, or minimize/restore it. Also, there is no need to use grabs or any strange bindings to let it work as a regular toplevel. (Sometimes, you may need to focus elsewhere then back to the shaped window and everything works.) The size of the container window was set to 800x600, so this is the greatest pixmap that may be used for the shape, so if you need something larger, change the code above (XCreateSimpleWindow). The actual (visible) size will be defined by your pixmap mask, so don't worry. If you need something fancier, try the Shape extension.
 #!/bin/sh
 #\
 exec wish "$0" "[email protected]"

 wm withdraw .
 load wmx.so
 package require Img

 set fw [shaped_container]
 toplevel .t -use $fw
 set bgimg [image create photo -file forma.png]
 label .t.l -image $bgimg
 .t config -width [image width $bgimg] -height [image height $bgimg]

 button .t.b -text saida -command exit
 text .t.t -width 40 -height 5
 button .t.b1 -text teste -command {puts teste}
 place .t.l -x 0 -y 0 -anchor nw
 pack propagate .t 0
 pack .t.t -pady 50
 pack .t.b .t.b1 -pady 10
 bind .t <1> {
    set g [getXwingeometry $fw]
    set x [expr [lindex $g 2]-%X]
    set y [expr [lindex $g 3]-%Y]
 }
 bind .t <B1-Motion> {
    moveXwin $fw [expr %X+$x] [expr %Y+$y]
 }
 bind all <Control-c> exit
 setXwinshape $fw @forma_mask.xbm

To run the example above you may want to write your own shape in the Gimp, or use this: .

PWQ 25 Jan 05, under fvwm2.4 under linux. the setXwinshape function causes the window to disappear. Also button 1 also causes the window to disappear on MouseDown. Xfree 4.? has the SHAPE extension.

RP 25 Jan 2005, Let's see if the changes made above make it to work. Now it requires a bitmap mask, which can easily generated by opening the above file with the Gimp, and saving again as forma.xbm, and selecting to save a mask bitmap as well (named forma_mask.xbm). This is much faster than the previous version. Anyway, both versions work for me under Slackware 10, in fvwm2, kde and gnome. Maybe I'm missing something... The only complaint I have is the apparent difficult to make it focus first. I have to click outside it and then inside, to have the text widget receiving focus. Does someone knows why?

DKF: The SHAPE extension has been around a long time; even ten years ago nearly every Xserver supported it properly (the exceptions were based on Win16, and even then they could do a creditable job in some modes). These days, I'd be surprised if anything didn't support it at all. (And no, I don't know anything about the focus problems, but I'd suspect window manager bugs...)

RP -- Ok, I found what I was looking for, and solved the focus problem. As PWQ pointed to me, it is really a bug with -use/-container as implemented in current Tk (even in 8.5 this bug is still there, and unfortunatelly I don't know how to solve). Here is the new, much simpler implementation with the modified test program. There is no need to create another window, just apply the shape to the window manager's frame. (Please copy wm_nodecorations, initXwmhandler, and Wmx_Init from the listing above, and remove the remaining commands, as we don't need them anymore).
 int setXwinshape(ClientData client_data, Tcl_Interp *interp, int objc,
        Tcl_Obj * CONST objv[]) {
    Window w;
    char *wpath;
    Pixmap pixmap, mask;
    char *bmap;
    XEvent ev;
    Tk_Window tkwin = Tk_MainWindow(interp);
    Window *children;
    int nchildren;
    Window dummy,parent;

    if (objc != 3) {
        Tcl_WrongNumArgs(interp,1,objv,"window xpm_data");
        return TCL_ERROR;
    }

    Tcl_GetLongFromObj(interp,objv[1], &w);
    wpath = Tcl_GetStringFromObj(objv[1], NULL);
    XQueryTree(dpy,Tk_WindowId(Tk_NameToWindow(interp,wpath,tkwin)),
        &dummy,&w,&children,&nchildren);
    bmap = Tcl_GetStringFromObj(objv[2], NULL);
    mask = Tk_GetBitmap(interp, tkwin, Tk_GetUid(bmap));
    if (mask == None) {
        return TCL_ERROR;
    }
    wm_nodecorations(w);

    XShapeCombineMask(dpy, w, ShapeBounding, 0, 0, mask, ShapeSet);

    XSetTransientForHint(dpy, w, rootw);
    XUnmapWindow(dpy, w);
    XMapWindow(dpy, w);
    XSync(dpy, False);
    Tk_FreeBitmap(dpy, mask);
    return TCL_OK;
 }

and now the modified script:
 package require Tk
 wm withdraw .
 package require wmx
 package require Img

 toplevel .t
 #wm geometry .t +5000+5000  ;# trying to reduce the annoying flicker at the creation time...
 wm state .t iconic
 set fw [winfo id .t]
 set bgimg [image create photo -file [file join $starkit::topdir forma.png]]
 label .t.l -image $bgimg -takefocus 1
 .t config -width [image width $bgimg] -height [image height $bgimg]

 button .t.b -text saida -command exit
 text .t.t -width 40 -height 5
 button .t.b1 -text teste -command {puts teste}
 place .t.l -x 0 -y 0 -anchor nw
 pack propagate .t 0
 pack .t.t -pady 50
 pack .t.b .t.b1 -pady 10

 bind .t <1> {
    set x [expr [winfo rootx .t]-%X]
    set y [expr [winfo rooty .t]-%Y]
 }

 bind .t <B1-Motion> {
     wm geometry .t +[expr %X+$x]+[expr %Y+$y]
 }

 bind all <Control-c> exit
 # this "update idletasks" is required, otherwise there is no repareting taking place!
 update idle
 setXwinshape .t @[file join $starkit::topdir forma_mask.xbm]
 wm deiconify .t

and voilà, a nice shaped and managed toplevel to make nice looking applications. Thanks Philip Quaife for his comments (e-mails).

La perfection n'est pas lorsqu'il n'y a plus rien à ajouter, mais lorsque qu'il n'y a plus rien à enlever. -- Perfection is not when there is nothing to add, but when there is nothing to remove. A. de Saint Exupery

SeS (26-10-2010): See also Custom Toplevel Frame which uses Ffidl to access DLL's and create customized toplevel frames in the Windows OS