--- open-vm-tools-10.1.10-6082533/services/plugins/resolutionKMS/resolutionKMS.c 2017-09-27 18:16:07.428992005 -0700 +++ open-vm-tools-10.1.10-modified/services/plugins/resolutionKMS/resolutionKMS.c 2017-09-27 16:38:40.000000000 -0700 @@ -1,5 +1,5 @@ /********************************************************* - * Copyright (C) 2008-2016 VMware, Inc. All rights reserved. + * Copyright (C) 2008-2017 VMware, Inc. All rights reserved. * * This program is free software; you can redistribute it and/or modify it * under the terms of the GNU Lesser General Public License as published @@ -279,6 +279,34 @@ g_warning("%s: Unable to set tools.capability.resolution_server\n", __FUNCTION__); } + + if (value == 1) { + /* + * Whenever resolutionKMS is enabled, send + * "tools.capability.resolution_server toolbox-dnd 0" to clear + * resolutionSet as resolution server. + * + * Note: The below rpc is sent to TOOLS_DND_NAME if rpcChannelName is + * TOOLS_DAEMON_NAME and vice versa (to clear the opposite channel). + * This is how rpcChannelName is selected in ToolsOnLoad(): + * + * if (strcmp(ctx->name, VMTOOLS_GUEST_SERVICE) == 0) { + * rpcChannelName = TOOLS_DAEMON_NAME; + * } else if (strcmp(ctx->name, VMTOOLS_USER_SERVICE) == 0) { + * rpcChannelName = TOOLS_DND_NAME; + * } + */ + gchar *msgClear; + msgClear = g_strdup_printf("tools.capability.resolution_server %s 0", + (strcmp(rpcChannelName, TOOLS_DAEMON_NAME) == 0 ? + TOOLS_DND_NAME : TOOLS_DAEMON_NAME)); + if (!RpcChannel_Send(chan, msgClear, strlen(msgClear), NULL, NULL)) { + g_warning("%s: Unable to clear tools.capability.resolution_server\n", + __FUNCTION__); + } + g_free(msgClear); + } + g_free(msg); } @@ -448,7 +476,7 @@ /* * Save the RPC channel name from the ToolsAppCtx so that we can use it later - * in calls to ResolutionSetServerCapability(). + * in calls to ResolutionKMSServerCapability(). */ if (strcmp(ctx->name, VMTOOLS_GUEST_SERVICE) == 0) { --- open-vm-tools-10.1.10-6082533/services/plugins/resolutionSet/resolutionCommon.c 2017-09-27 18:16:07.429992005 -0700 +++ open-vm-tools-10.1.10-modified/services/plugins/resolutionSet/resolutionCommon.c 2017-09-27 16:38:40.000000000 -0700 @@ -1,5 +1,5 @@ /********************************************************* - * Copyright (C) 2016 VMware, Inc. All rights reserved. + * Copyright (C) 2016-2017 VMware, Inc. All rights reserved. * * This program is free software; you can redistribute it and/or modify it * under the terms of the GNU Lesser General Public License as published @@ -38,6 +38,7 @@ /* The DRM device we are looking for */ #define RESOLUTION_VENDOR "0x15ad" #define RESOLUTION_DEVICE "0x0405" +#define RESOLUTION_KERNELNAME "vmwgfx" /* Required DRM version for resolutionKMS */ #define RESOLUTION_DRM_MAJOR 2 @@ -84,11 +85,19 @@ struct udev_list_entry *devices, *devListEntry; struct udev_device *dev; int fd = -1; + int drmFd; const char *devNode = NULL; + /* Force load the kernel module. */ + drmFd = drmOpen(RESOLUTION_KERNELNAME, NULL); + if (drmFd >= 0) { + (void) drmDropMaster(drmFd); + } + udev = udev_new(); - if (!udev) - return -1; + if (!udev) { + goto outNoUdev; + } /* * Udev error return codes that are not caught immediately are @@ -148,6 +157,10 @@ udev_enumerate_unref(enumerate); udev_unref(udev); + if (drmFd >= 0) { + drmClose(drmFd); + } + return fd; outFound: @@ -155,6 +168,10 @@ outErr: udev_enumerate_unref(enumerate); udev_unref(udev); + outNoUdev: + if (drmFd >= 0) { + drmClose(drmFd); + } return -1; } @@ -190,7 +207,7 @@ } if (ver->version_major != RESOLUTION_DRM_MAJOR || - ver->version_minor < RESOLUTION_DRM_MINOR) { + ver->version_minor < RESOLUTION_DRM_MINOR) { g_debug("%s: Insufficient DRM version %d.%d for resolutionKMS.\n", __func__, ver->version_major, ver->version_minor); drmFreeVersion(ver); --- open-vm-tools-10.1.10-6082533/services/plugins/resolutionSet/resolutionDL.c 2017-09-27 18:16:07.429992005 -0700 +++ open-vm-tools-10.1.10-modified/services/plugins/resolutionSet/resolutionDL.c 2017-09-27 16:38:40.000000000 -0700 @@ -66,6 +66,8 @@ }; static struct FuncToResolv drm2Table[] = { + LIBDRM_RESOLV(Open), + LIBDRM_RESOLV(Close), LIBDRM_RESOLV(GetVersion), LIBDRM_RESOLV(FreeVersion), LIBDRM_RESOLV(DropMaster), --- open-vm-tools-10.1.10-6082533/services/plugins/resolutionSet/resolutionDL.h 2017-09-27 18:16:07.429992005 -0700 +++ open-vm-tools-10.1.10-modified/services/plugins/resolutionSet/resolutionDL.h 2017-09-27 16:38:40.000000000 -0700 @@ -144,6 +144,8 @@ * However this struct is not subject to the license header of this file. */ struct Drm2Interface { + int (*Open)(const char *, const char *); + int (*Close)(int); drmVersionPtr (*GetVersion)(int fd); void (*FreeVersion)(drmVersionPtr); int (*DropMaster)(int fd); @@ -190,6 +192,10 @@ #define udev_list_entry_foreach(_a, _b)\ udevi_list_entry_foreach(udevi, _a, _b) +#define drmOpen \ + drmi->Open +#define drmClose \ + drmi->Close #define drmGetVersion \ drmi->GetVersion #define drmFreeVersion \ --- open-vm-tools-10.1.10-6082533/services/plugins/resolutionSet/resolutionSet.c 2017-09-27 18:16:07.429992005 -0700 +++ open-vm-tools-10.1.10-modified/services/plugins/resolutionSet/resolutionSet.c 2017-09-27 16:38:40.000000000 -0700 @@ -76,8 +76,7 @@ * * Initialize the guest resolution library. * - * @param[in] handle Back-end specific handle, if needed. E.g., in the X11 - case, this refers to the X11 display handle. + * @param[in] handle Back-end specific handle, if needed. * @return TRUE on success, FALSE on failure */ --- open-vm-tools-10.1.10-6082533/services/plugins/resolutionSet/resolutionX11.c 2017-09-27 18:16:07.429992005 -0700 +++ open-vm-tools-10.1.10-modified/services/plugins/resolutionSet/resolutionX11.c 2017-09-27 16:38:40.000000000 -0700 @@ -35,8 +35,6 @@ #ifndef NO_MULTIMON #include #endif -#include -#include #include "vmware.h" #include "debug.h" @@ -62,6 +60,8 @@ Bool canUseVMwareCtrlTopologySet; // TRUE if VMwareCtrl extension supports topology set Bool canUseRandR12; // TRUE if RandR extension >= 1.2 available + + Bool canUseResolutionKMS; // TRUE if backing off for resolutionKMS } ResolutionInfoX11Type; @@ -78,6 +78,7 @@ static Bool ResolutionCanSet(void); static Bool TopologyCanSet(void); static Bool SelectResolution(uint32 width, uint32 height); +static int ResolutionX11ErrorHandler(Display *d, XErrorEvent *e); /* @@ -89,29 +90,45 @@ * X11 back-end initializer. Records caller's X11 display, then determines * which capabilities are available. * - * @param[in] handle User's X11 display + * @param[in] handle (ResolutionInfoX11Type is used as backend specific handle) * @return TRUE on success, FALSE on failure. */ Bool ResolutionBackendInit(InitHandle handle) { - ResolutionInfoX11Type *resInfoX = &resolutionInfoX11; + ResolutionInfoX11Type *resInfoX = (ResolutionInfoX11Type *)handle; ResolutionInfoType *resInfo = &resolutionInfo; int dummy1; int dummy2; - memset(resInfoX, 0, sizeof *resInfoX); + if (resInfoX->canUseResolutionKMS == TRUE) { + resInfo->canSetResolution = FALSE; + resInfo->canSetTopology = FALSE; + return FALSE; + } - resInfoX->display = handle; + XSetErrorHandler(ResolutionX11ErrorHandler); + resInfoX->display = XOpenDisplay(NULL); + /* + * In case display is NULL, we do not load resolutionSet + * as it serve no purpose. Also avoids SEGFAULT issue + * like BZ1880932. + * + * VMX currently remembers the settings across a reboot, + * so let's say someone replaces our Xorg driver with + * xf86-video-modesetting, and then rebooted, we'd end up here, + * but the VMX would still send resolution / topology events + * and we'd hit the same segfault. + */ if (resInfoX->display == NULL) { + g_error("%s: Invalid display detected.\n", __func__); resInfo->canSetResolution = FALSE; resInfo->canSetTopology = FALSE; - return TRUE; + return FALSE; } - resInfoX->display = handle; resInfoX->rootWindow = DefaultRootWindow(resInfoX->display); resInfoX->canUseVMwareCtrl = VMwareCtrl_QueryVersion(resInfoX->display, &dummy1, &dummy2); @@ -132,6 +149,10 @@ void ResolutionBackendCleanup(void) { + ResolutionInfoX11Type *resInfoX = &resolutionInfoX11; + if (resInfoX->display) { + XCloseDisplay(resInfoX->display); + } return; } @@ -524,7 +545,7 @@ g_debug("Setting guest resolution to: %dx%d (requested: %d, %d)\n", xrrSizes[bestFitIndex].width, xrrSizes[bestFitIndex].height, width, height); rc = XRRSetScreenConfig(resInfoX->display, xrrConfig, resInfoX->rootWindow, - bestFitIndex, xrrCurRotation, GDK_CURRENT_TIME); + bestFitIndex, xrrCurRotation, CurrentTime); g_debug("XRRSetScreenConfig returned %d (result: %dx%d)\n", rc, xrrSizes[bestFitIndex].width, xrrSizes[bestFitIndex].height); } else { @@ -574,42 +595,28 @@ /** - * Obtain a "handle", which for X11, is a display pointer. + * Obtain a "handle". * * @note We will have to move this out of the resolution plugin soon, I am - * just landing this here now for convenience as I port resolution set over + * just landing this here now for convenience as I port resolution set over * to the new service architecture. * - * @return X server display + * @return ResolutionInfoX11Type as backend specific handle */ InitHandle ResolutionToolkitInit(ToolsAppCtx *ctx) // IN: For config database access { - int argc = 1; - char *argv[] = {"", NULL}; - GtkWidget *wnd; - Display *display; + ResolutionInfoX11Type *resInfoX = &resolutionInfoX11; int fd; + memset(resInfoX, 0, sizeof *resInfoX); + fd = resolutionCheckForKMS(ctx); if (fd >= 0) { resolutionDRMClose(fd); g_message("%s: Backing off for resolutionKMS.\n", __func__); - return (InitHandle) 0; + resInfoX->canUseResolutionKMS = TRUE; } - - XSetErrorHandler(ResolutionX11ErrorHandler); - gtk_init(&argc, (char ***) &argv); - wnd = gtk_invisible_new(); -#ifndef GTK3 - display = GDK_WINDOW_XDISPLAY(wnd->window); -#else - display = GDK_WINDOW_XDISPLAY(gtk_widget_get_window(wnd)); -#endif - - if (!display) - g_error("%s: Invalid display detected.\n", __func__); - - return (InitHandle) display; + return (InitHandle) resInfoX; } --- open-vm-tools-10.1.10-6082533/services/plugins/resolutionSet/resolutionRandR12.c 2017-07-28 15:19:20.000000000 -0700 +++ open-vm-tools-10.1.10-modified/services/plugins/resolutionSet/resolutionRandR12.c 2017-09-27 16:38:40.000000000 -0700 @@ -139,7 +139,7 @@ #define LOG_STOP fclose(_ofile) #else #define LOG_START -#include +#include #define LOG_STOP #endif @@ -1000,7 +1000,7 @@ info = RandR12GetInfo(dpy, rootWin); if (!info) { g_warning("%s: Setup info struct failed.\n", __func__); - goto out_ungrab; + return FALSE; } RandR12GetDpi(dpy, screen, info);