--- open-vm-tools-10.1.10-6082533/services/plugins/resolutionKMS/resolutionKMS.c	2017-09-27 18:16:07.428992005 -0700

+++ open-vm-tools-10.1.10-modified/services/plugins/resolutionKMS/resolutionKMS.c	2017-09-27 16:38:40.000000000 -0700

@@ -1,5 +1,5 @@

 /*********************************************************

- * Copyright (C) 2008-2016 VMware, Inc. All rights reserved.

+ * Copyright (C) 2008-2017 VMware, Inc. All rights reserved.

  *

  * This program is free software; you can redistribute it and/or modify it

  * under the terms of the GNU Lesser General Public License as published

@@ -279,6 +279,34 @@

       g_warning("%s: Unable to set tools.capability.resolution_server\n",

                 __FUNCTION__);

    }

+

+   if (value == 1) {

+      /*

+       * Whenever resolutionKMS is enabled, send

+       * "tools.capability.resolution_server toolbox-dnd 0" to clear

+       * resolutionSet as resolution server.

+       *

+       * Note: The below rpc is sent to TOOLS_DND_NAME if rpcChannelName is

+       * TOOLS_DAEMON_NAME and vice versa (to clear the opposite channel).

+       * This is how rpcChannelName is selected in ToolsOnLoad():

+       *

+       *    if (strcmp(ctx->name, VMTOOLS_GUEST_SERVICE) == 0) {

+       *       rpcChannelName = TOOLS_DAEMON_NAME;

+       *    } else if (strcmp(ctx->name, VMTOOLS_USER_SERVICE) == 0) {

+       *       rpcChannelName = TOOLS_DND_NAME;

+       *    }

+       */

+      gchar *msgClear;

+      msgClear = g_strdup_printf("tools.capability.resolution_server %s 0",

+                                 (strcmp(rpcChannelName, TOOLS_DAEMON_NAME) == 0 ?

+                                 TOOLS_DND_NAME : TOOLS_DAEMON_NAME));

+      if (!RpcChannel_Send(chan, msgClear, strlen(msgClear), NULL, NULL)) {

+         g_warning("%s: Unable to clear tools.capability.resolution_server\n",

+                   __FUNCTION__);

+      }

+      g_free(msgClear);

+   }

+

    g_free(msg);

 }

@@ -448,7 +476,7 @@

    /*

     * Save the RPC channel name from the ToolsAppCtx so that we can use it later

-    * in calls to ResolutionSetServerCapability().

+    * in calls to ResolutionKMSServerCapability().

     */

    if (strcmp(ctx->name, VMTOOLS_GUEST_SERVICE) == 0) {

--- open-vm-tools-10.1.10-6082533/services/plugins/resolutionSet/resolutionCommon.c	2017-09-27 18:16:07.429992005 -0700

+++ open-vm-tools-10.1.10-modified/services/plugins/resolutionSet/resolutionCommon.c	2017-09-27 16:38:40.000000000 -0700

@@ -1,5 +1,5 @@

 /*********************************************************

- * Copyright (C) 2016 VMware, Inc. All rights reserved.

+ * Copyright (C) 2016-2017 VMware, Inc. All rights reserved.

  *

  * This program is free software; you can redistribute it and/or modify it

  * under the terms of the GNU Lesser General Public License as published

@@ -38,6 +38,7 @@

 /* The DRM device we are looking for */

 #define RESOLUTION_VENDOR     "0x15ad"

 #define RESOLUTION_DEVICE     "0x0405"

+#define RESOLUTION_KERNELNAME "vmwgfx"

 /* Required DRM version for resolutionKMS */

 #define RESOLUTION_DRM_MAJOR  2

@@ -84,11 +85,19 @@

     struct udev_list_entry *devices, *devListEntry;

     struct udev_device *dev;

     int fd = -1;

+    int drmFd;

     const char *devNode = NULL;

+    /* Force load the kernel module. */

+    drmFd = drmOpen(RESOLUTION_KERNELNAME, NULL);

+    if (drmFd >= 0) {

+       (void) drmDropMaster(drmFd);

+    }

+

     udev = udev_new();

-    if (!udev)

-	return -1;

+    if (!udev) {

+        goto outNoUdev;

+    }

     /*

      * Udev error return codes that are not caught immediately are

@@ -148,6 +157,10 @@

     udev_enumerate_unref(enumerate);

     udev_unref(udev);

+    if (drmFd >= 0) {

+       drmClose(drmFd);

+    }

+

     return fd;

   outFound:

@@ -155,6 +168,10 @@

   outErr:

     udev_enumerate_unref(enumerate);

     udev_unref(udev);

+  outNoUdev:

+    if (drmFd >= 0) {

+       drmClose(drmFd);

+    }

     return -1;

 }

@@ -190,7 +207,7 @@

     }

     if (ver->version_major != RESOLUTION_DRM_MAJOR ||

-	ver->version_minor < RESOLUTION_DRM_MINOR) {

+        ver->version_minor < RESOLUTION_DRM_MINOR) {

        g_debug("%s: Insufficient DRM version %d.%d for resolutionKMS.\n",

                __func__, ver->version_major, ver->version_minor);

        drmFreeVersion(ver);

--- open-vm-tools-10.1.10-6082533/services/plugins/resolutionSet/resolutionDL.c	2017-09-27 18:16:07.429992005 -0700

+++ open-vm-tools-10.1.10-modified/services/plugins/resolutionSet/resolutionDL.c	2017-09-27 16:38:40.000000000 -0700

@@ -66,6 +66,8 @@

 };

 static struct FuncToResolv drm2Table[] = {

+    LIBDRM_RESOLV(Open),

+    LIBDRM_RESOLV(Close),

     LIBDRM_RESOLV(GetVersion),

     LIBDRM_RESOLV(FreeVersion),

     LIBDRM_RESOLV(DropMaster),

--- open-vm-tools-10.1.10-6082533/services/plugins/resolutionSet/resolutionDL.h	2017-09-27 18:16:07.429992005 -0700

+++ open-vm-tools-10.1.10-modified/services/plugins/resolutionSet/resolutionDL.h	2017-09-27 16:38:40.000000000 -0700

@@ -144,6 +144,8 @@

  * However this struct is not subject to the license header of this file.

  */

 struct Drm2Interface {

+    int (*Open)(const char *, const char *);

+    int (*Close)(int);

     drmVersionPtr (*GetVersion)(int fd);

     void (*FreeVersion)(drmVersionPtr);

     int (*DropMaster)(int fd);

@@ -190,6 +192,10 @@

 #define udev_list_entry_foreach(_a, _b)\

     udevi_list_entry_foreach(udevi, _a, _b)

+#define drmOpen \

+    drmi->Open

+#define drmClose \

+    drmi->Close

 #define drmGetVersion \

     drmi->GetVersion

 #define drmFreeVersion \

--- open-vm-tools-10.1.10-6082533/services/plugins/resolutionSet/resolutionSet.c	2017-09-27 18:16:07.429992005 -0700

+++ open-vm-tools-10.1.10-modified/services/plugins/resolutionSet/resolutionSet.c	2017-09-27 16:38:40.000000000 -0700

@@ -76,8 +76,7 @@

  *

  * Initialize the guest resolution library.

  *

- * @param[in] handle  Back-end specific handle, if needed.  E.g., in the X11

-                      case, this refers to the X11 display handle.

+ * @param[in] handle  Back-end specific handle, if needed.

  * @return TRUE on success, FALSE on failure

  */

--- open-vm-tools-10.1.10-6082533/services/plugins/resolutionSet/resolutionX11.c	2017-09-27 18:16:07.429992005 -0700

+++ open-vm-tools-10.1.10-modified/services/plugins/resolutionSet/resolutionX11.c	2017-09-27 16:38:40.000000000 -0700

@@ -35,8 +35,6 @@

 #ifndef NO_MULTIMON

 #include <X11/extensions/Xinerama.h>

 #endif

-#include <gtk/gtk.h>

-#include <gdk/gdkx.h>

 #include "vmware.h"

 #include "debug.h"

@@ -62,6 +60,8 @@

    Bool         canUseVMwareCtrlTopologySet;

                                 // TRUE if VMwareCtrl extension supports topology set

    Bool         canUseRandR12;  // TRUE if RandR extension >= 1.2 available

+

+   Bool         canUseResolutionKMS;    // TRUE if backing off for resolutionKMS

 } ResolutionInfoX11Type;

@@ -78,6 +78,7 @@

 static Bool ResolutionCanSet(void);

 static Bool TopologyCanSet(void);

 static Bool SelectResolution(uint32 width, uint32 height);

+static int ResolutionX11ErrorHandler(Display *d, XErrorEvent *e);

 /*

@@ -89,29 +90,45 @@

  * X11 back-end initializer.  Records caller's X11 display, then determines

  * which capabilities are available.

  *

- * @param[in] handle User's X11 display

+ * @param[in] handle (ResolutionInfoX11Type is used as backend specific handle)

  * @return TRUE on success, FALSE on failure.

  */

 Bool

 ResolutionBackendInit(InitHandle handle)

 {

-   ResolutionInfoX11Type *resInfoX = &resolutionInfoX11;

+   ResolutionInfoX11Type *resInfoX = (ResolutionInfoX11Type *)handle;

    ResolutionInfoType *resInfo = &resolutionInfo;

    int dummy1;

    int dummy2;

-   memset(resInfoX, 0, sizeof *resInfoX);

+   if (resInfoX->canUseResolutionKMS == TRUE) {

+      resInfo->canSetResolution = FALSE;

+      resInfo->canSetTopology = FALSE;

+      return FALSE;

+   }

-   resInfoX->display = handle;

+   XSetErrorHandler(ResolutionX11ErrorHandler);

+   resInfoX->display = XOpenDisplay(NULL);

+   /*

+    * In case display is NULL, we do not load resolutionSet

+    * as it serve no purpose. Also avoids SEGFAULT issue

+    * like BZ1880932.

+    *

+    * VMX currently remembers the settings across a reboot,

+    * so let's say someone replaces our Xorg driver with

+    * xf86-video-modesetting, and then rebooted, we'd end up here,

+    * but the VMX would still send resolution / topology events

+    * and we'd hit the same segfault.

+    */

    if (resInfoX->display == NULL) {

+      g_error("%s: Invalid display detected.\n", __func__);

       resInfo->canSetResolution = FALSE;

       resInfo->canSetTopology = FALSE;

-      return TRUE;

+      return FALSE;

    }

-   resInfoX->display = handle;

    resInfoX->rootWindow = DefaultRootWindow(resInfoX->display);

    resInfoX->canUseVMwareCtrl = VMwareCtrl_QueryVersion(resInfoX->display, &dummy1,

                                                         &dummy2);

@@ -132,6 +149,10 @@

 void

 ResolutionBackendCleanup(void)

 {

+   ResolutionInfoX11Type *resInfoX = &resolutionInfoX11;

+   if (resInfoX->display) {

+      XCloseDisplay(resInfoX->display);

+   }

    return;

 }

@@ -524,7 +545,7 @@

       g_debug("Setting guest resolution to: %dx%d (requested: %d, %d)\n",

               xrrSizes[bestFitIndex].width, xrrSizes[bestFitIndex].height, width, height);

       rc = XRRSetScreenConfig(resInfoX->display, xrrConfig, resInfoX->rootWindow,

-                              bestFitIndex, xrrCurRotation, GDK_CURRENT_TIME);

+                              bestFitIndex, xrrCurRotation, CurrentTime);

       g_debug("XRRSetScreenConfig returned %d (result: %dx%d)\n", rc,

               xrrSizes[bestFitIndex].width, xrrSizes[bestFitIndex].height);

    } else {

@@ -574,42 +595,28 @@

 /**

- * Obtain a "handle", which for X11, is a display pointer. 

+ * Obtain a "handle".

  *

  * @note We will have to move this out of the resolution plugin soon, I am

- * just landing this here now for convenience as I port resolution set over 

+ * just landing this here now for convenience as I port resolution set over

  * to the new service architecture.

  *

- * @return X server display 

+ * @return ResolutionInfoX11Type as backend specific handle

  */

 InitHandle

 ResolutionToolkitInit(ToolsAppCtx *ctx) // IN: For config database access

 {

-   int argc = 1;

-   char *argv[] = {"", NULL};

-   GtkWidget *wnd;

-   Display *display;

+   ResolutionInfoX11Type *resInfoX = &resolutionInfoX11;

    int fd;

+   memset(resInfoX, 0, sizeof *resInfoX);

+

    fd = resolutionCheckForKMS(ctx);

    if (fd >= 0) {

       resolutionDRMClose(fd);

       g_message("%s: Backing off for resolutionKMS.\n", __func__);

-      return (InitHandle) 0;

+      resInfoX->canUseResolutionKMS = TRUE;

    }

-

-   XSetErrorHandler(ResolutionX11ErrorHandler);

-   gtk_init(&argc, (char ***) &argv);

-   wnd = gtk_invisible_new();

-#ifndef GTK3

-   display = GDK_WINDOW_XDISPLAY(wnd->window);

-#else

-   display = GDK_WINDOW_XDISPLAY(gtk_widget_get_window(wnd));

-#endif

-

-   if (!display)

-      g_error("%s: Invalid display detected.\n", __func__);

-

-   return (InitHandle) display;

+   return (InitHandle) resInfoX;

 }

--- open-vm-tools-10.1.10-6082533/services/plugins/resolutionSet/resolutionRandR12.c	2017-07-28 15:19:20.000000000 -0700

+++ open-vm-tools-10.1.10-modified/services/plugins/resolutionSet/resolutionRandR12.c	2017-09-27 16:38:40.000000000 -0700

@@ -139,7 +139,7 @@

 #define LOG_STOP fclose(_ofile)

 #else

 #define LOG_START

-#include <gtk/gtk.h>

+#include <glib.h>

 #define LOG_STOP

 #endif

@@ -1000,7 +1000,7 @@

    info = RandR12GetInfo(dpy, rootWin);

    if (!info) {

       g_warning("%s: Setup info struct failed.\n", __func__);

-      goto out_ungrab;

+      return FALSE;

    }

    RandR12GetDpi(dpy, screen, info);