Fix dynamic mode edge cases

The VMware Xorg driver supports dynamic modelines that can be set from
userspace via an X extension. These are used to implement VM features
which need to automatically change the resolution of the guest OS.

This driver implements the feature using two modelines.  The driver
would alternately update one mode then the other, so that in typical
usage one mode is current and the other is available for the next mode
switch.

This usually worked, but there were many edge cases that could cause
this alternating pattern to get 'out of sync', so we'd end up changing
the resolution of the current video mode. This could end up putting
the X server in a state where the screen resolution has been changed,
but the hardware was never reprogrammed for the new resolution.

This patch fixes the problem by explicitly searching for a dynamic
mode that isn't currently in use. We no longer rely on the alternating
pattern.

3 files changed, 29 insertions, 17 deletions

diff --git a/src/vmware.c b/src/vmware.c
index 60a3004..5d29f72 100644
--- a/src/vmware.c
+++ b/src/vmware.c
@@ -1784,8 +1784,7 @@ VMWAREScreenInit(int scrnIndex, ScreenPtr pScreen, int argc, char **argv)
      * We will lazily add the dynamic modes as the are needed when new
      * modes are requested through the control extension.
      */
-    pVMWARE->dynMode1 = NULL;
-    pVMWARE->dynMode2 = NULL;
+    memset(&pVMWARE->dynModes, 0, sizeof pVMWARE->dynModes);
 
 #if VMWARE_DRIVER_FUNC
     pScrn->DriverFunc = VMWareDriverFunc;
diff --git a/src/vmware.h b/src/vmware.h
index 2b3c119..80393e6 100644
--- a/src/vmware.h
+++ b/src/vmware.h
@@ -46,6 +46,9 @@
 /* Arbitrarily choose max cursor dimensions.  The emulation doesn't care. */
 #define MAX_CURS        32
 
+#define NUM_DYN_MODES   2
+
+
 typedef struct {
     CARD32 svga_reg_enable;
     CARD32 svga_reg_width;
@@ -94,8 +97,7 @@ typedef struct {
     VMWARERegRec SavedReg;
     VMWARERegRec ModeReg;
 
-    DisplayModePtr dynMode1;
-    DisplayModePtr dynMode2;
+    DisplayModePtr dynModes[NUM_DYN_MODES];
 
     Bool* pvtSema;
 
diff --git a/src/vmwarectrl.c b/src/vmwarectrl.c
index 24865d6..634b9ca 100644
--- a/src/vmwarectrl.c
+++ b/src/vmwarectrl.c
@@ -116,19 +116,20 @@ VMwareCtrlDoSetRes(ScrnInfoPtr pScrn,
                    CARD32 y,
                    Bool resetXinerama)
 {
+   int modeIndex;
    DisplayModePtr mode;
    VMWAREPtr pVMWARE = VMWAREPTR(pScrn);
 
    if (pScrn && pScrn->modes) {
       VmwareLog(("DoSetRes: %d %d\n", x, y));
-  
+
       if (resetXinerama) {
          xfree(pVMWARE->xineramaNextState);
          pVMWARE->xineramaNextState = NULL;
          pVMWARE->xineramaNextNumOutputs = 0;
       }
 
-      /* 
+      /*
        * Don't resize larger than possible but don't
        * return an X Error either.
        */
@@ -138,20 +139,30 @@ VMwareCtrlDoSetRes(ScrnInfoPtr pScrn,
       }
 
       /*
-       * Switch the dynamic modes so that we alternate
-       * which one gets updated on each call.
+       * Find an dynamic mode which isn't current, and replace it with
+       * the requested mode. Normally this will cause us to alternate
+       * between two dynamic mode slots, but there are some important
+       * corner cases to consider. For example, adding the same mode
+       * multiple times, adding a mode that we never switch to, or
+       * adding a mode which is a duplicate of a built-in mode. The
+       * best way to handle all of these cases is to directly test the
+       * dynamic mode against the current mode pointer for this
+       * screen.
        */
-      mode = pVMWARE->dynMode1;
-      pVMWARE->dynMode1 = pVMWARE->dynMode2;
-      pVMWARE->dynMode2 = mode;
 
-      /*
-       * Initialise the dynamic mode if it hasn't been used before.
-       */
-      if (!pVMWARE->dynMode1) {
-         pVMWARE->dynMode1 = VMWAREAddDisplayMode(pScrn, "DynMode", 1, 1);
+      for (modeIndex = 0; modeIndex < NUM_DYN_MODES; modeIndex++) {
+         /*
+          * Initialise the dynamic mode if it hasn't been used before.
+          */
+         if (!pVMWARE->dynModes[modeIndex]) {
+            pVMWARE->dynModes[modeIndex] = VMWAREAddDisplayMode(pScrn, "DynMode", 1, 1);
+         }
+
+         mode = pVMWARE->dynModes[modeIndex];
+         if (mode != pScrn->currentMode) {
+            break;
+         }
       }
-      mode = pVMWARE->dynMode1;
 
       mode->HDisplay = x;
       mode->VDisplay = y;