diff --git a/Xi/exevents.c b/Xi/exevents.c
index 58fe49363..22bb5639a 100644
--- a/Xi/exevents.c
+++ b/Xi/exevents.c
@@ -1187,7 +1187,6 @@ TouchRejected(DeviceIntPtr sourcedev, TouchPointInfoPtr ti, XID resource,
               TouchOwnershipEvent *ev)
 {
     Bool was_owner = (resource == ti->listeners[0].listener);
-    void *grab;
     int i;
 
     /* Send a TouchEnd event to the resource being removed, but only if they
@@ -1202,11 +1201,7 @@ TouchRejected(DeviceIntPtr sourcedev, TouchPointInfoPtr ti, XID resource,
 
     /* Remove the resource from the listener list, updating
      * ti->num_listeners, as well as ti->num_grabs if it was a grab. */
-    if (TouchRemoveListener(ti, resource)) {
-        if (dixLookupResourceByType(&grab, resource, RT_PASSIVEGRAB,
-                                    serverClient, DixGetAttrAccess) == Success)
-            ti->num_grabs--;
-    }
+    TouchRemoveListener(ti, resource);
 
     /* If the current owner was removed and there are further listeners, deliver
      * the TouchOwnership or TouchBegin event to the new owner. */
@@ -1300,21 +1295,10 @@ RetrieveTouchDeliveryData(DeviceIntPtr dev, TouchPointInfoPtr ti,
 
     if (listener->type == LISTENER_GRAB ||
         listener->type == LISTENER_POINTER_GRAB) {
-        rc = dixLookupResourceByType((pointer *) grab, listener->listener,
-                                     RT_PASSIVEGRAB,
-                                     serverClient, DixSendAccess);
-        if (rc != Success) {
-            /* the grab doesn't exist but we have a grabbing listener - this
-             * is an implicit/active grab */
-            rc = dixLookupClient(client, listener->listener, serverClient,
-                                 DixSendAccess);
-            if (rc != Success)
-                return FALSE;
 
-            *grab = dev->deviceGrab.grab;
-            if (!*grab)
-                return FALSE;
-        }
+        *grab = listener->grab;
+
+        BUG_RETURN_VAL(!*grab, FALSE);
 
         *client = rClient(*grab);
         *win = (*grab)->window;
@@ -1467,6 +1451,7 @@ DeliverTouchEmulatedEvent(DeviceIntPtr dev, TouchPointInfoPtr ti,
              */
             l = &ti->listeners[ti->num_listeners - 1];
             l->listener = devgrab->resource;
+            l->grab = devgrab;
 
             if (devgrab->grabtype != XI2 || devgrab->type != XI_TouchBegin)
                 l->type = LISTENER_POINTER_GRAB;
diff --git a/dix/events.c b/dix/events.c
index 73593626e..bea68cc51 100644
--- a/dix/events.c
+++ b/dix/events.c
@@ -1438,6 +1438,7 @@ UpdateTouchesForGrab(DeviceIntPtr mouse)
                 ti->listeners[0].type = LISTENER_POINTER_GRAB;
             else
                 ti->listeners[0].type = LISTENER_GRAB;
+            ti->listeners[0].grab = grab;
         }
     }
 }
diff --git a/dix/touch.c b/dix/touch.c
index d890b6227..99f105b2e 100644
--- a/dix/touch.c
+++ b/dix/touch.c
@@ -677,13 +677,17 @@ TouchResourceIsOwner(TouchPointInfoPtr ti, XID resource)
 void
 TouchAddListener(TouchPointInfoPtr ti, XID resource, enum InputLevel level,
                  enum TouchListenerType type, enum TouchListenerState state,
-                 WindowPtr window)
+                 WindowPtr window,
+                 GrabPtr grab)
 {
     ti->listeners[ti->num_listeners].listener = resource;
     ti->listeners[ti->num_listeners].level = level;
     ti->listeners[ti->num_listeners].state = state;
     ti->listeners[ti->num_listeners].type = type;
     ti->listeners[ti->num_listeners].window = window;
+    ti->listeners[ti->num_listeners].grab = grab;
+    if (grab)
+        ti->num_grabs++;
     ti->num_listeners++;
 }
 
@@ -702,6 +706,11 @@ TouchRemoveListener(TouchPointInfoPtr ti, XID resource)
         if (ti->listeners[i].listener == resource) {
             int j;
 
+            if (ti->listeners[i].grab) {
+                ti->listeners[i].grab = NULL;
+                ti->num_grabs--;
+            }
+
             for (j = i; j < ti->num_listeners - 1; j++)
                 ti->listeners[j] = ti->listeners[j + 1];
             ti->num_listeners--;
@@ -733,8 +742,7 @@ TouchAddGrabListener(DeviceIntPtr dev, TouchPointInfoPtr ti,
     }
 
     TouchAddListener(ti, grab->resource, grab->grabtype,
-                     type, LISTENER_AWAITING_BEGIN, grab->window);
-    ti->num_grabs++;
+                     type, LISTENER_AWAITING_BEGIN, grab->window, grab);
 }
 
 /**
@@ -790,7 +798,7 @@ TouchAddRegularListener(DeviceIntPtr dev, TouchPointInfoPtr ti,
                 TouchEventHistoryAllocate(ti);
 
             TouchAddListener(ti, iclients->resource, XI2,
-                             type, LISTENER_AWAITING_BEGIN, win);
+                             type, LISTENER_AWAITING_BEGIN, win, NULL);
             return TRUE;
         }
     }
@@ -806,7 +814,7 @@ TouchAddRegularListener(DeviceIntPtr dev, TouchPointInfoPtr ti,
             TouchEventHistoryAllocate(ti);
             TouchAddListener(ti, iclients->resource, XI,
                              LISTENER_POINTER_REGULAR, LISTENER_AWAITING_BEGIN,
-                             win);
+                             win, NULL);
             return TRUE;
         }
     }
@@ -821,7 +829,7 @@ TouchAddRegularListener(DeviceIntPtr dev, TouchPointInfoPtr ti,
             TouchEventHistoryAllocate(ti);
             TouchAddListener(ti, win->drawable.id, CORE,
                              LISTENER_POINTER_REGULAR, LISTENER_AWAITING_BEGIN,
-                             win);
+                             win, NULL);
             return TRUE;
         }
 
@@ -832,7 +840,7 @@ TouchAddRegularListener(DeviceIntPtr dev, TouchPointInfoPtr ti,
 
             TouchEventHistoryAllocate(ti);
             TouchAddListener(ti, oclients->resource, CORE,
-                             type, LISTENER_AWAITING_BEGIN, win);
+                             type, LISTENER_AWAITING_BEGIN, win, NULL);
             return TRUE;
         }
     }
diff --git a/include/input.h b/include/input.h
index 23a20b59d..79739e274 100644
--- a/include/input.h
+++ b/include/input.h
@@ -567,7 +567,7 @@ extern void TouchEventHistoryReplay(TouchPointInfoPtr ti, DeviceIntPtr dev,
 extern Bool TouchResourceIsOwner(TouchPointInfoPtr ti, XID resource);
 extern void TouchAddListener(TouchPointInfoPtr ti, XID resource,
                              enum InputLevel level, enum TouchListenerType type,
-                             enum TouchListenerState state, WindowPtr window);
+                             enum TouchListenerState state, WindowPtr window, GrabPtr grab);
 extern Bool TouchRemoveListener(TouchPointInfoPtr ti, XID resource);
 extern void TouchSetupListeners(DeviceIntPtr dev, TouchPointInfoPtr ti,
                                 InternalEvent *ev);
diff --git a/include/inputstr.h b/include/inputstr.h
index a9d46cc32..32d7b621c 100644
--- a/include/inputstr.h
+++ b/include/inputstr.h
@@ -305,6 +305,7 @@ typedef struct _TouchListener {
     enum TouchListenerState state;
     enum InputLevel level;  /* matters only for emulating touches */
     WindowPtr window;
+    GrabPtr grab;
 } TouchListener;
 
 typedef struct _TouchPointInfo {