diff --git a/hw/xwayland/xwayland-input.c b/hw/xwayland/xwayland-input.c
index 010d28c43..0515eb950 100644
--- a/hw/xwayland/xwayland-input.c
+++ b/hw/xwayland/xwayland-input.c
@@ -462,12 +462,6 @@ keyboard_handle_modifiers(void *data, struct wl_keyboard *keyboard,
     xkbStateNotify sn;
     CARD16 changed;
 
-    /* We don't need any of this while we have keyboard focus since
-       the regular key event processing already takes care of setting
-       our internal state correctly. */
-    if (xwl_seat->keyboard_focus)
-        return;
-
     for (dev = inputInfo.devices; dev; dev = dev->next) {
         if (dev != xwl_seat->keyboard &&
             dev != GetMaster(xwl_seat->keyboard, MASTER_KEYBOARD))
@@ -476,10 +470,12 @@ keyboard_handle_modifiers(void *data, struct wl_keyboard *keyboard,
         old_state = dev->key->xkbInfo->state;
         new_state = &dev->key->xkbInfo->state;
 
+        if (!xwl_seat->keyboard_focus) {
+            new_state->locked_mods = mods_locked & XkbAllModifiersMask;
+            XkbLatchModifiers(dev, XkbAllModifiersMask,
+                              mods_latched & XkbAllModifiersMask);
+        }
         new_state->locked_group = group & XkbAllGroupsMask;
-        new_state->locked_mods = mods_locked & XkbAllModifiersMask;
-        XkbLatchModifiers(dev, XkbAllModifiersMask,
-                          mods_latched & XkbAllModifiersMask);
 
         XkbComputeDerivedState(dev->key->xkbInfo);