diff --git a/src/con.c b/src/con.c
index deb65558..b7168d41 100644
--- a/src/con.c
+++ b/src/con.c
@@ -2373,6 +2373,10 @@ bool con_swap(Con *first, Con *second) {
 
     /* Move first to second. */
     result &= _con_move_to_con(first, second, false, false, false, true, false);
+    /* If swapping the containers didn't work we don't need to mess with the focus. */
+    if (!result) {
+        goto swap_end;
+    }
 
     /* If we moved the container holding the focused window to another
      * workspace we need to ensure the visible workspace has the focused
@@ -2385,8 +2389,6 @@ bool con_swap(Con *first, Con *second) {
 
     /* Move second to where first has been originally. */
     result &= _con_move_to_con(second, fake, false, false, false, true, false);
-
-    /* If swapping the containers didn't work we don't need to mess with the focus. */
     if (!result) {
         goto swap_end;
     }