Implement tree flattening to automatically solve situations of redundant chains of split containers

This should fix the move problems. See comment of tree_flatten() for a little
example.

include/tree.h

@@ -84,4 +84,19 @@ void tree_close(Con *con, bool kill_window, bool dont_kill_parent);
  */
 bool tree_restore(const char *path);
 
+/**
+ * tree_flatten() removes pairs of redundant split containers, e.g.:
+ *       [workspace, horizontal]
+ *   [v-split]           [child3]
+ *   [h-split]
+ * [child1] [child2]
+ * In this example, the v-split and h-split container are redundant.
+ * Such a situation can be created by moving containers in a direction which is
+ * not the orientation of their parent container. i3 needs to create a new
+ * split container then and if you move containers this way multiple times,
+ * redundant chains of split-containers can be the result.
+ *
+ */
+void tree_flatten(Con *child);
+
 #endif

src/con.c

@@ -669,6 +669,8 @@ void con_set_layout(Con *con, int layout) {
         if (old_focused)
             con_focus(old_focused);
 
+        tree_flatten(croot);
+
         return;
     }
 

src/tree.c

@@ -518,4 +518,100 @@ void tree_move(char way, orientation_t orientation) {
         DLOG("Old container empty after moving. Let's close it\n");
         tree_close(old_parent, false, false);
     }
+
+    tree_flatten(croot);
+}
+
+/*
+ * tree_flatten() removes pairs of redundant split containers, e.g.:
+ *       [workspace, horizontal]
+ *   [v-split]           [child3]
+ *   [h-split]
+ * [child1] [child2]
+ * In this example, the v-split and h-split container are redundant.
+ * Such a situation can be created by moving containers in a direction which is
+ * not the orientation of their parent container. i3 needs to create a new
+ * split container then and if you move containers this way multiple times,
+ * redundant chains of split-containers can be the result.
+ *
+ */
+void tree_flatten(Con *con) {
+    Con *current, *child, *parent = con->parent;
+    DLOG("Checking if I can flatten con = %p / %s\n", con, con->name);
+
+    /* We only consider normal containers without windows */
+    if (con->type != CT_CON || con->window != NULL)
+        goto recurse;
+
+    /* Ensure it got only one child */
+    child = TAILQ_FIRST(&(con->nodes_head));
+    if (TAILQ_NEXT(child, nodes) != NULL)
+        goto recurse;
+
+    /* The child must have a different orientation than the con but the same as
+     * the con’s parent to be redundant */
+    if (con->orientation == NO_ORIENTATION ||
+        child->orientation == NO_ORIENTATION ||
+        con->orientation == child->orientation ||
+        child->orientation != parent->orientation)
+        goto recurse;
+
+    DLOG("Alright, I have to flatten this situation now. Stay calm.\n");
+    /* 1: save focus */
+    Con *focus_next = TAILQ_FIRST(&(child->focus_head));
+
+    DLOG("detaching...\n");
+    /* 2: re-attach the children to the parent before con */
+    while (!TAILQ_EMPTY(&(child->nodes_head))) {
+        current = TAILQ_FIRST(&(child->nodes_head));
+        DLOG("detaching current=%p / %s\n", current, current->name);
+        con_detach(current);
+        DLOG("re-attaching\n");
+        /* We don’t use con_attach() here because for a CT_CON, the special
+         * case handling of con_attach() does not trigger. So all it would do
+         * is calling TAILQ_INSERT_AFTER, but with the wrong container. So we
+         * directly use the TAILQ macros. */
+        current->parent = parent;
+        TAILQ_INSERT_BEFORE(con, current, nodes);
+        DLOG("attaching to focus list\n");
+        TAILQ_INSERT_TAIL(&(parent->focus_head), current, focused);
+    }
+    DLOG("re-attached all\n");
+
+    /* 3: restore focus, if con was focused */
+    if (focus_next != NULL &&
+        TAILQ_FIRST(&(parent->focus_head)) == con) {
+        DLOG("restoring focus to focus_next=%p\n", focus_next);
+        TAILQ_REMOVE(&(parent->focus_head), focus_next, focused);
+        TAILQ_INSERT_HEAD(&(parent->focus_head), focus_next, focused);
+        DLOG("restored focus.\n");
+    }
+
+    /* 4: close the redundant cons */
+    DLOG("closing redundant cons\n");
+    tree_close(con, false, true);
+
+    /* Well, we got to abort the recursion here because we destroyed the
+     * container. However, if tree_flatten() is called sufficiently often,
+     * there can’t be the situation of having two pairs of redundant containers
+     * at once. Therefore, we can safely abort the recursion on this level
+     * after flattening. */
+    return;
+
+recurse:
+    /* We cannot use normal foreach here because tree_flatten might close the
+     * current container. */
+    current = TAILQ_FIRST(&(con->nodes_head));
+    while (current != NULL) {
+        Con *next = TAILQ_NEXT(current, nodes);
+        tree_flatten(current);
+        current = next;
+    }
+
+    current = TAILQ_FIRST(&(con->floating_head));
+    while (current != NULL) {
+        Con *next = TAILQ_NEXT(current, floating_windows);
+        tree_flatten(current);
+        current = next;
+    }
 }

testcases/t/45-flattening.t

@@ -0,0 +1,33 @@
+#!perl
+# vim:ts=4:sw=4:expandtab
+#
+# by moving the window in the opposite orientation that its parent has, we
+# force i3 to create a new split container with the appropriate orientation.
+# However, when doing that two times in a row, we end up with two split
+# containers which are then redundant (workspace is horizontal, then v-split,
+# then h-split – we could just append the children of the latest h-split to the
+# workspace itself).
+#
+# This testcase checks that the tree is properly flattened after moving.
+#
+use X11::XCB qw(:all);
+use i3test tests => 2;
+
+my $x = X11::XCB::Connection->new;
+
+my $tmp = get_unused_workspace;
+cmd "workspace $tmp";
+
+my $left = open_standard_window($x);
+sleep 0.25;
+my $mid = open_standard_window($x);
+sleep 0.25;
+my $right = open_standard_window($x);
+sleep 0.25;
+
+cmd 'move before v';
+cmd 'move after h';
+my $ws = get_ws($tmp);
+
+is($ws->{orientation}, 'horizontal', 'workspace orientation is horizontal');
+is(@{$ws->{nodes}}, 3, 'all three windows on workspace level');