Description: Update cgmanager code to work under systemd
 . Since systemd-machined is now running, we get actual errors, not just a dbus
 method not available error.  ignore it since we are using cgmanager.
 . Keep track of comounted controllers so we avoid double-cgroup-enter.  (This
 isn't an issue for qemu where libvirt does the moving, but in lxc the task
 moves itself - relative to its current cgroup).
 . Use cgm_list_controllers rather than /proc/cgroups to get list of
 controllers, it makes comount detection easier.
 . Move libvirt to root cgroup so it can manipulate cgroups.
 . ignore failure from virCgroupNewMachineSystemd.  It used to fail with a
 dbus method not available error bc systemd was not running.  Now systemd is
 running, but we still want to try the manual creation.
 . cgm_remove - don't return failure if cgroup doesn't exist.  It was probably
 removed with its comount (i.e. cpu,cpuacct)
Author: Serge Hallyn <serge.hallyn@ubuntu.com>

Index: libvirt-1.2.12/src/util/vircgroup.c
===================================================================
--- libvirt-1.2.12.orig/src/util/vircgroup.c
+++ libvirt-1.2.12/src/util/vircgroup.c
@@ -337,6 +337,12 @@ virCgroupCopyMounts(virCgroupPtr group,
         if (!parent->controllers[i].mountPoint)
             continue;
 
+#ifdef HAVE_CGMANAGER
+        if (VIR_STRDUP(group->controllers[i].comount,
+                       parent->controllers[i].comount) < 0)
+            return -1;
+#endif
+
         if (VIR_STRDUP(group->controllers[i].mountPoint,
                        parent->controllers[i].mountPoint) < 0)
             return -1;
@@ -349,48 +355,50 @@ virCgroupCopyMounts(virCgroupPtr group,
 }
 
 #ifdef HAVE_CGMANAGER
-static void cg_add_cgroup(virCgroupPtr group, const char *g)
+static void do_add_cgroup(virCgroupPtr group, char *g, char *comount)
 {
     int i = virCgroupControllerTypeFromString(g);
     if (i < 0)
         return;
-    if (VIR_STRDUP(group->controllers[i].mountPoint, "/") < 0) {
-        VIR_WARN("Out of memory copying \"/\".  Proceeding without cgroup controll %s.", g);
-        return;
-    }
+    if (VIR_STRDUP(group->controllers[i].mountPoint, "/") < 0)
+        VIR_ERROR("Out of memory copying \"/\". for cgroup mountpoint %s", g);
     group->controllers[i].linkPoint = NULL;
+    if (comount && VIR_STRDUP(group->controllers[i].comount, comount) < 0)
+        VIR_ERROR("Out of memory copying cgroup comount %s for %s", comount, g);
+}
+
+static void cg_add_cgroup(virCgroupPtr group, char *g)
+{
+    char *first = NULL;
+    char *comma = strchr(g, ',');
+    while (comma) {
+        *comma = '\0';
+        if (!first)
+            first = g;
+        do_add_cgroup(group, g, first);
+        g = comma+1;
+        comma = strchr(g, ',');
+    }
+    do_add_cgroup(group, g, first ? first : g);
 }
 
 static bool cg_get_cgroups(virCgroupPtr group)
 {
-    FILE *fin = NULL;
-    char *line = NULL;
-    size_t len = 0;
+    char **list;
 
     if (!cgm_dbus_connect()) {
         return false;
     }
-    /* check to see if name=systemd is mounted */
-    if (cgm_controller_exists("name=systemd"))
-        cg_add_cgroup(group, "name=systemd");
-    cgm_dbus_disconnect();
-    fin = fopen("/proc/cgroups", "r");
-    if (fin == NULL) {
-        virReportSystemError(errno, "%s",
-                             _("Unable to open /proc/cgroups"));
-        return false;
+    if (!cgm_list_controllers(&list)) {
+	    cgm_dbus_disconnect();
+	    return false;
     }
-    while (getline(&line, &len, fin) > 0) {
-        char *p;
-        if (line[0] == '#')
-            continue;
-        p = strchr(line, '\t');
-        if (p)
-            *p = '\0';
-        cg_add_cgroup(group, line);
+    cgm_dbus_disconnect();
+
+    for (int i = 0; list[i]; i++) {
+        cg_add_cgroup(group, list[i]);
     }
-    VIR_FREE(line);
-    VIR_FORCE_FCLOSE(fin);
+    nih_free(list);
     return true;
 }
 #else
@@ -1083,11 +1091,6 @@ virCgroupSetMemoryUseHierarchy(virCgroup
     unsigned long long value;
     const char *filename = "memory.use_hierarchy";
 
-#ifdef HAVE_CGMANAGER
-    /* cgmanager will have set up inheritence for us */
-    if (cgm_running)
-        return 0;
-#endif
     if (virCgroupGetValueU64(group,
                              VIR_CGROUP_CONTROLLER_MEMORY,
                              filename, &value) < 0)
@@ -1142,7 +1145,7 @@ virCgroupMakeGroup(virCgroupPtr parent,
         if (usecgm) {
             int32_t existed;
             if (!cgm_create(virCgroupControllerTypeToString(i),
-                    group->path, &existed))
+                    group->path, &existed) && existed != 1)
                 goto cleanup;
             continue;
         }
@@ -1272,6 +1275,24 @@ virCgroupNew(pid_t pid,
     return -1;
 }
 
+static bool in_handled_controllers(char *list, char *which)
+{
+    char *comma;
+
+    if (!list)
+        return false;
+    comma = strchr(list, ',');
+    while (comma) {
+        if (strncmp(list, which, comma-list) == 0)
+            return true;
+        list = comma + 1;
+        comma = strchr(list, ',');
+    }
+    if (strcmp(list, which) == 0)
+        return true;
+    return false;
+}
+
 
 /**
  * virCgroupAddTask:
@@ -1286,6 +1307,9 @@ virCgroupAddTask(virCgroupPtr group, pid
 {
     int ret = -1;
     size_t i;
+#ifdef HAVE_CGMANAGER
+    nih_local char *handled_controllers = NULL;
+#endif
 
     for (i = 0; i < VIR_CGROUP_CONTROLLER_LAST; i++) {
         /* Skip over controllers not mounted */
@@ -1297,6 +1321,8 @@ virCgroupAddTask(virCgroupPtr group, pid
             continue;
 
 #ifdef HAVE_CGMANAGER
+        if (in_handled_controllers(handled_controllers, group->controllers[i].comount))
+            continue;
         if (cgm_dbus_connect()) {
             if (!cgm_enter(virCgroupControllerTypeToString(i), group->path, pid)) {
                 cgm_dbus_disconnect();
@@ -1307,6 +1333,10 @@ virCgroupAddTask(virCgroupPtr group, pid
             cgm_dbus_disconnect();
             VIR_INFO("Moved %d to %s:%s", pid,
                     virCgroupControllerTypeToString(i), group->path);
+            if (group->controllers[i].comount)
+		    NIH_MUST( nih_strcat_sprintf(&handled_controllers, NULL, "%s%s",
+			handled_controllers ? "," : "",
+			group->controllers[i].comount) );
         } else
 #endif
         if (virCgroupSetValueU64(group, i, "tasks", pid) < 0)
@@ -1318,7 +1348,6 @@ virCgroupAddTask(virCgroupPtr group, pid
     return ret;
 }
 
-
 /**
  * virCgroupAddTaskController:
  *
@@ -1423,6 +1452,7 @@ virCgroupMoveTask(virCgroupPtr src_group
     char *content = NULL;
     size_t i;
 #ifdef HAVE_CGMANAGER
+    nih_local char *handled_controllers = NULL;
     bool usecgm = false;
 
     if (cgm_dbus_connect())
@@ -1431,6 +1461,14 @@ virCgroupMoveTask(virCgroupPtr src_group
 
     for (i = 0; i < VIR_CGROUP_CONTROLLER_LAST; i++) {
         bool firstrun = true;
+#ifdef HAVE_CGMANAGER
+        const char *typestr = virCgroupControllerTypeToString(i);
+        if (in_handled_controllers(handled_controllers, typestr))
+            continue;
+        NIH_MUST( nih_strcat_sprintf(&handled_controllers, NULL, "%s%s",
+			handled_controllers ? "," : "",
+			src_group->controllers[i].comount) );
+#endif
         if (!src_group->controllers[i].mountPoint ||
             !dest_group->controllers[i].mountPoint)
             continue;
@@ -1445,7 +1483,6 @@ virCgroupMoveTask(virCgroupPtr src_group
          * until content is empty.  */
         while (1) {
 #ifdef HAVE_CGMANAGER
-            const char *typestr = virCgroupControllerTypeToString(i);
             pid_t *pids = NULL;
             size_t j, nrpids;
             if (usecgm) {
@@ -1958,6 +1995,14 @@ int virCgroupTerminateMachine(const char
                               const char *drivername,
                               bool privileged)
 {
+#ifdef HAVE_CGMANAGER
+    bool usecgm = false;
+    if (cgm_dbus_connect())
+        usecgm = true;
+    cgm_dbus_disconnect();
+    if (usecgm)
+        return 0;
+#endif
     return virSystemdTerminateMachine(name, drivername, privileged);
 }
 
@@ -2025,9 +2070,17 @@ virCgroupNewMachine(const char *name,
                     virCgroupPtr *group)
 {
     int rv;
+    bool usecgm = false;
+#ifdef HAVE_CGMANAGER
+    if (cgm_dbus_connect())
+        usecgm = true;
+    cgm_dbus_disconnect();
+#endif
 
     *group = NULL;
 
+    if (usecgm)
+        goto skip;
     if ((rv = virCgroupNewMachineSystemd(name,
                                          drivername,
                                          privileged,
@@ -2045,6 +2098,7 @@ virCgroupNewMachine(const char *name,
     if (rv == -1)
         return -1;
 
+skip:
     return virCgroupNewMachineManual(name,
                                      drivername,
                                      pidleader,
@@ -2085,6 +2139,7 @@ virCgroupFree(virCgroupPtr *group)
         VIR_FREE((*group)->controllers[i].mountPoint);
         VIR_FREE((*group)->controllers[i].linkPoint);
         VIR_FREE((*group)->controllers[i].placement);
+        VIR_FREE((*group)->controllers[i].comount);
     }
 
     VIR_FREE((*group)->path);
@@ -3830,7 +3885,7 @@ static int cgm_kill(virCgroupPtr group,
 
     if ((i = cgm_get_controller_path(group, -1)) < 0) {
         VIR_ERROR("cgm_kill: Could not get controller path");
-        return -1;
+        return 0;
     }
 
     controller = virCgroupControllerTypeToString(i);
@@ -3865,7 +3920,8 @@ static int cgm_kill(virCgroupPtr group,
             }
             ignore_value(virHashAddEntry(pidhash, (void*)upid, (void*)1));
         }
-        nih_free(pids);
+        if (nrpids)
+            nih_free(pids);
     }
 
  done:
@@ -3873,7 +3929,7 @@ static int cgm_kill(virCgroupPtr group,
 
 cleanup:
 
-    VIR_DEBUG("cgm_kill: returning %d, killedAny %d", ret, killedAny);
+    VIR_ERROR("cgm_kill: returning %d, killedAny %d", ret, killedAny);
     return ret;
 }
 
@@ -4526,16 +4582,6 @@ virCgroupHasEmptyTasks(virCgroupPtr cgro
     return ret;
 }
 
-void virCgroupEscape(void) {
-#ifdef HAVE_CGMANAGER
-    int i;
-    if (cgm_dbus_connect()) {
-        for (i = 0; i < VIR_CGROUP_CONTROLLER_LAST; i++)
-            cgm_escape(virCgroupControllerTypeToString(i));
-        cgm_dbus_disconnect();
-    }
-#endif
-}
 #else /* !VIR_CGROUP_SUPPORTED */
 
 bool
@@ -5274,7 +5320,4 @@ virCgroupHasEmptyTasks(virCgroupPtr cgro
                          _("Control groups not supported on this platform"));
     return -1;
 }
-
-void virCgroupEscape(void) {
-}
 #endif /* !VIR_CGROUP_SUPPORTED */
Index: libvirt-1.2.12/daemon/Makefile.am
===================================================================
--- libvirt-1.2.12.orig/daemon/Makefile.am
+++ libvirt-1.2.12/daemon/Makefile.am
@@ -40,6 +40,8 @@ DAEMON_SOURCES =					\
 		libvirtd.c libvirtd.h			\
 		remote.c remote.h			\
 		stream.c stream.h			\
+		$(top_srcdir)/src/util/cgmanager.c      \
+		$(top_srcdir)/src/util/cgmanager.h      \
 		$(DAEMON_GENERATED)
 
 LIBVIRTD_CONF_SOURCES = libvirtd-config.c libvirtd-config.h
@@ -148,6 +150,10 @@ libvirtd_CFLAGS = \
 	$(COVERAGE_CFLAGS) $(CGMANAGER_CFLAGS) \
 	-DQEMUD_PID_FILE="\"$(QEMUD_PID_FILE)\""
 
+if ENABLE_CGMANAGER
+libvirtd_CFLAGS += -DHAVE_CGMANAGER
+endif
+
 libvirtd_LDFLAGS =					\
 	$(RELRO_LDFLAGS)				\
 	$(PIE_LDFLAGS)					\
Index: libvirt-1.2.12/daemon/libvirtd.c
===================================================================
--- libvirt-1.2.12.orig/daemon/libvirtd.c
+++ libvirt-1.2.12/daemon/libvirtd.c
@@ -102,6 +102,7 @@
 #  include "nwfilter/nwfilter_driver.h"
 # endif
 #endif
+#include "util/cgmanager.h"
 
 #include "configmake.h"
 
@@ -1265,8 +1266,7 @@ int main(int argc, char **argv) {
     }
 
     /* move ourselves to root cgroup if necessary */
-    // XXX todo - figure out how to get the fn included
-    // virCgroupEscape();
+    cgm_escape();
 
     if (daemonSetupLogging(config, privileged, verbose, godaemon) < 0) {
         VIR_ERROR(_("Can't initialize logging"));
Index: libvirt-1.2.12/src/util/cgmanager.c
===================================================================
--- libvirt-1.2.12.orig/src/util/cgmanager.c
+++ libvirt-1.2.12/src/util/cgmanager.c
@@ -50,6 +50,7 @@ bool cgm_dbus_connect(void)
     DBusError dbus_error;
     DBusConnection *connection;
     dbus_error_init(&dbus_error);
+    static int32_t api_version;
 
     virMutexLock(&cgmanager_lock);
 
@@ -71,21 +72,29 @@ bool cgm_dbus_connect(void)
         nerr = nih_error_get();
         VIR_ERROR("cgmanager: Error opening proxy: %s", nerr->message);
         nih_free(nerr);
-	virMutexUnlock(&cgmanager_lock);
+        virMutexUnlock(&cgmanager_lock);
         return false;
     }
 
-    // force fd passing negotiation
-    if (cgmanager_ping_sync(NULL, cgroup_manager, 0) != 0) {
+    if (cgmanager_get_api_version_sync(NULL, cgroup_manager, &api_version) != 0) {
         NihError *nerr;
         nerr = nih_error_get();
-        VIR_ERROR("cgmanager: Error pinging manager: %s", nerr->message);
+        VIR_ERROR("cgmanager: Error getting cgmanager version: %s", nerr->message);
         nih_free(nerr);
         nih_free(cgroup_manager);
         cgroup_manager = NULL;
-	virMutexUnlock(&cgmanager_lock);
+        virMutexUnlock(&cgmanager_lock);
+        return false;
+    }
+
+    if (api_version < 8) {
+        VIR_ERROR("cgmanager version is too old");
+        nih_free(cgroup_manager);
+        cgroup_manager = NULL;
+        virMutexUnlock(&cgmanager_lock);
         return false;
     }
+
     cgm_running = true;
     return true;
 }
@@ -136,10 +145,6 @@ bool cgm_remove(const char *controller,
         return false;
     }
 
-    if (existed == -1) {
-        VIR_ERROR("cgmanager: cgm_remove failed: %s:%s did not exist", controller, cgroup_path);
-        return false;
-    }
     return true;
 }
 
@@ -251,10 +256,12 @@ bool cgm_get_tasks(const char *controlle
     if (ret) {
         NihError *nerr;
         nerr = nih_error_get();
-        VIR_ERROR("cgmanager: cgm_get_tasks for controller=%s, cgroup_path=%s failed: %s",
+        if (*nrpids != 0)
+            VIR_ERROR("cgmanager: cgm_get_tasks for controller=%s, cgroup_path=%s failed: %s",
                   controller, cgroup_path, nerr->message);
         nih_free(nerr);
-        return false;
+        if (*nrpids != 0)
+            return false;
     }
 
     return true;
@@ -313,6 +320,18 @@ bool cgm_controller_exists(const char *c
     return true;
 }
 
+bool cgm_list_controllers(char ***output)
+{
+    if ( cgmanager_list_controllers_sync(NULL, cgroup_manager, output) != 0) {
+        NihError *nerr;
+        nerr = nih_error_get();
+        nih_free(nerr);
+        return false;
+    }
+
+    return true;
+}
+
 bool cgm_enter(const char *controller, const char *cgroup_path, pid_t pid)
 {
     if (cgroup_path[0] == '/')
@@ -330,16 +349,21 @@ bool cgm_enter(const char *controller, c
     return true;
 }
 
-bool cgm_escape(const char *controller)
+void cgm_escape(void)
 {
-    if (cgmanager_move_pid_sync(NULL, cgroup_manager, controller, "/",
+    if (!cgm_dbus_connect()) {
+        VIR_ERROR("cgm_escape: failed to connect to cgmanager");
+        return;
+    }
+    if (cgmanager_move_pid_abs_sync(NULL, cgroup_manager, "all", "/",
              getpid()) != 0) {
         NihError *nerr;
         nerr = nih_error_get();
-        VIR_DEBUG("cgmanager: Failed escaping to root cgroup for controller %s: %s",
-                  controller, nerr->message);
+        VIR_DEBUG("cgmanager: Failed escaping to root cgroup for %s",
+                  nerr->message);
         nih_free(nerr);
-    }
-    return true;
+    } else
+        VIR_DEBUG("Escaped to root cgroup");
+    cgm_dbus_disconnect();
 }
 #endif
Index: libvirt-1.2.12/src/util/cgmanager.h
===================================================================
--- libvirt-1.2.12.orig/src/util/cgmanager.h
+++ libvirt-1.2.12/src/util/cgmanager.h
@@ -47,5 +47,6 @@ bool cgm_get_pid_cgroup(const char *cont
 bool cgm_get_pid_cgroup_abs(const char *controller, pid_t pid, char **cgpath);
 bool cgm_controller_exists(const char *controller);
 bool cgm_enter(const char *controller, const char *cgroup_path, pid_t pid);
-bool cgm_escape(const char *controller);
+void cgm_escape(void);
+bool cgm_list_controllers(char ***output);
 #endif
Index: libvirt-1.2.12/src/util/vircgrouppriv.h
===================================================================
--- libvirt-1.2.12.orig/src/util/vircgrouppriv.h
+++ libvirt-1.2.12/src/util/vircgrouppriv.h
@@ -33,6 +33,7 @@
 
 struct virCgroupController {
     int type;
+    char *comount;
     char *mountPoint;
     /* If mountPoint holds several controllers co-mounted,
      * then linkPoint is path of the symlink to the mountPoint
Index: libvirt-1.2.12/src/util/vircgroup.h
===================================================================
--- libvirt-1.2.12.orig/src/util/vircgroup.h
+++ libvirt-1.2.12/src/util/vircgroup.h
@@ -274,7 +274,6 @@ int virCgroupSetOwner(virCgroupPtr cgrou
 
 int virCgroupHasEmptyTasks(virCgroupPtr cgroup, int controller);
 
-void virCgroupEscape(void);
 int virCgroupDetectPlacement(virCgroupPtr group,
                              pid_t pid,
                              const char *path);
