commit 925de19ed7f13e0d12d0b993496d314bab886589
Author: Daniel P. Berrange <berrange@redhat.com>
Date:   Wed Jan 22 18:13:30 2014 +0000

    Add a mutex to serialize updates to firewall
    
    The nwfilter conf update mutex previously serialized
    updates to the internal data structures for firewall
    rules, and updates to the firewall itself. The latter
    was recently turned into a read/write lock, and filter
    instantiation allowed to proceed in parallel. It was
    believed that this was ok, since each filter is created
    on a separate iptables/ebtables chain.
    
    It turns out that there is a subtle lock ordering problem
    on virNWFilterObjPtr instances. __virNWFilterInstantiateFilter
    will hold a lock on the virNWFilterObjPtr it is instantiating.
    This in turn invokes virNWFilterInstantiate which then invokes
    virNWFilterDetermineMissingVarsRec which then invokes
    virNWFilterObjFindByName. This iterates over every single
    virNWFilterObjPtr in the list, locking them and checking their
    name. So if 2 or more threads try to instantiate a filter in
    parallel, they'll all hold 1 lock at the top level in the
    __virNWFilterInstantiateFilter method which will cause the
    other thread to deadlock in virNWFilterObjFindByName.
    
    The fix is to add an exclusive mutex to serialize the
    execution of __virNWFilterInstantiateFilter.
    
    Signed-off-by: Daniel P. Berrange <berrange@redhat.com>

Index: libvirt-1.1.1/src/nwfilter/nwfilter_driver.c
===================================================================
--- libvirt-1.1.1.orig/src/nwfilter/nwfilter_driver.c	2014-03-25 14:15:27.446627559 -0500
+++ libvirt-1.1.1/src/nwfilter/nwfilter_driver.c	2014-03-25 14:15:27.438627559 -0500
@@ -199,7 +199,8 @@ nwfilterStateInitialize(bool privileged,
     if (virNWFilterDHCPSnoopInit() < 0)
         goto err_exit_learnshutdown;
 
-    virNWFilterTechDriversInit(privileged);
+    if (virNWFilterTechDriversInit(privileged) < 0)
+        goto err_dhcpsnoop_shutdown;
 
     if (virNWFilterConfLayerInit(virNWFilterDomainFWUpdateCB,
                                  driverState) < 0)
@@ -247,6 +248,7 @@ error:
 
 err_techdrivers_shutdown:
     virNWFilterTechDriversShutdown();
+err_dhcpsnoop_shutdown:
     virNWFilterDHCPSnoopShutdown();
 err_exit_learnshutdown:
     virNWFilterLearnShutdown();
@@ -323,10 +325,10 @@ nwfilterStateCleanup(void) {
 
     if (driverState->privileged) {
         virNWFilterConfLayerShutdown();
-        virNWFilterTechDriversShutdown();
         virNWFilterDHCPSnoopShutdown();
         virNWFilterLearnShutdown();
         virNWFilterIPAddrMapShutdown();
+        virNWFilterTechDriversShutdown();
 
         nwfilterDriverLock(driverState);
 
Index: libvirt-1.1.1/src/nwfilter/nwfilter_gentech_driver.c
===================================================================
--- libvirt-1.1.1.orig/src/nwfilter/nwfilter_gentech_driver.c	2014-03-25 14:15:27.446627559 -0500
+++ libvirt-1.1.1/src/nwfilter/nwfilter_gentech_driver.c	2014-03-25 14:15:27.438627559 -0500
@@ -55,30 +55,53 @@ static virNWFilterTechDriverPtr filter_t
     NULL
 };
 
+/* Serializes instantiation of filters. This is necessary
+ * to avoid lock ordering deadlocks. eg __virNWFilterInstantiateFilter
+ * will hold a lock on a virNWFilterObjPtr. This in turn invokes
+ * virNWFilterInstantiate which invokes virNWFilterDetermineMissingVarsRec
+ * which invokes virNWFilterObjFindByName. This iterates over every single
+ * virNWFilterObjPtr in the list. So if 2 threads try to instantiate a
+ * filter in parallel, they'll both hold 1 lock at the top level in
+ * __virNWFilterInstantiateFilter which will cause the other thread
+ * to deadlock in virNWFilterObjFindByName.
+ *
+ * XXX better long term solution is to make virNWFilterObjList use a
+ * hash table as is done for virDomainObjList. You can then get
+ * lockless lookup of objects by name.
+ */
+static virMutex updateMutex;
 
-void virNWFilterTechDriversInit(bool privileged) {
+int virNWFilterTechDriversInit(bool privileged)
+{
     size_t i = 0;
     VIR_DEBUG("Initializing NWFilter technology drivers");
+    if (virMutexInitRecursive(&updateMutex) < 0)
+        return -1;
+
     while (filter_tech_drivers[i]) {
         if (!(filter_tech_drivers[i]->flags & TECHDRV_FLAG_INITIALIZED))
             filter_tech_drivers[i]->init(privileged);
         i++;
     }
+    return 0;
 }
 
 
-void virNWFilterTechDriversShutdown(void) {
+void virNWFilterTechDriversShutdown(void)
+{
     size_t i = 0;
     while (filter_tech_drivers[i]) {
         if ((filter_tech_drivers[i]->flags & TECHDRV_FLAG_INITIALIZED))
             filter_tech_drivers[i]->shutdown();
         i++;
     }
+    virMutexDestroy(&updateMutex);
 }
 
 
 virNWFilterTechDriverPtr
-virNWFilterTechDriverForName(const char *name) {
+virNWFilterTechDriverForName(const char *name)
+{
     size_t i = 0;
     while (filter_tech_drivers[i]) {
        if (STREQ(filter_tech_drivers[i]->name, name)) {
@@ -934,6 +957,8 @@ _virNWFilterInstantiateFilter(virNWFilte
     int ifindex;
     int rc;
 
+    virMutexLock(&updateMutex);
+
     /* after grabbing the filter update lock check for the interface; if
        it's not there anymore its filters will be or are being removed
        (while holding the lock) and we don't want to build new ones */
@@ -961,6 +986,8 @@ _virNWFilterInstantiateFilter(virNWFilte
                                         foundNewFilter);
 
 cleanup:
+    virMutexUnlock(&updateMutex);
+
     return rc;
 }
 
@@ -980,6 +1007,7 @@ virNWFilterInstantiateFilterLate(virNWFi
     bool foundNewFilter = false;
 
     virNWFilterReadLockFilterUpdates();
+    virMutexLock(&updateMutex);
 
     rc = __virNWFilterInstantiateFilter(driver,
                                         vmuuid,
@@ -1005,6 +1033,7 @@ virNWFilterInstantiateFilterLate(virNWFi
     }
 
     virNWFilterUnlockFilterUpdates();
+    virMutexUnlock(&updateMutex);
 
     return rc;
 }
@@ -1128,7 +1157,11 @@ _virNWFilterTeardownFilter(const char *i
 int
 virNWFilterTeardownFilter(const virDomainNetDefPtr net)
 {
-    return _virNWFilterTeardownFilter(net->ifname);
+    int ret;
+    virMutexLock(&updateMutex);
+    ret = _virNWFilterTeardownFilter(net->ifname);
+    virMutexUnlock(&updateMutex);
+    return ret;
 }
 
 
Index: libvirt-1.1.1/src/nwfilter/nwfilter_gentech_driver.h
===================================================================
--- libvirt-1.1.1.orig/src/nwfilter/nwfilter_gentech_driver.h	2014-03-25 14:15:27.446627559 -0500
+++ libvirt-1.1.1/src/nwfilter/nwfilter_gentech_driver.h	2014-03-25 14:15:27.442627559 -0500
@@ -30,7 +30,7 @@ virNWFilterTechDriverPtr virNWFilterTech
 int virNWFilterRuleInstAddData(virNWFilterRuleInstPtr res,
                                void *data);
 
-void virNWFilterTechDriversInit(bool privileged);
+int virNWFilterTechDriversInit(bool privileged);
 void virNWFilterTechDriversShutdown(void);
 
 enum instCase {
