Fix random issues when handling multiple cloudinit volumes

Sometimes the libvirt provider fails with cryptic messages when
multiple cloudinit volumes are being used.

libvirtd contains several messages like
`internal error: pool has asynchronous jobs running`. This seems to be
caused by invoking `pool.Refresh()` while a volume is being uploaded.

This commit introduces a new structure that can be used to synchronize
operations involving storage pools.

From my testing this solves the issues we have experiences so far.

5 files changed, 66 insertions, 1 deletions

diff --git a/libvirt/cloudinit_def.go b/libvirt/cloudinit_def.go
index da7e5f69..ea7c0dbe 100644
--- a/libvirt/cloudinit_def.go
+++ b/libvirt/cloudinit_def.go
@@ -58,6 +58,9 @@ func (ci *defCloudInit) CreateAndUpload(virConn *libvirt.VirConnection) (string,
 	}
 	defer pool.Free()
 
+	PoolSync.AcquireLock(ci.PoolName)
+	defer PoolSync.ReleaseLock(ci.PoolName)
+
 	// Refresh the pool of the volume so that libvirt knows it is
 	// not longer in use.
 	WaitForSuccess("Error refreshing pool for volume", func() error {
@@ -161,7 +164,7 @@ func (ci *defCloudInit) createISO() (string, error) {
 	if err = cmd.Run(); err != nil {
 		return "", fmt.Errorf("Error while starting the creation of CloudInit's ISO image: %s", err)
 	}
-	log.Print("ISO created at %s", isoDestination)
+	log.Printf("ISO created at %s", isoDestination)
 
 	return isoDestination, nil
 }
diff --git a/libvirt/pool_sync.go b/libvirt/pool_sync.go
new file mode 100644
index 00000000..8c53ac05
--- /dev/null
+++ b/libvirt/pool_sync.go
@@ -0,0 +1,49 @@
+package libvirt
+
+import (
+	"sync"
+)
+
+// LibVirtPoolSync makes possible to synchronize operations
+// against libvirt pools.
+// Doing pool.Refresh() operations while uploading or removing
+// a volume into the pool causes errors inside of libvirtd
+type LibVirtPoolSync struct {
+	PoolLocks     map[string]*sync.Mutex
+	internalMutex sync.Mutex
+}
+
+// Allocate a new instance of LibVirtPoolSync
+func NewLibVirtPoolSync() LibVirtPoolSync {
+	pool := LibVirtPoolSync{}
+	pool.PoolLocks = make(map[string]*sync.Mutex)
+
+	return pool
+}
+
+// Acquire a lock for the specified pool
+func (ps LibVirtPoolSync) AcquireLock(pool string) {
+	ps.internalMutex.Lock()
+	defer ps.internalMutex.Unlock()
+
+	lock, exists := ps.PoolLocks[pool]
+	if !exists {
+		lock = new(sync.Mutex)
+		ps.PoolLocks[pool] = lock
+	}
+
+	lock.Lock()
+}
+
+// Release the look for the specified pool
+func (ps LibVirtPoolSync) ReleaseLock(pool string) {
+	ps.internalMutex.Lock()
+	defer ps.internalMutex.Unlock()
+
+	lock, exists := ps.PoolLocks[pool]
+	if !exists {
+		return
+	}
+
+	lock.Unlock()
+}
diff --git a/libvirt/resource_libvirt_domain.go b/libvirt/resource_libvirt_domain.go
index 728c10ca..e7c86e10 100644
--- a/libvirt/resource_libvirt_domain.go
+++ b/libvirt/resource_libvirt_domain.go
@@ -14,6 +14,8 @@ import (
 	"github.com/hashicorp/terraform/helper/schema"
 )
 
+var PoolSync = NewLibVirtPoolSync()
+
 func init() {
 	spew.Config.Indent = "\t"
 }
diff --git a/libvirt/resource_libvirt_volume.go b/libvirt/resource_libvirt_volume.go
index 1e2de8a9..3f84b34c 100644
--- a/libvirt/resource_libvirt_volume.go
+++ b/libvirt/resource_libvirt_volume.go
@@ -181,6 +181,9 @@ func resourceLibvirtVolumeCreate(d *schema.ResourceData, meta interface{}) error
 		poolName = d.Get("pool").(string)
 	}
 
+	PoolSync.AcquireLock(poolName)
+	defer PoolSync.ReleaseLock(poolName)
+
 	pool, err := virConn.LookupStoragePoolByName(poolName)
 	if err != nil {
 		return fmt.Errorf("can't find storage pool '%s'", poolName)
diff --git a/libvirt/utils.go b/libvirt/utils.go
index a4110904..0700b756 100644
--- a/libvirt/utils.go
+++ b/libvirt/utils.go
@@ -70,6 +70,14 @@ func RemoveVolume(virConn *libvirt.VirConnection, key string) error {
 	}
 	defer volPool.Free()
 
+	poolName, err := volPool.GetName()
+	if err != nil {
+		return fmt.Errorf("Error retrieving name of volume: %s", err)
+	}
+
+	PoolSync.AcquireLock(poolName)
+	defer PoolSync.ReleaseLock(poolName)
+
 	WaitForSuccess("Error refreshing pool for volume", func() error {
 		return volPool.Refresh(0)
 	})