Add new method to get information on domain interfaces

We've found that, when executing destroy/create cycles on
a domain where the virtual network(s) to which it is attached
are persisted across these cycles then we get network failures
on recreation of the domain since the old entries for
the domain are still present in the virtual network(s). This issue
should (hopefully) be fixed by commit
a20d2a92668b2d5e077267c50074bf853ef371d.  We've also found that,
on occasion, we don't get complete information on a domain's
interfaces from ListAllInterfaceAddresses. This patch implements
a new method getDomainInterfacesFromNetworks which is called from
domainGetIfacesInfo after the qemu agent method has been used if
the qemu agent method doesn't return any information.  This new
method builds the interface information from information collected
from the networks themselves.

Change-Id: I5271ee191db93b2b1a0f14dc226d0b945f77a901

3 files changed, 200 insertions, 9 deletions

diff --git a/libvirt/domain.go b/libvirt/domain.go
index 9991dc42..d85a1473 100644
--- a/libvirt/domain.go
+++ b/libvirt/domain.go
@@ -21,8 +21,10 @@ const domWaitLeaseDone = "all-addresses-obtained"
 
 var errDomainInvalidState = errors.New("invalid state for domain")
 
-func domainWaitForLeases(domain *libvirt.Domain, waitForLeases map[libvirtxml.DomainInterface]struct{}, timeout time.Duration) error {
+func domainWaitForLeases(domain *libvirt.Domain, waitForLeases map[libvirtxml.DomainInterface]struct{},
+	timeout time.Duration, domainDef libvirtxml.Domain, virConn *libvirt.Connect) error {
 	waitFunc := func() (interface{}, string, error) {
+
 		state, err := domainGetState(*domain)
 		if err != nil {
 			return false, "", err
@@ -40,7 +42,7 @@ func domainWaitForLeases(domain *libvirt.Domain, waitForLeases map[libvirtxml.Do
 
 		// check we have IPs for all the interfaces we are waiting for
 		for iface := range waitForLeases {
-			found, ignore, err := domainIfaceHasAddress(*domain, iface)
+			found, ignore, err := domainIfaceHasAddress(*domain, iface, domainDef, virConn)
 			if err != nil {
 				return false, "", err
 			}
@@ -72,7 +74,8 @@ func domainWaitForLeases(domain *libvirt.Domain, waitForLeases map[libvirtxml.Do
 	return err
 }
 
-func domainIfaceHasAddress(domain libvirt.Domain, iface libvirtxml.DomainInterface) (found bool, ignore bool, err error) {
+func domainIfaceHasAddress(domain libvirt.Domain, iface libvirtxml.DomainInterface,
+	domainDef libvirtxml.Domain, virConn *libvirt.Connect) (found bool, ignore bool, err error) {
 
 	mac := strings.ToUpper(iface.MAC.Address)
 	if mac == "" {
@@ -82,7 +85,7 @@ func domainIfaceHasAddress(domain libvirt.Domain, iface libvirtxml.DomainInterfa
 	}
 
 	log.Printf("[DEBUG] waiting for network address for iface=%s\n", mac)
-	ifacesWithAddr, err := domainGetIfacesInfo(domain)
+	ifacesWithAddr, err := domainGetIfacesInfo(domain, domainDef, virConn)
 	if err != nil {
 		return false, false, fmt.Errorf("Error retrieving interface addresses: %s", err)
 	}
@@ -140,7 +143,8 @@ func domainIsRunning(domain libvirt.Domain) (bool, error) {
 	return state == libvirt.DOMAIN_RUNNING, nil
 }
 
-func domainGetIfacesInfo(domain libvirt.Domain) ([]libvirt.DomainInterface, error) {
+func domainGetIfacesInfo(domain libvirt.Domain, domainDef libvirtxml.Domain,
+	virConn *libvirt.Connect) ([]libvirt.DomainInterface, error) {
 
 	_, found := os.LookupEnv(skipQemuEnvVar)
 	if found {
@@ -159,6 +163,12 @@ func domainGetIfacesInfo(domain libvirt.Domain) ([]libvirt.DomainInterface, erro
 		}
 	}
 
+	log.Print("[DEBUG] getting domain addresses from networks")
+	interfaces := getDomainInterfacesFromNetworks(domainDef, virConn)
+	if len(interfaces) > 0 {
+		return interfaces, nil
+	}
+
 	// get all the interfaces attached to libvirt networks
 	log.Print("[DEBUG] no interfaces could be obtained with qemu-agent: falling back to the libvirt API")
 	interfaces, err := domain.ListAllInterfaceAddresses(libvirt.DOMAIN_INTERFACE_ADDRESSES_SRC_LEASE)
@@ -177,3 +187,56 @@ func domainGetIfacesInfo(domain libvirt.Domain) ([]libvirt.DomainInterface, erro
 
 	return interfaces, nil
 }
+
+func getDomainInterfacesFromNetworks(domain libvirtxml.Domain,
+	virConn *libvirt.Connect) []libvirt.DomainInterface {
+
+	var ifacesList []libvirt.DomainInterface
+	var networkNames []string
+	var macAddresses []string
+
+	for _, networkInterface := range domain.Devices.Interfaces {
+		networkNames = append(networkNames, networkInterface.Source.Network)
+		macAddresses = append(macAddresses, strings.ToUpper(networkInterface.MAC.Address))
+	}
+
+	networkMacAddresses := make(map[string]map[string][]string)
+	for _, networkName := range networkNames {
+		network, err := virConn.LookupNetworkByName(networkName)
+		if err != nil {
+			log.Printf("[ERROR] Error retrieving libvirt network: %s", err)
+			return ifacesList
+		}
+		defer network.Free()
+
+		networkDef, err := newDefNetworkfromLibvirt(network)
+		macAddresses := make(map[string][]string)
+		for _, ips := range networkDef.IPs {
+			for _, dhcpHost := range ips.DHCP.Hosts {
+				macAddresses[dhcpHost.MAC] = append(macAddresses[dhcpHost.MAC], dhcpHost.IP)
+			}
+		}
+		networkMacAddresses[networkName] = macAddresses
+	}
+
+	for name, networkMacAddress := range networkMacAddresses {
+		for mac, ips := range networkMacAddress {
+			for _, domMac := range macAddresses {
+				if mac == domMac {
+					virDom := libvirt.DomainInterface{}
+					virDom.Name = name
+					virDom.Hwaddr = mac
+					for _, ip := range ips {
+						virDomIP := libvirt.DomainIPAddress{}
+						virDomIP.Addr = ip
+						virDom.Addrs = append(virDom.Addrs, virDomIP)
+					}
+					ifacesList = append(ifacesList, virDom)
+				}
+			}
+		}
+	}
+
+	log.Printf("[DEBUG] Interfaces: %s", spew.Sdump(ifacesList))
+	return ifacesList
+}
diff --git a/libvirt/resource_libvirt_domain.go b/libvirt/resource_libvirt_domain.go
index 616f919b..25516f7b 100644
--- a/libvirt/resource_libvirt_domain.go
+++ b/libvirt/resource_libvirt_domain.go
@@ -593,6 +593,8 @@ func resourceLibvirtDomainCreate(d *schema.ResourceData, meta interface{}) error
 				continue
 			}
 
+			networkXMLDesc, err := network.GetXMLDesc(0)
+			log.Printf("[DEBUG] network def xml in create: %s", networkXMLDesc)
 			hostname := domainDef.Name
 			if hostnameI, ok := d.GetOk(prefix + ".hostname"); ok {
 				hostname = hostnameI.(string)
@@ -605,7 +607,6 @@ func resourceLibvirtDomainCreate(d *schema.ResourceData, meta interface{}) error
 					if ip == nil {
 						return fmt.Errorf("Could not parse addresses '%s'", address)
 					}
-
 					log.Printf("[INFO] Adding IP/MAC/host=%s/%s/%s to %s", ip.String(), mac, hostname, networkName)
 					if err := updateOrAddHost(network, ip.String(), mac, hostname); err != nil {
 						return err
@@ -711,7 +712,7 @@ func resourceLibvirtDomainCreate(d *schema.ResourceData, meta interface{}) error
 	log.Printf("[INFO] Domain ID: %s", d.Id())
 
 	if len(waitForLeases) > 0 {
-		err = domainWaitForLeases(domain, waitForLeases, d.Timeout(schema.TimeoutCreate))
+		err = domainWaitForLeases(domain, waitForLeases, d.Timeout(schema.TimeoutCreate), domainDef, virConn)
 		if err != nil {
 			return err
 		}
@@ -986,8 +987,8 @@ func resourceLibvirtDomainRead(d *schema.ResourceData, meta interface{}) error {
 	}
 	d.Set("filesystems", filesystems)
 
-	// look interfaces with addresses
-	ifacesWithAddr, err := domainGetIfacesInfo(*domain)
+	// lookup interfaces with addresses
+	ifacesWithAddr, err := domainGetIfacesInfo(*domain, domainDef, virConn)
 	if err != nil {
 		return fmt.Errorf("Error retrieving interface addresses: %s", err)
 	}
diff --git a/libvirt/resource_libvirt_domain_test.go b/libvirt/resource_libvirt_domain_test.go
index 43ce2478..ac05c494 100644
--- a/libvirt/resource_libvirt_domain_test.go
+++ b/libvirt/resource_libvirt_domain_test.go
@@ -335,6 +335,65 @@ func TestAccLibvirtDomain_NetworkInterface(t *testing.T) {
 	})
 }
 
+func TestAccLibvirtDomain_CheckDHCPEntries(t *testing.T) {
+	var domain libvirt.Domain
+	var network libvirt.Network
+
+	var configWithDomain = fmt.Sprintf(`
+	    resource "libvirt_network" "acceptance-test-network" {
+		    name = "acceptance-test-network"
+		    mode = "nat"
+		    domain = "acceptance-test-network-local"
+		    addresses = ["192.0.0.0/24"]
+	    }
+
+            resource "libvirt_domain" "acceptance-test-domain" {
+                    name = "terraform-test"
+                    network_interface {
+                            network_id = "${libvirt_network.acceptance-test-network.id}"
+                            hostname = "terraform-test"
+                            addresses = ["192.0.0.2"]
+                    }
+            }`)
+
+	var configWithoutDomain = fmt.Sprintf(`
+	    resource "libvirt_network" "acceptance-test-network" {
+		    name = "acceptance-test-network"
+		    mode = "nat"
+		    domain = "acceptance-test-network-local"
+		    addresses = ["192.0.0.0/24"]
+	    }`)
+
+	resource.Test(t, resource.TestCase{
+		PreCheck:     func() { testAccPreCheck(t) },
+		Providers:    testAccProviders,
+		CheckDestroy: testAccCheckLibvirtDomainDestroy,
+		Steps: []resource.TestStep{
+			resource.TestStep{
+				Config: configWithDomain,
+				Check: resource.ComposeTestCheckFunc(
+					testAccCheckLibvirtDomainExists("libvirt_domain.acceptance-test-domain", &domain),
+					testAccCheckLibvirtNetworkExists("libvirt_network.acceptance-test-network", &network),
+				),
+			},
+			resource.TestStep{
+				Config: configWithoutDomain,
+				Check: resource.ComposeTestCheckFunc(
+					testAccCheckLibvirtDestroyLeavesIPs("libvirt_network.acceptance-test-network",
+						"192.0.0.2", &network),
+				),
+			},
+			resource.TestStep{
+				Config:             configWithDomain,
+				ExpectNonEmptyPlan: true,
+				Check: resource.ComposeTestCheckFunc(
+					testAccCheckLibvirtDomainExists("libvirt_domain.acceptance-test-domain", &domain),
+				),
+			},
+		},
+	})
+}
+
 func TestAccLibvirtDomain_Graphics(t *testing.T) {
 	var domain libvirt.Domain
 
@@ -717,6 +776,38 @@ func testAccCheckLibvirtURLDisk(u *url.URL, domain *libvirt.Domain) resource.Tes
 	}
 }
 
+func testAccCheckLibvirtDestroyLeavesIPs(n string, ip string, network *libvirt.Network) resource.TestCheckFunc {
+	return func(s *terraform.State) error {
+		rs, ok := s.RootModule().Resources[n]
+		if !ok {
+			return fmt.Errorf("Not found: %s", n)
+		}
+
+		if rs.Primary.ID == "" {
+			return fmt.Errorf("No libvirt network ID is set")
+		}
+
+		virConn := testAccProvider.Meta().(*Client).libvirt
+
+		retrieveNetwork, err := virConn.LookupNetworkByUUIDString(rs.Primary.ID)
+
+		if err != nil {
+			return err
+		}
+
+		networkDef, err := newDefNetworkfromLibvirt(retrieveNetwork)
+
+		for _, ips := range networkDef.IPs {
+			for _, dhcpHost := range ips.DHCP.Hosts {
+				if dhcpHost.IP == ip {
+					return nil
+				}
+			}
+		}
+		return fmt.Errorf("Hostname with ip '%s' does not have a dhcp entry in network", ip)
+	}
+}
+
 func testAccCheckLibvirtDomainKernelInitrdCmdline(domain *libvirt.Domain, kernel *libvirt.StorageVol, initrd *libvirt.StorageVol) resource.TestCheckFunc {
 	return func(s *terraform.State) error {
 		xmlDesc, err := domain.GetXMLDesc(0)
@@ -923,3 +1014,39 @@ func TestAccLibvirtDomain_ArchType(t *testing.T) {
 		},
 	})
 }
+
+func testAccCheckLibvirtNetworkExists(n string, network *libvirt.Network) resource.TestCheckFunc {
+	return func(s *terraform.State) error {
+		rs, ok := s.RootModule().Resources[n]
+		if !ok {
+			return fmt.Errorf("Not found: %s", n)
+		}
+
+		if rs.Primary.ID == "" {
+			return fmt.Errorf("No libvirt network ID is set")
+		}
+
+		virConn := testAccProvider.Meta().(*Client).libvirt
+
+		retrieveNetwork, err := virConn.LookupNetworkByUUIDString(rs.Primary.ID)
+
+		if err != nil {
+			return err
+		}
+
+		log.Printf("The ID is %s", rs.Primary.ID)
+
+		realID, err := retrieveNetwork.GetUUIDString()
+		if err != nil {
+			return err
+		}
+
+		if realID != rs.Primary.ID {
+			return fmt.Errorf("Libvirt network not found")
+		}
+
+		network = retrieveNetwork
+
+		return nil
+	}
+}