From f40ab554c0bc16ec68c0a088019a2eca32889752 Mon Sep 17 00:00:00 2001 From: Rob Dobson Date: Tue, 18 Nov 2014 18:18:39 +0000 Subject: [PATCH] Use the Host Internal Management interface to retrieve an instances IP as opposed to relying on the tools being installed. Signed-off-by: Rob Dobson --- builder/xenserver/builder.go | 28 +++-- builder/xenserver/client.go | 79 +++++++++++- builder/xenserver/ssh.go | 113 +++++++++++++++++ builder/xenserver/step_create_instance.go | 3 +- .../xenserver/step_forward_port_over_ssh.go | 79 ++++++++++++ .../step_forward_vnc_port_over_ssh.go | 116 ++---------------- builder/xenserver/step_start_on_himn.go | 96 +++++++++++++++ builder/xenserver/step_wait.go | 42 +------ examples/http/centos6-ks.cfg | 4 - 9 files changed, 403 insertions(+), 157 deletions(-) create mode 100644 builder/xenserver/step_forward_port_over_ssh.go create mode 100644 builder/xenserver/step_start_on_himn.go diff --git a/builder/xenserver/builder.go b/builder/xenserver/builder.go index 6bac879..3284c50 100644 --- a/builder/xenserver/builder.go +++ b/builder/xenserver/builder.go @@ -33,8 +33,8 @@ type config struct { SrUuid string `mapstructure:"sr_uuid"` NetworkUuid string `mapstructure:"network_uuid"` - VncPortMin uint `mapstructure:"vnc_port_min"` - VncPortMax uint `mapstructure:"vnc_port_max"` + HostPortMin uint `mapstructure:"host_port_min"` + HostPortMax uint `mapstructure:"host_port_max"` BootCommand []string `mapstructure:"boot_command"` RawBootWait string `mapstructure:"boot_wait"` @@ -92,12 +92,12 @@ func (self *Builder) Prepare (raws ...interface{}) (params []string, retErr erro // Set default vaules - if self.config.VncPortMin == 0 { - self.config.VncPortMin = 5900 + if self.config.HostPortMin == 0 { + self.config.HostPortMin = 5900 } - if self.config.VncPortMax == 0 { - self.config.VncPortMax = 6000 + if self.config.HostPortMax == 0 { + self.config.HostPortMax = 6000 } if self.config.RawBootWait == "" { @@ -259,9 +259,9 @@ func (self *Builder) Prepare (raws ...interface{}) (params []string, retErr erro errs, errors.New("the HTTP min port must be less than the max")) } - if self.config.VncPortMin > self.config.VncPortMax { + if self.config.HostPortMin > self.config.HostPortMax { errs = packer.MultiErrorAppend( - errs, errors.New("the VNC min port must be less than the max")) + errs, errors.New("the host min port must be less than the max")) } if self.config.ISOChecksumType == "" { @@ -340,15 +340,23 @@ func (self *Builder) Run(ui packer.Ui, hook packer.Hook, cache packer.Cache) (pa }, new(stepPrepareOutputDir), new(stepHTTPServer), - new(stepUploadIso), +// new(stepUploadIso), new(stepCreateInstance), new(stepStartVmPaused), new(stepForwardVncPortOverSsh), new(stepBootWait), new(stepTypeBootCommand), new(stepWait), + new(stepStartOnHIMN), + &stepForwardPortOverSSH{ + RemotePort: himnSSHPort, + RemoteDest: himnSSHIP, + HostPortMin: self.config.HostPortMin, + HostPortMax: self.config.HostPortMax, + ResultKey: "local_ssh_port", + }, &common.StepConnectSSH{ - SSHAddress: sshAddress, + SSHAddress: sshLocalAddress, SSHConfig: sshConfig, SSHWaitTimeout: self.config.sshWaitTimeout, }, diff --git a/builder/xenserver/client.go b/builder/xenserver/client.go index 11117a9..7e40b4a 100644 --- a/builder/xenserver/client.go +++ b/builder/xenserver/client.go @@ -55,6 +55,11 @@ type VBD struct { Client *XenAPIClient } +type VIF struct { + Ref string + Client *XenAPIClient +} + func (c *XenAPIClient) RPCCall (result interface{}, method string, params []interface{}) (err error) { fmt.Println(params) p := new(xmlrpc.Params) @@ -148,6 +153,26 @@ func (client *XenAPIClient) GetNetworkByUuid (network_uuid string) (network *Net return } + +func (client *XenAPIClient) GetNetworkByNameLabel (name_label string) (networks []*Network, err error) { + networks = make([]*Network, 0) + result := APIResult{} + err = client.APICall(&result, "network.get_by_name_label", name_label) + if err != nil { + return networks, err + } + + for _, elem := range result.Value.([]interface{}) { + network := new(Network) + network.Ref = elem.(string) + network.Client = client + networks = append(networks, network) + } + + return networks, nil +} + + func (client *XenAPIClient) GetSRByUuid (sr_uuid string) (sr *SR, err error) { sr = new(SR) result := APIResult{} @@ -276,6 +301,24 @@ func (self *VM) GetVBDs() (vbds []VBD, err error) { return vbds, nil } + +func (self *VM) GetVIFs() (vifs []VIF, err error) { + vifs = make([]VIF, 0) + result := APIResult{} + err = self.Client.APICall(&result, "VM.get_VIFs", self.Ref) + if err != nil { + return vifs, err + } + for _, elem := range result.Value.([]interface{}) { + vif := VIF{} + vif.Ref = elem.(string) + vif.Client = self.Client + vifs = append(vifs, vif) + } + + return vifs, nil +} + func (self *VM) GetDisks() (vdis []*VDI, err error) { // Return just data disks (non-isos) vdis = make([]*VDI, 0) @@ -406,7 +449,7 @@ func (self *VM) SetPlatform(params map[string]string) (err error) { } -func (self *VM) ConnectNetwork (network *Network, device string) (err error) { +func (self *VM) ConnectNetwork (network *Network, device string) (vif *VIF, err error) { // Create the VIF vif_rec := make(xmlrpc.Struct) @@ -423,10 +466,14 @@ func (self *VM) ConnectNetwork (network *Network, device string) (err error) { err = self.Client.APICall(&result, "VIF.create", vif_rec) if err != nil { - return err + return nil, err } - return nil + vif = new(VIF) + vif.Ref = result.Value.(string) + vif.Client = self.Client + + return vif, nil } // Setters @@ -470,6 +517,21 @@ func (self *SR) CreateVdi (name_label, size string) (vdi *VDI, err error) { return } +// Network associated functions + +func (self *Network) GetAssignedIPs () (ip_map map[string]string, err error) { + ip_map = make(map[string]string, 0) + result := APIResult{} + err = self.Client.APICall(&result, "network.get_assigned_ips", self.Ref) + if err != nil { + return ip_map, err + } + for k, v := range result.Value.(xmlrpc.Struct) { + ip_map[k] = v.(string) + } + return ip_map, nil +} + // VBD associated functions func (self *VBD) GetRecord () (record map[string]interface{}, err error) { record = make(map[string]interface{}) @@ -506,6 +568,17 @@ func (self *VBD) Eject () (err error) { return nil } +// VIF associated functions + +func (self *VIF) Destroy () (err error) { + result := APIResult{} + err = self.Client.APICall(&result, "VIF.destroy", self.Ref) + if err != nil { + return err + } + return nil +} + // VDI associated functions func (self *VDI) GetUuid () (vdi_uuid string, err error) { diff --git a/builder/xenserver/ssh.go b/builder/xenserver/ssh.go index 143349d..52ac08c 100644 --- a/builder/xenserver/ssh.go +++ b/builder/xenserver/ssh.go @@ -6,6 +6,11 @@ import ( "github.com/mitchellh/multistep" commonssh "github.com/mitchellh/packer/common/ssh" "github.com/mitchellh/packer/communicator/ssh" + "strings" + "log" + "bytes" + "net" + "io" ) func sshAddress(state multistep.StateBag) (string, error) { @@ -14,6 +19,13 @@ func sshAddress(state multistep.StateBag) (string, error) { return fmt.Sprintf("%s:%d", sshIP, sshHostPort), nil } +func sshLocalAddress(state multistep.StateBag) (string, error) { + sshLocalPort := state.Get("local_ssh_port").(uint) + conn_str := fmt.Sprintf("%s:%d", "127.0.0.1", sshLocalPort) + log.Printf("sshLocalAddress: %s", conn_str) + return conn_str, nil +} + func sshConfig(state multistep.StateBag) (*gossh.ClientConfig, error) { config := state.Get("config").(config) @@ -37,3 +49,104 @@ func sshConfig(state multistep.StateBag) (*gossh.ClientConfig, error) { Auth: auth, }, nil } + + + + +func execute_ssh_cmd (cmd, host, port, username, password string) (stdout string, err error) { + // Setup connection config + config := &gossh.ClientConfig { + User: username, + Auth: []gossh.AuthMethod { + gossh.Password(password), + }, + } + + client, err := gossh.Dial("tcp", host + ":" + port, config) + + if err != nil { + return "", err + } + + //Create session + session, err := client.NewSession() + + if err != nil { + return "", err + } + + defer session.Close() + + var b bytes.Buffer + session.Stdout = &b + if err := session.Run(cmd); err != nil { + return "", err + } + + session.Close() + return strings.Trim(b.String(), "\n"), nil +} + +func forward(local_conn net.Conn, config *gossh.ClientConfig, server, remote_dest string, remote_port uint) { + ssh_client_conn, err := gossh.Dial("tcp", server + ":22", config) + if err != nil { + log.Fatalf("local ssh.Dial error: %s", err) + } + + remote_loc := fmt.Sprintf("%s:%d", remote_dest, remote_port) + ssh_conn, err := ssh_client_conn.Dial("tcp", remote_loc) + if err != nil { + log.Fatalf("ssh.Dial error: %s", err) + } + + go func() { + _, err = io.Copy(ssh_conn, local_conn) + if err != nil { + log.Fatalf("io.copy failed: %v", err) + } + }() + + go func() { + _, err = io.Copy(local_conn, ssh_conn) + if err != nil { + log.Fatalf("io.copy failed: %v", err) + } + }() +} + +func ssh_port_forward(local_port uint, remote_port uint, remote_dest, host, username, password string) (err error) { + + config := &gossh.ClientConfig { + User: username, + Auth: []gossh.AuthMethod{ + gossh.Password(password), + }, + } + + // Listen on a local port + local_listener, err := net.Listen("tcp", + fmt.Sprintf("%s:%d", + "127.0.0.1", + local_port)) + if err != nil { + log.Fatalf("Local listen failed: %s", err) + return err + } + + for { + local_connection, err := local_listener.Accept() + + if err != nil { + log.Fatalf("Local accept failed: %s", err) + return err + } + + + // Forward to a remote port + go forward(local_connection, config, host, remote_dest, remote_port) + } + + + + return nil +} diff --git a/builder/xenserver/step_create_instance.go b/builder/xenserver/step_create_instance.go index 007768a..ed7993d 100644 --- a/builder/xenserver/step_create_instance.go +++ b/builder/xenserver/step_create_instance.go @@ -40,7 +40,7 @@ func (self *stepCreateInstance) Run(state multistep.StateBag) multistep.StepActi if err != nil { ui.Say(err.Error()) } - err = instance.ConnectNetwork(network, "0") + _, err = instance.ConnectNetwork(network, "0") if err != nil { ui.Say(err.Error()) @@ -57,6 +57,7 @@ func (self *stepCreateInstance) Run(state multistep.StateBag) multistep.StepActi // Stash the VM reference self.InstanceId, _ = instance.GetUuid() state.Put("instance_uuid", self.InstanceId) + state.Put("instance", instance) ui.Say(fmt.Sprintf("Created instance '%s'", self.InstanceId)) return multistep.ActionContinue diff --git a/builder/xenserver/step_forward_port_over_ssh.go b/builder/xenserver/step_forward_port_over_ssh.go new file mode 100644 index 0000000..00e8302 --- /dev/null +++ b/builder/xenserver/step_forward_port_over_ssh.go @@ -0,0 +1,79 @@ +package xenserver + +import ( + "github.com/mitchellh/multistep" + "github.com/mitchellh/packer/packer" + "log" + "net" + "fmt" + "time" +) + +type stepForwardPortOverSSH struct { + + RemotePort func (state multistep.StateBag) (uint, error) + RemoteDest func (state multistep.StateBag) (string, error) + + HostPortMin uint + HostPortMax uint + + ResultKey string +} + + +func (self *stepForwardPortOverSSH) Run(state multistep.StateBag) multistep.StepAction { + + config := state.Get("config").(config) + ui := state.Get("ui").(packer.Ui) + + // Find a free local port: + + log.Printf("Looking for an available port between %d and %d", + self.HostPortMin, + self.HostPortMax) + + + var sshHostPort uint + var foundPort bool + + foundPort = false + + for i := self.HostPortMin; i < self.HostPortMax; i++ { + sshHostPort = i + log.Printf("Trying port: %d", sshHostPort) + l, err := net.Listen("tcp", fmt.Sprintf("127.0.0.1:%d", sshHostPort)) + if err == nil { + l.Close() + foundPort = true + break + } + + } + + if !foundPort { + log.Fatal("Error: unable to find free host port. Try providing a larger range") + return multistep.ActionHalt + } + + + ui.Say(fmt.Sprintf("Creating a local port forward over SSH on local port %d", sshHostPort)) + + remotePort, _ := self.RemotePort(state) + remoteDest, _ := self.RemoteDest(state) + + + go ssh_port_forward(sshHostPort, remotePort, remoteDest, config.HostIp, config.Username, config.Password) + ui.Say(fmt.Sprintf("Port forward setup. %d ---> %s:%d on %s", sshHostPort, remoteDest, remotePort, config.HostIp)) + + // Provide the local port to future steps. + state.Put(self.ResultKey, sshHostPort) + + // Need to wait before connecting + // @todo: figure out why and check for the correct conditions. + ui.Message("Sleeping 30...") + time.Sleep(30 * time.Second) + + return multistep.ActionContinue +} + +func (self *stepForwardPortOverSSH) Cleanup(state multistep.StateBag) {} diff --git a/builder/xenserver/step_forward_vnc_port_over_ssh.go b/builder/xenserver/step_forward_vnc_port_over_ssh.go index 72d64ca..7fa4f16 100644 --- a/builder/xenserver/step_forward_vnc_port_over_ssh.go +++ b/builder/xenserver/step_forward_vnc_port_over_ssh.go @@ -5,112 +5,13 @@ import ( "github.com/mitchellh/multistep" "github.com/mitchellh/packer/packer" "fmt" - "code.google.com/p/go.crypto/ssh" - "bytes" - "net" - "io" "strconv" "log" - "strings" ) type stepForwardVncPortOverSsh struct {} -func execute_ssh_cmd (cmd, host, port, username, password string) (stdout string, err error) { - // Setup connection config - config := &ssh.ClientConfig { - User: username, - Auth: []ssh.AuthMethod { - ssh.Password(password), - }, - } - - client, err := ssh.Dial("tcp", host + ":" + port, config) - - if err != nil { - return "", err - } - - //Create session - session, err := client.NewSession() - - if err != nil { - return "", err - } - - defer session.Close() - - var b bytes.Buffer - session.Stdout = &b - if err := session.Run(cmd); err != nil { - return "", err - } - - session.Close() - return strings.Trim(b.String(), "\n"), nil -} - -func forward(local_conn net.Conn, config *ssh.ClientConfig, server, remote_port string) { - ssh_client_conn, err := ssh.Dial("tcp", server + ":22", config) - if err != nil { - log.Fatalf("local ssh.Dial error: %s", err) - } - - ssh_conn, err := ssh_client_conn.Dial("tcp", "127.0.0.1:" + remote_port) - if err != nil { - log.Fatalf("ssh.Dial error: %s", err) - } - - go func() { - _, err = io.Copy(ssh_conn, local_conn) - if err != nil { - log.Fatalf("io.copy failed: %v", err) - } - }() - - go func() { - _, err = io.Copy(local_conn, ssh_conn) - if err != nil { - log.Fatalf("io.copy failed: %v", err) - } - }() -} - -func ssh_port_forward(local_port, remote_port, host, username, password string) (err error) { - - config := &ssh.ClientConfig { - User: username, - Auth: []ssh.AuthMethod{ - ssh.Password(password), - }, - } - - // Listen on a local port - local_listener, err := net.Listen("tcp", "127.0.0.1:" + local_port) - if err != nil { - log.Fatalf("Local listen failed: %s", err) - return err - } - - for { - local_connection, err := local_listener.Accept() - - if err != nil { - log.Fatalf("Local accept failed: %s", err) - return err - } - - - // Forward to a remote port - go forward(local_connection, config, host, remote_port) - } - - - - return nil -} - func (self *stepForwardVncPortOverSsh) Run(state multistep.StateBag) multistep.StepAction { // client := state.Get("client").(XenAPIClient) @@ -125,14 +26,23 @@ func (self *stepForwardVncPortOverSsh) Run(state multistep.StateBag) multistep.S remote_vncport, _ := execute_ssh_cmd(cmd, config.HostIp, "22", config.Username, config.Password) + remote_port, err := strconv.ParseUint(remote_vncport, 10, 16) + remote_port_uint := uint(remote_port) + if err != nil { + log.Fatal(err.Error()) + log.Fatal(fmt.Sprintf("Unable to convert '%s' to an int", remote_vncport)) + return multistep.ActionHalt + } + ui.Say("The VNC port is " + remote_vncport) // Just take the min port for the moment - state.Put("local_vnc_port", config.VncPortMin) - local_port := strconv.Itoa(int(config.VncPortMin)) - ui.Say("About to setup SSH Port forward setup on local port " + local_port) + state.Put("local_vnc_port", config.HostPortMin) + //local_port := strconv.Itoa(int(config.HostPortMin)) + local_port := config.HostPortMin + ui.Say(fmt.Sprintf("About to setup SSH Port forward setup on local port %d",local_port)) - go ssh_port_forward(local_port, remote_vncport, config.HostIp, config.Username, config.Password) + go ssh_port_forward(local_port, remote_port_uint, "127.0.0.1", config.HostIp, config.Username, config.Password) ui.Say("Port forward setup.") return multistep.ActionContinue diff --git a/builder/xenserver/step_start_on_himn.go b/builder/xenserver/step_start_on_himn.go new file mode 100644 index 0000000..d3c03d6 --- /dev/null +++ b/builder/xenserver/step_start_on_himn.go @@ -0,0 +1,96 @@ +package xenserver + +import ( + "github.com/mitchellh/multistep" + "github.com/mitchellh/packer/packer" + "time" + "log" +) + +type stepStartOnHIMN struct{} + + +/* + * This step starts the installed guest on the Host Internal Management Network + * as there exists an API to obtain the IP allocated to the VM by XAPI. + * This in turn will allow Packer to SSH into the VM, provided NATing has been + * enabled on the host. + * + */ + +func (self *stepStartOnHIMN) Run(state multistep.StateBag) multistep.StepAction { + + ui := state.Get("ui").(packer.Ui) + client := state.Get("client").(XenAPIClient) + + ui.Say("Step: Start VM on the Host Internal Mangement Network") + + instance := state.Get("instance").(*VM) + + // Find the HIMN Ref + networks, err := client.GetNetworkByNameLabel("Host internal management network") + if err != nil || len(networks) == 0 { + log.Fatal("Unable to find a host internal management network") + log.Fatal(err.Error()) + return multistep.ActionHalt + } + + + himn := networks[0] + + // Create a VIF for the HIMN + himn_vif, err := instance.ConnectNetwork(himn, "0") + if err != nil { + log.Fatal("Error creating VIF") + log.Fatal(err.Error()) + return multistep.ActionHalt + } + + // Start the VM + instance.Start(false, false) + + + var himn_iface_ip string = "" + + // Obtain the allocated IP + for i:=0; i < 10; i++ { + ips, _ := himn.GetAssignedIPs() + log.Printf("IPs: %s", ips) + log.Printf("Ref: %s", instance.Ref) + + //Check for instance.Ref in map + if vm_ip, ok := ips[himn_vif.Ref]; ok { + ui.Say("Found the VM's IP " + vm_ip) + himn_iface_ip = vm_ip + break + } + + ui.Say("Wait for IP address...") + time.Sleep(10*time.Second) + + } + + if himn_iface_ip != "" { + state.Put("himn_ssh_address", himn_iface_ip) + ui.Say("Stored VM's IP " + himn_iface_ip) + } else { + log.Fatal("Unable to find an IP on the Host-internal management interface") + return multistep.ActionHalt + } + + return multistep.ActionContinue + +} + +func (self *stepStartOnHIMN) Cleanup(state multistep.StateBag) {} + + +func himnSSHIP (state multistep.StateBag) (string, error) { + ip := state.Get("himn_ssh_address").(string) + return ip, nil +} + + +func himnSSHPort (state multistep.StateBag) (uint, error) { + return 22, nil +} diff --git a/builder/xenserver/step_wait.go b/builder/xenserver/step_wait.go index ab53e97..b056bc4 100644 --- a/builder/xenserver/step_wait.go +++ b/builder/xenserver/step_wait.go @@ -5,7 +5,6 @@ import ( "github.com/mitchellh/packer/packer" "time" "reflect" - "github.com/nilshell/xmlrpc" ) @@ -34,7 +33,7 @@ func (self *stepWait) Run(state multistep.StateBag) multistep.StepAction { } } - //Eject ISO from drive and start VM + // Eject ISO from drive vbds, _ := instance.GetVBDs() for _, vbd := range vbds { rec, _ := vbd.GetRecord() @@ -53,42 +52,13 @@ func (self *stepWait) Run(state multistep.StateBag) multistep.StepAction { } } - ui.Say("Starting VM...") - instance.Start(false, false) - - - vm_ip := "" - for i:=0; i < 10; i++ { - ref, _ := instance.GetGuestMetricsRef() - - if ref != "OpaqueRef:NULL" { - metrics, _ := instance.GetGuestMetrics() - // todo: xmlrpc shouldn't be needed here - networks := metrics["networks"].(xmlrpc.Struct) - for k, v := range networks { - if k == "0/ip" && v.(string) != "" { - vm_ip = v.(string) - break - } - } - - } - - // Check if an IP has been returned yet - if vm_ip != "" { - break - } - - ui.Say("Wait for IP address...") - time.Sleep(10*time.Second) + // Destroy all connected VIFs + vifs, _ := instance.GetVIFs() + for _, vif := range vifs { + ui.Message("Destroying VIF " + vif.Ref) + vif.Destroy() } - - // Pass on the VM's IP - state.Put("ssh_address", vm_ip) - ui.Say("Found the VM's IP " + vm_ip) - - return multistep.ActionContinue } diff --git a/examples/http/centos6-ks.cfg b/examples/http/centos6-ks.cfg index 4259930..26d933c 100644 --- a/examples/http/centos6-ks.cfg +++ b/examples/http/centos6-ks.cfg @@ -38,10 +38,6 @@ yum -y update # update root certs wget -O/etc/pki/tls/certs/ca-bundle.crt http://curl.haxx.se/ca/cacert.pem -# xe-guest-utilities -rpm -Uhv http://cam-st05.uk.xensource.com/xe-guest-utilities-xenstore-6.4.93-1292.x86_64.rpm -rpm -Uvh http://cam-st05.uk.xensource.com/xe-guest-utilities-6.4.93-1292.x86_64.rpm - # vagrant groupadd vagrant -g 999 useradd vagrant -g vagrant -G wheel -u 900 -s /bin/bash