From 1130b3a864432adf2d17c9821c8ed84ba10ffc36 Mon Sep 17 00:00:00 2001 From: Yongkun Anfernee Gui Date: Tue, 24 Oct 2017 13:29:58 -0700 Subject: [PATCH] Add vmware driver which supports Fusion and Workstation VMware Fusion is available on MacOS. VMware Workstation is available on both Linux and Windows. vmwarefusion is still supported, but it only works for fusion. It should be deprecated in future. --- pkg/drivers/vmware/driver.go | 775 ++++++++++++++++++ pkg/drivers/vmware/driver_test.go | 133 +++ pkg/drivers/vmware/vmrun.go | 94 +++ pkg/drivers/vmware/vmware.go | 40 + pkg/drivers/vmware/vmware_darwin.go | 53 ++ pkg/drivers/vmware/vmware_linux.go | 42 + pkg/drivers/vmware/vmware_windows.go | 53 ++ pkg/drivers/vmware/vmx.go | 88 ++ pkg/minikube/cluster/cluster.go | 22 +- pkg/minikube/machine/client_darwin.go | 3 + pkg/minikube/machine/client_linux.go | 3 + pkg/minikube/machine/client_windows.go | 3 + .../drivers/plugin/localbinary/plugin.go | 2 +- 13 files changed, 1309 insertions(+), 2 deletions(-) create mode 100644 pkg/drivers/vmware/driver.go create mode 100644 pkg/drivers/vmware/driver_test.go create mode 100644 pkg/drivers/vmware/vmrun.go create mode 100644 pkg/drivers/vmware/vmware.go create mode 100644 pkg/drivers/vmware/vmware_darwin.go create mode 100644 pkg/drivers/vmware/vmware_linux.go create mode 100644 pkg/drivers/vmware/vmware_windows.go create mode 100644 pkg/drivers/vmware/vmx.go diff --git a/pkg/drivers/vmware/driver.go b/pkg/drivers/vmware/driver.go new file mode 100644 index 000000000000..db728015f124 --- /dev/null +++ b/pkg/drivers/vmware/driver.go @@ -0,0 +1,775 @@ +/* +Copyright 2017 The Kubernetes Authors All rights reserved. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +/* + * Copyright 2017 VMware, Inc. All rights reserved. Licensed under the Apache v2 License. + */ + +package vmware + +import ( + "archive/tar" + "bytes" + "fmt" + "io/ioutil" + "net" + "os" + "path/filepath" + "regexp" + "runtime" + "strings" + "text/template" + "time" + + "github.com/docker/machine/libmachine/drivers" + "github.com/docker/machine/libmachine/log" + "github.com/docker/machine/libmachine/mcnflag" + "github.com/docker/machine/libmachine/mcnutils" + "github.com/docker/machine/libmachine/ssh" + "github.com/docker/machine/libmachine/state" + cryptossh "golang.org/x/crypto/ssh" +) + +const ( + B2DUser = "docker" + B2DPass = "tcuser" + isoFilename = "boot2docker.iso" + isoConfigDrive = "configdrive.iso" +) + +// Driver for VMware +type Driver struct { + *drivers.BaseDriver + Memory int + DiskSize int + CPU int + ISO string + Boot2DockerURL string + + SSHPassword string + ConfigDriveISO string + ConfigDriveURL string + NoShare bool +} + +const ( + defaultSSHUser = B2DUser + defaultSSHPass = B2DPass + defaultDiskSize = 20000 + defaultCPU = 1 + defaultMemory = 1024 +) + +// GetCreateFlags registers the flags this driver adds to +// "docker hosts create" +func (d *Driver) GetCreateFlags() []mcnflag.Flag { + return []mcnflag.Flag{ + mcnflag.StringFlag{ + EnvVar: "VMWARE_BOOT2DOCKER_URL", + Name: "vmware-boot2docker-url", + Usage: "URL for boot2docker image", + Value: "", + }, + mcnflag.StringFlag{ + EnvVar: "VMWARE_CONFIGDRIVE_URL", + Name: "vmware-configdrive-url", + Usage: "URL for cloud-init configdrive", + Value: "", + }, + mcnflag.IntFlag{ + EnvVar: "VMWARE_CPU_COUNT", + Name: "vmware-cpu-count", + Usage: "number of CPUs for the machine (-1 to use the number of CPUs available)", + Value: defaultCPU, + }, + mcnflag.IntFlag{ + EnvVar: "VMWARE_MEMORY_SIZE", + Name: "vmware-memory-size", + Usage: "size of memory for host VM (in MB)", + Value: defaultMemory, + }, + mcnflag.IntFlag{ + EnvVar: "VMWARE_DISK_SIZE", + Name: "vmware-disk-size", + Usage: "size of disk for host VM (in MB)", + Value: defaultDiskSize, + }, + mcnflag.StringFlag{ + EnvVar: "VMWARE_SSH_USER", + Name: "vmware-ssh-user", + Usage: "SSH user", + Value: defaultSSHUser, + }, + mcnflag.StringFlag{ + EnvVar: "VMWARE_SSH_PASSWORD", + Name: "vmware-ssh-password", + Usage: "SSH password", + Value: defaultSSHPass, + }, + mcnflag.BoolFlag{ + EnvVar: "VMWARE_NO_SHARE", + Name: "vmware-no-share", + Usage: "Disable the mount of your home directory", + }, + } +} + +func NewDriver(hostName, storePath string) drivers.Driver { + return &Driver{ + CPU: defaultCPU, + Memory: defaultMemory, + DiskSize: defaultDiskSize, + SSHPassword: defaultSSHPass, + BaseDriver: &drivers.BaseDriver{ + SSHUser: defaultSSHUser, + MachineName: hostName, + StorePath: storePath, + }, + } +} + +func (d *Driver) GetSSHHostname() (string, error) { + return d.GetIP() +} + +func (d *Driver) GetSSHUsername() string { + if d.SSHUser == "" { + d.SSHUser = "docker" + } + + return d.SSHUser +} + +// DriverName returns the name of the driver +func (d *Driver) DriverName() string { + return "vmware" +} + +func (d *Driver) SetConfigFromFlags(flags drivers.DriverOptions) error { + d.Memory = flags.Int("vmware-memory-size") + d.CPU = flags.Int("vmware-cpu-count") + d.DiskSize = flags.Int("vmware-disk-size") + d.Boot2DockerURL = flags.String("vmware-boot2docker-url") + d.ConfigDriveURL = flags.String("vmware-configdrive-url") + d.ISO = d.ResolveStorePath(isoFilename) + d.ConfigDriveISO = d.ResolveStorePath(isoConfigDrive) + d.SetSwarmConfigFromFlags(flags) + d.SSHUser = flags.String("vmware-ssh-user") + d.SSHPassword = flags.String("vmware-ssh-password") + d.SSHPort = 22 + d.NoShare = flags.Bool("vmware-no-share") + + // We support a maximum of 16 cpu to be consistent with Virtual Hardware 10 + // specs. + if d.CPU < 1 { + d.CPU = int(runtime.NumCPU()) + } + if d.CPU > 16 { + d.CPU = 16 + } + + return nil +} + +func (d *Driver) GetURL() (string, error) { + ip, err := d.GetIP() + if err != nil { + return "", err + } + if ip == "" { + return "", nil + } + return fmt.Sprintf("tcp://%s", net.JoinHostPort(ip, "2376")), nil +} + +func (d *Driver) GetIP() (string, error) { + s, err := d.GetState() + if err != nil { + return "", err + } + + if s != state.Running { + return "", drivers.ErrHostIsNotRunning + } + + // determine MAC address for VM + macaddr, err := d.getMacAddressFromVmx() + if err != nil { + return "", err + } + + // attempt to find the address in the vmnet configuration + if ip, err := d.getIPfromVmnetConfiguration(macaddr); err == nil { + return ip, err + } + + // address not found in vmnet so look for a DHCP lease + ip, err := d.getIPfromDHCPLease(macaddr) + if err != nil { + return "", err + } + + return ip, nil +} + +func (d *Driver) GetState() (state.State, error) { + // VMRUN only tells use if the vm is running or not + vmxp, err := filepath.EvalSymlinks(d.vmxPath()) + if err != nil { + return state.Error, err + } + + if stdout, _, _ := vmrun("list"); strings.Contains(stdout, vmxp) { + return state.Running, nil + } + return state.Stopped, nil +} + +// PreCreateCheck checks that the machine creation process can be started safely. +func (d *Driver) PreCreateCheck() error { + // Downloading boot2docker to cache should be done here to make sure + // that a download failure will not leave a machine half created. + b2dutils := mcnutils.NewB2dUtils(d.StorePath) + return b2dutils.UpdateISOCache(d.Boot2DockerURL) +} + +func (d *Driver) Create() error { + os.MkdirAll(filepath.Join(d.StorePath, "machines", d.GetMachineName()), 0755) + + b2dutils := mcnutils.NewB2dUtils(d.StorePath) + if err := b2dutils.CopyIsoToMachineDir(d.Boot2DockerURL, d.MachineName); err != nil { + return err + } + + // download cloud-init config drive + if d.ConfigDriveURL != "" { + if err := b2dutils.DownloadISO(d.ResolveStorePath("."), isoConfigDrive, d.ConfigDriveURL); err != nil { + return err + } + } + + log.Infof("Creating SSH key...") + if err := ssh.GenerateSSHKey(d.GetSSHKeyPath()); err != nil { + return err + } + + log.Infof("Creating VM...") + if err := os.MkdirAll(d.ResolveStorePath("."), 0755); err != nil { + return err + } + + if _, err := os.Stat(d.vmxPath()); err == nil { + return ErrMachineExist + } + + // Generate vmx config file from template + vmxt := template.Must(template.New("vmx").Parse(vmx)) + vmxfile, err := os.Create(d.vmxPath()) + if err != nil { + return err + } + vmxt.Execute(vmxfile, d) + + // Generate vmdk file + diskImg := d.ResolveStorePath(fmt.Sprintf("%s.vmdk", d.MachineName)) + if _, err := os.Stat(diskImg); err != nil { + if !os.IsNotExist(err) { + return err + } + + if err := vdiskmanager(diskImg, d.DiskSize); err != nil { + return err + } + } + + log.Infof("Starting %s...", d.MachineName) + vmrun("start", d.vmxPath(), "nogui") + + var ip string + + log.Infof("Waiting for VM to come online...") + for i := 1; i <= 60; i++ { + ip, err = d.GetIP() + if err != nil { + log.Debugf("Not there yet %d/%d, error: %s", i, 60, err) + time.Sleep(2 * time.Second) + continue + } + + if ip != "" { + log.Debugf("Got an ip: %s", ip) + conn, err := net.DialTimeout("tcp", fmt.Sprintf("%s:%d", ip, 22), time.Duration(2*time.Second)) + if err != nil { + log.Debugf("SSH Daemon not responding yet: %s", err) + time.Sleep(2 * time.Second) + continue + } + conn.Close() + break + } + } + + if ip == "" { + return fmt.Errorf("Machine didn't return an IP after 120 seconds, aborting") + } + + // we got an IP, let's copy ssh keys over + d.IPAddress = ip + + // Do not execute the rest of boot2docker specific configuration + // The upload of the public ssh key uses a ssh connection, + // this works without installed vmware client tools + if d.ConfigDriveURL != "" { + var keyfh *os.File + var keycontent []byte + + log.Infof("Copy public SSH key to %s [%s]", d.MachineName, d.IPAddress) + + // create .ssh folder in users home + if err := executeSSHCommand(fmt.Sprintf("mkdir -p /home/%s/.ssh", d.SSHUser), d); err != nil { + return err + } + + // read generated public ssh key + if keyfh, err = os.Open(d.publicSSHKeyPath()); err != nil { + return err + } + defer keyfh.Close() + + if keycontent, err = ioutil.ReadAll(keyfh); err != nil { + return err + } + + // add public ssh key to authorized_keys + if err := executeSSHCommand(fmt.Sprintf("echo '%s' > /home/%s/.ssh/authorized_keys", string(keycontent), d.SSHUser), d); err != nil { + return err + } + + // make it secure + if err := executeSSHCommand(fmt.Sprintf("chmod 600 /home/%s/.ssh/authorized_keys", d.SSHUser), d); err != nil { + return err + } + + log.Debugf("Leaving create sequence early, configdrive found") + return nil + } + + // Generate a tar keys bundle + if err := d.generateKeyBundle(); err != nil { + return err + } + + // Test if /var/lib/boot2docker exists + vmrun("-gu", B2DUser, "-gp", B2DPass, "directoryExistsInGuest", d.vmxPath(), "/var/lib/boot2docker") + + // Copy SSH keys bundle + vmrun("-gu", B2DUser, "-gp", B2DPass, "CopyFileFromHostToGuest", d.vmxPath(), d.ResolveStorePath("userdata.tar"), "/home/docker/userdata.tar") + + // Expand tar file. + vmrun("-gu", B2DUser, "-gp", B2DPass, "runScriptInGuest", d.vmxPath(), "/bin/sh", "sudo sh -c \"tar xvf /home/docker/userdata.tar -C /home/docker > /var/log/userdata.log 2>&1 && chown -R docker:staff /home/docker\"") + + // copy to /var/lib/boot2docker + vmrun("-gu", B2DUser, "-gp", B2DPass, "runScriptInGuest", d.vmxPath(), "/bin/sh", "sudo /bin/mv /home/docker/userdata.tar /var/lib/boot2docker/userdata.tar") + + // Enable Shared Folders + vmrun("-gu", B2DUser, "-gp", B2DPass, "enableSharedFolders", d.vmxPath()) + + shareName := "Users" + shareDir := "/Users" + + if shareDir != "" && !d.NoShare { + if _, err := os.Stat(shareDir); err != nil && !os.IsNotExist(err) { + return err + } else if !os.IsNotExist(err) { + // add shared folder, create mountpoint and mount it. + vmrun("-gu", B2DUser, "-gp", B2DPass, "addSharedFolder", d.vmxPath(), shareName, shareDir) + command := "[ ! -d " + shareDir + " ]&& sudo mkdir " + shareDir + "; sudo mount --bind /mnt/hgfs/" + shareDir + " " + shareDir + " || [ -f /usr/local/bin/vmhgfs-fuse ]&& sudo /usr/local/bin/vmhgfs-fuse -o allow_other .host:/" + shareName + " " + shareDir + " || sudo mount -t vmhgfs -o uid=$(id -u),gid=$(id -g) .host:/" + shareName + " " + shareDir + vmrun("-gu", B2DUser, "-gp", B2DPass, "runScriptInGuest", d.vmxPath(), "/bin/sh", command) + } + } + return nil +} + +func (d *Driver) Start() error { + vmrun("start", d.vmxPath(), "nogui") + + // Do not execute the rest of boot2docker specific configuration, exit here + if d.ConfigDriveURL != "" { + log.Debugf("Leaving start sequence early, configdrive found") + return nil + } + + log.Debugf("Mounting Shared Folders...") + var shareName, shareDir string // TODO configurable at some point + switch runtime.GOOS { + case "darwin": + shareName = "Users" + shareDir = "/Users" + // TODO "linux" and "windows" + } + + if shareDir != "" { + if _, err := os.Stat(shareDir); err != nil && !os.IsNotExist(err) { + return err + } else if !os.IsNotExist(err) { + // create mountpoint and mount shared folder + command := "[ ! -d " + shareDir + " ]&& sudo mkdir " + shareDir + "; sudo mount --bind /mnt/hgfs/" + shareDir + " " + shareDir + " || [ -f /usr/local/bin/vmhgfs-fuse ]&& sudo /usr/local/bin/vmhgfs-fuse -o allow_other .host:/" + shareName + " " + shareDir + " || sudo mount -t vmhgfs -o uid=$(id -u),gid=$(id -g) .host:/" + shareName + " " + shareDir + vmrun("-gu", B2DUser, "-gp", B2DPass, "runScriptInGuest", d.vmxPath(), "/bin/sh", command) + } + } + + return nil +} + +func (d *Driver) Stop() error { + _, _, err := vmrun("stop", d.vmxPath(), "nogui") + return err +} + +func (d *Driver) Restart() error { + // Stop VM gracefully + if err := d.Stop(); err != nil { + return err + } + // Start it again and mount shared folder + return d.Start() +} + +func (d *Driver) Kill() error { + _, _, err := vmrun("stop", d.vmxPath(), "hard nogui") + return err +} + +func (d *Driver) Remove() error { + s, _ := d.GetState() + if s == state.Running { + if err := d.Kill(); err != nil { + return fmt.Errorf("Error stopping VM before deletion") + } + } + log.Infof("Deleting %s...", d.MachineName) + vmrun("deleteVM", d.vmxPath(), "nogui") + return nil +} + +func (d *Driver) Upgrade() error { + return fmt.Errorf("VMware does not currently support the upgrade operation") +} + +func (d *Driver) vmxPath() string { + return d.ResolveStorePath(fmt.Sprintf("%s.vmx", d.MachineName)) +} + +func (d *Driver) vmdkPath() string { + return d.ResolveStorePath(fmt.Sprintf("%s.vmdk", d.MachineName)) +} + +func (d *Driver) getMacAddressFromVmx() (string, error) { + var vmxfh *os.File + var vmxcontent []byte + var err error + + if vmxfh, err = os.Open(d.vmxPath()); err != nil { + return "", err + } + defer vmxfh.Close() + + if vmxcontent, err = ioutil.ReadAll(vmxfh); err != nil { + return "", err + } + + // Look for generatedAddress as we're passing a VMX with addressType = "generated". + var macaddr string + vmxparse := regexp.MustCompile(`^ethernet0.generatedAddress\s*=\s*"(.*?)"\s*$`) + for _, line := range strings.Split(string(vmxcontent), "\n") { + if matches := vmxparse.FindStringSubmatch(line); matches == nil { + continue + } else { + macaddr = strings.ToLower(matches[1]) + } + } + + if macaddr == "" { + return "", fmt.Errorf("couldn't find MAC address in VMX file %s", d.vmxPath()) + } + + log.Debugf("MAC address in VMX: %s", macaddr) + + return macaddr, nil +} + +func (d *Driver) getIPfromVmnetConfiguration(macaddr string) (string, error) { + + // DHCP lease table for NAT vmnet interface + confFiles, _ := filepath.Glob(DhcpConfigFiles()) + for _, conffile := range confFiles { + log.Debugf("Trying to find IP address in configuration file: %s", conffile) + if ipaddr, err := d.getIPfromVmnetConfigurationFile(conffile, macaddr); err == nil { + return ipaddr, err + } + } + + return "", fmt.Errorf("IP not found for MAC %s in vmnet configuration files", macaddr) +} + +func (d *Driver) getIPfromVmnetConfigurationFile(conffile, macaddr string) (string, error) { + var conffh *os.File + var confcontent []byte + + var currentip string + var lastipmatch string + var lastmacmatch string + + var err error + + if conffh, err = os.Open(conffile); err != nil { + return "", err + } + defer conffh.Close() + + if confcontent, err = ioutil.ReadAll(conffh); err != nil { + return "", err + } + + // find all occurrences of 'host .* { .. }' and extract + // out of the inner block the MAC and IP addresses + + // key = MAC, value = IP + m := make(map[string]string) + + // Begin of a host block, that contains the IP, MAC + hostbegin := regexp.MustCompile(`^host (.+?) {`) + // End of a host block + hostend := regexp.MustCompile(`^}`) + + // Get the IP address. + ip := regexp.MustCompile(`^\s*fixed-address (.+?);\r?$`) + // Get the MAC address associated. + mac := regexp.MustCompile(`^\s*hardware ethernet (.+?);\r?$`) + + // we use a block depth so that just in case inner blocks exists + // we are not being fooled by them + blockdepth := 0 + for _, line := range strings.Split(string(confcontent), "\n") { + + if matches := hostbegin.FindStringSubmatch(line); matches != nil { + blockdepth = blockdepth + 1 + continue + } + + // we are only in interested in endings if we in a block. Otherwise we will count + // ending of non host blocks as well + if matches := hostend.FindStringSubmatch(line); blockdepth > 0 && matches != nil { + blockdepth = blockdepth - 1 + + if blockdepth == 0 { + // add data + m[lastmacmatch] = lastipmatch + + // reset all temp var holders + lastipmatch = "" + lastmacmatch = "" + } + + continue + } + + // only if we are within the first level of a block + // we are looking for addresses to extract + if blockdepth == 1 { + if matches := ip.FindStringSubmatch(line); matches != nil { + lastipmatch = matches[1] + continue + } + + if matches := mac.FindStringSubmatch(line); matches != nil { + lastmacmatch = strings.ToLower(matches[1]) + continue + } + } + } + + log.Debugf("Following IPs found %s", m) + + // map is filled to now lets check if we have a MAC associated to an IP + currentip, ok := m[strings.ToLower(macaddr)] + + if !ok { + return "", fmt.Errorf("IP not found for MAC %s in vmnet configuration", macaddr) + } + + log.Debugf("IP found in vmnet configuration file: %s", currentip) + + return currentip, nil + +} + +func (d *Driver) getIPfromDHCPLease(macaddr string) (string, error) { + + // DHCP lease table for NAT vmnet interface + leasesFiles, _ := filepath.Glob(DhcpLeaseFiles()) + for _, dhcpfile := range leasesFiles { + log.Debugf("Trying to find IP address in leases file: %s", dhcpfile) + if ipaddr, err := d.getIPfromDHCPLeaseFile(dhcpfile, macaddr); err == nil { + return ipaddr, err + } + } + + return "", fmt.Errorf("IP not found for MAC %s in DHCP leases", macaddr) +} + +func (d *Driver) getIPfromDHCPLeaseFile(dhcpfile, macaddr string) (string, error) { + var dhcpfh *os.File + var dhcpcontent []byte + var lastipmatch string + var currentip string + var lastleaseendtime time.Time + var currentleadeendtime time.Time + var err error + + if dhcpfh, err = os.Open(dhcpfile); err != nil { + return "", err + } + defer dhcpfh.Close() + + if dhcpcontent, err = ioutil.ReadAll(dhcpfh); err != nil { + return "", err + } + + // Get the IP from the lease table. + leaseip := regexp.MustCompile(`^lease (.+?) {\r?$`) + // Get the lease end date time. + leaseend := regexp.MustCompile(`^\s*ends \d (.+?);\r?$`) + // Get the MAC address associated. + leasemac := regexp.MustCompile(`^\s*hardware ethernet (.+?);\r?$`) + + for _, line := range strings.Split(string(dhcpcontent), "\n") { + + if matches := leaseip.FindStringSubmatch(line); matches != nil { + lastipmatch = matches[1] + continue + } + + if matches := leaseend.FindStringSubmatch(line); matches != nil { + lastleaseendtime, _ = time.Parse("2006/01/02 15:04:05", matches[1]) + continue + } + + if matches := leasemac.FindStringSubmatch(line); matches != nil && matches[1] == macaddr && currentleadeendtime.Before(lastleaseendtime) { + currentip = lastipmatch + currentleadeendtime = lastleaseendtime + } + } + + if currentip == "" { + return "", fmt.Errorf("IP not found for MAC %s in DHCP leases", macaddr) + } + + log.Debugf("IP found in DHCP lease table: %s", currentip) + + return currentip, nil +} + +func (d *Driver) publicSSHKeyPath() string { + return d.GetSSHKeyPath() + ".pub" +} + +// Make a boot2docker userdata.tar key bundle +func (d *Driver) generateKeyBundle() error { + log.Debugf("Creating Tar key bundle...") + + magicString := "boot2docker, this is vmware speaking" + + tf, err := os.Create(d.ResolveStorePath("userdata.tar")) + if err != nil { + return err + } + defer tf.Close() + var fileWriter = tf + + tw := tar.NewWriter(fileWriter) + defer tw.Close() + + // magicString first so we can figure out who originally wrote the tar. + file := &tar.Header{Name: magicString, Size: int64(len(magicString))} + if err := tw.WriteHeader(file); err != nil { + return err + } + if _, err := tw.Write([]byte(magicString)); err != nil { + return err + } + // .ssh/key.pub => authorized_keys + file = &tar.Header{Name: ".ssh", Typeflag: tar.TypeDir, Mode: 0700} + if err := tw.WriteHeader(file); err != nil { + return err + } + pubKey, err := ioutil.ReadFile(d.publicSSHKeyPath()) + if err != nil { + return err + } + file = &tar.Header{Name: ".ssh/authorized_keys", Size: int64(len(pubKey)), Mode: 0644} + if err := tw.WriteHeader(file); err != nil { + return err + } + if _, err := tw.Write([]byte(pubKey)); err != nil { + return err + } + file = &tar.Header{Name: ".ssh/authorized_keys2", Size: int64(len(pubKey)), Mode: 0644} + if err := tw.WriteHeader(file); err != nil { + return err + } + if _, err := tw.Write([]byte(pubKey)); err != nil { + return err + } + + return tw.Close() +} + +// execute command over SSH with user / password authentication +func executeSSHCommand(command string, d *Driver) error { + log.Debugf("Execute executeSSHCommand: %s", command) + + config := &cryptossh.ClientConfig{ + User: d.SSHUser, + Auth: []cryptossh.AuthMethod{ + cryptossh.Password(d.SSHPassword), + }, + } + + client, err := cryptossh.Dial("tcp", fmt.Sprintf("%s:%d", d.IPAddress, d.SSHPort), config) + if err != nil { + log.Debugf("Failed to dial:", err) + return err + } + + session, err := client.NewSession() + if err != nil { + log.Debugf("Failed to create session: " + err.Error()) + return err + } + defer session.Close() + + var b bytes.Buffer + session.Stdout = &b + + if err := session.Run(command); err != nil { + log.Debugf("Failed to run: " + err.Error()) + return err + } + log.Debugf("Stdout from executeSSHCommand: %s", b.String()) + + return nil +} diff --git a/pkg/drivers/vmware/driver_test.go b/pkg/drivers/vmware/driver_test.go new file mode 100644 index 000000000000..af1580516119 --- /dev/null +++ b/pkg/drivers/vmware/driver_test.go @@ -0,0 +1,133 @@ +/* +Copyright 2017 The Kubernetes Authors All rights reserved. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package vmware + +import ( + "io/ioutil" + "log" + "os" + "testing" + + "github.com/docker/machine/libmachine/drivers" + "github.com/docker/machine/libmachine/state" +) + +var skip = !check(vmrunbin) || !check(vdiskmanbin) + +func check(path string) bool { + _, err := os.Stat(path) + if err != nil { + log.Printf("%q is missing", path) + return false + } + + return true +} + +func TestSetConfigFromFlags(t *testing.T) { + driver := NewDriver("default", "path") + + checkFlags := &drivers.CheckDriverOptions{ + FlagsValues: map[string]interface{}{}, + CreateFlags: driver.GetCreateFlags(), + } + + err := driver.SetConfigFromFlags(checkFlags) + if err != nil { + t.Fatal(err) + } + + if len(checkFlags.InvalidFlags) != 0 { + t.Fatalf("expect len(checkFlags.InvalidFlags) == 0; got %d", len(checkFlags.InvalidFlags)) + } +} + +func TestDriver(t *testing.T) { + if skip { + t.Skip() + } + + path, err := ioutil.TempDir("", "vmware-driver-test") + if err != nil { + t.Fatal(err) + } + + defer os.RemoveAll(path) + + driver := NewDriver("default", path) + + checkFlags := &drivers.CheckDriverOptions{ + FlagsValues: map[string]interface{}{}, + CreateFlags: driver.GetCreateFlags(), + } + + err = driver.SetConfigFromFlags(checkFlags) + if err != nil { + t.Fatal(err) + } + + driver.(*Driver).Boot2DockerURL = "https://github.com/boot2docker/boot2docker/releases/download/v17.10.0-ce-rc2/boot2docker.iso" + + err = driver.Create() + if err != nil { + t.Fatal(err) + } + + defer driver.Remove() + + st, err := driver.GetState() + if err != nil { + t.Fatal(err) + } + if st != state.Running { + t.Fatalf("expect state == Running; got %s", st.String()) + } + + ip, err := driver.GetIP() + if err != nil { + t.Fatal(err) + } + if ip == "" { + t.Fatal("expect ip non-zero; got ''") + } + + username := driver.GetSSHUsername() + if username == "" { + t.Fatal("expect username non-zero; got ''") + } + + key := driver.GetSSHKeyPath() + if key == "" { + t.Fatal("expect key non-zero; got ''") + } + + port, err := driver.GetSSHPort() + if err != nil { + t.Fatal(err) + } + if port == 0 { + t.Fatal("expect port not 0; got 0") + } + + host, err := driver.GetSSHHostname() + if err != nil { + t.Fatal(err) + } + if host == "" { + t.Fatal("expect host non-zero; got ''") + } +} diff --git a/pkg/drivers/vmware/vmrun.go b/pkg/drivers/vmware/vmrun.go new file mode 100644 index 000000000000..5661c39da121 --- /dev/null +++ b/pkg/drivers/vmware/vmrun.go @@ -0,0 +1,94 @@ +/* +Copyright 2017 The Kubernetes Authors All rights reserved. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +/* + * Copyright 2017 VMware, Inc. All rights reserved. Licensed under the Apache v2 License. + */ + +package vmware + +import ( + "bytes" + "errors" + "fmt" + "io" + "os" + "os/exec" + "strings" + + "github.com/docker/machine/libmachine/log" +) + +var ( + vmrunbin = setVmwareCmd("vmrun") + vdiskmanbin = setVmwareCmd("vmware-vdiskmanager") +) + +var ( + ErrMachineExist = errors.New("machine already exists") + ErrMachineNotExist = errors.New("machine does not exist") + ErrVMRUNNotFound = errors.New("VMRUN not found") +) + +func init() { + // vmrun with nogui on VMware Fusion through at least 8.0.1 doesn't work right + // if the umask is set to not allow world-readable permissions + SetUmask() +} + +func isMachineDebugEnabled() bool { + return os.Getenv("MACHINE_DEBUG") != "" +} + +func vmrun(args ...string) (string, string, error) { + cmd := exec.Command(vmrunbin, args...) + + var stdout bytes.Buffer + var stderr bytes.Buffer + cmd.Stdout, cmd.Stderr = &stdout, &stderr + + if isMachineDebugEnabled() { + cmd.Stdout = io.MultiWriter(os.Stdout, cmd.Stdout) + cmd.Stderr = io.MultiWriter(os.Stderr, cmd.Stderr) + } + + log.Debugf("executing: %v %v", vmrunbin, strings.Join(args, " ")) + + err := cmd.Run() + if err != nil { + if ee, ok := err.(*exec.Error); ok && ee == exec.ErrNotFound { + err = ErrVMRUNNotFound + } + } + + return stdout.String(), stderr.String(), err +} + +// Make a vmdk disk image with the given size (in MB). +func vdiskmanager(dest string, size int) error { + cmd := exec.Command(vdiskmanbin, "-c", "-t", "0", "-s", fmt.Sprintf("%dMB", size), "-a", "lsilogic", dest) + if isMachineDebugEnabled() { + cmd.Stdout = os.Stdout + cmd.Stderr = os.Stderr + } + + if stdout := cmd.Run(); stdout != nil { + if ee, ok := stdout.(*exec.Error); ok && ee == exec.ErrNotFound { + return ErrVMRUNNotFound + } + } + return nil +} diff --git a/pkg/drivers/vmware/vmware.go b/pkg/drivers/vmware/vmware.go new file mode 100644 index 000000000000..b57e1b66e3a1 --- /dev/null +++ b/pkg/drivers/vmware/vmware.go @@ -0,0 +1,40 @@ +// +build !darwin,!linux,!windows + +/* +Copyright 2017 The Kubernetes Authors All rights reserved. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package vmware + +import "github.com/docker/machine/libmachine/drivers" + +func NewDriver(hostName, storePath string) drivers.Driver { + return drivers.NewDriverNotSupported("vmware", hostName, storePath) +} + +func DhcpConfigFiles() string { + return "" +} + +func DhcpLeaseFiles() string { + return "" +} + +func SetUmask() { +} + +func setVmwareCmd(cmd string) string { + return "" +} diff --git a/pkg/drivers/vmware/vmware_darwin.go b/pkg/drivers/vmware/vmware_darwin.go new file mode 100644 index 000000000000..be52dcdcde3e --- /dev/null +++ b/pkg/drivers/vmware/vmware_darwin.go @@ -0,0 +1,53 @@ +/* +Copyright 2017 The Kubernetes Authors All rights reserved. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package vmware + +import ( + "os" + "os/exec" + "path/filepath" + "syscall" +) + +func DhcpConfigFiles() string { + return "/Library/Preferences/VMware Fusion/vmnet*/dhcpd.conf" +} + +func DhcpLeaseFiles() string { + return "/var/db/vmware/*.leases" +} + +func SetUmask() { + _ = syscall.Umask(022) +} + +// detect the vmrun and vmware-vdiskmanager cmds' path if needed +func setVmwareCmd(cmd string) string { + if path, err := exec.LookPath(cmd); err == nil { + return path + } + for _, fp := range []string{ + "/Applications/VMware Fusion.app/Contents/Library/", + } { + p := filepath.Join(fp, cmd) + _, err := os.Stat(p) + if err == nil { + return p + } + } + return cmd +} diff --git a/pkg/drivers/vmware/vmware_linux.go b/pkg/drivers/vmware/vmware_linux.go new file mode 100644 index 000000000000..661fac4b7d49 --- /dev/null +++ b/pkg/drivers/vmware/vmware_linux.go @@ -0,0 +1,42 @@ +/* +Copyright 2017 The Kubernetes Authors All rights reserved. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package vmware + +import ( + "os/exec" + "syscall" +) + +func DhcpConfigFiles() string { + return "/etc/vmware/vmnet*/dhcpd/dhcpd.conf" +} + +func DhcpLeaseFiles() string { + return "/etc/vmware/vmnet*/dhcpd/dhcpd.leases" +} + +func SetUmask() { + _ = syscall.Umask(022) +} + +// detect the vmrun and vmware-vdiskmanager cmds' path if needed +func setVmwareCmd(cmd string) string { + if path, err := exec.LookPath(cmd); err == nil { + return path + } + return cmd +} diff --git a/pkg/drivers/vmware/vmware_windows.go b/pkg/drivers/vmware/vmware_windows.go new file mode 100644 index 000000000000..c3b2ed74a79d --- /dev/null +++ b/pkg/drivers/vmware/vmware_windows.go @@ -0,0 +1,53 @@ +/* +Copyright 2017 The Kubernetes Authors All rights reserved. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package vmware + +import ( + "os" + "os/exec" + "path/filepath" +) + +func DhcpConfigFiles() string { + return `C:\ProgramData\VMware\vmnetdhcp.conf` +} + +func DhcpLeaseFiles() string { + return `C:\ProgramData\VMware\vmnetdhcp.leases` +} + +func SetUmask() { +} + +func setVmwareCmd(cmd string) string { + cmd = cmd + ".exe" + + if path, err := exec.LookPath(cmd); err == nil { + return path + } + for _, fp := range []string{ + `C:\Program Files (x86)\VMware\VMware Workstation`, + `C:\Program Files\VMware\VMware Workstation`, + } { + p := filepath.Join(fp, cmd) + _, err := os.Stat(p) + if err == nil { + return p + } + } + return cmd +} diff --git a/pkg/drivers/vmware/vmx.go b/pkg/drivers/vmware/vmx.go new file mode 100644 index 000000000000..73677d3b8809 --- /dev/null +++ b/pkg/drivers/vmware/vmx.go @@ -0,0 +1,88 @@ +/* +Copyright 2017 The Kubernetes Authors All rights reserved. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +/* + * Copyright 2017 VMware, Inc. All rights reserved. Licensed under the Apache v2 License. + */ + +package vmware + +const vmx = ` +.encoding = "UTF-8" +config.version = "8" +displayName = "{{.MachineName}}" +ethernet0.present = "TRUE" +ethernet0.connectionType = "nat" +ethernet0.virtualDev = "vmxnet3" +ethernet0.wakeOnPcktRcv = "FALSE" +ethernet0.addressType = "generated" +ethernet0.linkStatePropagation.enable = "TRUE" +pciBridge0.present = "TRUE" +pciBridge4.present = "TRUE" +pciBridge4.virtualDev = "pcieRootPort" +pciBridge4.functions = "8" +pciBridge5.present = "TRUE" +pciBridge5.virtualDev = "pcieRootPort" +pciBridge5.functions = "8" +pciBridge6.present = "TRUE" +pciBridge6.virtualDev = "pcieRootPort" +pciBridge6.functions = "8" +pciBridge7.present = "TRUE" +pciBridge7.virtualDev = "pcieRootPort" +pciBridge7.functions = "8" +pciBridge0.pciSlotNumber = "17" +pciBridge4.pciSlotNumber = "21" +pciBridge5.pciSlotNumber = "22" +pciBridge6.pciSlotNumber = "23" +pciBridge7.pciSlotNumber = "24" +scsi0.pciSlotNumber = "160" +usb.pciSlotNumber = "32" +ethernet0.pciSlotNumber = "192" +sound.pciSlotNumber = "33" +vmci0.pciSlotNumber = "35" +sata0.pciSlotNumber = "36" +floppy0.present = "FALSE" +guestOS = "other3xlinux-64" +hpet0.present = "TRUE" +sata0.present = "TRUE" +sata0:1.present = "TRUE" +sata0:1.fileName = "{{.ISO}}" +sata0:1.deviceType = "cdrom-image" +{{ if .ConfigDriveURL }} +sata0:2.present = "TRUE" +sata0:2.fileName = "{{.ConfigDriveISO}}" +sata0:2.deviceType = "cdrom-image" +{{ end }} +vmci0.present = "TRUE" +mem.hotadd = "TRUE" +memsize = "{{.Memory}}" +powerType.powerOff = "soft" +powerType.powerOn = "soft" +powerType.reset = "soft" +powerType.suspend = "soft" +scsi0.present = "TRUE" +scsi0.virtualDev = "pvscsi" +scsi0:0.fileName = "{{.MachineName}}.vmdk" +scsi0:0.present = "TRUE" +tools.synctime = "TRUE" +virtualHW.productCompatibility = "hosted" +virtualHW.version = "10" +msg.autoanswer = "TRUE" +uuid.action = "create" +numvcpus = "{{.CPU}}" +hgfs.mapRootShare = "FALSE" +hgfs.linkRootShare = "FALSE" +` diff --git a/pkg/minikube/cluster/cluster.go b/pkg/minikube/cluster/cluster.go index bbc6dcf6dc6e..3b96a4a1dd6e 100644 --- a/pkg/minikube/cluster/cluster.go +++ b/pkg/minikube/cluster/cluster.go @@ -38,7 +38,7 @@ import ( "github.com/docker/machine/libmachine/state" "github.com/golang/glog" "github.com/pkg/errors" - + "k8s.io/minikube/pkg/drivers/vmware" cfg "k8s.io/minikube/pkg/minikube/config" "k8s.io/minikube/pkg/minikube/constants" @@ -185,6 +185,18 @@ func createVirtualboxHost(config MachineConfig) drivers.Driver { return d } +func createVMwareHost(config MachineConfig) drivers.Driver { + d := vmware.NewDriver(cfg.GetMachineName(), constants.GetMinipath()).(*vmware.Driver) + d.Boot2DockerURL = config.Downloader.GetISOFileURI(config.MinikubeISO) + d.Memory = config.Memory + d.CPU = config.CPUs + d.DiskSize = config.DiskSize + d.SSHPort = 22 + d.ISO = d.ResolveStorePath("boot2docker.iso") + + return d +} + func createHost(api libmachine.API, config MachineConfig) (*host.Host, error) { var driver interface{} @@ -197,6 +209,8 @@ func createHost(api libmachine.API, config MachineConfig) (*host.Host, error) { switch config.VMDriver { case "virtualbox": driver = createVirtualboxHost(config) + case "vmware": + driver = createVMwareHost(config) case "vmwarefusion": driver = createVMwareFusionHost(config) case "kvm", "kvm2": @@ -313,6 +327,12 @@ func GetVMHostIP(host *host.Host) (net.IP, error) { return ip, nil case "xhyve": return net.ParseIP("192.168.64.1"), nil + case "vmware": + ip, err := host.Driver.GetIP() + if err != nil { + return []byte{}, nil + } + return net.ParseIP(ip), nil default: return []byte{}, errors.New("Error, attempted to get host ip address for unsupported driver") } diff --git a/pkg/minikube/machine/client_darwin.go b/pkg/minikube/machine/client_darwin.go index 04dfbc85b38d..cdfed8382159 100644 --- a/pkg/minikube/machine/client_darwin.go +++ b/pkg/minikube/machine/client_darwin.go @@ -25,6 +25,7 @@ import ( "github.com/docker/machine/libmachine/drivers/plugin" "github.com/golang/glog" "github.com/pkg/errors" + "k8s.io/minikube/pkg/drivers/vmware" ) var driverMap = map[string]driverGetter{ @@ -56,6 +57,8 @@ func registerDriver(driverName string) { plugin.RegisterDriver(virtualbox.NewDriver("", "")) case "vmwarefusion": plugin.RegisterDriver(vmwarefusion.NewDriver("", "")) + case "vmware": + plugin.RegisterDriver(vmware.NewDriver("", "")) default: glog.Exitf("Unsupported driver: %s\n", driverName) } diff --git a/pkg/minikube/machine/client_linux.go b/pkg/minikube/machine/client_linux.go index 306c254e4151..ff621c5936a1 100644 --- a/pkg/minikube/machine/client_linux.go +++ b/pkg/minikube/machine/client_linux.go @@ -25,6 +25,7 @@ import ( "github.com/golang/glog" "github.com/pkg/errors" "k8s.io/minikube/pkg/drivers/none" + "k8s.io/minikube/pkg/drivers/vmware" ) var driverMap = map[string]driverGetter{ @@ -48,6 +49,8 @@ func registerDriver(driverName string) { plugin.RegisterDriver(virtualbox.NewDriver("", "")) case "none": plugin.RegisterDriver(none.NewDriver("", "")) + case "vmware": + plugin.RegisterDriver(vmware.NewDriver("", "")) default: glog.Exitf("Unsupported driver: %s\n", driverName) } diff --git a/pkg/minikube/machine/client_windows.go b/pkg/minikube/machine/client_windows.go index f2b50c1471d1..254216353a33 100644 --- a/pkg/minikube/machine/client_windows.go +++ b/pkg/minikube/machine/client_windows.go @@ -25,6 +25,7 @@ import ( "github.com/docker/machine/libmachine/drivers/plugin" "github.com/golang/glog" "github.com/pkg/errors" + "k8s.io/minikube/pkg/drivers/vmware" ) var driverMap = map[string]driverGetter{ @@ -48,6 +49,8 @@ func registerDriver(driverName string) { plugin.RegisterDriver(virtualbox.NewDriver("", "")) case "hyperv": plugin.RegisterDriver(hyperv.NewDriver("", "")) + case "vmware": + plugin.RegisterDriver(vmware.NewDriver("", "")) default: glog.Exitf("Unsupported driver: %s\n", driverName) } diff --git a/vendor/github.com/docker/machine/libmachine/drivers/plugin/localbinary/plugin.go b/vendor/github.com/docker/machine/libmachine/drivers/plugin/localbinary/plugin.go index 59116dd8eda3..875dac056e86 100644 --- a/vendor/github.com/docker/machine/libmachine/drivers/plugin/localbinary/plugin.go +++ b/vendor/github.com/docker/machine/libmachine/drivers/plugin/localbinary/plugin.go @@ -20,7 +20,7 @@ var ( CoreDrivers = []string{"amazonec2", "azure", "digitalocean", "exoscale", "generic", "google", "hyperv", "none", "openstack", "rackspace", "softlayer", "virtualbox", "vmwarefusion", - "vmwarevcloudair", "vmwarevsphere"} + "vmwarevcloudair", "vmwarevsphere", "vmware"} ) const (