diff --git a/test-vfkit/basic_test.go b/test-vfkit/basic_test.go new file mode 100644 index 00000000..1c6d4ee7 --- /dev/null +++ b/test-vfkit/basic_test.go @@ -0,0 +1,223 @@ +package e2e_vfkit + +import ( + "context" + "net" + "net/http" + + gvproxyclient "github.com/containers/gvisor-tap-vsock/pkg/client" + + "github.com/containers/gvisor-tap-vsock/pkg/types" + "github.com/onsi/ginkgo" + "github.com/onsi/gomega" +) + +var _ = ginkgo.Describe("connectivity with vfkit", func() { + ginkgo.It("should configure the interface", func() { + out, err := sshExec("ifconfig $(route | grep '^default' | grep -o '[^ ]*$')") + gomega.Expect(err).ShouldNot(gomega.HaveOccurred()) + gomega.Expect(string(out)).To(gomega.ContainSubstring("mtu 1500")) + gomega.Expect(string(out)).To(gomega.ContainSubstring("inet 192.168.127.2")) + gomega.Expect(string(out)).To(gomega.ContainSubstring("netmask 255.255.255.0")) + gomega.Expect(string(out)).To(gomega.ContainSubstring("broadcast 192.168.127.255")) + }) + + ginkgo.It("should configure the default route", func() { + out, err := sshExec("ip route show") + gomega.Expect(err).ShouldNot(gomega.HaveOccurred()) + gomega.Expect(string(out)).To(gomega.MatchRegexp(`default via 192\.168\.127\.1 dev (.*?) proto dhcp (src 192\.168\.127\.2 )?metric 100`)) + }) + + ginkgo.It("should configure dns settings", func() { + out, err := sshExec("cat /etc/resolv.conf") + gomega.Expect(err).ShouldNot(gomega.HaveOccurred()) + gomega.Expect(string(out)).To(gomega.ContainSubstring("nameserver 192.168.127.1")) + }) + + ginkgo.It("should ping the tap device", func() { + out, err := sshExec("ping -c2 192.168.127.2") + gomega.Expect(err).ShouldNot(gomega.HaveOccurred()) + gomega.Expect(string(out)).To(gomega.ContainSubstring("2 packets transmitted, 2 received, 0% packet loss")) + }) + + ginkgo.It("should ping the gateway", func() { + out, err := sshExec("ping -c2 192.168.127.1") + gomega.Expect(err).ShouldNot(gomega.HaveOccurred()) + gomega.Expect(string(out)).To(gomega.ContainSubstring("2 packets transmitted, 2 received, 0% packet loss")) + }) +}) + +var _ = ginkgo.Describe("dns with vfkit", func() { + ginkgo.It("should resolve redhat.com", func() { + out, err := sshExec("nslookup redhat.com") + gomega.Expect(err).ShouldNot(gomega.HaveOccurred()) + gomega.Expect(string(out)).To(gomega.ContainSubstring("Address: 52.200.142.250")) + }) + + ginkgo.It("should resolve CNAME record for docs.crc.dev", func() { + out, err := sshExec("nslookup -query=cname docs.crc.dev") + gomega.Expect(err).ShouldNot(gomega.HaveOccurred()) + gomega.Expect(string(out)).To(gomega.ContainSubstring("docs.crc.dev canonical name = webredir.gandi.net.")) + }) + ginkgo.It("should resolve MX record for crc.dev", func() { + out, err := sshExec("nslookup -query=mx crc.dev") + gomega.Expect(err).ShouldNot(gomega.HaveOccurred()) + gomega.Expect(string(out)).To(gomega.ContainSubstring("crc.dev mail exchanger = 10 spool.mail.gandi.net.")) + }) + + ginkgo.It("should resolve NS record for wikipedia.org", func() { + out, err := sshExec("nslookup -query=ns wikipedia.org") + gomega.Expect(err).ShouldNot(gomega.HaveOccurred()) + gomega.Expect(string(out)).To(gomega.ContainSubstring("wikipedia.org nameserver = ns0.wikimedia.org.")) + }) + ginkgo.It("should resolve IMAPS SRV record for crc.dev", func() { + out, err := sshExec("nslookup -query=srv _imaps._tcp.crc.dev") + gomega.Expect(err).ShouldNot(gomega.HaveOccurred()) + gomega.Expect(string(out)).To(gomega.ContainSubstring(`_imaps._tcp.crc.dev service = 0 1 993 mail.gandi.net.`)) + }) + ginkgo.It("should resolve TXT for crc.dev", func() { + out, err := sshExec("nslookup -query=txt crc.dev") + gomega.Expect(err).ShouldNot(gomega.HaveOccurred()) + gomega.Expect(string(out)).To(gomega.ContainSubstring(`text = "v=spf1`)) + }) + + ginkgo.It("should resolve gateway.containers.internal", func() { + out, err := sshExec("nslookup gateway.containers.internal") + gomega.Expect(err).ShouldNot(gomega.HaveOccurred()) + gomega.Expect(string(out)).To(gomega.ContainSubstring("Address: 192.168.127.1")) + }) + + ginkgo.It("should resolve host.containers.internal", func() { + out, err := sshExec("nslookup host.containers.internal") + gomega.Expect(err).ShouldNot(gomega.HaveOccurred()) + gomega.Expect(string(out)).To(gomega.ContainSubstring("Address: 192.168.127.254")) + }) + + ginkgo.It("should resolve dynamically added dns entry test.dynamic.internal", func() { + client := gvproxyclient.New(&http.Client{ + Transport: &http.Transport{ + DialContext: func(_ context.Context, _, _ string) (net.Conn, error) { + return net.Dial("unix", sock) + }, + }, + }, "http://base") + err := client.AddDNS(&types.Zone{ + Name: "dynamic.internal.", + Records: []types.Record{ + { + Name: "test", + IP: net.ParseIP("192.168.127.254"), + }, + }, + }) + gomega.Expect(err).ShouldNot(gomega.HaveOccurred()) + + out, err := sshExec("nslookup test.dynamic.internal") + + gomega.Expect(err).ShouldNot(gomega.HaveOccurred()) + gomega.Expect(string(out)).To(gomega.ContainSubstring("Address: 192.168.127.254")) + }) + + ginkgo.It("should resolve recently added dns entry test.dynamic.internal", func() { + client := gvproxyclient.New(&http.Client{ + Transport: &http.Transport{ + DialContext: func(_ context.Context, _, _ string) (net.Conn, error) { + return net.Dial("unix", sock) + }, + }, + }, "http://base") + err := client.AddDNS(&types.Zone{ + Name: "dynamic.internal.", + Records: []types.Record{ + { + Name: "test", + IP: net.ParseIP("192.168.127.254"), + }, + }, + }) + gomega.Expect(err).ShouldNot(gomega.HaveOccurred()) + + err = client.AddDNS(&types.Zone{ + Name: "dynamic.internal.", + Records: []types.Record{ + { + Name: "test", + IP: net.ParseIP("192.168.127.253"), + }, + }, + }) + gomega.Expect(err).ShouldNot(gomega.HaveOccurred()) + + out, err := sshExec("nslookup test.dynamic.internal") + + gomega.Expect(err).ShouldNot(gomega.HaveOccurred()) + gomega.Expect(string(out)).To(gomega.ContainSubstring("Address: 192.168.127.253")) + }) + + ginkgo.It("should retain order of existing zone", func() { + client := gvproxyclient.New(&http.Client{ + Transport: &http.Transport{ + DialContext: func(_ context.Context, _, _ string) (net.Conn, error) { + return net.Dial("unix", sock) + }, + }, + }, "http://base") + _ = client.AddDNS(&types.Zone{ + Name: "dynamic.testing.", + DefaultIP: net.ParseIP("192.168.127.2"), + }) + _ = client.AddDNS(&types.Zone{ + Name: "testing.", + Records: []types.Record{ + { + Name: "host", + IP: net.ParseIP("192.168.127.3"), + }, + }, + }) + out, err := sshExec("nslookup test.dynamic.internal") + gomega.Expect(err).ShouldNot(gomega.HaveOccurred()) + gomega.Expect(string(out)).To(gomega.ContainSubstring("Address: 192.168.127.2")) + + _ = client.AddDNS(&types.Zone{ + Name: "testing.", + Records: []types.Record{ + { + Name: "gateway", + IP: net.ParseIP("192.168.127.1"), + }, + }, + }) + out, err = sshExec("nslookup *.dynamic.testing") + gomega.Expect(err).ShouldNot(gomega.HaveOccurred()) + gomega.Expect(string(out)).To(gomega.ContainSubstring("Address: 192.168.127.2")) + + out, err = sshExec("nslookup gateway.testing") + gomega.Expect(err).ShouldNot(gomega.HaveOccurred()) + gomega.Expect(string(out)).To(gomega.ContainSubstring("Address: 192.168.127.1")) + }) +}) + +var _ = ginkgo.Describe("command-line format with vfkit", func() { + ginkgo.It("should convert Command to command line format", func() { + command := types.NewGvproxyCommand() + command.AddEndpoint("unix:///tmp/network.sock") + command.Debug = true + command.AddQemuSocket("tcp://0.0.0.0:1234") + command.PidFile = "~/gv-pidfile.txt" + command.LogFile = "~/gv.log" + command.AddForwardUser("demouser") + + cmd := command.ToCmdline() + gomega.Expect(cmd).To(gomega.Equal([]string{ + "-listen", "unix:///tmp/network.sock", + "-debug", + "-mtu", "1500", + "-ssh-port", "2222", + "-listen-qemu", "tcp://0.0.0.0:1234", + "-forward-user", "demouser", + "-pid-file", "~/gv-pidfile.txt", + "-log-file", "~/gv.log", + })) + }) +}) diff --git a/test-vfkit/vfkit_suite_test.go b/test-vfkit/vfkit_suite_test.go new file mode 100644 index 00000000..b7e977a1 --- /dev/null +++ b/test-vfkit/vfkit_suite_test.go @@ -0,0 +1,238 @@ +package e2e_vfkit + +import ( + "flag" + "fmt" + "io" + "os" + "os/exec" + "path/filepath" + "strconv" + "strings" + "testing" + "time" + + e2e_utils "github.com/containers/gvisor-tap-vsock/test-utils" + + "github.com/onsi/ginkgo" + "github.com/onsi/gomega" + "github.com/pkg/errors" + log "github.com/sirupsen/logrus" +) + +func TestSuite(t *testing.T) { + gomega.RegisterFailHandler(ginkgo.Fail) + ginkgo.RunSpecs(t, "gvisor-tap-vsock suite") +} + +const ( + sock = "/tmp/gvproxy-api-vfkit.sock" + vfkitSock = "/tmp/vfkit.sock" + sshPort = 2222 + ignitionUser = "test" + vfkitConLog = "vfkitcon.log" + // #nosec "test" (for manual usage) + ignitionPasswordHash = "$y$j9T$TqJWt3/mKJbH0sYi6B/LD1$QjVRuUgntjTHjAdAkqhkr4F73m.Be4jBXdAaKw98sPC" +) + +var ( + tmpDir string + binDir string + host *exec.Cmd + client *exec.Cmd + privateKeyFile string + publicKeyFile string + ignFile string +) + +func init() { + flag.StringVar(&tmpDir, "tmpDir", "../tmp", "temporary working directory") + flag.StringVar(&binDir, "bin", "../bin", "directory with compiled binaries") + privateKeyFile = filepath.Join(tmpDir, "id_test") + publicKeyFile = privateKeyFile + ".pub" + ignFile = filepath.Join(tmpDir, "test.ign") + +} + +var _ = ginkgo.BeforeSuite(func() { + gomega.Expect(os.MkdirAll(filepath.Join(tmpDir, "disks"), os.ModePerm)).Should(gomega.Succeed()) + + downloader, err := e2e_utils.NewFcosDownloader(filepath.Join(tmpDir, "disks")) + gomega.Expect(err).ShouldNot(gomega.HaveOccurred()) + fcosImage, err := downloader.DownloadImage("applehv", "raw.gz") + gomega.Expect(err).ShouldNot(gomega.HaveOccurred()) + + publicKey, err := createSSHKeys() + gomega.Expect(err).ShouldNot(gomega.HaveOccurred()) + + err = e2e_utils.CreateIgnition(ignFile, publicKey, ignitionUser, ignitionPasswordHash) + gomega.Expect(err).ShouldNot(gomega.HaveOccurred()) + +outer: + for panics := 0; ; panics++ { + _ = os.Remove(sock) + + // #nosec + host = exec.Command(filepath.Join(binDir, "gvproxy"), fmt.Sprintf("--listen=unix://%s", sock), fmt.Sprintf("--listen-vfkit=unixgram://%s", vfkitSock)) + + host.Stderr = os.Stderr + host.Stdout = os.Stdout + gomega.Expect(host.Start()).Should(gomega.Succeed()) + go func() { + if err := host.Wait(); err != nil { + log.Error(err) + } + }() + + for { + _, err := os.Stat(sock) + if os.IsNotExist(err) { + log.Info("waiting for socket") + time.Sleep(100 * time.Millisecond) + continue + } + _, err = os.Stat(vfkitSock) + if os.IsNotExist(err) { + log.Info("waiting for vfkit socket") + time.Sleep(100 * time.Millisecond) + continue + } + break + } + + vfkitArgs := `--cpus 2 --memory 2048 --bootloader efi,variable-store=efi-variable-store,create --device virtio-blk,path=%s --ignition %s --device virtio-net,unixSocketPath=%s,mac=5a:94:ef:e4:0c:ee` + // #nosec + client = exec.Command(vfkitExecutable(), strings.Split(fmt.Sprintf(vfkitArgs, fcosImage, ignFile, vfkitSock), " ")...) + client.Stderr = os.Stderr + client.Stdout = os.Stdout + gomega.Expect(client.Start()).Should(gomega.Succeed()) + go func() { + if err := client.Wait(); err != nil { + log.Error(err) + } + }() + + for { + _, err := sshExec("whoami") + if err == nil { + break outer + } + + // Check for panic + didPanic, err := panicCheck(vfkitConLog) + gomega.Expect(err).ShouldNot(gomega.HaveOccurred()) + + if didPanic { + gomega.Expect(panics).ToNot(gomega.BeNumerically(">", 15), "No more than 15 panics allowed") + log.Info("Detected Kernel panic, retrying...") + _ = client.Process.Kill() + _ = host.Process.Kill() + _ = os.Remove(vfkitConLog) + continue outer + } + + log.Infof("waiting for client to connect: %v", err) + time.Sleep(time.Second) + } + } + + /* err = scp(filepath.Join(binDir, "test-companion"), "/tmp/test-companion") + gomega.Expect(err).ShouldNot(gomega.HaveOccurred()) + + // start an embedded DNS and http server in the VM. Wait a bit for the server to start. + cmd := sshCommand("sudo /tmp/test-companion") + gomega.Expect(cmd.Start()).ShouldNot(gomega.HaveOccurred()) */ + time.Sleep(5 * time.Second) +}) + +func vfkitExecutable() string { + qemuBinaries := []string{"vfkit"} + for _, binary := range qemuBinaries { + path, err := exec.LookPath(binary) + if err == nil && path != "" { + return path + } + } + + return "" +} + +func createSSHKeys() (string, error) { + _ = os.Remove(publicKeyFile) + _ = os.Remove(privateKeyFile) + err := exec.Command("ssh-keygen", "-N", "", "-t", "ed25519", "-f", privateKeyFile).Run() + if err != nil { + return "", errors.Wrap(err, "Could not generate ssh keys") + } + + return readPublicKey() +} + +func readPublicKey() (string, error) { + publicKey, err := os.ReadFile(publicKeyFile) + if err != nil { + return "", nil + } + + return strings.TrimSpace(string(publicKey)), nil +} + +func scp(src, dst string) error { + sshCmd := exec.Command("scp", + "-o", "UserKnownHostsFile=/dev/null", + "-o", "StrictHostKeyChecking=no", + "-o", "IdentitiesOnly=yes", + "-i", privateKeyFile, + "-P", strconv.Itoa(sshPort), + src, + fmt.Sprintf("%s@127.0.0.1:%s", ignitionUser, dst)) // #nosec G204 + sshCmd.Stderr = os.Stderr + sshCmd.Stdout = os.Stdout + return sshCmd.Run() +} + +func sshExec(cmd ...string) ([]byte, error) { + return sshCommand(cmd...).Output() +} + +func sshCommand(cmd ...string) *exec.Cmd { + sshCmd := exec.Command("ssh", + "-o", "UserKnownHostsFile=/dev/null", + "-o", "StrictHostKeyChecking=no", + "-o", "IdentitiesOnly=yes", + "-i", privateKeyFile, + "-p", strconv.Itoa(sshPort), + fmt.Sprintf("%s@127.0.0.1", ignitionUser), "--", strings.Join(cmd, " ")) // #nosec G204 + return sshCmd +} + +func panicCheck(con string) (bool, error) { + file, err := os.Open(con) + if err != nil { + return false, err + } + + _, _ = file.Seek(-500, io.SeekEnd) + // Ignore seek errors (not enough content yet) + + contents := make([]byte, 500) + _, err = io.ReadAtLeast(file, contents, len(contents)) + if err != nil && err != io.ErrUnexpectedEOF { + return false, err + } + + return strings.Contains(string(contents), "end Kernel panic"), nil +} + +var _ = ginkgo.AfterSuite(func() { + if host != nil { + if err := host.Process.Kill(); err != nil { + log.Error(err) + } + } + if client != nil { + if err := client.Process.Kill(); err != nil { + log.Error(err) + } + } +})