testiso: improve error when QEMU is terminated

mantle/cmd/kola/testiso.go

@@ -537,8 +537,12 @@ func awaitCompletion(ctx context.Context, inst *platform.QemuInstance, outdir st
 	}
 	go func() {
 		err := inst.Wait()
+		// only one Wait() gets process data, so also manually check for signal
+		if err == nil && inst.Signaled() {
+			err = errors.New("process killed")
+		}
 		if err != nil {
-			errchan <- err
+			errchan <- errors.Wrapf(err, "QEMU unexpectedly exited while awaiting completion")
 		}
 		time.Sleep(1 * time.Minute)
 		errchan <- fmt.Errorf("QEMU exited; timed out waiting for completion")
@@ -549,6 +553,10 @@ func awaitCompletion(ctx context.Context, inst *platform.QemuInstance, outdir st
 			l, err := r.ReadString('\n')
 			if err != nil {
 				if err == io.EOF {
+					// this may be from QEMU getting killed or exiting; wait a bit
+					// to give a chance for .Wait() above to feed the channel with a
+					// better error
+					time.Sleep(1 * time.Second)
 					errchan <- fmt.Errorf("Got EOF from completion channel, %s expected", exp)
 				} else {
 					errchan <- errors.Wrapf(err, "reading from completion channel")

mantle/platform/metal.go

@@ -24,6 +24,7 @@ import (
 	"os"
 	"path/filepath"
 	"strings"
+	"time"
 
 	coreosarch "github.com/coreos/stream-metadata-go/arch"
 	"github.com/pkg/errors"
@@ -435,11 +436,25 @@ func (t *installerRun) completePxeSetup(kargs []string) error {
 
 func switchBootOrderSignal(qinst *QemuInstance, bootstartedchan *os.File, booterrchan *chan error) {
 	*booterrchan = make(chan error)
+	go func() {
+		err := qinst.Wait()
+		// only one Wait() gets process data, so also manually check for signal
+		if err == nil && qinst.Signaled() {
+			err = errors.New("process killed")
+		}
+		if err != nil {
+			*booterrchan <- errors.Wrapf(err, "QEMU unexpectedly exited while waiting for %s", bootStartedSignal)
+		}
+	}()
 	go func() {
 		r := bufio.NewReader(bootstartedchan)
 		l, err := r.ReadString('\n')
 		if err != nil {
 			if err == io.EOF {
+				// this may be from QEMU getting killed or exiting; wait a bit
+				// to give a chance for .Wait() above to feed the channel with a
+				// better error
+				time.Sleep(1 * time.Second)
 				*booterrchan <- fmt.Errorf("Got EOF from boot started channel, %s expected", bootStartedSignal)
 			} else {
 				*booterrchan <- errors.Wrapf(err, "reading from boot started channel")

mantle/platform/qemu.go

@@ -147,6 +147,11 @@ type QemuInstance struct {
 	qmpSocketPath string
 }
 
+// Signaled returns whether QEMU process was signaled.
+func (inst *QemuInstance) Signaled() bool {
+	return inst.qemu.Signaled()
+}
+
 // Pid returns the PID of QEMU process.
 func (inst *QemuInstance) Pid() int {
 	return inst.qemu.Pid()

mantle/system/exec/exec.go

@@ -47,6 +47,9 @@ type Cmd interface {
 
 	// Simplified wrapper for Process.Pid
 	Pid() int
+
+	// Simplified wrapper to know if a process was signaled
+	Signaled() bool
 }
 
 // Basic Cmd implementation based on exec.Cmd
@@ -93,6 +96,14 @@ func (cmd *ExecCmd) Kill() error {
 	return err
 }
 
+func (cmd *ExecCmd) Signaled() bool {
+	if cmd.ProcessState == nil {
+		return false
+	}
+	status := cmd.ProcessState.Sys().(syscall.WaitStatus)
+	return status.Signaled()
+}
+
 func (cmd *ExecCmd) Pid() int {
 	return cmd.Process.Pid
 }