packer_test: make packer test suite modular

Having only one test suite for the whole of Packer makes it harder to
segregate between test types, and makes for a longer runtime as no tests
run in parallel by default.

This commit splits the packer_test suite into several components in
order to make extension easier.

First we have `lib`: this package embeds the core for running Packer
test suites. This ships facilities to build your own test suite for
Packer core, and exposes convenience methods and structures for building
plugins, packer core, and use it to run a test suite in a temporary
directory.

Then we have two separate test suites: one for plugins, and one for core
itself, the latter of which does not depend on plugins being compiled at
all.

This sets the stage for more specialised test suites in the future, each
of which can run in parallel on different parts of the code.

packer_test/local_eval_test.go → packer_test/core_tests/local_eval_test.go

@@ -1,8 +1,12 @@
-package packer_test
+package core_test
 
-import "fmt"
+import (
+	"fmt"
 
-func (ts *PackerTestSuite) TestEvalLocalsOrder() {
+	"github.com/hashicorp/packer/packer_test/lib"
+)
+
+func (ts *PackerCoreTestSuite) TestEvalLocalsOrder() {
 	ts.SkipNoAcc()
 
 	pluginDir, cleanup := ts.MakePluginDir()
@@ -12,20 +16,21 @@ func (ts *PackerTestSuite) TestEvalLocalsOrder() {
 		Runs(10).
 		Stdin("local.test_local\n").
 		SetArgs("console", "./templates/locals_no_order.pkr.hcl").
-		Assert(MustSucceed(), Grep("\\[\\]", grepStdout, grepInvert))
+		Assert(lib.MustSucceed(),
+			lib.Grep("\\[\\]", lib.GrepStdout, lib.GrepInvert))
 }
 
-func (ts *PackerTestSuite) TestLocalDuplicates() {
+func (ts *PackerCoreTestSuite) TestLocalDuplicates() {
 	pluginDir, cleanup := ts.MakePluginDir()
 	defer cleanup()
 
 	for _, cmd := range []string{"console", "validate", "build"} {
 		ts.Run(fmt.Sprintf("duplicate local detection with %s command - expect error", cmd), func() {
 			ts.PackerCommand().UsePluginDir(pluginDir).
 				SetArgs(cmd, "./templates/locals_duplicate.pkr.hcl").
-				Assert(MustFail(),
-					Grep("Duplicate local definition"),
-					Grep("Local variable \"test\" is defined twice"))
+				Assert(lib.MustFail(),
+					lib.Grep("Duplicate local definition"),
+					lib.Grep("Local variable \"test\" is defined twice"))
 		})
 	}
 }

packer_test/core_tests/suite_test.go

@@ -0,0 +1,23 @@
+package core_test
+
+import (
+	"testing"
+
+	"github.com/hashicorp/packer/packer_test/lib"
+	"github.com/stretchr/testify/suite"
+)
+
+type PackerCoreTestSuite struct {
+	*lib.PackerTestSuite
+}
+
+func Test_PackerPluginSuite(t *testing.T) {
+	baseSuite, cleanup := lib.PackerCoreSuite(t)
+	defer cleanup()
+
+	ts := &PackerCoreTestSuite{
+		baseSuite,
+	}
+
+	suite.Run(t, ts)
+}

packer_test/base_test.go → packer_test/lib/base.go

@@ -1,4 +1,4 @@
-package packer_test
+package lib
 
 import (
 	"fmt"
@@ -19,7 +19,7 @@ func BuildTestPacker(t *testing.T) (string, error) {
 		return "", fmt.Errorf("failed to compile packer binary: %s", err)
 	}
 
-	packerCoreDir := filepath.Dir(testDir)
+	packerCoreDir := filepath.Dir(filepath.Dir(testDir))
 
 	outBin := filepath.Join(os.TempDir(), fmt.Sprintf("packer_core-%d", rand.Int()))
 	if runtime.GOOS == "windows" {

packer_test/commands_test.go → packer_test/lib/commands.go

@@ -1,4 +1,4 @@
-package packer_test
+package lib
 
 import (
 	"fmt"

packer_test/gadgets_test.go → packer_test/lib/gadgets.go

@@ -1,4 +1,4 @@
-package packer_test
+package lib
 
 import (
 	"fmt"
@@ -86,21 +86,21 @@ func (_ mustFail) Check(stdout, stderr string, err error) error {
 	return nil
 }
 
-type grepOpts int
+type GrepOpts int
 
 const (
 	// Invert the check, i.e. by default an empty grep fails, if this is set, a non-empty grep fails
-	grepInvert grepOpts = iota
+	GrepInvert GrepOpts = iota
 	// Only grep stderr
-	grepStderr
+	GrepStderr
 	// Only grep stdout
-	grepStdout
+	GrepStdout
 )
 
 // Grep returns a checker that performs a regexp match on the command's output and returns an error if it failed
 //
 // Note: by default both streams will be checked by the grep
-func Grep(expression string, opts ...grepOpts) Checker {
+func Grep(expression string, opts ...GrepOpts) Checker {
 	pc := PipeChecker{
 		name:   fmt.Sprintf("command | grep -E %q", expression),
 		stream: BothStreams,
@@ -111,11 +111,11 @@ func Grep(expression string, opts ...grepOpts) Checker {
 	}
 	for _, opt := range opts {
 		switch opt {
-		case grepInvert:
+		case GrepInvert:
 			pc.check = ExpectEmptyInput()
-		case grepStderr:
+		case GrepStderr:
 			pc.stream = OnlyStderr
-		case grepStdout:
+		case GrepStdout:
 			pc.stream = OnlyStdout
 		}
 	}
@@ -165,6 +165,6 @@ func (c CustomCheck) Name() string {
 // returned pipe checker.
 func LineCountCheck(lines int) *PipeChecker {
 	return MkPipeCheck(fmt.Sprintf("line count (%d)", lines), LineCount()).
-		SetTester(IntCompare(eq, lines)).
+		SetTester(IntCompare(Eq, lines)).
 		SetStream(OnlyStdout)
 }

packer_test/pipe_checker_test.go → packer_test/lib/pipe_checker.go

@@ -1,4 +1,4 @@
-package packer_test
+package lib
 
 import (
 	"fmt"
@@ -101,30 +101,30 @@ func ExpectEmptyInput() Tester {
 	})
 }
 
-type op int
+type Op int
 
 const (
-	eq op = iota
-	ne
-	gt
-	ge
-	lt
-	le
+	Eq Op = iota
+	Ne
+	Gt
+	Ge
+	Lt
+	Le
 )
 
-func (op op) String() string {
+func (op Op) String() string {
 	switch op {
-	case eq:
+	case Eq:
 		return "=="
-	case ne:
+	case Ne:
 		return "!="
-	case gt:
+	case Gt:
 		return ">"
-	case ge:
+	case Ge:
 		return ">="
-	case lt:
+	case Lt:
 		return "<"
-	case le:
+	case Le:
 		return "<="
 	}
 
@@ -134,7 +134,7 @@ func (op op) String() string {
 // IntCompare reads the input from the pipeline and compares it to a value using `op`
 //
 // If the input is not an int, this fails.
-func IntCompare(op op, value int) Tester {
+func IntCompare(op Op, value int) Tester {
 	return CustomTester(func(in string) error {
 		n, err := strconv.Atoi(strings.TrimSpace(in))
 		if err != nil {
@@ -143,17 +143,17 @@ func IntCompare(op op, value int) Tester {
 
 		var result bool
 		switch op {
-		case eq:
+		case Eq:
 			result = n == value
-		case ne:
+		case Ne:
 			result = n != value
-		case gt:
+		case Gt:
 			result = n > value
-		case ge:
+		case Ge:
 			result = n >= value
-		case lt:
+		case Lt:
 			result = n < value
-		case le:
+		case Le:
 			result = n <= value
 		default:
 			panic(fmt.Sprintf("Unsupported operator %d, make sure the operation is implemented for IntCompare", op))

packer_test/plugin_test.go → packer_test/lib/plugin.go

@@ -1,11 +1,10 @@
-package packer_test
+package lib
 
 import (
 	"crypto/sha256"
 	"fmt"
 	"io"
 	"os"
-	"os/exec"
 	"path/filepath"
 	"runtime"
 	"strings"
@@ -37,25 +36,6 @@ func LoadPluginVersion(pluginVersion string) (string, bool) {
 	return path, ok
 }
 
-var tempPluginBinaryPath = struct {
-	path string
-	once sync.Once
-}{}
-
-// PluginBinaryDir returns the path to the directory where temporary binaries will be compiled
-func PluginBinaryDir() string {
-	tempPluginBinaryPath.once.Do(func() {
-		tempDir, err := os.MkdirTemp("", "packer-core-acc-test-")
-		if err != nil {
-			panic(fmt.Sprintf("failed to create temporary directory for compiled plugins: %s", err))
-		}
-
-		tempPluginBinaryPath.path = tempDir
-	})
-
-	return tempPluginBinaryPath.path
-}
-
 // LDFlags compiles the ldflags for the plugin to compile based on the information provided.
 func LDFlags(version *version.Version) string {
 	pluginPackage := "github.com/hashicorp/packer-plugin-tester"
@@ -104,48 +84,6 @@ func ExpectedInstalledName(versionStr string) string {
 		runtime.GOOS, runtime.GOARCH, ext)
 }
 
-// BuildSimplePlugin creates a plugin that essentially does nothing.
-//
-// The plugin's code is contained in a subdirectory of this, and lets us
-// change the attributes of the plugin binary itself, like the SDK version,
-// the plugin's version, etc.
-//
-// The plugin is functional, and can be used to run builds with.
-// There won't be anything substantial created though, its goal is only
-// to validate the core functionality of Packer.
-//
-// The path to the plugin is returned, it won't be removed automatically
-// though, deletion is the caller's responsibility.
-func BuildSimplePlugin(versionString string, t *testing.T) string {
-	// Only build plugin binary if not already done beforehand
-	path, ok := LoadPluginVersion(versionString)
-	if ok {
-		return path
-	}
-
-	v := version.Must(version.NewSemver(versionString))
-
-	t.Logf("Building plugin in version %v", v)
-
-	testDir, err := currentDir()
-	if err != nil {
-		t.Fatalf("failed to compile plugin binary: %s", err)
-	}
-
-	testerPluginDir := filepath.Join(testDir, "plugin_tester")
-	outBin := filepath.Join(PluginBinaryDir(), BinaryName(v))
-
-	compileCommand := exec.Command("go", "build", "-C", testerPluginDir, "-o", outBin, "-ldflags", LDFlags(v), ".")
-	logs, err := compileCommand.CombinedOutput()
-	if err != nil {
-		t.Fatalf("failed to compile plugin binary: %s\ncompiler logs: %s", err, logs)
-	}
-
-	StorePluginVersion(v.String(), outBin)
-
-	return outBin
-}
-
 // currentDir returns the directory in which the current file is located.
 //
 // Since we're in tests it's reliable as they're supposed to run on the same
@@ -171,7 +109,7 @@ func (ts *PackerTestSuite) MakePluginDir(pluginVersions ...string) (pluginTempDi
 	t := ts.T()
 
 	for _, ver := range pluginVersions {
-		BuildSimplePlugin(ver, t)
+		ts.BuildSimplePlugin(ver, t)
 	}
 
 	var err error