packer: pick protobuf/gob for serialisation

As we're trying to move away from gob for serialising data over the
wire, this commit adds the capability for Packer to pick dynamically
between gob or protobuf for the serialisation format to communicate with
plugins.

As it stands, if all the plugins discovered are compatible with
protobuf, and we have not forced gob usage, protobuf will be the
serialisation format picked.

If any plugin is not compatible with protobuf, gob will eb used for
communicating with all the plugins that will be used over the course of
a command.

command/execute.go

@@ -5,13 +5,14 @@
 package command
 
 import (
+	"flag"
 	"fmt"
-	"log"
 	"regexp"
 	"strings"
 
 	packersdk "github.com/hashicorp/packer-plugin-sdk/packer"
 	"github.com/hashicorp/packer-plugin-sdk/plugin"
+	"github.com/hashicorp/packer-plugin-sdk/rpc"
 
 	filebuilder "github.com/hashicorp/packer/builder/file"
 	nullbuilder "github.com/hashicorp/packer/builder/null"
@@ -75,18 +76,49 @@ var Datasources = map[string]packersdk.Datasource{
 
 var pluginRegexp = regexp.MustCompile("packer-(builder|post-processor|provisioner|datasource)-(.+)")
 
-func (c *ExecuteCommand) Run(args []string) int {
-	// This is an internal call (users should not call this directly) so we're
-	// not going to do much input validation. If there's a problem we'll often
-	// just crash. Error handling should be added to facilitate debugging.
-	log.Printf("args: %#v", args)
+type ExecuteArgs struct {
+	UseProtobuf bool
+	CommandType string
+}
+
+func (ea *ExecuteArgs) AddFlagSets(flags *flag.FlagSet) {
+	flags.BoolVar(&ea.UseProtobuf, "protobuf", false, "Use protobuf for serialising data over the wire instead of gob")
+}
+
+func (c *ExecuteCommand) ParseArgs(args []string) (*ExecuteArgs, int) {
+	var cfg ExecuteArgs
+	flags := c.Meta.FlagSet("")
+	flags.Usage = func() { c.Ui.Say(c.Help()) }
+	cfg.AddFlagSets(flags)
+	if err := flags.Parse(args); err != nil {
+		return &cfg, 1
+	}
+
+	args = flags.Args()
 	if len(args) != 1 {
-		c.Ui.Error(c.Help())
-		return 1
+		flags.Usage()
+		return &cfg, 1
+	}
+	cfg.CommandType = args[0]
+	return &cfg, 0
+}
+
+func (c *ExecuteCommand) Run(args []string) int {
+	cfg, ret := c.ParseArgs(args)
+	if ret != 0 {
+		return ret
+	}
+
+	return c.RunContext(cfg)
+}
+
+func (c *ExecuteCommand) RunContext(args *ExecuteArgs) int {
+	if args.UseProtobuf {
+		rpc.UseProto = true
 	}
 
 	// Plugin will match something like "packer-builder-amazon-ebs"
-	parts := pluginRegexp.FindStringSubmatch(args[0])
+	parts := pluginRegexp.FindStringSubmatch(args.CommandType)
 	if len(parts) != 3 {
 		c.Ui.Error(c.Help())
 		return 1
@@ -138,11 +170,15 @@ func (c *ExecuteCommand) Run(args []string) int {
 
 func (*ExecuteCommand) Help() string {
 	helpText := `
-Usage: packer execute PLUGIN
+Usage: packer execute [options] PLUGIN
 
   Runs an internally-compiled version of a plugin from the packer binary.
 
   NOTE: this is an internal command and you should not call it yourself.
+
+Options:
+
+  --protobuf: use protobuf for serialising data over-the-wire instead of gob.
 `
 
 	return strings.TrimSpace(helpText)

config.go

@@ -16,10 +16,6 @@ import (
 	"github.com/hashicorp/packer/packer"
 )
 
-// PACKERSPACE is used to represent the spaces that separate args for a command
-// without being confused with spaces in the path to the command itself.
-const PACKERSPACE = "-PACKERSPACE-"
-
 type config struct {
 	DisableCheckpoint          bool              `json:"disable_checkpoint"`
 	DisableCheckpointSignature bool              `json:"disable_checkpoint_signature"`
@@ -109,43 +105,67 @@ func (c *config) discoverInternalComponents() error {
 	for builder := range command.Builders {
 		builder := builder
 		if !c.Plugins.Builders.Has(builder) {
-			bin := fmt.Sprintf("%s%sexecute%spacker-builder-%s",
-				packerPath, PACKERSPACE, PACKERSPACE, builder)
 			c.Plugins.Builders.Set(builder, func() (packersdk.Builder, error) {
-				return c.Plugins.Client(bin).Builder()
+				args := []string{"execute"}
+
+				if packer.PackerUseProto {
+					args = append(args, "--protobuf")
+				}
+
+				args = append(args, fmt.Sprintf("packer-builder-%s", builder))
+
+				return c.Plugins.Client(packerPath, args...).Builder()
 			})
 		}
 	}
 
 	for provisioner := range command.Provisioners {
 		provisioner := provisioner
 		if !c.Plugins.Provisioners.Has(provisioner) {
-			bin := fmt.Sprintf("%s%sexecute%spacker-provisioner-%s",
-				packerPath, PACKERSPACE, PACKERSPACE, provisioner)
 			c.Plugins.Provisioners.Set(provisioner, func() (packersdk.Provisioner, error) {
-				return c.Plugins.Client(bin).Provisioner()
+				args := []string{"execute"}
+
+				if packer.PackerUseProto {
+					args = append(args, "--protobuf")
+				}
+
+				args = append(args, fmt.Sprintf("packer-provisioner-%s", provisioner))
+
+				return c.Plugins.Client(packerPath, args...).Provisioner()
 			})
 		}
 	}
 
 	for postProcessor := range command.PostProcessors {
 		postProcessor := postProcessor
 		if !c.Plugins.PostProcessors.Has(postProcessor) {
-			bin := fmt.Sprintf("%s%sexecute%spacker-post-processor-%s",
-				packerPath, PACKERSPACE, PACKERSPACE, postProcessor)
 			c.Plugins.PostProcessors.Set(postProcessor, func() (packersdk.PostProcessor, error) {
-				return c.Plugins.Client(bin).PostProcessor()
+				args := []string{"execute"}
+
+				if packer.PackerUseProto {
+					args = append(args, "--protobuf")
+				}
+
+				args = append(args, fmt.Sprintf("packer-post-processor-%s", postProcessor))
+
+				return c.Plugins.Client(packerPath, args...).PostProcessor()
 			})
 		}
 	}
 
 	for dataSource := range command.Datasources {
 		dataSource := dataSource
 		if !c.Plugins.DataSources.Has(dataSource) {
-			bin := fmt.Sprintf("%s%sexecute%spacker-datasource-%s",
-				packerPath, PACKERSPACE, PACKERSPACE, dataSource)
 			c.Plugins.DataSources.Set(dataSource, func() (packersdk.Datasource, error) {
-				return c.Plugins.Client(bin).Datasource()
+				args := []string{"execute"}
+
+				if packer.PackerUseProto {
+					args = append(args, "--protobuf")
+				}
+
+				args = append(args, fmt.Sprintf("packer-datasource-%s", dataSource))
+
+				return c.Plugins.Client(packerPath, args...).Datasource()
 			})
 		}
 	}

go.mod

@@ -24,7 +24,7 @@ require (
 	github.com/hashicorp/hcl/v2 v2.19.1
 	github.com/hashicorp/hcp-sdk-go v0.96.0
 	github.com/hashicorp/packer-plugin-amazon v1.2.1
-	github.com/hashicorp/packer-plugin-sdk v0.5.3
+	github.com/hashicorp/packer-plugin-sdk v0.5.4-0.20240606193218-b9a810803331
 	github.com/jehiah/go-strftime v0.0.0-20171201141054-1d33003b3869
 	github.com/klauspost/compress v1.13.6 // indirect
 	github.com/klauspost/pgzip v1.2.5
@@ -48,12 +48,12 @@ require (
 	golang.org/x/mod v0.13.0
 	golang.org/x/net v0.24.0
 	golang.org/x/oauth2 v0.15.0
-	golang.org/x/sync v0.4.0
+	golang.org/x/sync v0.5.0
 	golang.org/x/sys v0.19.0 // indirect
 	golang.org/x/term v0.19.0 // indirect
-	golang.org/x/text v0.14.0 // indirect
+	golang.org/x/text v0.14.0
 	golang.org/x/tools v0.14.0
-	google.golang.org/api v0.128.0 // indirect
+	google.golang.org/api v0.150.0 // indirect
 	google.golang.org/grpc v1.59.0
 )
 
@@ -65,10 +65,10 @@ require (
 )
 
 require (
-	cloud.google.com/go/compute v1.23.0 // indirect
+	cloud.google.com/go/compute v1.23.1 // indirect
 	cloud.google.com/go/compute/metadata v0.2.3 // indirect
-	cloud.google.com/go/iam v1.1.2 // indirect
-	cloud.google.com/go/storage v1.30.1 // indirect
+	cloud.google.com/go/iam v1.1.3 // indirect
+	cloud.google.com/go/storage v1.35.1 // indirect
 	dario.cat/mergo v1.0.0 // indirect
 	github.com/Azure/go-ntlmssp v0.0.0-20200615164410-66371956d46c // indirect
 	github.com/ChrisTrenkamp/goxpath v0.0.0-20210404020558-97928f7e12b6 // indirect
@@ -84,7 +84,7 @@ require (
 	github.com/armon/go-metrics v0.4.1 // indirect
 	github.com/armon/go-radix v1.0.0 // indirect
 	github.com/asaskevich/govalidator v0.0.0-20230301143203-a9d515a09cc2 // indirect
-	github.com/aws/aws-sdk-go v1.44.114 // indirect
+	github.com/aws/aws-sdk-go v1.45.6 // indirect
 	github.com/bgentry/go-netrc v0.0.0-20140422174119-9fd32a8b3d3d // indirect
 	github.com/bgentry/speakeasy v0.1.0 // indirect
 	github.com/bmatcuk/doublestar v1.1.5 // indirect
@@ -112,15 +112,15 @@ require (
 	github.com/go-openapi/validate v0.22.4 // indirect
 	github.com/gofrs/uuid v4.0.0+incompatible // indirect
 	github.com/golang/protobuf v1.5.3 // indirect
-	github.com/google/s2a-go v0.1.4 // indirect
+	github.com/google/s2a-go v0.1.7 // indirect
 	github.com/google/uuid v1.4.0 // indirect
-	github.com/googleapis/enterprise-certificate-proxy v0.2.4 // indirect
+	github.com/googleapis/enterprise-certificate-proxy v0.3.2 // indirect
 	github.com/googleapis/gax-go/v2 v2.12.0 // indirect
 	github.com/hashicorp/aws-sdk-go-base v0.7.1 // indirect
 	github.com/hashicorp/consul/api v1.25.1 // indirect
 	github.com/hashicorp/errwrap v1.1.0 // indirect
 	github.com/hashicorp/go-cleanhttp v0.5.2 // indirect
-	github.com/hashicorp/go-getter/gcs/v2 v2.2.1 // indirect
+	github.com/hashicorp/go-getter/gcs/v2 v2.2.2 // indirect
 	github.com/hashicorp/go-getter/s3/v2 v2.2.2 // indirect
 	github.com/hashicorp/go-hclog v1.5.0 // indirect
 	github.com/hashicorp/go-immutable-radix v1.3.1 // indirect
@@ -169,6 +169,8 @@ require (
 	github.com/tklauser/go-sysconf v0.3.11 // indirect
 	github.com/tklauser/numcpus v0.6.0 // indirect
 	github.com/ugorji/go/codec v1.2.6 // indirect
+	github.com/vmihailenco/msgpack/v5 v5.3.5 // indirect
+	github.com/vmihailenco/tagparser/v2 v2.0.0 // indirect
 	github.com/xanzy/ssh-agent v0.3.3 // indirect
 	github.com/xeipuuv/gojsonpointer v0.0.0-20190905194746-02993c407bfb // indirect
 	github.com/yusufpapurcu/wmi v1.2.2 // indirect
@@ -178,12 +180,12 @@ require (
 	go.opentelemetry.io/otel/metric v1.17.0 // indirect
 	go.opentelemetry.io/otel/trace v1.17.0 // indirect
 	golang.org/x/exp v0.0.0-20230321023759-10a507213a29 // indirect
-	golang.org/x/time v0.0.0-20220210224613-90d013bbcef8 // indirect
+	golang.org/x/time v0.3.0 // indirect
 	golang.org/x/xerrors v0.0.0-20220907171357-04be3eba64a2 // indirect
 	google.golang.org/appengine v1.6.7 // indirect
-	google.golang.org/genproto v0.0.0-20231012201019-e917dd12ba7a // indirect
-	google.golang.org/genproto/googleapis/api v0.0.0-20231002182017-d307bd883b97 // indirect
-	google.golang.org/genproto/googleapis/rpc v0.0.0-20231016165738-49dd2c1f3d0b // indirect
+	google.golang.org/genproto v0.0.0-20231016165738-49dd2c1f3d0b // indirect
+	google.golang.org/genproto/googleapis/api v0.0.0-20231016165738-49dd2c1f3d0b // indirect
+	google.golang.org/genproto/googleapis/rpc v0.0.0-20231030173426-d783a09b4405 // indirect
 	google.golang.org/protobuf v1.33.0 // indirect
 	gopkg.in/cheggaaa/pb.v1 v1.0.28 // indirect
 	gopkg.in/warnings.v0 v0.1.2 // indirect