Improve error reporting

* Add ConfigType to ConfigData for improved errors
* Add validation errors to DataFromProto
* Improve config parsing and  validation message

artifact_source_config/artifact_source_config_base.go

@@ -14,7 +14,6 @@ type ArtifactSourceConfigBase struct {
 }
 
 func (b *ArtifactSourceConfigBase) Validate() error {
-	// TODO #config #valiate
 	return nil
 }
 

config_data/config_data.go

@@ -14,16 +14,19 @@ type ConfigData interface {
 	GetHcl() []byte
 	GetRange() hcl.Range
 	Identifier() string
+	GetConfigType() string
 }
 
 type ConfigDataImpl struct {
 	Hcl   []byte
 	Range hcl.Range
 	// Id represent the type of the config:
-	// - if this is a table config, this will be the table name
+	// - if this is a partition config, this will be the table name
 	// - if this is a source config, this will be the source type
 	// - if this is a connection config, this will be the connection type (i.e. plugin name)
 	Id string
+	// ConfigType is the type of the config data, i.e. connection, source, partition
+	ConfigType string
 }
 
 // GetHcl returns the HCL config data
@@ -41,6 +44,11 @@ func (c *ConfigDataImpl) Identifier() string {
 	return c.Id
 }
 
+// GetConfigType returns the type of the config data
+func (c *ConfigDataImpl) GetConfigType() string {
+	return c.ConfigType
+}
+
 func DataFromProto[T ConfigData](data *proto.ConfigData) (T, error) {
 	/* the target field is the name ofd the target - this will be either:
 	- a source: source.<source_type>
@@ -52,20 +60,28 @@ func DataFromProto[T ConfigData](data *proto.ConfigData) (T, error) {
 	parts := strings.Split(data.Target, ".")
 	switch any(empty).(type) {
 	case *SourceConfigData:
-		// TODO assert len
+		if len(parts) != 2 {
+			return empty, fmt.Errorf("invalid source config target %s: expected a name of format source.<type>", data.Target)
+		}
 		if parts[0] != "source" {
 			return empty, fmt.Errorf("invalid source config target %s: expected a source", data.Target)
 		}
 		d := NewSourceConfigData(data.Hcl, proto.RangeFromProto(data.Range), parts[1])
 		return ConfigData(d).(T), nil
 
 	case *PartitionConfigData:
+		if len(parts) != 3 {
+			return empty, fmt.Errorf("invalid source config target %s: expected a name of format partition.<table>.<partition>", data.Target)
+		}
 		if parts[0] != "partition" {
 			return empty, fmt.Errorf("invalid source config target %s: expected a partition", data.Target)
 		}
 		d := NewPartitionConfigData(data.Hcl, proto.RangeFromProto(data.Range), parts[1], parts[2])
 		return ConfigData(d).(T), nil
 	case *ConnectionConfigData:
+		if len(parts) != 2 {
+			return empty, fmt.Errorf("invalid source config target %s: expected a name of format connection.<type>", data.Target)
+		}
 		if parts[0] != "connection" {
 			return empty, fmt.Errorf("invalid source config target %s: expected a connection", data.Target)
 		}

config_data/connection.go

@@ -10,9 +10,10 @@ type ConnectionConfigData struct {
 func NewConnectionConfigData(hcl []byte, decRange hcl.Range, ty string) *ConnectionConfigData {
 	return &ConnectionConfigData{
 		ConfigDataImpl: &ConfigDataImpl{
-			Hcl:   hcl,
-			Range: decRange,
-			Id:    ty,
+			Hcl:        hcl,
+			Range:      decRange,
+			Id:         ty,
+			ConfigType: "connection",
 		},
 		Type: ty,
 	}

config_data/partition.go

@@ -11,9 +11,10 @@ type PartitionConfigData struct {
 func NewPartitionConfigData(hcl []byte, decRange hcl.Range, table string, partition string) *PartitionConfigData {
 	return &PartitionConfigData{
 		ConfigDataImpl: &ConfigDataImpl{
-			Hcl:   hcl,
-			Range: decRange,
-			Id:    table,
+			Hcl:        hcl,
+			Range:      decRange,
+			Id:         table,
+			ConfigType: "partition",
 		},
 		Table:     table,
 		Partition: partition,

config_data/source.go

@@ -10,9 +10,10 @@ type SourceConfigData struct {
 func NewSourceConfigData(hcl []byte, decRange hcl.Range, sourceType string) *SourceConfigData {
 	return &SourceConfigData{
 		ConfigDataImpl: &ConfigDataImpl{
-			Hcl:   hcl,
-			Range: decRange,
-			Id:    sourceType,
+			Hcl:        hcl,
+			Range:      decRange,
+			Id:         sourceType,
+			ConfigType: "source",
 		},
 		Type: sourceType,
 	}

design/interaction/plugin events.txt

@@ -0,0 +1,26 @@
+title Plugin Events
+
+
+participant CLI
+participant Collector
+participant PluginManager
+participant Plugin
+
+
+
+[->CLI:Collect
+activate CLI
+CLI->Collector:Collect
+activate Collector
+Collector->>PluginManager:Collect
+activate PluginManager
+Collector->Collector:handlePluginEvent
+activate Collector
+PluginManager->Plugin:Collect
+activate Plugin
+PluginManager ->PluginManager:readCollectionEvents
+activate PluginManager
+Plugin -->>PluginManager:event
+PluginManager -->> Collector:event
+Plugin -->>PluginManager:event
+PluginManager -->> Collector:event

go.mod

@@ -7,29 +7,22 @@ toolchain go1.23.2
 replace github.com/turbot/pipe-fittings => ../pipe-fittings
 
 require (
-	cloud.google.com/go/storage v1.42.0
-	github.com/aws/aws-sdk-go-v2 v1.30.3
-	github.com/aws/aws-sdk-go-v2/config v1.27.11
-	github.com/aws/aws-sdk-go-v2/credentials v1.17.26
-	github.com/aws/aws-sdk-go-v2/service/cloudwatchlogs v1.37.3
-	github.com/aws/aws-sdk-go-v2/service/s3 v1.58.2
 	github.com/hashicorp/go-hclog v1.6.2
 	github.com/hashicorp/go-plugin v1.6.1
 	github.com/hashicorp/hcl/v2 v2.20.1
 	github.com/iancoleman/strcase v0.3.0
-	github.com/mitchellh/go-homedir v1.1.0
-	github.com/rs/dnscache v0.0.0-20230804202142-fc85eb664529
+	github.com/marcboeker/go-duckdb v1.8.3
+	github.com/rs/xid v1.5.0
 	github.com/satyrius/gonx v1.4.0
 	github.com/stretchr/testify v1.9.0
 	github.com/turbot/go-kit v0.10.0-rc.0
 	github.com/turbot/pipe-fittings v1.4.1
 	github.com/zclconf/go-cty v1.14.4
-	golang.org/x/exp v0.0.0-20231006140011-7918f672742d
+	golang.org/x/exp v0.0.0-20240909161429-701f63a606c0
 	golang.org/x/sync v0.8.0
 	golang.org/x/time v0.5.0
-	google.golang.org/api v0.189.0
-	google.golang.org/grpc v1.66.0
-	google.golang.org/protobuf v1.34.2
+	google.golang.org/grpc v1.67.1
+	google.golang.org/protobuf v1.35.1
 )
 
 require (
@@ -38,24 +31,25 @@ require (
 	cloud.google.com/go/auth/oauth2adapt v0.2.3 // indirect
 	cloud.google.com/go/compute/metadata v0.5.0 // indirect
 	cloud.google.com/go/iam v1.1.10 // indirect
+	cloud.google.com/go/storage v1.42.0 // indirect
 	github.com/AdaLogics/go-fuzz-headers v0.0.0-20230811130428-ced1acdcaa24 // indirect
 	github.com/Masterminds/semver/v3 v3.2.1 // indirect
 	github.com/Microsoft/hcsshim v0.11.5 // indirect
 	github.com/acarl005/stripansi v0.0.0-20180116102854-5a71ef0e047d // indirect
 	github.com/agext/levenshtein v1.2.3 // indirect
+	github.com/apache/arrow-go/v18 v18.0.0 // indirect
 	github.com/apparentlymart/go-cidr v1.1.0 // indirect
 	github.com/apparentlymart/go-textseg/v15 v15.0.0 // indirect
 	github.com/aws/aws-sdk-go v1.44.183 // indirect
-	github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.6.3 // indirect
+	github.com/aws/aws-sdk-go-v2 v1.30.3 // indirect
+	github.com/aws/aws-sdk-go-v2/config v1.27.11 // indirect
+	github.com/aws/aws-sdk-go-v2/credentials v1.17.26 // indirect
 	github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.16.11 // indirect
 	github.com/aws/aws-sdk-go-v2/internal/configsources v1.3.15 // indirect
 	github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.6.15 // indirect
 	github.com/aws/aws-sdk-go-v2/internal/ini v1.8.0 // indirect
-	github.com/aws/aws-sdk-go-v2/internal/v4a v1.3.15 // indirect
 	github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.11.3 // indirect
-	github.com/aws/aws-sdk-go-v2/service/internal/checksum v1.3.17 // indirect
 	github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.11.17 // indirect
-	github.com/aws/aws-sdk-go-v2/service/internal/s3shared v1.17.15 // indirect
 	github.com/aws/aws-sdk-go-v2/service/sso v1.22.3 // indirect
 	github.com/aws/aws-sdk-go-v2/service/ssooidc v1.26.4 // indirect
 	github.com/aws/aws-sdk-go-v2/service/sts v1.30.3 // indirect
@@ -82,9 +76,11 @@ require (
 	github.com/go-playground/locales v0.14.1 // indirect
 	github.com/go-playground/universal-translator v0.18.1 // indirect
 	github.com/go-playground/validator/v10 v10.20.0 // indirect
+	github.com/goccy/go-json v0.10.3 // indirect
 	github.com/goccy/go-yaml v1.11.2 // indirect
 	github.com/golang/groupcache v0.0.0-20210331224755-41bb18bfe9da // indirect
 	github.com/golang/protobuf v1.5.4 // indirect
+	github.com/google/flatbuffers v24.3.25+incompatible // indirect
 	github.com/google/go-cmp v0.6.0 // indirect
 	github.com/google/s2a-go v0.1.7 // indirect
 	github.com/google/uuid v1.6.0 // indirect
@@ -113,14 +109,16 @@ require (
 	github.com/jackc/puddle/v2 v2.2.1 // indirect
 	github.com/jmespath/go-jmespath v0.4.0 // indirect
 	github.com/karrick/gows v0.3.0 // indirect
-	github.com/klauspost/compress v1.17.2 // indirect
+	github.com/klauspost/compress v1.17.11 // indirect
+	github.com/klauspost/cpuid/v2 v2.2.8 // indirect
 	github.com/leodido/go-urn v1.4.0 // indirect
 	github.com/logrusorgru/aurora v2.0.3+incompatible // indirect
 	github.com/magiconair/properties v1.8.7 // indirect
 	github.com/mattn/go-colorable v0.1.13 // indirect
 	github.com/mattn/go-isatty v0.0.20 // indirect
 	github.com/mattn/go-runewidth v0.0.15 // indirect
 	github.com/mitchellh/colorstring v0.0.0-20190213212951-d06e56a500db // indirect
+	github.com/mitchellh/go-homedir v1.1.0 // indirect
 	github.com/mitchellh/go-testing-interface v1.14.1 // indirect
 	github.com/mitchellh/go-wordwrap v1.0.0 // indirect
 	github.com/mitchellh/mapstructure v1.5.0 // indirect
@@ -130,10 +128,10 @@ require (
 	github.com/opencontainers/go-digest v1.0.0 // indirect
 	github.com/opencontainers/image-spec v1.1.0 // indirect
 	github.com/pelletier/go-toml/v2 v2.2.2 // indirect
+	github.com/pierrec/lz4/v4 v4.1.21 // indirect
 	github.com/pjbgf/sha1cd v0.3.0 // indirect
 	github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2 // indirect
 	github.com/rivo/uniseg v0.2.0 // indirect
-	github.com/rs/xid v1.5.0 // indirect
 	github.com/sagikazarmark/locafero v0.4.0 // indirect
 	github.com/sagikazarmark/slog-shim v0.1.0 // indirect
 	github.com/shiena/ansicolor v0.0.0-20230509054315-a9deabde6e02 // indirect
@@ -158,6 +156,7 @@ require (
 	github.com/xlab/treeprint v1.2.0 // indirect
 	github.com/yusufpapurcu/wmi v1.2.2 // indirect
 	github.com/zclconf/go-cty-yaml v1.0.3 // indirect
+	github.com/zeebo/xxh3 v1.0.2 // indirect
 	go.opencensus.io v0.24.0 // indirect
 	go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc v0.49.0 // indirect
 	go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.49.0 // indirect
@@ -166,18 +165,19 @@ require (
 	go.opentelemetry.io/otel/trace v1.26.0 // indirect
 	go.uber.org/atomic v1.9.0 // indirect
 	go.uber.org/multierr v1.9.0 // indirect
-	golang.org/x/crypto v0.25.0 // indirect
-	golang.org/x/mod v0.17.0 // indirect
-	golang.org/x/net v0.27.0 // indirect
-	golang.org/x/oauth2 v0.21.0 // indirect
-	golang.org/x/sys v0.22.0 // indirect
-	golang.org/x/term v0.22.0 // indirect
-	golang.org/x/text v0.17.0 // indirect
-	golang.org/x/tools v0.21.1-0.20240508182429-e35e4ccd0d2d // indirect
+	golang.org/x/crypto v0.28.0 // indirect
+	golang.org/x/mod v0.21.0 // indirect
+	golang.org/x/net v0.30.0 // indirect
+	golang.org/x/oauth2 v0.22.0 // indirect
+	golang.org/x/sys v0.26.0 // indirect
+	golang.org/x/term v0.25.0 // indirect
+	golang.org/x/text v0.19.0 // indirect
+	golang.org/x/tools v0.26.0 // indirect
 	golang.org/x/xerrors v0.0.0-20231012003039-104605ab7028 // indirect
+	google.golang.org/api v0.189.0 // indirect
 	google.golang.org/genproto v0.0.0-20240722135656-d784300faade // indirect
-	google.golang.org/genproto/googleapis/api v0.0.0-20240722135656-d784300faade // indirect
-	google.golang.org/genproto/googleapis/rpc v0.0.0-20240722135656-d784300faade // indirect
+	google.golang.org/genproto/googleapis/api v0.0.0-20240814211410-ddb44dafa142 // indirect
+	google.golang.org/genproto/googleapis/rpc v0.0.0-20240903143218-8af14fe29dc1 // indirect
 	gopkg.in/ini.v1 v1.67.0 // indirect
 	gopkg.in/yaml.v3 v3.0.1 // indirect
 	oras.land/oras-go/v2 v2.5.0 // indirect