Skip to content

Commit

Permalink
Migrating datadog exporter to use aws sdk v2
Browse files Browse the repository at this point in the history
  • Loading branch information
LZiHaN committed Dec 12, 2024
1 parent ca10fa7 commit 1a27699
Show file tree
Hide file tree
Showing 12 changed files with 401 additions and 145 deletions.
16 changes: 15 additions & 1 deletion exporter/datadogexporter/go.mod
Original file line number Diff line number Diff line change
Expand Up @@ -38,7 +38,7 @@ require (
github.com/DataDog/opentelemetry-mapping-go/pkg/quantile v0.21.0
github.com/DataDog/sketches-go v1.4.6 // indirect
github.com/GoogleCloudPlatform/opentelemetry-operations-go/detectors/gcp v1.25.0
github.com/aws/aws-sdk-go v1.55.5
github.com/aws/aws-sdk-go v1.55.5 // indirect
github.com/cenkalti/backoff/v4 v4.3.0
github.com/google/go-cmp v0.6.0
github.com/open-telemetry/opentelemetry-collector-contrib/connector/datadogconnector v0.115.0
Expand Down Expand Up @@ -91,6 +91,10 @@ require (
)

require (
github.com/aws/aws-sdk-go-v2 v1.32.6
github.com/aws/aws-sdk-go-v2/config v1.28.6
github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.16.21
github.com/aws/aws-sdk-go-v2/service/ec2 v1.196.0
github.com/open-telemetry/opentelemetry-collector-contrib/pkg/datadog v0.115.0
go.opentelemetry.io/collector/component/componenttest v0.115.1-0.20241206185113-3f3e208e71b8
go.opentelemetry.io/collector/consumer/consumererror v0.115.1-0.20241206185113-3f3e208e71b8
Expand Down Expand Up @@ -158,6 +162,16 @@ require (
github.com/antchfx/xmlquery v1.4.2 // indirect
github.com/antchfx/xpath v1.3.2 // indirect
github.com/armon/go-metrics v0.4.1 // indirect
github.com/aws/aws-sdk-go-v2/credentials v1.17.47 // indirect
github.com/aws/aws-sdk-go-v2/internal/configsources v1.3.25 // indirect
github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.6.25 // indirect
github.com/aws/aws-sdk-go-v2/internal/ini v1.8.1 // indirect
github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.12.1 // indirect
github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.12.6 // indirect
github.com/aws/aws-sdk-go-v2/service/sso v1.24.7 // indirect
github.com/aws/aws-sdk-go-v2/service/ssooidc v1.28.6 // indirect
github.com/aws/aws-sdk-go-v2/service/sts v1.33.2 // indirect
github.com/aws/smithy-go v1.22.1 // indirect
github.com/benbjohnson/clock v1.3.5 // indirect
github.com/beorn7/perks v1.0.1 // indirect
github.com/bmatcuk/doublestar/v4 v4.7.1 // indirect
Expand Down
28 changes: 28 additions & 0 deletions exporter/datadogexporter/go.sum

Some generated files are not rendered by default. Learn more about how customized files appear on GitHub.

70 changes: 41 additions & 29 deletions exporter/datadogexporter/internal/hostmetadata/internal/ec2/ec2.go
Original file line number Diff line number Diff line change
Expand Up @@ -7,14 +7,18 @@ package ec2 // import "github.com/open-telemetry/opentelemetry-collector-contrib
import (
"context"
"fmt"
"io"
"strings"
"sync"

"github.com/aws/aws-sdk-go-v2/config"
"github.com/aws/aws-sdk-go-v2/feature/ec2/imds"
"github.com/aws/aws-sdk-go-v2/service/ec2/types"

"github.com/DataDog/opentelemetry-mapping-go/pkg/otlp/attributes/source"
"github.com/aws/aws-sdk-go/aws"
"github.com/aws/aws-sdk-go/aws/ec2metadata"
"github.com/aws/aws-sdk-go/aws/session"
"github.com/aws/aws-sdk-go/service/ec2"
"github.com/aws/aws-sdk-go-v2/aws"
"github.com/aws/aws-sdk-go-v2/service/ec2"

"go.uber.org/zap"

"github.com/open-telemetry/opentelemetry-collector-contrib/exporter/datadogexporter/internal/hostmetadata/provider"
Expand Down Expand Up @@ -42,31 +46,42 @@ func isDefaultHostname(hostname string) bool {

// GetHostInfo gets the hostname info from EC2 metadata
func GetHostInfo(ctx context.Context, logger *zap.Logger) (hostInfo *HostInfo) {
sess, err := session.NewSession()
hostInfo = &HostInfo{}

cfg, err := config.LoadDefaultConfig(ctx)
if err != nil {
logger.Warn("Failed to build AWS session", zap.Error(err))
logger.Warn("Failed to build AWS config", zap.Error(err))
return
}

meta := ec2metadata.New(sess)
client := imds.NewFromConfig(cfg)

if !meta.AvailableWithContext(ctx) {
logger.Debug("EC2 Metadata not available")
// Check if metadata service is available by trying to retrieve instance ID
_, err = client.GetMetadata(ctx, &imds.GetMetadataInput{
Path: "instance-id",
})
if err != nil {
logger.Debug("EC2 Metadata service is not available", zap.Error(err))
return
}

if idDoc, err := meta.GetInstanceIdentityDocumentWithContext(ctx); err == nil {
if idDoc, err := client.GetInstanceIdentityDocument(ctx, &imds.GetInstanceIdentityDocumentInput{}); err == nil {
hostInfo.InstanceID = idDoc.InstanceID
} else {
logger.Warn("Failed to get EC2 instance id document", zap.Error(err))
}

if ec2Hostname, err := meta.GetMetadataWithContext(ctx, "hostname"); err == nil {
hostInfo.EC2Hostname = ec2Hostname
metadataOutput, err := client.GetMetadata(ctx, &imds.GetMetadataInput{Path: "hostname"})
if err != nil {
logger.Warn("Failed to retrieve EC2 hostname", zap.Error(err))
} else {
logger.Warn("Failed to get EC2 hostname", zap.Error(err))
defer metadataOutput.Content.Close()
hostnameBytes, readErr := io.ReadAll(metadataOutput.Content)
if readErr != nil {
logger.Warn("Failed to read EC2 hostname content", zap.Error(readErr))
} else {
hostInfo.EC2Hostname = string(hostnameBytes)
}
}

return
Expand Down Expand Up @@ -94,13 +109,13 @@ type Provider struct {
}

func NewProvider(logger *zap.Logger) (*Provider, error) {
sess, err := session.NewSession()
cfg, err := config.LoadDefaultConfig(context.Background())
if err != nil {
return nil, err
}
return &Provider{
logger: logger,
detector: ec2provider.NewProvider(sess),
detector: ec2provider.NewProvider(cfg),
}, nil
}

Expand Down Expand Up @@ -129,23 +144,20 @@ func (p *Provider) instanceTags(ctx context.Context) (*ec2.DescribeTagsOutput, e
// Similar to:
// - https://github.com/open-telemetry/opentelemetry-collector-contrib/blob/39dbc1ac8/processor/resourcedetectionprocessor/internal/aws/ec2/ec2.go#L118-L151
// - https://github.com/DataDog/datadog-agent/blob/1b4afdd6a03e8fabcc169b924931b2bb8935dab9/pkg/util/ec2/ec2_tags.go#L104-L134
sess, err := session.NewSession(&aws.Config{
Region: aws.String(meta.Region),
})
cfg, err := config.LoadDefaultConfig(ctx,
config.WithRegion(meta.Region),
)
if err != nil {
return nil, fmt.Errorf("failed to build AWS session: %w", err)
return nil, fmt.Errorf("failed to load AWS config: %w", err)
}

svc := ec2.New(sess)
return svc.DescribeTagsWithContext(ctx,
&ec2.DescribeTagsInput{
Filters: []*ec2.Filter{{
Name: aws.String("resource-id"),
Values: []*string{
aws.String(meta.InstanceID),
},
}},
})
client := ec2.NewFromConfig(cfg)
return client.DescribeTags(ctx, &ec2.DescribeTagsInput{
Filters: []types.Filter{{
Name: aws.String("resource-id"),
Values: []string{meta.InstanceID},
}},
})
}

// clusterNameFromTags gets the AWS EC2 Cluster name from the tags on an EC2 instance.
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -5,7 +5,9 @@ package ec2
import (
"testing"

"github.com/aws/aws-sdk-go/service/ec2"
"github.com/aws/aws-sdk-go-v2/service/ec2"
"github.com/aws/aws-sdk-go-v2/service/ec2/types"

"github.com/stretchr/testify/assert"
"go.uber.org/zap"
)
Expand Down Expand Up @@ -56,7 +58,7 @@ func TestClusterNameFromEC2Tags(t *testing.T) {
name: "missing cluster name tag",
ec2Tags: &ec2.DescribeTagsOutput{
NextToken: strp("NextToken"),
Tags: []*ec2.TagDescription{
Tags: []types.TagDescription{
{Key: strp("some key"), Value: strp("some value")},
},
},
Expand All @@ -66,7 +68,7 @@ func TestClusterNameFromEC2Tags(t *testing.T) {
name: "cluster name tag only has the prefix",
ec2Tags: &ec2.DescribeTagsOutput{
NextToken: strp("NextToken"),
Tags: []*ec2.TagDescription{
Tags: []types.TagDescription{
{Key: strp("some key"), Value: strp("some value")},
{Key: strp("kubernetes.io/cluster/"), Value: strp("some value")},
},
Expand All @@ -77,7 +79,7 @@ func TestClusterNameFromEC2Tags(t *testing.T) {
name: "cluster name is available",
ec2Tags: &ec2.DescribeTagsOutput{
NextToken: strp("NextToken"),
Tags: []*ec2.TagDescription{
Tags: []types.TagDescription{
{Key: strp("some key"), Value: strp("some value")},
{Key: strp("kubernetes.io/cluster/myclustername"), Value: strp("some value")},
},
Expand Down
42 changes: 32 additions & 10 deletions internal/metadataproviders/aws/ec2/metadata.go
Original file line number Diff line number Diff line change
Expand Up @@ -5,37 +5,59 @@ package ec2 // import "github.com/open-telemetry/opentelemetry-collector-contrib

import (
"context"
"fmt"
"io"

"github.com/aws/aws-sdk-go/aws/ec2metadata"
"github.com/aws/aws-sdk-go/aws/session"
"github.com/aws/aws-sdk-go-v2/aws"
"github.com/aws/aws-sdk-go-v2/feature/ec2/imds"
)

type Provider interface {
Get(ctx context.Context) (ec2metadata.EC2InstanceIdentityDocument, error)
Get(ctx context.Context) (imds.InstanceIdentityDocument, error)
Hostname(ctx context.Context) (string, error)
InstanceID(ctx context.Context) (string, error)
}

type metadataClient struct {
metadata *ec2metadata.EC2Metadata
client *imds.Client
}

var _ Provider = (*metadataClient)(nil)

func NewProvider(sess *session.Session) Provider {
func NewProvider(cfg aws.Config) Provider {
return &metadataClient{
metadata: ec2metadata.New(sess),
client: imds.NewFromConfig(cfg),
}
}

func (c *metadataClient) getMetadata(ctx context.Context, path string) (string, error) {
output, err := c.client.GetMetadata(ctx, &imds.GetMetadataInput{Path: path})
if err != nil {
return "", fmt.Errorf("failed to get %s from IMDS: %w", path, err)
}
defer output.Content.Close()

data, err := io.ReadAll(output.Content)
if err != nil {
return "", fmt.Errorf("failed to read %s response: %w", path, err)
}

return string(data), nil
}

func (c *metadataClient) InstanceID(ctx context.Context) (string, error) {
return c.metadata.GetMetadataWithContext(ctx, "instance-id")
return c.getMetadata(ctx, "instance-id")
}

func (c *metadataClient) Hostname(ctx context.Context) (string, error) {
return c.metadata.GetMetadataWithContext(ctx, "hostname")
return c.getMetadata(ctx, "hostname")
}

func (c *metadataClient) Get(ctx context.Context) (ec2metadata.EC2InstanceIdentityDocument, error) {
return c.metadata.GetInstanceIdentityDocumentWithContext(ctx)
func (c *metadataClient) Get(ctx context.Context) (imds.InstanceIdentityDocument, error) {
output, err := c.client.GetInstanceIdentityDocument(ctx, &imds.GetInstanceIdentityDocumentInput{})
if err != nil {
return imds.InstanceIdentityDocument{}, fmt.Errorf("failed to get instance identity document: %w", err)
}

return output.InstanceIdentityDocument, nil
}
Loading

0 comments on commit 1a27699

Please sign in to comment.