-
Notifications
You must be signed in to change notification settings - Fork 5
Commit
This commit does not belong to any branch on this repository, and may belong to a fork outside of the repository.
Merge pull request #229 from practo/redshift-table-use-metric
Collect Redshift Metrics
- Loading branch information
Showing
10 changed files
with
283 additions
and
51 deletions.
There are no files selected for viewing
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,106 @@ | ||
| package controllers | ||
|
|
||
| import ( | ||
| "context" | ||
| "fmt" | ||
| "github.com/practo/klog/v2" | ||
| "github.com/practo/tipoca-stream/redshiftsink/pkg/redshift" | ||
| "sigs.k8s.io/controller-runtime/pkg/client" | ||
| "sync" | ||
| "time" | ||
| ) | ||
|
|
||
| type RedshiftCollector struct { | ||
| redshifter *redshift.Redshift | ||
| } | ||
|
|
||
| func NewRedshiftCollector(client client.Client, secretName, secretNamespace string) (*RedshiftCollector, error) { | ||
| secret := make(map[string]string) | ||
| k8sSecret, err := getSecret(context.Background(), client, secretName, secretNamespace) | ||
| if err != nil { | ||
| return nil, fmt.Errorf("Error getting secret, %v", err) | ||
| } | ||
| for key, value := range k8sSecret.Data { | ||
| secret[key] = string(value) | ||
| } | ||
|
|
||
| redshifter, err := NewRedshiftConnection(secret, "") | ||
| if err != nil { | ||
| return nil, err | ||
| } | ||
|
|
||
| return &RedshiftCollector{ | ||
| redshifter: redshifter, | ||
| }, nil | ||
| } | ||
|
|
||
| func (r *RedshiftCollector) Collect(ctx context.Context, wg *sync.WaitGroup) error { | ||
| defer wg.Done() | ||
| // TODO: make it possible to remove below comment, create the view | ||
| // expects the view to be present redshiftsink_operator.scan_query_total | ||
| err := r.redshifter.CollectQueryTotal(ctx) | ||
| if err != nil { | ||
| klog.Errorf("Redshift Collector shutdown due to error: %v", err) | ||
| return err | ||
| } | ||
| for { | ||
| select { | ||
| case <-ctx.Done(): | ||
| klog.V(2).Infof("ctx cancelled, bye collector") | ||
| return nil | ||
| case <-time.After(time.Second * 1800): | ||
| err := r.redshifter.CollectQueryTotal(ctx) | ||
| if err != nil { | ||
| klog.Errorf("Redshift Collector shutdown due to error: %v", err) | ||
| return err | ||
| } | ||
| } | ||
| } | ||
|
|
||
| return nil | ||
| } | ||
|
|
||
| func NewRedshiftConnection( | ||
| secret map[string]string, | ||
| schema string, | ||
| ) ( | ||
| *redshift.Redshift, | ||
| error, | ||
| ) { | ||
| redshiftSecret := make(map[string]string) | ||
| redshiftSecretKeys := []string{ | ||
| "redshiftHost", | ||
| "redshiftPort", | ||
| "redshiftDatabase", | ||
| "redshiftUser", | ||
| "redshiftPassword", | ||
| } | ||
| for _, key := range redshiftSecretKeys { | ||
| value, err := secretByKey(secret, key) | ||
| if err != nil { | ||
| return nil, err | ||
| } | ||
| redshiftSecret[key] = value | ||
| } | ||
| config := redshift.RedshiftConfig{ | ||
| Schema: schema, | ||
| Host: redshiftSecret["redshiftHost"], | ||
| Port: redshiftSecret["redshiftPort"], | ||
| Database: redshiftSecret["redshiftDatabase"], | ||
| User: redshiftSecret["redshiftUser"], | ||
| Password: redshiftSecret["redshiftPassword"], | ||
| Timeout: 10, | ||
| Stats: true, | ||
| MaxOpenConns: 3, | ||
| MaxIdleConns: 3, | ||
| } | ||
|
|
||
| conn, err := redshift.NewRedshift(config) | ||
| if err != nil { | ||
| return nil, fmt.Errorf( | ||
| "Error creating redshift connecton, config: %+v, err: %v", | ||
| config, err) | ||
| } | ||
|
|
||
| return conn, nil | ||
| } |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Oops, something went wrong.