diff --git a/aggregator.go b/aggregator.go index 67edbdb5..331b4ded 100644 --- a/aggregator.go +++ b/aggregator.go @@ -37,11 +37,11 @@ import ( "syscall" "time" + "github.com/RedHatInsights/insights-operator-utils/logger" "github.com/rs/zerolog/log" "github.com/RedHatInsights/insights-results-aggregator/conf" "github.com/RedHatInsights/insights-results-aggregator/consumer" - "github.com/RedHatInsights/insights-results-aggregator/logger" "github.com/RedHatInsights/insights-results-aggregator/metrics" "github.com/RedHatInsights/insights-results-aggregator/migration" "github.com/RedHatInsights/insights-results-aggregator/server" diff --git a/conf/configuration.go b/conf/configuration.go index aae71cf6..4aa0056e 100644 --- a/conf/configuration.go +++ b/conf/configuration.go @@ -42,12 +42,12 @@ import ( "strings" "github.com/BurntSushi/toml" + "github.com/RedHatInsights/insights-operator-utils/logger" mapset "github.com/deckarep/golang-set" "github.com/rs/zerolog/log" "github.com/spf13/viper" "github.com/RedHatInsights/insights-results-aggregator/broker" - "github.com/RedHatInsights/insights-results-aggregator/logger" "github.com/RedHatInsights/insights-results-aggregator/server" "github.com/RedHatInsights/insights-results-aggregator/storage" "github.com/RedHatInsights/insights-results-aggregator/types" diff --git a/conf/configuration_test.go b/conf/configuration_test.go index 8398c480..d296a910 100644 --- a/conf/configuration_test.go +++ b/conf/configuration_test.go @@ -23,13 +23,13 @@ import ( "testing" "time" + "github.com/RedHatInsights/insights-operator-utils/logger" "github.com/RedHatInsights/insights-operator-utils/tests/helpers" mapset "github.com/deckarep/golang-set" "github.com/rs/zerolog" "github.com/stretchr/testify/assert" "github.com/RedHatInsights/insights-results-aggregator/conf" - "github.com/RedHatInsights/insights-results-aggregator/logger" "github.com/RedHatInsights/insights-results-aggregator/server" "github.com/RedHatInsights/insights-results-aggregator/storage" "github.com/RedHatInsights/insights-results-aggregator/types" diff --git a/docs/packages/aggregator.html b/docs/packages/aggregator.html index 6522fed3..81a2b62f 100644 --- a/docs/packages/aggregator.html +++ b/docs/packages/aggregator.html @@ -161,11 +161,11 @@
"syscall"
"time"
+
"github.com/RedHatInsights/insights-operator-utils/logger"
"github.com/rs/zerolog/log"
"github.com/RedHatInsights/insights-results-aggregator/conf"
"github.com/RedHatInsights/insights-results-aggregator/consumer"
-
"github.com/RedHatInsights/insights-results-aggregator/logger"
"github.com/RedHatInsights/insights-results-aggregator/metrics"
"github.com/RedHatInsights/insights-results-aggregator/migration"
"github.com/RedHatInsights/insights-results-aggregator/server"
diff --git a/docs/packages/conf/configuration.html b/docs/packages/conf/configuration.html index 1fb67f60..5d67d3f9 100644 --- a/docs/packages/conf/configuration.html +++ b/docs/packages/conf/configuration.html @@ -166,12 +166,12 @@
"strings"
"github.com/BurntSushi/toml"
+
"github.com/RedHatInsights/insights-operator-utils/logger"
mapset
"github.com/deckarep/golang-set"
"github.com/rs/zerolog/log"
"github.com/spf13/viper"
"github.com/RedHatInsights/insights-results-aggregator/broker"
-
"github.com/RedHatInsights/insights-results-aggregator/logger"
"github.com/RedHatInsights/insights-results-aggregator/server"
"github.com/RedHatInsights/insights-results-aggregator/storage"
"github.com/RedHatInsights/insights-results-aggregator/types"
diff --git a/docs/packages/conf/configuration_test.html b/docs/packages/conf/configuration_test.html index 1dcad295..173291ec 100644 --- a/docs/packages/conf/configuration_test.html +++ b/docs/packages/conf/configuration_test.html @@ -142,13 +142,13 @@
"testing"
"time"
+
"github.com/RedHatInsights/insights-operator-utils/logger"
"github.com/RedHatInsights/insights-operator-utils/tests/helpers"
mapset
"github.com/deckarep/golang-set"
"github.com/rs/zerolog"
"github.com/stretchr/testify/assert"
"github.com/RedHatInsights/insights-results-aggregator/conf"
-
"github.com/RedHatInsights/insights-results-aggregator/logger"
"github.com/RedHatInsights/insights-results-aggregator/server"
"github.com/RedHatInsights/insights-results-aggregator/storage"
"github.com/RedHatInsights/insights-results-aggregator/types"
diff --git a/docs/packages/logger/configuration.html b/docs/packages/logger/configuration.html deleted file mode 100644 index 16f0d626..00000000 --- a/docs/packages/logger/configuration.html +++ /dev/null @@ -1,209 +0,0 @@ - - - - - -configuration.go - - - - -
-
- - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -

configuration.go

/* -Copyright © 2020 Red Hat, Inc. - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/
- -
package
logger
- -

LoggingConfiguration represents configuration for logging in general

-
type
LoggingConfiguration
struct
{
-

Debug enables pretty colored logging

-
	
Debug
bool
`mapstructure:"debug" toml:"debug"`
- -

LogLevel sets logging level to show. Possible values are: -"debug" -"info" -"warn", "warning" -"error" -"fatal"

- -

logging level won't be changed if value is not one of listed above

-
	
LogLevel
string
`mapstructure:"log_level" toml:"log_level"`
- -

LoggingToCloudWatchEnabled enables logging to CloudWatch -(configuration for CloudWatch is in CloudWatchConfiguration)

-
	
LoggingToCloudWatchEnabled
bool
`mapstructure:"logging_to_cloud_watch_enabled" toml:"logging_to_cloud_watch_enabled"`
-
}
- -

CloudWatchConfiguration represents configuration of CloudWatch logger

-
type
CloudWatchConfiguration
struct
{
-
AWSAccessID
string
`mapstructure:"aws_access_id" toml:"aws_access_id"`
-
AWSSecretKey
string
`mapstructure:"aws_secret_key" toml:"aws_secret_key"`
-
AWSSessionToken
string
`mapstructure:"aws_session_token" toml:"aws_session_token"`
-
AWSRegion
string
`mapstructure:"aws_region" toml:"aws_region"`
-
LogGroup
string
`mapstructure:"log_group" toml:"log_group"`
-
StreamName
string
`mapstructure:"stream_name" toml:"stream_name"`
-
CreateStreamIfNotExists
bool
`mapstructure:"create_stream_if_not_exists" toml:"create_stream_if_not_exists"`
- -

enable debug logs for debugging aws client itself

-
	
Debug
bool
`mapstructure:"debug" toml:"debug"`
-
}
- -
-
- - diff --git a/docs/packages/logger/logger.html b/docs/packages/logger/logger.html deleted file mode 100644 index 59c7dcf3..00000000 --- a/docs/packages/logger/logger.html +++ /dev/null @@ -1,393 +0,0 @@ - - - - - -logger.go - - - - -
-
- - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -

logger.go

/* -Copyright © 2020 Red Hat, Inc. - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/
- -

Package logger contains the configuration structures needed to configure -the access to CloudWatch server to sending the log messages there.

-
package
logger
- -
import
(
-
"encoding/json"
-
"fmt"
-
"io"
-
"os"
-
"strings"
- -
"github.com/RedHatInsights/cloudwatch"
-
"github.com/Shopify/sarama"
-
"github.com/aws/aws-sdk-go/aws"
-
"github.com/aws/aws-sdk-go/aws/credentials"
-
"github.com/aws/aws-sdk-go/aws/session"
-
"github.com/aws/aws-sdk-go/service/cloudwatchlogs"
-
"github.com/rs/zerolog"
-
"github.com/rs/zerolog/log"
-
)
- -

WorkaroundForRHIOPS729 keeps only those fields that are currently getting their way to Kibana -TODO: delete when RHIOPS-729 is fixed

-
type
WorkaroundForRHIOPS729
struct
{
-
io
.
Writer
-
}
- -
func
(
writer
WorkaroundForRHIOPS729
)
Write
(
bytes
[
]
byte
)
(
int
,
error
)
{
-
var
obj
map
[
string
]
interface
{
}
- -
err
:=
json
.
Unmarshal
(
bytes
,
&
obj
)
-
if
err
!=
nil
{
-

it's not JSON object, so we don't do anything

-
		
return
writer
.
Writer
.
Write
(
bytes
)
-
}
- -

lowercase the keys

-
	
for
key
:=
range
obj
{
-
val
:=
obj
[
key
]
-
delete
(
obj
,
key
)
-
obj
[
strings
.
ToUpper
(
key
)
]
=
val
-
}
- -
resultBytes
,
err
:=
json
.
Marshal
(
obj
)
-
if
err
!=
nil
{
-
return
0
,
err
-
}
- -
written
,
err
:=
writer
.
Writer
.
Write
(
resultBytes
)
-
if
err
!=
nil
{
-
return
written
,
err
-
}
- -
if
written
<
len
(
resultBytes
)
{
-
return
written
,
fmt
.
Errorf
(
"too few bytes were written"
)
-
}
- -
return
len
(
bytes
)
,
nil
-
}
- -

AWSCloudWatchEndpoint allows you to mock cloudwatch client by redirecting requests to a local proxy

-
var
AWSCloudWatchEndpoint
string
- -

InitZerolog initializes zerolog with provided configs to use proper stdout and/or CloudWatch logging

-
func
InitZerolog
(
-
loggingConf
LoggingConfiguration
,
cloudWatchConf
CloudWatchConfiguration
,
additionalWriters
...
io
.
Writer
,
-
)
error
{
-
setGlobalLogLevel
(
loggingConf
)
- -
var
writers
[
]
io
.
Writer
- -
writers
=
append
(
writers
,
additionalWriters
...
)
- -
if
loggingConf
.
Debug
{
-

nice colored output

-
		
writers
=
append
(
writers
,
zerolog
.
ConsoleWriter
{
Out
:
os
.
Stdout
}
)
-
}
else
{
-
writers
=
append
(
writers
,
os
.
Stdout
)
-
}
- -
cloudWatchConf
.
StreamName
=
strings
.
ReplaceAll
(
cloudWatchConf
.
StreamName
,
"$HOSTNAME"
,
os
.
Getenv
(
"HOSTNAME"
)
)
- -
if
loggingConf
.
LoggingToCloudWatchEnabled
{
-
awsLogLevel
:=
aws
.
LogOff
-
if
cloudWatchConf
.
Debug
{
-
awsLogLevel
=
aws
.
LogDebugWithSigning
|
-
aws
.
LogDebugWithSigning
|
-
aws
.
LogDebugWithHTTPBody
|
-
aws
.
LogDebugWithEventStreamBody
-
}
- -
awsConf
:=
aws
.
NewConfig
(
)
.
-
WithCredentials
(
credentials
.
NewStaticCredentials
(
-
cloudWatchConf
.
AWSAccessID
,
cloudWatchConf
.
AWSSecretKey
,
cloudWatchConf
.
AWSSessionToken
,
-
)
)
.
-
WithRegion
(
cloudWatchConf
.
AWSRegion
)
.
-
WithLogLevel
(
awsLogLevel
)
- -
if
len
(
AWSCloudWatchEndpoint
)
>
0
{
-
awsConf
=
awsConf
.
WithEndpoint
(
AWSCloudWatchEndpoint
)
-
}
- -
cloudWatchSession
:=
session
.
Must
(
session
.
NewSession
(
awsConf
)
)
-
CloudWatchClient
:=
cloudwatchlogs
.
New
(
cloudWatchSession
)
- -
var
cloudWatchWriter
io
.
Writer
-
if
cloudWatchConf
.
CreateStreamIfNotExists
{
-
group
:=
cloudwatch
.
NewGroup
(
cloudWatchConf
.
LogGroup
,
CloudWatchClient
)
- -
var
err
error
-
cloudWatchWriter
,
err
=
group
.
Create
(
cloudWatchConf
.
StreamName
)
-
if
err
!=
nil
{
-
return
err
-
}
-
}
else
{
-
cloudWatchWriter
=
cloudwatch
.
NewWriter
(
-
cloudWatchConf
.
LogGroup
,
cloudWatchConf
.
StreamName
,
CloudWatchClient
,
-
)
-
}
- -
writers
=
append
(
writers
,
&
WorkaroundForRHIOPS729
{
Writer
:
cloudWatchWriter
}
)
-
}
- -
logsWriter
:=
io
.
MultiWriter
(
writers
...
)
- -
log
.
Logger
=
zerolog
.
New
(
logsWriter
)
.
With
(
)
.
Timestamp
(
)
.
Logger
(
)
- -

zerolog doesn't implement Println required by sarama

-
	
sarama
.
Logger
=
&
SaramaZerologger
{
zerologger
:
log
.
Logger
}
- -
return
nil
-
}
- -
func
setGlobalLogLevel
(
configuration
LoggingConfiguration
)
{
-
logLevel
:=
strings
.
ToLower
(
strings
.
TrimSpace
(
configuration
.
LogLevel
)
)
- -
switch
logLevel
{
-
case
"debug"
:
-
zerolog
.
SetGlobalLevel
(
zerolog
.
DebugLevel
)
-
case
"info"
:
-
zerolog
.
SetGlobalLevel
(
zerolog
.
InfoLevel
)
-
case
"warn"
,
"warning"
:
-
zerolog
.
SetGlobalLevel
(
zerolog
.
WarnLevel
)
-
case
"error"
:
-
zerolog
.
SetGlobalLevel
(
zerolog
.
ErrorLevel
)
-
case
"fatal"
:
-
zerolog
.
SetGlobalLevel
(
zerolog
.
FatalLevel
)
-
}
-
}
- -
const
kafkaErrorPrefix
=
"kafka: error"
- -

SaramaZerologger is a wrapper to make sarama log to zerolog -those logs can be filtered by key "package" with value "sarama"

-
type
SaramaZerologger
struct
{
zerologger
zerolog
.
Logger
}
- -

Print wraps print method

-
func
(
logger
*
SaramaZerologger
)
Print
(
params
...
interface
{
}
)
{
-
var
messages
[
]
string
-
for
_
,
item
:=
range
params
{
-
messages
=
append
(
messages
,
fmt
.
Sprint
(
item
)
)
-
}
- -
logger
.
logMessage
(
"%v"
,
strings
.
Join
(
messages
,
" "
)
)
-
}
- -

Printf wraps printf method

-
func
(
logger
*
SaramaZerologger
)
Printf
(
format
string
,
params
...
interface
{
}
)
{
-
logger
.
logMessage
(
format
,
params
...
)
-
}
- -

Println wraps println method

-
func
(
logger
*
SaramaZerologger
)
Println
(
v
...
interface
{
}
)
{
-
logger
.
Print
(
v
...
)
-
}
- -
func
(
logger
*
SaramaZerologger
)
logMessage
(
format
string
,
params
...
interface
{
}
)
{
-
var
event
*
zerolog
.
Event
-
messageStr
:=
fmt
.
Sprintf
(
format
,
params
...
)
- -
if
strings
.
HasPrefix
(
messageStr
,
kafkaErrorPrefix
)
{
-
event
=
logger
.
zerologger
.
Error
(
)
-
}
else
{
-
event
=
logger
.
zerologger
.
Info
(
)
-
}
- -
event
=
event
.
Str
(
"package"
,
"sarama"
)
-
event
.
Msg
(
messageStr
)
-
}
- -
-
- - diff --git a/docs/packages/logger/logger_test.html b/docs/packages/logger/logger_test.html deleted file mode 100644 index 3f18523b..00000000 --- a/docs/packages/logger/logger_test.html +++ /dev/null @@ -1,414 +0,0 @@ - - - - - -logger_test.go - - - - -
-
- - - - - - - - - - -

logger_test.go

Copyright 2020 Red Hat, Inc

- -

Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at

- -
 http://www.apache.org/licenses/LICENSE-2.0
-
- -

Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License.

-
package
logger_test
- -
import
(
-
"bytes"
-
"net/http"
-
"strings"
-
"testing"
-
"time"
- -
"github.com/Shopify/sarama"
-
"github.com/rs/zerolog"
-
"github.com/rs/zerolog/log"
-
"github.com/stretchr/testify/assert"
- -
"github.com/RedHatInsights/insights-results-aggregator/logger"
-
"github.com/RedHatInsights/insights-results-aggregator/tests/helpers"
-
)
- -
const
(
-
testTimeout
=
10
*
time
.
Second
-
)
- -
var
(
-
cloudWatchConf
=
logger
.
CloudWatchConfiguration
{
-
AWSAccessID
:
"access ID"
,
-
AWSSecretKey
:
"secret"
,
-
AWSSessionToken
:
"sess token"
,
-
AWSRegion
:
"aws region"
,
-
LogGroup
:
"log group"
,
-
StreamName
:
"stream name"
,
-
CreateStreamIfNotExists
:
true
,
-
Debug
:
false
,
-
}
-
describeLogStreamsEvent
=
CloudWatchExpect
{
-
http
.
MethodPost
,
-
"Logs_20140328.DescribeLogStreams"
,
-
`{ - "descending": false, - "logGroupName": "`
+
cloudWatchConf
.
LogGroup
+
`", - "logStreamNamePrefix": "`
+
cloudWatchConf
.
StreamName
+
`", - "orderBy": "LogStreamName" - }`
,
-
http
.
StatusOK
,
-
`{ - "logStreams": [ - { - "arn": "arn:aws:logs:`
+
-
cloudWatchConf
.
AWSRegion
+
`:012345678910:log-group:`
+
cloudWatchConf
.
LogGroup
+
-
`:log-stream:`
+
cloudWatchConf
.
StreamName
+
`", - "creationTime": 1, - "firstEventTimestamp": 2, - "lastEventTimestamp": 3, - "lastIngestionTime": 4, - "logStreamName": "`
+
cloudWatchConf
.
StreamName
+
`", - "storedBytes": 100, - "uploadSequenceToken": "1" - } - ], - "nextToken": "token1" - }`
,
-
}
-
)
- -
type
CloudWatchExpect
struct
{
-
ExpectedMethod
string
-
ExpectedTarget
string
-
ExpectedBody
string
-
ResultStatusCode
int
-
ResultBody
string
-
}
- -
func
TestSaramaZerologger
(
t
*
testing
.
T
)
{
-
const
expectedStrInfoLevel
=
"some random message"
-
const
expectedErrStrErrorLevel
=
"kafka: error test error"
- -
buf
:=
new
(
bytes
.
Buffer
)
- -
err
:=
logger
.
InitZerolog
(
-
logger
.
LoggingConfiguration
{
-
Debug
:
false
,
-
LogLevel
:
"debug"
,
-
LoggingToCloudWatchEnabled
:
false
,
-
}
,
-
logger
.
CloudWatchConfiguration
{
}
,
-
zerolog
.
New
(
buf
)
,
-
)
-
helpers
.
FailOnError
(
t
,
err
)
- -
t
.
Run
(
"InfoLevel"
,
func
(
t
*
testing
.
T
)
{
-
buf
.
Reset
(
)
- -
sarama
.
Logger
.
Printf
(
expectedStrInfoLevel
)
- -
assert
.
Contains
(
t
,
buf
.
String
(
)
,
`\"level\":\"info\"`
)
-
assert
.
Contains
(
t
,
buf
.
String
(
)
,
expectedStrInfoLevel
)
-
}
)
- -
t
.
Run
(
"ErrorLevel"
,
func
(
t
*
testing
.
T
)
{
-
buf
.
Reset
(
)
- -
sarama
.
Logger
.
Print
(
expectedErrStrErrorLevel
)
- -
assert
.
Contains
(
t
,
buf
.
String
(
)
,
`\"level\":\"error\"`
)
-
assert
.
Contains
(
t
,
buf
.
String
(
)
,
expectedErrStrErrorLevel
)
-
}
)
-
}
- -
func
TestLoggerSetLogLevel
(
t
*
testing
.
T
)
{
-
logLevels
:=
[
]
string
{
"debug"
,
"info"
,
"warning"
,
"error"
}
-
for
logLevelIndex
,
logLevel
:=
range
logLevels
{
-
t
.
Run
(
logLevel
,
func
(
t
*
testing
.
T
)
{
-
buf
:=
new
(
bytes
.
Buffer
)
- -
err
:=
logger
.
InitZerolog
(
-
logger
.
LoggingConfiguration
{
-
Debug
:
false
,
-
LogLevel
:
logLevel
,
-
LoggingToCloudWatchEnabled
:
false
,
-
}
,
-
logger
.
CloudWatchConfiguration
{
}
,
-
zerolog
.
New
(
buf
)
,
-
)
-
helpers
.
FailOnError
(
t
,
err
)
- -
log
.
Debug
(
)
.
Msg
(
"debug level"
)
-
log
.
Info
(
)
.
Msg
(
"info level"
)
-
log
.
Warn
(
)
.
Msg
(
"warning level"
)
-
log
.
Error
(
)
.
Msg
(
"error level"
)
- -
for
i
:=
0
;
i
<
len
(
logLevels
)
;
i
++
{
-
if
i
<
logLevelIndex
{
-
assert
.
NotContains
(
t
,
buf
.
String
(
)
,
logLevels
[
i
]
+
" level"
)
-
}
else
{
-
assert
.
Contains
(
t
,
buf
.
String
(
)
,
logLevels
[
i
]
+
" level"
)
-
}
-
}
-
}
)
-
}
-
}
- -
func
TestWorkaroundForRHIOPS729_Write
(
t
*
testing
.
T
)
{
-
for
_
,
testCase
:=
range
[
]
struct
{
-
Name
string
-
StrToWrite
string
-
ExpectedStr
string
-
IsJSON
bool
-
}
{
-
{
"NotJSON"
,
"some expected string"
,
"some expected string"
,
false
}
,
-
{
-
"JSON"
,
-
`{"level": "error", "is_something": true}`
,
-
`{"LEVEL":"error", "IS_SOMETHING": true}`
,
-
true
,
-
}
,
-
}
{
-
t
.
Run
(
testCase
.
Name
,
func
(
t
*
testing
.
T
)
{
-
buf
:=
new
(
bytes
.
Buffer
)
-
unJSONWriter
:=
logger
.
WorkaroundForRHIOPS729
{
Writer
:
buf
}
- -
writtenBytes
,
err
:=
unJSONWriter
.
Write
(
[
]
byte
(
testCase
.
StrToWrite
)
)
-
helpers
.
FailOnError
(
t
,
err
)
- -
assert
.
Equal
(
t
,
writtenBytes
,
len
(
testCase
.
StrToWrite
)
)
-
if
testCase
.
IsJSON
{
-
helpers
.
AssertStringsAreEqualJSON
(
t
,
testCase
.
ExpectedStr
,
buf
.
String
(
)
)
-
}
else
{
-
assert
.
Equal
(
t
,
testCase
.
ExpectedStr
,
strings
.
TrimSpace
(
buf
.
String
(
)
)
)
-
}
-
}
)
-
}
-
}
- -
func
TestInitZerolog_LogToCloudWatch
(
t
*
testing
.
T
)
{
-
err
:=
logger
.
InitZerolog
(
-
logger
.
LoggingConfiguration
{
-
Debug
:
false
,
-
LogLevel
:
"debug"
,
-
LoggingToCloudWatchEnabled
:
true
,
-
}
,
-
logger
.
CloudWatchConfiguration
{
}
,
-
)
-
helpers
.
FailOnError
(
t
,
err
)
-
}
- -
func
TestLoggingToCloudwatch
(
t
*
testing
.
T
)
{
-
helpers
.
RunTestWithTimeout
(
t
,
func
(
t
testing
.
TB
)
{
-
defer
helpers
.
CleanAfterGock
(
t
)
- -
const
baseURL
=
"http://localhost:9999"
-
logger
.
AWSCloudWatchEndpoint
=
baseURL
+
"/cloudwatch"
- -
expects
:=
[
]
CloudWatchExpect
{
-
{
-
http
.
MethodPost
,
-
"Logs_20140328.CreateLogStream"
,
-
`{ - "logGroupName": "`
+
cloudWatchConf
.
LogGroup
+
`", - "logStreamName": "`
+
cloudWatchConf
.
StreamName
+
`" - }`
,
-
http
.
StatusBadRequest
,
-
`{ - "__type": "ResourceAlreadyExistsException", - "message": "The specified log stream already exists" - }`
,
-
}
,
-
describeLogStreamsEvent
,
-
{
-
http
.
MethodPost
,
-
"Logs_20140328.PutLogEvents"
,
-
`{ - "logEvents": [ - { - "message": "test message text goes right here", - "timestamp": 1 - } - ], - "logGroupName": "`
+
cloudWatchConf
.
LogGroup
+
`", - "logStreamName":"`
+
cloudWatchConf
.
StreamName
+
`", - "sequenceToken":"1" - }`
,
-
http
.
StatusOK
,
-
`{"nextSequenceToken":"2"}`
,
-
}
,
-
{
-
http
.
MethodPost
,
-
"Logs_20140328.PutLogEvents"
,
-
`{ - "logEvents": [ - { - "message": "second test message text goes right here", - "timestamp": 2 - } - ], - "logGroupName": "`
+
cloudWatchConf
.
LogGroup
+
`", - "logStreamName":"`
+
cloudWatchConf
.
StreamName
+
`", - "sequenceToken":"2" - }`
,
-
http
.
StatusOK
,
-
`{"nextSequenceToken":"3"}`
,
-
}
,
-
describeLogStreamsEvent
,
-
describeLogStreamsEvent
,
-
describeLogStreamsEvent
,
-
}
- -
for
_
,
expect
:=
range
expects
{
-
helpers
.
GockExpectAPIRequest
(
t
,
baseURL
,
&
helpers
.
APIRequest
{
-
Method
:
expect
.
ExpectedMethod
,
-
Body
:
expect
.
ExpectedBody
,
-
Endpoint
:
"cloudwatch/"
,
-
ExtraHeaders
:
http
.
Header
{
-
"X-Amz-Target"
:
[
]
string
{
expect
.
ExpectedTarget
}
,
-
}
,
-
}
,
&
helpers
.
APIResponse
{
-
StatusCode
:
expect
.
ResultStatusCode
,
-
Body
:
expect
.
ResultBody
,
-
Headers
:
map
[
string
]
string
{
-
"Content-Type"
:
"application/x-amz-json-1.1"
,
-
}
,
-
}
)
-
}
- -
err
:=
logger
.
InitZerolog
(
logger
.
LoggingConfiguration
{
-
Debug
:
false
,
-
LogLevel
:
"debug"
,
-
LoggingToCloudWatchEnabled
:
true
,
-
}
,
cloudWatchConf
)
-
helpers
.
FailOnError
(
t
,
err
)
- -
log
.
Error
(
)
.
Msg
(
"test message"
)
-
}
,
testTimeout
)
-
}
- -
-
- - diff --git a/go.mod b/go.mod index 787e7b2d..36c87575 100644 --- a/go.mod +++ b/go.mod @@ -7,7 +7,7 @@ require ( github.com/DATA-DOG/go-sqlmock v1.4.1 github.com/RedHatInsights/cloudwatch v0.0.0-20200512151223-b0b55757a24b github.com/RedHatInsights/insights-content-service v0.0.0-20200619153839-23a428468a08 - github.com/RedHatInsights/insights-operator-utils v1.5.1 + github.com/RedHatInsights/insights-operator-utils v1.6.0 github.com/RedHatInsights/insights-results-aggregator-data v0.0.0-20200826060335-2d341f371b98 github.com/RedHatInsights/insights-results-aggregator-utils v0.0.0-20200616074815-67f30b0e724d // indirect github.com/RedHatInsights/insights-results-smart-proxy v0.0.0-20200619163313-7d5e376de430 // indirect diff --git a/go.sum b/go.sum index 6c2158ab..2abecd71 100644 --- a/go.sum +++ b/go.sum @@ -38,6 +38,8 @@ github.com/RedHatInsights/insights-operator-utils v1.5.0 h1:tuMocnJt943aE0ULZcFG github.com/RedHatInsights/insights-operator-utils v1.5.0/go.mod h1:n5NNAOPIv1DfnexF9Zr4LXCC3J+uBLuwgnuk1SqOw6Y= github.com/RedHatInsights/insights-operator-utils v1.5.1 h1:dfZsB9ZFr+l02XCJ8ss9IpxDnBn3DC0iPzFlAmoe87M= github.com/RedHatInsights/insights-operator-utils v1.5.1/go.mod h1:wlJ8nuJm5jjTfm5H+dbdDH7T+3AV3roXwZUs8S09+lw= +github.com/RedHatInsights/insights-operator-utils v1.6.0 h1:jRg9v+KzPezsq7++PBmI+P+4OxnT/TFraea3M0CPUUM= +github.com/RedHatInsights/insights-operator-utils v1.6.0/go.mod h1:RW9Jq4LgqIkV3WY6AS2EkopYyZDIr5BNGiU5I75HryM= github.com/RedHatInsights/insights-results-aggregator v0.0.0-20200604090056-3534f6dd9c1c/go.mod h1:7Pc15NYXErx7BMJ4rF1Hacm+29G6atzjhwBpXNFMt+0= github.com/RedHatInsights/insights-results-aggregator-data v0.0.0-20200622112651-e5a7c1d79015 h1:NvNKLp20xelGO7Q8gfPshoRKd5Rhq9yZJJm009mlWPY= github.com/RedHatInsights/insights-results-aggregator-data v0.0.0-20200622112651-e5a7c1d79015/go.mod h1:1pHaSnEh5Bhc8IZNqP6vbBI3jtWd5/rv4x2Y2/zG90w= diff --git a/logger/configuration.go b/logger/configuration.go deleted file mode 100644 index 055f23e3..00000000 --- a/logger/configuration.go +++ /dev/null @@ -1,51 +0,0 @@ -/* -Copyright © 2020 Red Hat, Inc. - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/ - -package logger - -// LoggingConfiguration represents configuration for logging in general -type LoggingConfiguration struct { - // Debug enables pretty colored logging - Debug bool `mapstructure:"debug" toml:"debug"` - - // LogLevel sets logging level to show. Possible values are: - // "debug" - // "info" - // "warn", "warning" - // "error" - // "fatal" - // - // logging level won't be changed if value is not one of listed above - LogLevel string `mapstructure:"log_level" toml:"log_level"` - - // LoggingToCloudWatchEnabled enables logging to CloudWatch - // (configuration for CloudWatch is in CloudWatchConfiguration) - LoggingToCloudWatchEnabled bool `mapstructure:"logging_to_cloud_watch_enabled" toml:"logging_to_cloud_watch_enabled"` -} - -// CloudWatchConfiguration represents configuration of CloudWatch logger -type CloudWatchConfiguration struct { - AWSAccessID string `mapstructure:"aws_access_id" toml:"aws_access_id"` - AWSSecretKey string `mapstructure:"aws_secret_key" toml:"aws_secret_key"` - AWSSessionToken string `mapstructure:"aws_session_token" toml:"aws_session_token"` - AWSRegion string `mapstructure:"aws_region" toml:"aws_region"` - LogGroup string `mapstructure:"log_group" toml:"log_group"` - StreamName string `mapstructure:"stream_name" toml:"stream_name"` - CreateStreamIfNotExists bool `mapstructure:"create_stream_if_not_exists" toml:"create_stream_if_not_exists"` - - // enable debug logs for debugging aws client itself - Debug bool `mapstructure:"debug" toml:"debug"` -} diff --git a/logger/logger.go b/logger/logger.go deleted file mode 100644 index 3d5e2753..00000000 --- a/logger/logger.go +++ /dev/null @@ -1,205 +0,0 @@ -/* -Copyright © 2020 Red Hat, Inc. - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/ - -// Package logger contains the configuration structures needed to configure -// the access to CloudWatch server to sending the log messages there. -package logger - -import ( - "encoding/json" - "fmt" - "io" - "os" - "strings" - - "github.com/RedHatInsights/cloudwatch" - "github.com/Shopify/sarama" - "github.com/aws/aws-sdk-go/aws" - "github.com/aws/aws-sdk-go/aws/credentials" - "github.com/aws/aws-sdk-go/aws/session" - "github.com/aws/aws-sdk-go/service/cloudwatchlogs" - "github.com/rs/zerolog" - "github.com/rs/zerolog/log" -) - -// WorkaroundForRHIOPS729 keeps only those fields that are currently getting their way to Kibana -// TODO: delete when RHIOPS-729 is fixed -type WorkaroundForRHIOPS729 struct { - io.Writer -} - -func (writer WorkaroundForRHIOPS729) Write(bytes []byte) (int, error) { - var obj map[string]interface{} - - err := json.Unmarshal(bytes, &obj) - if err != nil { - // it's not JSON object, so we don't do anything - return writer.Writer.Write(bytes) - } - - // lowercase the keys - for key := range obj { - val := obj[key] - delete(obj, key) - obj[strings.ToUpper(key)] = val - } - - resultBytes, err := json.Marshal(obj) - if err != nil { - return 0, err - } - - written, err := writer.Writer.Write(resultBytes) - if err != nil { - return written, err - } - - if written < len(resultBytes) { - return written, fmt.Errorf("too few bytes were written") - } - - return len(bytes), nil -} - -// AWSCloudWatchEndpoint allows you to mock cloudwatch client by redirecting requests to a local proxy -var AWSCloudWatchEndpoint string - -// InitZerolog initializes zerolog with provided configs to use proper stdout and/or CloudWatch logging -func InitZerolog( - loggingConf LoggingConfiguration, cloudWatchConf CloudWatchConfiguration, additionalWriters ...io.Writer, -) error { - setGlobalLogLevel(loggingConf) - - var writers []io.Writer - - writers = append(writers, additionalWriters...) - - if loggingConf.Debug { - // nice colored output - writers = append(writers, zerolog.ConsoleWriter{Out: os.Stdout}) - } else { - writers = append(writers, os.Stdout) - } - - cloudWatchConf.StreamName = strings.ReplaceAll(cloudWatchConf.StreamName, "$HOSTNAME", os.Getenv("HOSTNAME")) - - if loggingConf.LoggingToCloudWatchEnabled { - awsLogLevel := aws.LogOff - if cloudWatchConf.Debug { - awsLogLevel = aws.LogDebugWithSigning | - aws.LogDebugWithSigning | - aws.LogDebugWithHTTPBody | - aws.LogDebugWithEventStreamBody - } - - awsConf := aws.NewConfig(). - WithCredentials(credentials.NewStaticCredentials( - cloudWatchConf.AWSAccessID, cloudWatchConf.AWSSecretKey, cloudWatchConf.AWSSessionToken, - )). - WithRegion(cloudWatchConf.AWSRegion). - WithLogLevel(awsLogLevel) - - if len(AWSCloudWatchEndpoint) > 0 { - awsConf = awsConf.WithEndpoint(AWSCloudWatchEndpoint) - } - - cloudWatchSession := session.Must(session.NewSession(awsConf)) - CloudWatchClient := cloudwatchlogs.New(cloudWatchSession) - - var cloudWatchWriter io.Writer - if cloudWatchConf.CreateStreamIfNotExists { - group := cloudwatch.NewGroup(cloudWatchConf.LogGroup, CloudWatchClient) - - var err error - cloudWatchWriter, err = group.Create(cloudWatchConf.StreamName) - if err != nil { - return err - } - } else { - cloudWatchWriter = cloudwatch.NewWriter( - cloudWatchConf.LogGroup, cloudWatchConf.StreamName, CloudWatchClient, - ) - } - - writers = append(writers, &WorkaroundForRHIOPS729{Writer: cloudWatchWriter}) - } - - logsWriter := io.MultiWriter(writers...) - - log.Logger = zerolog.New(logsWriter).With().Timestamp().Logger() - - // zerolog doesn't implement Println required by sarama - sarama.Logger = &SaramaZerologger{zerologger: log.Logger} - - return nil -} - -func setGlobalLogLevel(configuration LoggingConfiguration) { - logLevel := strings.ToLower(strings.TrimSpace(configuration.LogLevel)) - - switch logLevel { - case "debug": - zerolog.SetGlobalLevel(zerolog.DebugLevel) - case "info": - zerolog.SetGlobalLevel(zerolog.InfoLevel) - case "warn", "warning": - zerolog.SetGlobalLevel(zerolog.WarnLevel) - case "error": - zerolog.SetGlobalLevel(zerolog.ErrorLevel) - case "fatal": - zerolog.SetGlobalLevel(zerolog.FatalLevel) - } -} - -const kafkaErrorPrefix = "kafka: error" - -// SaramaZerologger is a wrapper to make sarama log to zerolog -// those logs can be filtered by key "package" with value "sarama" -type SaramaZerologger struct{ zerologger zerolog.Logger } - -// Print wraps print method -func (logger *SaramaZerologger) Print(params ...interface{}) { - var messages []string - for _, item := range params { - messages = append(messages, fmt.Sprint(item)) - } - - logger.logMessage("%v", strings.Join(messages, " ")) -} - -// Printf wraps printf method -func (logger *SaramaZerologger) Printf(format string, params ...interface{}) { - logger.logMessage(format, params...) -} - -// Println wraps println method -func (logger *SaramaZerologger) Println(v ...interface{}) { - logger.Print(v...) -} - -func (logger *SaramaZerologger) logMessage(format string, params ...interface{}) { - var event *zerolog.Event - messageStr := fmt.Sprintf(format, params...) - - if strings.HasPrefix(messageStr, kafkaErrorPrefix) { - event = logger.zerologger.Error() - } else { - event = logger.zerologger.Info() - } - - event = event.Str("package", "sarama") - event.Msg(messageStr) -} diff --git a/logger/logger_test.go b/logger/logger_test.go deleted file mode 100644 index 00ad6933..00000000 --- a/logger/logger_test.go +++ /dev/null @@ -1,286 +0,0 @@ -// Copyright 2020 Red Hat, Inc -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -package logger_test - -import ( - "bytes" - "net/http" - "strings" - "testing" - "time" - - "github.com/Shopify/sarama" - "github.com/rs/zerolog" - "github.com/rs/zerolog/log" - "github.com/stretchr/testify/assert" - - "github.com/RedHatInsights/insights-results-aggregator/logger" - "github.com/RedHatInsights/insights-results-aggregator/tests/helpers" -) - -const ( - testTimeout = 10 * time.Second -) - -var ( - cloudWatchConf = logger.CloudWatchConfiguration{ - AWSAccessID: "access ID", - AWSSecretKey: "secret", - AWSSessionToken: "sess token", - AWSRegion: "aws region", - LogGroup: "log group", - StreamName: "stream name", - CreateStreamIfNotExists: true, - Debug: false, - } - describeLogStreamsEvent = CloudWatchExpect{ - http.MethodPost, - "Logs_20140328.DescribeLogStreams", - `{ - "descending": false, - "logGroupName": "` + cloudWatchConf.LogGroup + `", - "logStreamNamePrefix": "` + cloudWatchConf.StreamName + `", - "orderBy": "LogStreamName" - }`, - http.StatusOK, - `{ - "logStreams": [ - { - "arn": "arn:aws:logs:` + - cloudWatchConf.AWSRegion + `:012345678910:log-group:` + cloudWatchConf.LogGroup + - `:log-stream:` + cloudWatchConf.StreamName + `", - "creationTime": 1, - "firstEventTimestamp": 2, - "lastEventTimestamp": 3, - "lastIngestionTime": 4, - "logStreamName": "` + cloudWatchConf.StreamName + `", - "storedBytes": 100, - "uploadSequenceToken": "1" - } - ], - "nextToken": "token1" - }`, - } -) - -type CloudWatchExpect struct { - ExpectedMethod string - ExpectedTarget string - ExpectedBody string - ResultStatusCode int - ResultBody string -} - -func TestSaramaZerologger(t *testing.T) { - const expectedStrInfoLevel = "some random message" - const expectedErrStrErrorLevel = "kafka: error test error" - - buf := new(bytes.Buffer) - - err := logger.InitZerolog( - logger.LoggingConfiguration{ - Debug: false, - LogLevel: "debug", - LoggingToCloudWatchEnabled: false, - }, - logger.CloudWatchConfiguration{}, - zerolog.New(buf), - ) - helpers.FailOnError(t, err) - - t.Run("InfoLevel", func(t *testing.T) { - buf.Reset() - - sarama.Logger.Printf(expectedStrInfoLevel) - - assert.Contains(t, buf.String(), `\"level\":\"info\"`) - assert.Contains(t, buf.String(), expectedStrInfoLevel) - }) - - t.Run("ErrorLevel", func(t *testing.T) { - buf.Reset() - - sarama.Logger.Print(expectedErrStrErrorLevel) - - assert.Contains(t, buf.String(), `\"level\":\"error\"`) - assert.Contains(t, buf.String(), expectedErrStrErrorLevel) - }) -} - -func TestLoggerSetLogLevel(t *testing.T) { - logLevels := []string{"debug", "info", "warning", "error"} - for logLevelIndex, logLevel := range logLevels { - t.Run(logLevel, func(t *testing.T) { - buf := new(bytes.Buffer) - - err := logger.InitZerolog( - logger.LoggingConfiguration{ - Debug: false, - LogLevel: logLevel, - LoggingToCloudWatchEnabled: false, - }, - logger.CloudWatchConfiguration{}, - zerolog.New(buf), - ) - helpers.FailOnError(t, err) - - log.Debug().Msg("debug level") - log.Info().Msg("info level") - log.Warn().Msg("warning level") - log.Error().Msg("error level") - - for i := 0; i < len(logLevels); i++ { - if i < logLevelIndex { - assert.NotContains(t, buf.String(), logLevels[i]+" level") - } else { - assert.Contains(t, buf.String(), logLevels[i]+" level") - } - } - }) - } -} - -func TestWorkaroundForRHIOPS729_Write(t *testing.T) { - for _, testCase := range []struct { - Name string - StrToWrite string - ExpectedStr string - IsJSON bool - }{ - {"NotJSON", "some expected string", "some expected string", false}, - { - "JSON", - `{"level": "error", "is_something": true}`, - `{"LEVEL":"error", "IS_SOMETHING": true}`, - true, - }, - } { - t.Run(testCase.Name, func(t *testing.T) { - buf := new(bytes.Buffer) - unJSONWriter := logger.WorkaroundForRHIOPS729{Writer: buf} - - writtenBytes, err := unJSONWriter.Write([]byte(testCase.StrToWrite)) - helpers.FailOnError(t, err) - - assert.Equal(t, writtenBytes, len(testCase.StrToWrite)) - if testCase.IsJSON { - helpers.AssertStringsAreEqualJSON(t, testCase.ExpectedStr, buf.String()) - } else { - assert.Equal(t, testCase.ExpectedStr, strings.TrimSpace(buf.String())) - } - }) - } -} - -func TestInitZerolog_LogToCloudWatch(t *testing.T) { - err := logger.InitZerolog( - logger.LoggingConfiguration{ - Debug: false, - LogLevel: "debug", - LoggingToCloudWatchEnabled: true, - }, - logger.CloudWatchConfiguration{}, - ) - helpers.FailOnError(t, err) -} - -func TestLoggingToCloudwatch(t *testing.T) { - helpers.RunTestWithTimeout(t, func(t testing.TB) { - defer helpers.CleanAfterGock(t) - - const baseURL = "http://localhost:9999" - logger.AWSCloudWatchEndpoint = baseURL + "/cloudwatch" - - expects := []CloudWatchExpect{ - { - http.MethodPost, - "Logs_20140328.CreateLogStream", - `{ - "logGroupName": "` + cloudWatchConf.LogGroup + `", - "logStreamName": "` + cloudWatchConf.StreamName + `" - }`, - http.StatusBadRequest, - `{ - "__type": "ResourceAlreadyExistsException", - "message": "The specified log stream already exists" - }`, - }, - describeLogStreamsEvent, - { - http.MethodPost, - "Logs_20140328.PutLogEvents", - `{ - "logEvents": [ - { - "message": "test message text goes right here", - "timestamp": 1 - } - ], - "logGroupName": "` + cloudWatchConf.LogGroup + `", - "logStreamName":"` + cloudWatchConf.StreamName + `", - "sequenceToken":"1" - }`, - http.StatusOK, - `{"nextSequenceToken":"2"}`, - }, - { - http.MethodPost, - "Logs_20140328.PutLogEvents", - `{ - "logEvents": [ - { - "message": "second test message text goes right here", - "timestamp": 2 - } - ], - "logGroupName": "` + cloudWatchConf.LogGroup + `", - "logStreamName":"` + cloudWatchConf.StreamName + `", - "sequenceToken":"2" - }`, - http.StatusOK, - `{"nextSequenceToken":"3"}`, - }, - describeLogStreamsEvent, - describeLogStreamsEvent, - describeLogStreamsEvent, - } - - for _, expect := range expects { - helpers.GockExpectAPIRequest(t, baseURL, &helpers.APIRequest{ - Method: expect.ExpectedMethod, - Body: expect.ExpectedBody, - Endpoint: "cloudwatch/", - ExtraHeaders: http.Header{ - "X-Amz-Target": []string{expect.ExpectedTarget}, - }, - }, &helpers.APIResponse{ - StatusCode: expect.ResultStatusCode, - Body: expect.ResultBody, - Headers: map[string]string{ - "Content-Type": "application/x-amz-json-1.1", - }, - }) - } - - err := logger.InitZerolog(logger.LoggingConfiguration{ - Debug: false, - LogLevel: "debug", - LoggingToCloudWatchEnabled: true, - }, cloudWatchConf) - helpers.FailOnError(t, err) - - log.Error().Msg("test message") - }, testTimeout) -}