From c5deb61da23e0d5bf8b0d5f49793737a34571a86 Mon Sep 17 00:00:00 2001 From: RRashmit Date: Wed, 13 Nov 2024 16:25:57 +0000 Subject: [PATCH 01/12] feat: add syslog receiver --- internal/collector/factories.go | 2 ++ internal/collector/factories_test.go | 2 +- .../collector/otel_collector_plugin_test.go | 10 +++++++++ internal/collector/otelcol.tmpl | 22 +++++++++++++++++++ internal/config/types.go | 7 ++++++ .../test-opentelemetry-collector-agent.yaml | 12 ++++++++++ test/docker/nginx-plus/deb/Dockerfile | 19 +++++++++------- test/types/config.go | 5 +++++ 8 files changed, 70 insertions(+), 9 deletions(-) diff --git a/internal/collector/factories.go b/internal/collector/factories.go index df3a431f64..75e6e50198 100644 --- a/internal/collector/factories.go +++ b/internal/collector/factories.go @@ -32,6 +32,7 @@ import ( "github.com/open-telemetry/opentelemetry-collector-contrib/processor/tailsamplingprocessor" "github.com/open-telemetry/opentelemetry-collector-contrib/processor/transformprocessor" "github.com/open-telemetry/opentelemetry-collector-contrib/receiver/hostmetricsreceiver" + "github.com/open-telemetry/opentelemetry-collector-contrib/receiver/syslogreceiver" "go.opentelemetry.io/collector/component" "go.opentelemetry.io/collector/connector" "go.opentelemetry.io/collector/exporter" @@ -110,6 +111,7 @@ func createReceiverFactories() (map[component.Type]receiver.Factory, error) { hostmetricsreceiver.NewFactory(), nginxreceiver.NewFactory(), nginxplusreceiver.NewFactory(), + syslogreceiver.NewFactory(), } return receiver.MakeFactoryMap(receiverList...) diff --git a/internal/collector/factories_test.go b/internal/collector/factories_test.go index 5cefda51eb..ca3d9423d5 100644 --- a/internal/collector/factories_test.go +++ b/internal/collector/factories_test.go @@ -17,7 +17,7 @@ func TestOTelComponentFactories(t *testing.T) { require.NoError(t, err, "OTelComponentFactories should not return an error") assert.NotNil(t, factories, "factories should not be nil") - assert.Len(t, factories.Receivers, 4) + assert.Len(t, factories.Receivers, 5) assert.Len(t, factories.Processors, 20) assert.Len(t, factories.Exporters, 4) assert.Len(t, factories.Extensions, 3) diff --git a/internal/collector/otel_collector_plugin_test.go b/internal/collector/otel_collector_plugin_test.go index 9e14c73cf4..99ae3c3055 100644 --- a/internal/collector/otel_collector_plugin_test.go +++ b/internal/collector/otel_collector_plugin_test.go @@ -120,6 +120,11 @@ func TestCollector_ProcessNginxConfigUpdateTopic(t *testing.T) { Network: &config.NetworkScraper{}, }, }, + SyslogReceiver: &config.SyslogReceiver{ + Host: "127.0.0.1", + Port: "515", + Protocol: "rfc5424", + }, OtlpReceivers: types.OtlpReceivers(), NginxPlusReceivers: []config.NginxPlusReceiver{ { @@ -157,6 +162,11 @@ func TestCollector_ProcessNginxConfigUpdateTopic(t *testing.T) { }, }, OtlpReceivers: types.OtlpReceivers(), + SyslogReceiver: &config.SyslogReceiver{ + Host: "127.0.0.1", + Port: "515", + Protocol: "rfc5424", + }, NginxReceivers: []config.NginxReceiver{ { InstanceID: "123", diff --git a/internal/collector/otelcol.tmpl b/internal/collector/otelcol.tmpl index e6b3baca83..b49e5f9f6f 100644 --- a/internal/collector/otelcol.tmpl +++ b/internal/collector/otelcol.tmpl @@ -1,4 +1,10 @@ receivers: +{{- if ne .Receivers.SyslogReceiver nil }} + syslog: + tcp: + listen_address: "{{- .Receivers.SyslogReceiver.Host -}}:{{- .Receivers.SyslogReceiver.Port -}}" + protocol: {{ .Receivers.SyslogReceiver.Protocol }} +{{- end }} {{- if ne .Receivers.HostMetrics nil }} hostmetrics: collection_interval: {{ .Receivers.HostMetrics.CollectionInterval }} @@ -233,3 +239,19 @@ service: {{- if ne .Exporters.Debug nil }} - debug {{- end }} + logs: + receivers: + {{- if ne .Receivers.SyslogReceiver nil }} + - syslog + {{- end }} + processors: + {{- if ne .Processors.Batch nil }} + - batch + {{- end }} + exporters: + {{- range $index, $otlpExporter := .Exporters.OtlpExporters }} + - otlp/{{$index}} + {{- end }} + {{- if ne .Exporters.Debug nil }} + - debug + {{- end }} diff --git a/internal/config/types.go b/internal/config/types.go index 0e36ae0819..2ec27c880b 100644 --- a/internal/config/types.go +++ b/internal/config/types.go @@ -163,6 +163,7 @@ type ( OtlpReceivers []OtlpReceiver `yaml:"-" mapstructure:"otlp_receivers"` NginxReceivers []NginxReceiver `yaml:"-" mapstructure:"nginx_receivers"` NginxPlusReceivers []NginxPlusReceiver `yaml:"-" mapstructure:"nginx_plus_receivers"` + SyslogReceiver *SyslogReceiver `yaml:"-" mapstructure:"syslog_receiver"` } OtlpReceiver struct { @@ -171,6 +172,12 @@ type ( OtlpTLSConfig *OtlpTLSConfig `yaml:"-" mapstructure:"tls"` } + SyslogReceiver struct { + Host string `yaml:"-" mapstructure:"host"` + Port string `yaml:"-" mapstructure:"port"` + Protocol string `yaml:"-" mapstructure:"protocol"` + } + NginxReceiver struct { InstanceID string `yaml:"-" mapstructure:"instance_id"` StubStatus string `yaml:"-" mapstructure:"stub_status"` diff --git a/test/config/collector/test-opentelemetry-collector-agent.yaml b/test/config/collector/test-opentelemetry-collector-agent.yaml index a4f5084a26..c88d3929ee 100644 --- a/test/config/collector/test-opentelemetry-collector-agent.yaml +++ b/test/config/collector/test-opentelemetry-collector-agent.yaml @@ -1,4 +1,8 @@ receivers: + syslog: + tcp: + listen_address: "127.0.0.1:515" + protocol: rfc5424 hostmetrics: collection_interval: 1m0s initial_delay: 1s @@ -84,3 +88,11 @@ service: - otlp/0 - prometheus - debug + logs: + receivers: + - syslog + processors: + - batch + exporters: + - otlp/0 + - debug diff --git a/test/docker/nginx-plus/deb/Dockerfile b/test/docker/nginx-plus/deb/Dockerfile index 0754e4bd02..6fa083c6fb 100644 --- a/test/docker/nginx-plus/deb/Dockerfile +++ b/test/docker/nginx-plus/deb/Dockerfile @@ -14,7 +14,7 @@ COPY $ENTRY_POINT /agent/entrypoint.sh RUN --mount=type=secret,id=nginx-crt,dst=nginx-repo.crt \ --mount=type=secret,id=nginx-key,dst=nginx-repo.key \ set -x \ -# Create nginx user/group first, to be consistent throughout Docker variants + # Create nginx user/group first, to be consistent throughout Docker variants && groupadd --system --gid 101 nginx \ && useradd --system --gid nginx --no-create-home --home-dir /nonexistent --uid 101 nginx \ && apt-get update \ @@ -37,20 +37,23 @@ RUN --mount=type=secret,id=nginx-crt,dst=nginx-repo.crt \ done; \ test -z "$found" && echo >&2 "error: failed to fetch GPG key $NGINX_GPGKEY" && exit 1; \ apt-get remove --purge --auto-remove -y gnupg1 && rm -rf /var/lib/apt/lists/* \ -# Install the latest release of NGINX Plus and/or NGINX Plus modules -# Uncomment individual modules if necessary -# Use versioned packages over defaults to specify a release - && nginxPackages=" \ - nginx-plus \ - " \ + # Install the latest release of NGINX Plus and/or NGINX Plus modules + # Uncomment individual modules if necessary + # Use versioned packages over defaults to specify a release + && nginxPackages="nginx-plus app-protect" \ + # Enable certificate-based authentication for the NGINX repository && echo "Acquire::https::pkgs.nginx.com::Verify-Peer \"true\";" > /etc/apt/apt.conf.d/90nginx \ && echo "Acquire::https::pkgs.nginx.com::Verify-Host \"true\";" >> /etc/apt/apt.conf.d/90nginx \ && echo "Acquire::https::pkgs.nginx.com::SslCert \"/etc/ssl/nginx/nginx-repo.crt\";" >> /etc/apt/apt.conf.d/90nginx \ && echo "Acquire::https::pkgs.nginx.com::SslKey \"/etc/ssl/nginx/nginx-repo.key\";" >> /etc/apt/apt.conf.d/90nginx \ + # Set up repository sources for NGINX Plus and App Protect && printf "deb https://pkgs.nginx.com/plus/ubuntu `lsb_release -cs` nginx-plus\n" > /etc/apt/sources.list.d/nginx-plus.list \ + && printf "deb https://pkgs.nginx.com/app-protect/ubuntu `lsb_release -cs` nginx-plus\n" > /etc/apt/sources.list.d/nginx-app-protect.list \ + # Copy certificates and keys for repository access && mkdir -p /etc/ssl/nginx \ && cat nginx-repo.crt > /etc/ssl/nginx/nginx-repo.crt \ && cat nginx-repo.key > /etc/ssl/nginx/nginx-repo.key \ + # Update and install NGINX packages && apt-get update \ && apt-get install --no-install-recommends --no-install-suggests -y \ $nginxPackages \ @@ -59,7 +62,7 @@ RUN --mount=type=secret,id=nginx-crt,dst=nginx-repo.crt \ jq \ gnupg2 \ && apt-get remove --purge -y lsb-release \ - && apt-get remove --purge --auto-remove -y && rm -rf /var/lib/apt/lists/* /etc/apt/sources.list.d/nginx-plus.list \ + && apt-get remove --purge --auto-remove -y && rm -rf /var/lib/apt/lists/* /etc/apt/sources.list.d/nginx-plus.list /etc/apt/sources.list.d/nginx-app-protect.list \ && rm -rf /etc/apt/apt.conf.d/90nginx /etc/ssl/nginx EXPOSE 80 diff --git a/test/types/config.go b/test/types/config.go index 3d570c9152..ecc97c8608 100644 --- a/test/types/config.go +++ b/test/types/config.go @@ -67,6 +67,11 @@ func AgentConfig() *config.Config { }, Receivers: config.Receivers{ OtlpReceivers: OtlpReceivers(), + SyslogReceiver: &config.SyslogReceiver{ + Host: "127.0.0.1", + Port: "515", + Protocol: "rfc5424", + }, HostMetrics: &config.HostMetrics{ CollectionInterval: time.Minute, InitialDelay: time.Second, From 016275714d8d11f0dc54b70bbe11e4157c776248 Mon Sep 17 00:00:00 2001 From: RRashmit Date: Wed, 13 Nov 2024 16:55:41 +0000 Subject: [PATCH 02/12] chore: update port to non priviledged --- internal/collector/otel_collector_plugin_test.go | 4 ++-- test/config/collector/test-opentelemetry-collector-agent.yaml | 2 +- test/types/config.go | 2 +- 3 files changed, 4 insertions(+), 4 deletions(-) diff --git a/internal/collector/otel_collector_plugin_test.go b/internal/collector/otel_collector_plugin_test.go index 99ae3c3055..608a7a73fa 100644 --- a/internal/collector/otel_collector_plugin_test.go +++ b/internal/collector/otel_collector_plugin_test.go @@ -122,7 +122,7 @@ func TestCollector_ProcessNginxConfigUpdateTopic(t *testing.T) { }, SyslogReceiver: &config.SyslogReceiver{ Host: "127.0.0.1", - Port: "515", + Port: "1515", Protocol: "rfc5424", }, OtlpReceivers: types.OtlpReceivers(), @@ -164,7 +164,7 @@ func TestCollector_ProcessNginxConfigUpdateTopic(t *testing.T) { OtlpReceivers: types.OtlpReceivers(), SyslogReceiver: &config.SyslogReceiver{ Host: "127.0.0.1", - Port: "515", + Port: "1515", Protocol: "rfc5424", }, NginxReceivers: []config.NginxReceiver{ diff --git a/test/config/collector/test-opentelemetry-collector-agent.yaml b/test/config/collector/test-opentelemetry-collector-agent.yaml index c88d3929ee..29bd572f54 100644 --- a/test/config/collector/test-opentelemetry-collector-agent.yaml +++ b/test/config/collector/test-opentelemetry-collector-agent.yaml @@ -1,7 +1,7 @@ receivers: syslog: tcp: - listen_address: "127.0.0.1:515" + listen_address: "127.0.0.1:1515" protocol: rfc5424 hostmetrics: collection_interval: 1m0s diff --git a/test/types/config.go b/test/types/config.go index ecc97c8608..bd88cfc19a 100644 --- a/test/types/config.go +++ b/test/types/config.go @@ -69,7 +69,7 @@ func AgentConfig() *config.Config { OtlpReceivers: OtlpReceivers(), SyslogReceiver: &config.SyslogReceiver{ Host: "127.0.0.1", - Port: "515", + Port: "1515", Protocol: "rfc5424", }, HostMetrics: &config.HostMetrics{ From 036aba93c7bfc2786dd754eab5703452bbf65ad8 Mon Sep 17 00:00:00 2001 From: RRashmit Date: Mon, 18 Nov 2024 10:00:04 +0000 Subject: [PATCH 03/12] chore: update the syslog type and template --- internal/collector/otel_collector_plugin.go | 11 +++ .../collector/otel_collector_plugin_test.go | 18 ++-- internal/collector/otelcol.tmpl | 12 +-- internal/config/defaults.go | 1 + internal/config/types.go | 8 +- internal/model/config.go | 7 ++ .../watcher/instance/nginx_config_parser.go | 17 ++++ .../test-opentelemetry-collector-agent.yaml | 6 +- test/docker/nginx-plus/deb/Dockerfile | 96 ++++++++++--------- test/types/config.go | 10 +- 10 files changed, 115 insertions(+), 71 deletions(-) diff --git a/internal/collector/otel_collector_plugin.go b/internal/collector/otel_collector_plugin.go index d2879dbfbc..822071d8cd 100644 --- a/internal/collector/otel_collector_plugin.go +++ b/internal/collector/otel_collector_plugin.go @@ -365,6 +365,16 @@ func (oc *Collector) checkForNewNginxReceivers(nginxConfigContext *model.NginxCo PlusAPI: nginxConfigContext.PlusAPI, }, ) + if nginxConfigContext.Syslog != nil && nginxConfigContext.Syslog.SyslogServer != "" { + oc.config.Collector.Receivers.SyslogReceivers = append( + oc.config.Collector.Receivers.SyslogReceivers, + config.SyslogReceiver{ + InstanceID: nginxConfigContext.InstanceID, + Server: nginxConfigContext.Syslog.SyslogServer, + Protocol: "rfc3164", // default value, need to get from the agent conf + }, + ) + } reloadCollector = true } else if nginxConfigContext.PlusAPI == "" { @@ -387,6 +397,7 @@ func (oc *Collector) checkForNewNginxReceivers(nginxConfigContext *model.NginxCo return reloadCollector } +// need to update new syslogreceivers func (oc *Collector) updateExistingNginxPlusReceiver( nginxConfigContext *model.NginxConfigContext, ) (nginxReceiverFound, reloadCollector bool) { diff --git a/internal/collector/otel_collector_plugin_test.go b/internal/collector/otel_collector_plugin_test.go index 608a7a73fa..5727e764dc 100644 --- a/internal/collector/otel_collector_plugin_test.go +++ b/internal/collector/otel_collector_plugin_test.go @@ -120,10 +120,11 @@ func TestCollector_ProcessNginxConfigUpdateTopic(t *testing.T) { Network: &config.NetworkScraper{}, }, }, - SyslogReceiver: &config.SyslogReceiver{ - Host: "127.0.0.1", - Port: "1515", - Protocol: "rfc5424", + SyslogReceivers: []config.SyslogReceiver{ + { + Server: "127.0.0.1:1515", + InstanceID: "1", + Protocol: "rfc3164"}, }, OtlpReceivers: types.OtlpReceivers(), NginxPlusReceivers: []config.NginxPlusReceiver{ @@ -162,10 +163,11 @@ func TestCollector_ProcessNginxConfigUpdateTopic(t *testing.T) { }, }, OtlpReceivers: types.OtlpReceivers(), - SyslogReceiver: &config.SyslogReceiver{ - Host: "127.0.0.1", - Port: "1515", - Protocol: "rfc5424", + SyslogReceivers: []config.SyslogReceiver{ + { + Server: "127.0.0.1:1515", + InstanceID: "1", + Protocol: "rfc3164"}, }, NginxReceivers: []config.NginxReceiver{ { diff --git a/internal/collector/otelcol.tmpl b/internal/collector/otelcol.tmpl index b49e5f9f6f..8d5de0fdfc 100644 --- a/internal/collector/otelcol.tmpl +++ b/internal/collector/otelcol.tmpl @@ -1,9 +1,9 @@ receivers: -{{- if ne .Receivers.SyslogReceiver nil }} - syslog: +{{- range .Receivers.SyslogReceivers }} + syslog/{{- .InstanceID -}}: tcp: - listen_address: "{{- .Receivers.SyslogReceiver.Host -}}:{{- .Receivers.SyslogReceiver.Port -}}" - protocol: {{ .Receivers.SyslogReceiver.Protocol }} + listen_address: "{{- .Server -}}" + protocol: rfc3164 {{- end }} {{- if ne .Receivers.HostMetrics nil }} hostmetrics: @@ -241,8 +241,8 @@ service: {{- end }} logs: receivers: - {{- if ne .Receivers.SyslogReceiver nil }} - - syslog + {{- range .Receivers.SyslogReceivers }} + - syslog/{{- .InstanceID -}} {{- end }} processors: {{- if ne .Processors.Batch nil }} diff --git a/internal/config/defaults.go b/internal/config/defaults.go index 526ea914c9..b5de7ffcb0 100644 --- a/internal/config/defaults.go +++ b/internal/config/defaults.go @@ -12,6 +12,7 @@ import ( ) const ( + // Add default nap parameter DefGracefulShutdownPeriod = 5 * time.Second DefNginxReloadMonitoringPeriod = 10 * time.Second DefTreatErrorsAsWarnings = false diff --git a/internal/config/types.go b/internal/config/types.go index 2ec27c880b..996c188168 100644 --- a/internal/config/types.go +++ b/internal/config/types.go @@ -163,7 +163,7 @@ type ( OtlpReceivers []OtlpReceiver `yaml:"-" mapstructure:"otlp_receivers"` NginxReceivers []NginxReceiver `yaml:"-" mapstructure:"nginx_receivers"` NginxPlusReceivers []NginxPlusReceiver `yaml:"-" mapstructure:"nginx_plus_receivers"` - SyslogReceiver *SyslogReceiver `yaml:"-" mapstructure:"syslog_receiver"` + SyslogReceivers []SyslogReceiver `yaml:"-" mapstructure:"syslog_receiver"` } OtlpReceiver struct { @@ -173,9 +173,9 @@ type ( } SyslogReceiver struct { - Host string `yaml:"-" mapstructure:"host"` - Port string `yaml:"-" mapstructure:"port"` - Protocol string `yaml:"-" mapstructure:"protocol"` + InstanceID string `yaml:"-" mapstructure:"instance_id"` + Server string `yaml:"-" mapstructure:"host"` + Protocol string `yaml:"-" mapstructure:"protocol"` } NginxReceiver struct { diff --git a/internal/model/config.go b/internal/model/config.go index 3f37428912..ef60d0e568 100644 --- a/internal/model/config.go +++ b/internal/model/config.go @@ -18,6 +18,7 @@ type NginxConfigContext struct { Files []*v1.File AccessLogs []*AccessLog ErrorLogs []*ErrorLog + Syslog *NAP } func (ncc *NginxConfigContext) Equal(otherNginxConfigContext *NginxConfigContext) bool { @@ -85,6 +86,12 @@ type ErrorLog struct { Readable bool } +type NAP struct { + Enable bool + Syslog bool + SyslogServer string +} + type ( WriteStatus int ) diff --git a/internal/watcher/instance/nginx_config_parser.go b/internal/watcher/instance/nginx_config_parser.go index ab147e2bd5..e0f7621e1d 100644 --- a/internal/watcher/instance/nginx_config_parser.go +++ b/internal/watcher/instance/nginx_config_parser.go @@ -16,6 +16,7 @@ import ( "net/http" "os" "path/filepath" + "regexp" "slices" "strconv" "strings" @@ -36,6 +37,7 @@ const ( stubStatusAPIDirective = "stub_status" apiFormat = "http://%s%s" locationDirective = "location" + napDirective = "app_protect_security_log" ) type ( @@ -104,6 +106,7 @@ func (ncp *NginxConfigParser) createNginxConfigContext( case "log_format": formatMap = ncp.formatMap(directive) case "access_log": + fmt.Println("am here") if !ncp.ignoreLog(directive.Args[0]) { accessLog := ncp.accessLog(directive.Args[0], ncp.accessLogDirectiveFormat(directive), formatMap) @@ -120,6 +123,20 @@ func (ncp *NginxConfigParser) createNginxConfigContext( case "ssl_certificate", "proxy_ssl_certificate", "ssl_client_certificate", "ssl_trusted_certificate": sslCertFile := ncp.sslCert(ctx, directive.Args[0], rootDir) nginxConfigContext.Files = append(nginxConfigContext.Files, sslCertFile) + case "app_protect_security_log": + // Extract the syslog:server part of the app_protect_security_log directive + if len(directive.Args) > 1 { + syslogArg := directive.Args[1] + re := regexp.MustCompile(`syslog:server=([\S]+)`) + matches := re.FindStringSubmatch(syslogArg) + if len(matches) > 1 { + syslogServer := matches[1] + nginxConfigContext.Syslog.SyslogServer = syslogServer + slog.InfoContext(ctx, "Captured syslog server", "syslog_server", syslogServer) + fmt.Println("syslogServer") + fmt.Println(syslogServer) + } + } } return nil diff --git a/test/config/collector/test-opentelemetry-collector-agent.yaml b/test/config/collector/test-opentelemetry-collector-agent.yaml index 29bd572f54..8d6be2900c 100644 --- a/test/config/collector/test-opentelemetry-collector-agent.yaml +++ b/test/config/collector/test-opentelemetry-collector-agent.yaml @@ -1,8 +1,8 @@ receivers: - syslog: + syslog/1: tcp: listen_address: "127.0.0.1:1515" - protocol: rfc5424 + protocol: rfc3164 hostmetrics: collection_interval: 1m0s initial_delay: 1s @@ -90,7 +90,7 @@ service: - debug logs: receivers: - - syslog + - syslog/1 processors: - batch exporters: diff --git a/test/docker/nginx-plus/deb/Dockerfile b/test/docker/nginx-plus/deb/Dockerfile index 6fa083c6fb..9951d5114c 100644 --- a/test/docker/nginx-plus/deb/Dockerfile +++ b/test/docker/nginx-plus/deb/Dockerfile @@ -1,79 +1,83 @@ +# Base Image Argument ARG BASE_IMAGE FROM ${BASE_IMAGE} as install-nginx LABEL maintainer="NGINX Docker Maintainers " +# Define arguments for flexibility ARG DEBIAN_FRONTEND=noninteractive ARG ENTRY_POINT ARG PACKAGE_NAME ARG PACKAGES_REPO +# Set working directory WORKDIR /agent COPY ./build /agent/build COPY $ENTRY_POINT /agent/entrypoint.sh +# Install NGINX Plus and NGINX App Protect RUN --mount=type=secret,id=nginx-crt,dst=nginx-repo.crt \ --mount=type=secret,id=nginx-key,dst=nginx-repo.key \ - set -x \ - # Create nginx user/group first, to be consistent throughout Docker variants + set -eux \ && groupadd --system --gid 101 nginx \ && useradd --system --gid nginx --no-create-home --home-dir /nonexistent --uid 101 nginx \ + \ && apt-get update \ && apt-get install --no-install-recommends --no-install-suggests -y \ - ca-certificates \ - gnupg1 \ - lsb-release \ - git \ - wget \ - make \ - && \ - NGINX_GPGKEY=573BFD6B3D8FBC641079A6ABABF5BD827BD9BF62; \ - found=''; \ - for server in \ - hkp://keyserver.ubuntu.com:80 \ - pgp.mit.edu \ - ; do \ - echo "Fetching GPG key $NGINX_GPGKEY from $server"; \ - apt-key adv --keyserver "$server" --keyserver-options timeout=10 --recv-keys "$NGINX_GPGKEY" && found=yes && break; \ - done; \ - test -z "$found" && echo >&2 "error: failed to fetch GPG key $NGINX_GPGKEY" && exit 1; \ - apt-get remove --purge --auto-remove -y gnupg1 && rm -rf /var/lib/apt/lists/* \ - # Install the latest release of NGINX Plus and/or NGINX Plus modules - # Uncomment individual modules if necessary - # Use versioned packages over defaults to specify a release - && nginxPackages="nginx-plus app-protect" \ - # Enable certificate-based authentication for the NGINX repository - && echo "Acquire::https::pkgs.nginx.com::Verify-Peer \"true\";" > /etc/apt/apt.conf.d/90nginx \ - && echo "Acquire::https::pkgs.nginx.com::Verify-Host \"true\";" >> /etc/apt/apt.conf.d/90nginx \ - && echo "Acquire::https::pkgs.nginx.com::SslCert \"/etc/ssl/nginx/nginx-repo.crt\";" >> /etc/apt/apt.conf.d/90nginx \ - && echo "Acquire::https::pkgs.nginx.com::SslKey \"/etc/ssl/nginx/nginx-repo.key\";" >> /etc/apt/apt.conf.d/90nginx \ - # Set up repository sources for NGINX Plus and App Protect - && printf "deb https://pkgs.nginx.com/plus/ubuntu `lsb_release -cs` nginx-plus\n" > /etc/apt/sources.list.d/nginx-plus.list \ - && printf "deb https://pkgs.nginx.com/app-protect/ubuntu `lsb_release -cs` nginx-plus\n" > /etc/apt/sources.list.d/nginx-app-protect.list \ - # Copy certificates and keys for repository access + ca-certificates \ + gnupg2 \ + lsb-release \ + git \ + wget \ + make \ + apt-transport-https \ + ubuntu-keyring \ + \ && mkdir -p /etc/ssl/nginx \ && cat nginx-repo.crt > /etc/ssl/nginx/nginx-repo.crt \ && cat nginx-repo.key > /etc/ssl/nginx/nginx-repo.key \ - # Update and install NGINX packages + \ + # Add NGINX repositories securely + && wget -qO - https://cs.nginx.com/static/keys/nginx_signing.key | gpg --dearmor > /usr/share/keyrings/nginx-archive-keyring.gpg \ + && wget -qO - https://cs.nginx.com/static/keys/app-protect-security-updates.key | gpg --dearmor > /usr/share/keyrings/app-protect-security-updates.gpg \ + \ + && printf "deb [signed-by=/usr/share/keyrings/nginx-archive-keyring.gpg] https://pkgs.nginx.com/plus/ubuntu $(lsb_release -cs) nginx-plus\n" > /etc/apt/sources.list.d/nginx-plus.list \ + && printf "deb [signed-by=/usr/share/keyrings/nginx-archive-keyring.gpg] https://pkgs.nginx.com/app-protect/ubuntu $(lsb_release -cs) nginx-plus\n" > /etc/apt/sources.list.d/nginx-app-protect.list \ + && printf "deb [signed-by=/usr/share/keyrings/app-protect-security-updates.gpg] https://pkgs.nginx.com/app-protect-security-updates/ubuntu $(lsb_release -cs) nginx-plus\n" >> /etc/apt/sources.list.d/nginx-app-protect.list \ + \ + && wget -P /etc/apt/apt.conf.d https://cs.nginx.com/static/files/90pkgs-nginx \ + \ && apt-get update \ + && apt-cache show nginx-plus || echo "nginx-plus package not found" \ + && apt-cache show app-protect || echo "app-protect package not found" \ && apt-get install --no-install-recommends --no-install-suggests -y \ - $nginxPackages \ - curl \ - gettext-base \ - jq \ - gnupg2 \ + nginx-plus \ + app-protect \ + app-protect-attack-signatures \ + curl \ + gettext-base \ + jq \ + \ && apt-get remove --purge -y lsb-release \ - && apt-get remove --purge --auto-remove -y && rm -rf /var/lib/apt/lists/* /etc/apt/sources.list.d/nginx-plus.list /etc/apt/sources.list.d/nginx-app-protect.list \ - && rm -rf /etc/apt/apt.conf.d/90nginx /etc/ssl/nginx + && apt-get remove --purge --auto-remove -y \ + \ + # Clean up + && rm -rf /var/lib/apt/lists/* \ + /etc/apt/sources.list.d/nginx-plus.list \ + /etc/apt/sources.list.d/nginx-app-protect.list \ + /etc/apt/apt.conf.d/90nginx \ + /etc/ssl/nginx -EXPOSE 80 +# Expose ports +EXPOSE 80 443 +# Set stop signal STOPSIGNAL SIGQUIT +# Set executable permissions for entrypoint RUN chmod +x /agent/entrypoint.sh -RUN apt install -y /agent/build/${PACKAGE_NAME}.deb - -STOPSIGNAL SIGTERM -EXPOSE 80 443 +# Install NGINX Agent package +RUN apt install -y /agent/build/${PACKAGE_NAME}.deb || { echo "Failed to install ${PACKAGE_NAME}"; exit 1; } +# Define entrypoint ENTRYPOINT ["/agent/entrypoint.sh"] diff --git a/test/types/config.go b/test/types/config.go index bd88cfc19a..d1e5cfccf3 100644 --- a/test/types/config.go +++ b/test/types/config.go @@ -67,10 +67,12 @@ func AgentConfig() *config.Config { }, Receivers: config.Receivers{ OtlpReceivers: OtlpReceivers(), - SyslogReceiver: &config.SyslogReceiver{ - Host: "127.0.0.1", - Port: "1515", - Protocol: "rfc5424", + SyslogReceivers: []config.SyslogReceiver{ + { + Server: "127.0.0.1:1515", + InstanceID: "1", + Protocol: "rfc3164", + }, }, HostMetrics: &config.HostMetrics{ CollectionInterval: time.Minute, From 0e138e7ccb330b248603591c947b5486c4bae498 Mon Sep 17 00:00:00 2001 From: RRashmit Date: Mon, 18 Nov 2024 10:14:28 +0000 Subject: [PATCH 04/12] chore: update comments --- internal/collector/otel_collector_plugin.go | 2 +- internal/config/defaults.go | 2 +- .../watcher/instance/nginx_config_parser.go | 1 - test/docker/nginx-plus/deb/Dockerfile | 67 ++++++------------- 4 files changed, 23 insertions(+), 49 deletions(-) diff --git a/internal/collector/otel_collector_plugin.go b/internal/collector/otel_collector_plugin.go index 822071d8cd..f0d8c813ca 100644 --- a/internal/collector/otel_collector_plugin.go +++ b/internal/collector/otel_collector_plugin.go @@ -397,7 +397,7 @@ func (oc *Collector) checkForNewNginxReceivers(nginxConfigContext *model.NginxCo return reloadCollector } -// need to update new syslogreceivers +// Todo: consider update scenario for new syslogreceivers func (oc *Collector) updateExistingNginxPlusReceiver( nginxConfigContext *model.NginxConfigContext, ) (nginxReceiverFound, reloadCollector bool) { diff --git a/internal/config/defaults.go b/internal/config/defaults.go index b5de7ffcb0..b386a236b6 100644 --- a/internal/config/defaults.go +++ b/internal/config/defaults.go @@ -12,7 +12,7 @@ import ( ) const ( - // Add default nap parameter + //Todo: Add default Protocol for nap DefGracefulShutdownPeriod = 5 * time.Second DefNginxReloadMonitoringPeriod = 10 * time.Second DefTreatErrorsAsWarnings = false diff --git a/internal/watcher/instance/nginx_config_parser.go b/internal/watcher/instance/nginx_config_parser.go index e0f7621e1d..52f0c09eec 100644 --- a/internal/watcher/instance/nginx_config_parser.go +++ b/internal/watcher/instance/nginx_config_parser.go @@ -124,7 +124,6 @@ func (ncp *NginxConfigParser) createNginxConfigContext( sslCertFile := ncp.sslCert(ctx, directive.Args[0], rootDir) nginxConfigContext.Files = append(nginxConfigContext.Files, sslCertFile) case "app_protect_security_log": - // Extract the syslog:server part of the app_protect_security_log directive if len(directive.Args) > 1 { syslogArg := directive.Args[1] re := regexp.MustCompile(`syslog:server=([\S]+)`) diff --git a/test/docker/nginx-plus/deb/Dockerfile b/test/docker/nginx-plus/deb/Dockerfile index 9951d5114c..1cd29f0b37 100644 --- a/test/docker/nginx-plus/deb/Dockerfile +++ b/test/docker/nginx-plus/deb/Dockerfile @@ -1,15 +1,12 @@ -# Base Image Argument ARG BASE_IMAGE FROM ${BASE_IMAGE} as install-nginx LABEL maintainer="NGINX Docker Maintainers " -# Define arguments for flexibility ARG DEBIAN_FRONTEND=noninteractive ARG ENTRY_POINT ARG PACKAGE_NAME ARG PACKAGES_REPO -# Set working directory WORKDIR /agent COPY ./build /agent/build COPY $ENTRY_POINT /agent/entrypoint.sh @@ -17,67 +14,45 @@ COPY $ENTRY_POINT /agent/entrypoint.sh # Install NGINX Plus and NGINX App Protect RUN --mount=type=secret,id=nginx-crt,dst=nginx-repo.crt \ --mount=type=secret,id=nginx-key,dst=nginx-repo.key \ - set -eux \ + set -x \ && groupadd --system --gid 101 nginx \ && useradd --system --gid nginx --no-create-home --home-dir /nonexistent --uid 101 nginx \ - \ && apt-get update \ && apt-get install --no-install-recommends --no-install-suggests -y \ - ca-certificates \ - gnupg2 \ - lsb-release \ - git \ - wget \ - make \ - apt-transport-https \ - ubuntu-keyring \ - \ + ca-certificates \ + gnupg2 \ + lsb-release \ + git \ + wget \ + make \ + apt-transport-https \ + ubuntu-keyring \ && mkdir -p /etc/ssl/nginx \ && cat nginx-repo.crt > /etc/ssl/nginx/nginx-repo.crt \ && cat nginx-repo.key > /etc/ssl/nginx/nginx-repo.key \ - \ - # Add NGINX repositories securely && wget -qO - https://cs.nginx.com/static/keys/nginx_signing.key | gpg --dearmor > /usr/share/keyrings/nginx-archive-keyring.gpg \ && wget -qO - https://cs.nginx.com/static/keys/app-protect-security-updates.key | gpg --dearmor > /usr/share/keyrings/app-protect-security-updates.gpg \ - \ - && printf "deb [signed-by=/usr/share/keyrings/nginx-archive-keyring.gpg] https://pkgs.nginx.com/plus/ubuntu $(lsb_release -cs) nginx-plus\n" > /etc/apt/sources.list.d/nginx-plus.list \ - && printf "deb [signed-by=/usr/share/keyrings/nginx-archive-keyring.gpg] https://pkgs.nginx.com/app-protect/ubuntu $(lsb_release -cs) nginx-plus\n" > /etc/apt/sources.list.d/nginx-app-protect.list \ - && printf "deb [signed-by=/usr/share/keyrings/app-protect-security-updates.gpg] https://pkgs.nginx.com/app-protect-security-updates/ubuntu $(lsb_release -cs) nginx-plus\n" >> /etc/apt/sources.list.d/nginx-app-protect.list \ - \ + && printf "deb [signed-by=/usr/share/keyrings/nginx-archive-keyring.gpg] https://pkgs.nginx.com/plus/ubuntu `lsb_release -cs` nginx-plus\n" > /etc/apt/sources.list.d/nginx-plus.list \ + && printf "deb [signed-by=/usr/share/keyrings/nginx-archive-keyring.gpg] https://pkgs.nginx.com/app-protect/ubuntu `lsb_release -cs` nginx-plus\n" > /etc/apt/sources.list.d/nginx-app-protect.list \ + && printf "deb [signed-by=/usr/share/keyrings/app-protect-security-updates.gpg] https://pkgs.nginx.com/app-protect-security-updates/ubuntu `lsb_release -cs` nginx-plus\n" >> /etc/apt/sources.list.d/nginx-app-protect.list \ && wget -P /etc/apt/apt.conf.d https://cs.nginx.com/static/files/90pkgs-nginx \ - \ && apt-get update \ - && apt-cache show nginx-plus || echo "nginx-plus package not found" \ - && apt-cache show app-protect || echo "app-protect package not found" \ && apt-get install --no-install-recommends --no-install-suggests -y \ - nginx-plus \ - app-protect \ - app-protect-attack-signatures \ - curl \ - gettext-base \ - jq \ - \ + nginx-plus \ + app-protect \ + app-protect-attack-signatures \ + curl \ + gettext-base \ + jq \ && apt-get remove --purge -y lsb-release \ - && apt-get remove --purge --auto-remove -y \ - \ - # Clean up - && rm -rf /var/lib/apt/lists/* \ - /etc/apt/sources.list.d/nginx-plus.list \ - /etc/apt/sources.list.d/nginx-app-protect.list \ - /etc/apt/apt.conf.d/90nginx \ - /etc/ssl/nginx + && apt-get remove --purge --auto-remove -y && rm -rf /var/lib/apt/lists/* /etc/apt/sources.list.d/nginx-plus.list /etc/apt/sources.list.d/nginx-app-protect.list \ + && rm -rf /etc/apt/apt.conf.d/90nginx /etc/ssl/nginx -# Expose ports EXPOSE 80 443 -# Set stop signal STOPSIGNAL SIGQUIT -# Set executable permissions for entrypoint RUN chmod +x /agent/entrypoint.sh +RUN apt install -y /agent/build/${PACKAGE_NAME}.deb -# Install NGINX Agent package -RUN apt install -y /agent/build/${PACKAGE_NAME}.deb || { echo "Failed to install ${PACKAGE_NAME}"; exit 1; } - -# Define entrypoint ENTRYPOINT ["/agent/entrypoint.sh"] From 4f6db4ae8b55624d629377e89135ffafccfa3ce8 Mon Sep 17 00:00:00 2001 From: RRashmit Date: Mon, 18 Nov 2024 15:49:36 +0000 Subject: [PATCH 05/12] chore: update the docker --- test/docker/nginx-plus/deb/Dockerfile | 39 ++++++++++++++++++--------- 1 file changed, 26 insertions(+), 13 deletions(-) diff --git a/test/docker/nginx-plus/deb/Dockerfile b/test/docker/nginx-plus/deb/Dockerfile index 1cd29f0b37..9b1aa04047 100644 --- a/test/docker/nginx-plus/deb/Dockerfile +++ b/test/docker/nginx-plus/deb/Dockerfile @@ -11,10 +11,10 @@ WORKDIR /agent COPY ./build /agent/build COPY $ENTRY_POINT /agent/entrypoint.sh -# Install NGINX Plus and NGINX App Protect RUN --mount=type=secret,id=nginx-crt,dst=nginx-repo.crt \ --mount=type=secret,id=nginx-key,dst=nginx-repo.key \ set -x \ + # Create nginx user/group first, to be consistent throughout Docker variants && groupadd --system --gid 101 nginx \ && useradd --system --gid nginx --no-create-home --home-dir /nonexistent --uid 101 nginx \ && apt-get update \ @@ -25,34 +25,47 @@ RUN --mount=type=secret,id=nginx-crt,dst=nginx-repo.crt \ git \ wget \ make \ - apt-transport-https \ - ubuntu-keyring \ && mkdir -p /etc/ssl/nginx \ - && cat nginx-repo.crt > /etc/ssl/nginx/nginx-repo.crt \ - && cat nginx-repo.key > /etc/ssl/nginx/nginx-repo.key \ - && wget -qO - https://cs.nginx.com/static/keys/nginx_signing.key | gpg --dearmor > /usr/share/keyrings/nginx-archive-keyring.gpg \ - && wget -qO - https://cs.nginx.com/static/keys/app-protect-security-updates.key | gpg --dearmor > /usr/share/keyrings/app-protect-security-updates.gpg \ + && cat nginx-repo.crt > /etc/ssl/nginx/nginx-repo.crt \ + && cat nginx-repo.key > /etc/ssl/nginx/nginx-repo.key \ + && echo "Acquire::https::pkgs.nginx.com::Verify-Peer \"true\";" > /etc/apt/apt.conf.d/90nginx \ + && echo "Acquire::https::pkgs.nginx.com::Verify-Host \"true\";" >> /etc/apt/apt.conf.d/90nginx \ + && echo "Acquire::https::pkgs.nginx.com::SslCert \"/etc/ssl/nginx/nginx-repo.crt\";" >> /etc/apt/apt.conf.d/90nginx \ + && echo "Acquire::https::pkgs.nginx.com::SslKey \"/etc/ssl/nginx/nginx-repo.key\";" >> /etc/apt/apt.conf.d/90nginx \ + # Add GPG keys for nginx-plus and app-protect + && wget -qO - https://cs.nginx.com/static/keys/nginx_signing.key | gpg --dearmor | tee /usr/share/keyrings/nginx-archive-keyring.gpg >/dev/null \ + && wget -qO - https://cs.nginx.com/static/keys/app-protect-security-updates.key | gpg --dearmor | tee /usr/share/keyrings/app-protect-security-updates.gpg >/dev/null \ + #&& ls -l /usr/share/keyrings/nginx-archive-keyring.gpg /usr/share/keyrings/app-protect-security-updates.gpg \ + + + # Install the latest release of NGINX Plus and/or NGINX Plus modules + # Set up repository sources for NGINX Plus and App Protect && printf "deb [signed-by=/usr/share/keyrings/nginx-archive-keyring.gpg] https://pkgs.nginx.com/plus/ubuntu `lsb_release -cs` nginx-plus\n" > /etc/apt/sources.list.d/nginx-plus.list \ - && printf "deb [signed-by=/usr/share/keyrings/nginx-archive-keyring.gpg] https://pkgs.nginx.com/app-protect/ubuntu `lsb_release -cs` nginx-plus\n" > /etc/apt/sources.list.d/nginx-app-protect.list \ - && printf "deb [signed-by=/usr/share/keyrings/app-protect-security-updates.gpg] https://pkgs.nginx.com/app-protect-security-updates/ubuntu `lsb_release -cs` nginx-plus\n" >> /etc/apt/sources.list.d/nginx-app-protect.list \ - && wget -P /etc/apt/apt.conf.d https://cs.nginx.com/static/files/90pkgs-nginx \ + && printf "deb [signed-by=/usr/share/keyrings/nginx-archive-keyring.gpg] https://pkgs.nginx.com/app-protect/ubuntu `lsb_release -cs` nginx-plus\n" | tee /etc/apt/sources.list.d/nginx-app-protect.list \ + && printf "deb [signed-by=/usr/share/keyrings/app-protect-security-updates.gpg] https://pkgs.nginx.com/app-protect-security-updates/ubuntu `lsb_release -cs` nginx-plus\n" | tee -a /etc/apt/sources.list.d/nginx-app-protect.list \ + # && wget -P /etc/apt/apt.conf.d https://cs.nginx.com/static/files/90pkgs-nginx \ + # && printf "deb [signed-by=/usr/share/keyrings/app-protect-security-updates.gpg] https://pkgs.nginx.com/app-protect/ubuntu `lsb_release -cs` nginx-plus\n" > /etc/apt/sources.list.d/nginx-app-protect.list \ && apt-get update \ && apt-get install --no-install-recommends --no-install-suggests -y \ nginx-plus \ app-protect \ - app-protect-attack-signatures \ curl \ gettext-base \ jq \ && apt-get remove --purge -y lsb-release \ - && apt-get remove --purge --auto-remove -y && rm -rf /var/lib/apt/lists/* /etc/apt/sources.list.d/nginx-plus.list /etc/apt/sources.list.d/nginx-app-protect.list \ + && apt-get remove --purge --auto-remove -y \ + && rm -rf /var/lib/apt/lists/* /etc/apt/sources.list.d/nginx-plus.list /etc/apt/sources.list.d/nginx-app-protect.list \ && rm -rf /etc/apt/apt.conf.d/90nginx /etc/ssl/nginx -EXPOSE 80 443 +EXPOSE 80 STOPSIGNAL SIGQUIT RUN chmod +x /agent/entrypoint.sh RUN apt install -y /agent/build/${PACKAGE_NAME}.deb +STOPSIGNAL SIGTERM + +EXPOSE 80 443 + ENTRYPOINT ["/agent/entrypoint.sh"] From 409242c1f959cabf71674c86f4320b21acc1de7d Mon Sep 17 00:00:00 2001 From: RRashmit Date: Mon, 25 Nov 2024 11:06:46 +0000 Subject: [PATCH 06/12] chore: updated the otel collector pipeline --- internal/collector/otelcol.tmpl | 2 ++ internal/config/types.go | 2 +- internal/watcher/instance/nginx_config_parser.go | 4 ---- test/mock/collector/otel-collector.yaml | 8 ++++++++ 4 files changed, 11 insertions(+), 5 deletions(-) diff --git a/internal/collector/otelcol.tmpl b/internal/collector/otelcol.tmpl index 8d5de0fdfc..94524705be 100644 --- a/internal/collector/otelcol.tmpl +++ b/internal/collector/otelcol.tmpl @@ -239,6 +239,7 @@ service: {{- if ne .Exporters.Debug nil }} - debug {{- end }} + {{- if and (ne .Receivers.SyslogReceivers nil) (ne .Exporters.OtlpExporters nil) }} logs: receivers: {{- range .Receivers.SyslogReceivers }} @@ -255,3 +256,4 @@ service: {{- if ne .Exporters.Debug nil }} - debug {{- end }} + {{- end }} diff --git a/internal/config/types.go b/internal/config/types.go index 996c188168..20e27d9b94 100644 --- a/internal/config/types.go +++ b/internal/config/types.go @@ -174,7 +174,7 @@ type ( SyslogReceiver struct { InstanceID string `yaml:"-" mapstructure:"instance_id"` - Server string `yaml:"-" mapstructure:"host"` + Server string `yaml:"-" mapstructure:"server"` Protocol string `yaml:"-" mapstructure:"protocol"` } diff --git a/internal/watcher/instance/nginx_config_parser.go b/internal/watcher/instance/nginx_config_parser.go index 52f0c09eec..541f5c4251 100644 --- a/internal/watcher/instance/nginx_config_parser.go +++ b/internal/watcher/instance/nginx_config_parser.go @@ -106,7 +106,6 @@ func (ncp *NginxConfigParser) createNginxConfigContext( case "log_format": formatMap = ncp.formatMap(directive) case "access_log": - fmt.Println("am here") if !ncp.ignoreLog(directive.Args[0]) { accessLog := ncp.accessLog(directive.Args[0], ncp.accessLogDirectiveFormat(directive), formatMap) @@ -132,12 +131,9 @@ func (ncp *NginxConfigParser) createNginxConfigContext( syslogServer := matches[1] nginxConfigContext.Syslog.SyslogServer = syslogServer slog.InfoContext(ctx, "Captured syslog server", "syslog_server", syslogServer) - fmt.Println("syslogServer") - fmt.Println(syslogServer) } } } - return nil }, ) diff --git a/test/mock/collector/otel-collector.yaml b/test/mock/collector/otel-collector.yaml index 1fb86a2560..b1e801a9cc 100644 --- a/test/mock/collector/otel-collector.yaml +++ b/test/mock/collector/otel-collector.yaml @@ -11,6 +11,10 @@ exporters: resource_to_telemetry_conversion: enabled: true add_metric_suffixes: false + debug: + verbosity: detailed + sampling_initial: 5 + sampling_thereafter: 200 processors: batch: @@ -28,3 +32,7 @@ service: receivers: [otlp] processors: [batch] exporters: [prometheus] + logs: + receivers: [otlp] + processors: [batch] + exporters: [debug] From 3fde54ce306e5156bc25abb64f45bc087363fa4d Mon Sep 17 00:00:00 2001 From: RRashmit Date: Mon, 25 Nov 2024 17:11:57 +0000 Subject: [PATCH 07/12] chore: update multiple syslog servers --- api/grpc/mpi/v1/command.pb.go | 2 + internal/collector/otel_collector_plugin.go | 7 ++- .../collector/otel_collector_plugin_test.go | 8 +-- internal/collector/otelcol.tmpl | 9 ++-- internal/config/types.go | 6 +-- internal/model/config.go | 2 +- .../watcher/instance/nginx_config_parser.go | 6 ++- .../instance/nginx_config_parser_test.go | 4 ++ .../test-opentelemetry-collector-agent.yaml | 4 +- test/docker/nginx-plus/deb/Dockerfile | 49 ++++++++++--------- test/model/config.go | 2 + test/protos/instances.go | 3 ++ test/types/config.go | 4 +- 13 files changed, 66 insertions(+), 40 deletions(-) diff --git a/api/grpc/mpi/v1/command.pb.go b/api/grpc/mpi/v1/command.pb.go index 7f6e52f857..9645d9c5e8 100644 --- a/api/grpc/mpi/v1/command.pb.go +++ b/api/grpc/mpi/v1/command.pb.go @@ -1737,6 +1737,8 @@ type NGINXPlusRuntimeInfo struct { DynamicModules []string `protobuf:"bytes,5,rep,name=dynamic_modules,json=dynamicModules,proto3" json:"dynamic_modules,omitempty"` // the plus API location PlusApi string `protobuf:"bytes,6,opt,name=plus_api,json=plusApi,proto3" json:"plus_api,omitempty"` + // a list of App Protect directives + AppProtectDirectives []string `protobuf:"bytes,7,rep,name=app_protect_directives,json=appProtectDirectives,proto3" json:"app_protect_directives,omitempty"` } func (x *NGINXPlusRuntimeInfo) Reset() { diff --git a/internal/collector/otel_collector_plugin.go b/internal/collector/otel_collector_plugin.go index f0d8c813ca..aa329dab81 100644 --- a/internal/collector/otel_collector_plugin.go +++ b/internal/collector/otel_collector_plugin.go @@ -365,17 +365,20 @@ func (oc *Collector) checkForNewNginxReceivers(nginxConfigContext *model.NginxCo PlusAPI: nginxConfigContext.PlusAPI, }, ) - if nginxConfigContext.Syslog != nil && nginxConfigContext.Syslog.SyslogServer != "" { + + if nginxConfigContext.Syslog != nil { oc.config.Collector.Receivers.SyslogReceivers = append( oc.config.Collector.Receivers.SyslogReceivers, config.SyslogReceiver{ InstanceID: nginxConfigContext.InstanceID, - Server: nginxConfigContext.Syslog.SyslogServer, + Server: nginxConfigContext.Syslog, Protocol: "rfc3164", // default value, need to get from the agent conf }, ) } + // slog.Error("hello", oc.config.Collector.Receivers) + reloadCollector = true } else if nginxConfigContext.PlusAPI == "" { nginxReceiverFound, reloadCollector = oc.updateExistingNginxOSSReceiver(nginxConfigContext) diff --git a/internal/collector/otel_collector_plugin_test.go b/internal/collector/otel_collector_plugin_test.go index 5727e764dc..f9138b3db9 100644 --- a/internal/collector/otel_collector_plugin_test.go +++ b/internal/collector/otel_collector_plugin_test.go @@ -122,8 +122,8 @@ func TestCollector_ProcessNginxConfigUpdateTopic(t *testing.T) { }, SyslogReceivers: []config.SyslogReceiver{ { - Server: "127.0.0.1:1515", - InstanceID: "1", + Server: []string{"127.0.0.1:1515"}, + InstanceID: "00", Protocol: "rfc3164"}, }, OtlpReceivers: types.OtlpReceivers(), @@ -165,8 +165,8 @@ func TestCollector_ProcessNginxConfigUpdateTopic(t *testing.T) { OtlpReceivers: types.OtlpReceivers(), SyslogReceivers: []config.SyslogReceiver{ { - Server: "127.0.0.1:1515", - InstanceID: "1", + Server: []string{"127.0.0.1:1515"}, + InstanceID: "00", Protocol: "rfc3164"}, }, NginxReceivers: []config.NginxReceiver{ diff --git a/internal/collector/otelcol.tmpl b/internal/collector/otelcol.tmpl index 94524705be..8850551407 100644 --- a/internal/collector/otelcol.tmpl +++ b/internal/collector/otelcol.tmpl @@ -1,9 +1,12 @@ receivers: -{{- range .Receivers.SyslogReceivers }} - syslog/{{- .InstanceID -}}: +{{- range $index, $otlpReceiver := .Receivers.SyslogReceivers }} + {{- $instanceID := .InstanceID }} + {{- range $serverIndex, $server := .Server }} + syslog/{{$index}}{{$serverIndex}}: tcp: - listen_address: "{{- .Server -}}" + listen_address: "{{ $server }}" protocol: rfc3164 + {{- end }} {{- end }} {{- if ne .Receivers.HostMetrics nil }} hostmetrics: diff --git a/internal/config/types.go b/internal/config/types.go index 20e27d9b94..36bb3fae9f 100644 --- a/internal/config/types.go +++ b/internal/config/types.go @@ -173,9 +173,9 @@ type ( } SyslogReceiver struct { - InstanceID string `yaml:"-" mapstructure:"instance_id"` - Server string `yaml:"-" mapstructure:"server"` - Protocol string `yaml:"-" mapstructure:"protocol"` + InstanceID string `yaml:"-" mapstructure:"instance_id"` + Server []string `yaml:"-" mapstructure:"server"` + Protocol string `yaml:"-" mapstructure:"protocol"` } NginxReceiver struct { diff --git a/internal/model/config.go b/internal/model/config.go index ef60d0e568..faeb3c121e 100644 --- a/internal/model/config.go +++ b/internal/model/config.go @@ -18,7 +18,7 @@ type NginxConfigContext struct { Files []*v1.File AccessLogs []*AccessLog ErrorLogs []*ErrorLog - Syslog *NAP + Syslog []string } func (ncc *NginxConfigContext) Equal(otherNginxConfigContext *NginxConfigContext) bool { diff --git a/internal/watcher/instance/nginx_config_parser.go b/internal/watcher/instance/nginx_config_parser.go index 541f5c4251..e93bd8bbde 100644 --- a/internal/watcher/instance/nginx_config_parser.go +++ b/internal/watcher/instance/nginx_config_parser.go @@ -100,6 +100,7 @@ func (ncp *NginxConfigParser) createNginxConfigContext( for _, conf := range payload.Config { formatMap := make(map[string]string) + syslogMap := make(map[string]bool) err := ncp.crossplaneConfigTraverse(ctx, &conf, func(ctx context.Context, parent, directive *crossplane.Directive) error { switch directive.Directive { @@ -129,7 +130,10 @@ func (ncp *NginxConfigParser) createNginxConfigContext( matches := re.FindStringSubmatch(syslogArg) if len(matches) > 1 { syslogServer := matches[1] - nginxConfigContext.Syslog.SyslogServer = syslogServer + if !syslogMap[syslogServer] { + nginxConfigContext.Syslog = append(nginxConfigContext.Syslog, syslogServer) + syslogMap[syslogServer] = true + } slog.InfoContext(ctx, "Captured syslog server", "syslog_server", syslogServer) } } diff --git a/internal/watcher/instance/nginx_config_parser_test.go b/internal/watcher/instance/nginx_config_parser_test.go index e4c493e530..f3ac1790af 100644 --- a/internal/watcher/instance/nginx_config_parser_test.go +++ b/internal/watcher/instance/nginx_config_parser_test.go @@ -279,6 +279,7 @@ func TestNginxConfigParser_Parse(t *testing.T) { tests := []struct { instance *mpi.Instance name string + syslog []string }{ { name: "Test 1: Valid response", @@ -287,6 +288,7 @@ func TestNginxConfigParser_Parse(t *testing.T) { { name: "Test 2: Error response", instance: protos.GetNginxPlusInstance([]string{}), + syslog: []string{"127.0.0.1:1515"}, }, } @@ -298,7 +300,9 @@ func TestNginxConfigParser_Parse(t *testing.T) { ltsvAccessLog.Name(), errorLog.Name(), test.instance.GetInstanceMeta().GetInstanceId(), + test.syslog, ) + expectedConfigContext.Files = append(expectedConfigContext.Files, &mpi.File{ FileMeta: fileMeta, }) diff --git a/test/config/collector/test-opentelemetry-collector-agent.yaml b/test/config/collector/test-opentelemetry-collector-agent.yaml index 8d6be2900c..f8970a4dec 100644 --- a/test/config/collector/test-opentelemetry-collector-agent.yaml +++ b/test/config/collector/test-opentelemetry-collector-agent.yaml @@ -1,5 +1,5 @@ receivers: - syslog/1: + syslog/00: tcp: listen_address: "127.0.0.1:1515" protocol: rfc3164 @@ -90,7 +90,7 @@ service: - debug logs: receivers: - - syslog/1 + - syslog/00 processors: - batch exporters: diff --git a/test/docker/nginx-plus/deb/Dockerfile b/test/docker/nginx-plus/deb/Dockerfile index 9b1aa04047..d160ff0cbc 100644 --- a/test/docker/nginx-plus/deb/Dockerfile +++ b/test/docker/nginx-plus/deb/Dockerfile @@ -14,47 +14,52 @@ COPY $ENTRY_POINT /agent/entrypoint.sh RUN --mount=type=secret,id=nginx-crt,dst=nginx-repo.crt \ --mount=type=secret,id=nginx-key,dst=nginx-repo.key \ set -x \ - # Create nginx user/group first, to be consistent throughout Docker variants +# Create nginx user/group first, to be consistent throughout Docker variants && groupadd --system --gid 101 nginx \ && useradd --system --gid nginx --no-create-home --home-dir /nonexistent --uid 101 nginx \ && apt-get update \ && apt-get install --no-install-recommends --no-install-suggests -y \ ca-certificates \ - gnupg2 \ + gnupg1 \ lsb-release \ git \ wget \ make \ - && mkdir -p /etc/ssl/nginx \ - && cat nginx-repo.crt > /etc/ssl/nginx/nginx-repo.crt \ - && cat nginx-repo.key > /etc/ssl/nginx/nginx-repo.key \ + && \ + NGINX_GPGKEY=573BFD6B3D8FBC641079A6ABABF5BD827BD9BF62; \ + found=''; \ + for server in \ + hkp://keyserver.ubuntu.com:80 \ + pgp.mit.edu \ + ; do \ + echo "Fetching GPG key $NGINX_GPGKEY from $server"; \ + apt-key adv --keyserver "$server" --keyserver-options timeout=10 --recv-keys "$NGINX_GPGKEY" && found=yes && break; \ + done; \ + test -z "$found" && echo >&2 "error: failed to fetch GPG key $NGINX_GPGKEY" && exit 1; \ + apt-get remove --purge --auto-remove -y gnupg1 && rm -rf /var/lib/apt/lists/* \ +# Install the latest release of NGINX Plus and/or NGINX Plus modules +# Uncomment individual modules if necessary +# Use versioned packages over defaults to specify a release + && nginxPackages=" \ + nginx-plus \ + " \ && echo "Acquire::https::pkgs.nginx.com::Verify-Peer \"true\";" > /etc/apt/apt.conf.d/90nginx \ && echo "Acquire::https::pkgs.nginx.com::Verify-Host \"true\";" >> /etc/apt/apt.conf.d/90nginx \ && echo "Acquire::https::pkgs.nginx.com::SslCert \"/etc/ssl/nginx/nginx-repo.crt\";" >> /etc/apt/apt.conf.d/90nginx \ && echo "Acquire::https::pkgs.nginx.com::SslKey \"/etc/ssl/nginx/nginx-repo.key\";" >> /etc/apt/apt.conf.d/90nginx \ - # Add GPG keys for nginx-plus and app-protect - && wget -qO - https://cs.nginx.com/static/keys/nginx_signing.key | gpg --dearmor | tee /usr/share/keyrings/nginx-archive-keyring.gpg >/dev/null \ - && wget -qO - https://cs.nginx.com/static/keys/app-protect-security-updates.key | gpg --dearmor | tee /usr/share/keyrings/app-protect-security-updates.gpg >/dev/null \ - #&& ls -l /usr/share/keyrings/nginx-archive-keyring.gpg /usr/share/keyrings/app-protect-security-updates.gpg \ - - - # Install the latest release of NGINX Plus and/or NGINX Plus modules - # Set up repository sources for NGINX Plus and App Protect - && printf "deb [signed-by=/usr/share/keyrings/nginx-archive-keyring.gpg] https://pkgs.nginx.com/plus/ubuntu `lsb_release -cs` nginx-plus\n" > /etc/apt/sources.list.d/nginx-plus.list \ - && printf "deb [signed-by=/usr/share/keyrings/nginx-archive-keyring.gpg] https://pkgs.nginx.com/app-protect/ubuntu `lsb_release -cs` nginx-plus\n" | tee /etc/apt/sources.list.d/nginx-app-protect.list \ - && printf "deb [signed-by=/usr/share/keyrings/app-protect-security-updates.gpg] https://pkgs.nginx.com/app-protect-security-updates/ubuntu `lsb_release -cs` nginx-plus\n" | tee -a /etc/apt/sources.list.d/nginx-app-protect.list \ - # && wget -P /etc/apt/apt.conf.d https://cs.nginx.com/static/files/90pkgs-nginx \ - # && printf "deb [signed-by=/usr/share/keyrings/app-protect-security-updates.gpg] https://pkgs.nginx.com/app-protect/ubuntu `lsb_release -cs` nginx-plus\n" > /etc/apt/sources.list.d/nginx-app-protect.list \ + && printf "deb https://pkgs.nginx.com/plus/R32/ubuntu `lsb_release -cs` nginx-plus\n" > /etc/apt/sources.list.d/nginx-plus.list \ + && mkdir -p /etc/ssl/nginx \ + && cat nginx-repo.crt > /etc/ssl/nginx/nginx-repo.crt \ + && cat nginx-repo.key > /etc/ssl/nginx/nginx-repo.key \ && apt-get update \ && apt-get install --no-install-recommends --no-install-suggests -y \ - nginx-plus \ - app-protect \ + $nginxPackages \ curl \ gettext-base \ jq \ + gnupg2 \ && apt-get remove --purge -y lsb-release \ - && apt-get remove --purge --auto-remove -y \ - && rm -rf /var/lib/apt/lists/* /etc/apt/sources.list.d/nginx-plus.list /etc/apt/sources.list.d/nginx-app-protect.list \ + && apt-get remove --purge --auto-remove -y && rm -rf /var/lib/apt/lists/* /etc/apt/sources.list.d/nginx-plus.list \ && rm -rf /etc/apt/apt.conf.d/90nginx /etc/ssl/nginx EXPOSE 80 diff --git a/test/model/config.go b/test/model/config.go index 8d8cdc5594..c5a242a49d 100644 --- a/test/model/config.go +++ b/test/model/config.go @@ -20,6 +20,7 @@ func GetConfigContextWithNames( ltsvAccessLogName, errorLogName string, instanceID string, + syslog []string, ) *model.NginxConfigContext { return &model.NginxConfigContext{ AccessLogs: []*model.AccessLog{ @@ -52,5 +53,6 @@ func GetConfigContextWithNames( }, }, InstanceID: instanceID, + Syslog: syslog, } } diff --git a/test/protos/instances.go b/test/protos/instances.go index 6ea7c9b1eb..fa6a3eb6ee 100644 --- a/test/protos/instances.go +++ b/test/protos/instances.go @@ -102,6 +102,9 @@ func GetNginxPlusInstance(expectedModules []string) *v1.Instance { "stream_proxy_protocol_vendor_module", "stream_realip_module", "stream_ssl_module", "stream_ssl_preread_module", }, + AppProtectDirectives: []string{ + `app_protect_security_log "/etc/app_protect/conf/log_default.json" syslog:server=127.0.0.1:514`, + }, PlusApi: "", }, }, diff --git a/test/types/config.go b/test/types/config.go index d1e5cfccf3..109298ebfb 100644 --- a/test/types/config.go +++ b/test/types/config.go @@ -69,8 +69,8 @@ func AgentConfig() *config.Config { OtlpReceivers: OtlpReceivers(), SyslogReceivers: []config.SyslogReceiver{ { - Server: "127.0.0.1:1515", - InstanceID: "1", + Server: []string{"127.0.0.1:1515"}, + InstanceID: "00", Protocol: "rfc3164", }, }, From 7ddc9b836d8b414a0ad749b1326b60a47224b233 Mon Sep 17 00:00:00 2001 From: dhurley Date: Thu, 28 Nov 2024 14:22:26 +0000 Subject: [PATCH 08/12] Replace syslog receiver with tcplog receiver --- api/grpc/mpi/v1/command.pb.go | 2 - go.mod | 1 + go.sum | 2 + internal/collector/factories.go | 4 +- internal/collector/otel_collector_plugin.go | 75 ++++++++++++++----- .../collector/otel_collector_plugin_test.go | 14 +--- internal/collector/otelcol.tmpl | 26 ++++--- internal/collector/settings_test.go | 21 ++++++ internal/config/defaults.go | 9 --- internal/config/types.go | 17 +++-- internal/model/config.go | 24 +++--- .../watcher/instance/nginx_config_parser.go | 16 ++-- .../instance/nginx_config_parser_test.go | 19 ++--- .../test-opentelemetry-collector-agent.yaml | 14 ++-- .../nginx-with-multiple-access-logs.conf | 3 + test/model/config.go | 7 +- test/protos/instances.go | 3 - test/types/config.go | 7 -- 18 files changed, 158 insertions(+), 106 deletions(-) diff --git a/api/grpc/mpi/v1/command.pb.go b/api/grpc/mpi/v1/command.pb.go index 8a572c8e63..e390eb5ac1 100644 --- a/api/grpc/mpi/v1/command.pb.go +++ b/api/grpc/mpi/v1/command.pb.go @@ -1746,8 +1746,6 @@ type NGINXPlusRuntimeInfo struct { DynamicModules []string `protobuf:"bytes,5,rep,name=dynamic_modules,json=dynamicModules,proto3" json:"dynamic_modules,omitempty"` // the plus API location PlusApi string `protobuf:"bytes,6,opt,name=plus_api,json=plusApi,proto3" json:"plus_api,omitempty"` - // a list of App Protect directives - AppProtectDirectives []string `protobuf:"bytes,7,rep,name=app_protect_directives,json=appProtectDirectives,proto3" json:"app_protect_directives,omitempty"` } func (x *NGINXPlusRuntimeInfo) Reset() { diff --git a/go.mod b/go.mod index e19b6d469a..ff91a835e8 100644 --- a/go.mod +++ b/go.mod @@ -45,6 +45,7 @@ require ( github.com/open-telemetry/opentelemetry-collector-contrib/processor/tailsamplingprocessor v0.108.0 github.com/open-telemetry/opentelemetry-collector-contrib/processor/transformprocessor v0.108.0 github.com/open-telemetry/opentelemetry-collector-contrib/receiver/hostmetricsreceiver v0.108.0 + github.com/open-telemetry/opentelemetry-collector-contrib/receiver/tcplogreceiver v0.108.0 github.com/open-telemetry/opentelemetry-collector-contrib/testbed v0.108.0 github.com/shirou/gopsutil/v4 v4.24.7 github.com/spf13/pflag v1.0.5 diff --git a/go.sum b/go.sum index dd098cbcba..a2af99d112 100644 --- a/go.sum +++ b/go.sum @@ -693,6 +693,8 @@ github.com/open-telemetry/opentelemetry-collector-contrib/receiver/prometheusrec github.com/open-telemetry/opentelemetry-collector-contrib/receiver/prometheusreceiver v0.108.0/go.mod h1:VUCVWsYWUlxut3MQbH3ADOz7bQBrtC3NlzJl9PQpKAk= github.com/open-telemetry/opentelemetry-collector-contrib/receiver/syslogreceiver v0.108.0 h1:FyiVuycmSmI4pg5PKvRjpUT0qaBVdFW9AeX1giolQG8= github.com/open-telemetry/opentelemetry-collector-contrib/receiver/syslogreceiver v0.108.0/go.mod h1:bmgqS6N2W31oEVg82zAkCJxmMrVamkODmZWr9qpCdK8= +github.com/open-telemetry/opentelemetry-collector-contrib/receiver/tcplogreceiver v0.108.0 h1:zm8uZlzIgK/QajEte0ccj9LZhgSAakhy0P+WPDo5DVc= +github.com/open-telemetry/opentelemetry-collector-contrib/receiver/tcplogreceiver v0.108.0/go.mod h1:KQshgnBLGVFaRtmEiCIEzYa+DFHTzfDxRjzMx5KIH2Y= github.com/open-telemetry/opentelemetry-collector-contrib/receiver/zipkinreceiver v0.108.0 h1:fR/TiCF1jkTSLIpvxJ8ycBYx3wAIMo5sOjyna28qNPc= github.com/open-telemetry/opentelemetry-collector-contrib/receiver/zipkinreceiver v0.108.0/go.mod h1:VdIAkr+2oTSwkl+CQ9UW3ZiV0TXobqOznSV5I5CB7Rc= github.com/open-telemetry/opentelemetry-collector-contrib/testbed v0.108.0 h1:rZlnwqE9x/MX+13+tgqx5tizLoRpJ4m3qD6ww1hdf5k= diff --git a/internal/collector/factories.go b/internal/collector/factories.go index cac41fd248..74d9a5f7f1 100644 --- a/internal/collector/factories.go +++ b/internal/collector/factories.go @@ -32,7 +32,7 @@ import ( "github.com/open-telemetry/opentelemetry-collector-contrib/processor/tailsamplingprocessor" "github.com/open-telemetry/opentelemetry-collector-contrib/processor/transformprocessor" "github.com/open-telemetry/opentelemetry-collector-contrib/receiver/hostmetricsreceiver" - "github.com/open-telemetry/opentelemetry-collector-contrib/receiver/syslogreceiver" + "github.com/open-telemetry/opentelemetry-collector-contrib/receiver/tcplogreceiver" "go.opentelemetry.io/collector/component" "go.opentelemetry.io/collector/connector" "go.opentelemetry.io/collector/exporter" @@ -111,7 +111,7 @@ func createReceiverFactories() (map[component.Type]receiver.Factory, error) { hostmetricsreceiver.NewFactory(), nginxreceiver.NewFactory(), nginxplusreceiver.NewFactory(), - syslogreceiver.NewFactory(), + tcplogreceiver.NewFactory(), } return receiver.MakeFactoryMap(receiverList...) diff --git a/internal/collector/otel_collector_plugin.go b/internal/collector/otel_collector_plugin.go index 6b513705c5..72d74ee3f6 100644 --- a/internal/collector/otel_collector_plugin.go +++ b/internal/collector/otel_collector_plugin.go @@ -24,8 +24,10 @@ import ( ) const ( - maxTimeToWaitForShutdown = 30 * time.Second - filePermission = 0o600 + maxTimeToWaitForShutdown = 30 * time.Second + filePermission = 0o600 + timestampConversionExpression = `EXPR(split(body, ">")[0] + ">" + ` + + `date(split(split(body, ">")[1], " ")[0]).Format("Jan 02 15:04:05") + " " + split(body, " ", 2)[1])` ) type ( @@ -237,7 +239,7 @@ func (oc *Collector) handleNginxConfigUpdate(ctx context.Context, msg *bus.Messa return } - reloadCollector := oc.checkForNewNginxReceivers(nginxConfigContext) + reloadCollector := oc.checkForNewReceivers(nginxConfigContext) if reloadCollector { slog.InfoContext(ctx, "Reloading OTel collector config") @@ -362,7 +364,7 @@ func (oc *Collector) restartCollector(ctx context.Context) { } } -func (oc *Collector) checkForNewNginxReceivers(nginxConfigContext *model.NginxConfigContext) bool { +func (oc *Collector) checkForNewReceivers(nginxConfigContext *model.NginxConfigContext) bool { nginxReceiverFound, reloadCollector := oc.updateExistingNginxPlusReceiver(nginxConfigContext) if !nginxReceiverFound && nginxConfigContext.PlusAPI != "" { @@ -374,19 +376,6 @@ func (oc *Collector) checkForNewNginxReceivers(nginxConfigContext *model.NginxCo }, ) - if nginxConfigContext.Syslog != nil { - oc.config.Collector.Receivers.SyslogReceivers = append( - oc.config.Collector.Receivers.SyslogReceivers, - config.SyslogReceiver{ - InstanceID: nginxConfigContext.InstanceID, - Server: nginxConfigContext.Syslog, - Protocol: "rfc3164", // default value, need to get from the agent conf - }, - ) - } - - // slog.Error("hello", oc.config.Collector.Receivers) - reloadCollector = true } else if nginxConfigContext.PlusAPI == "" { nginxReceiverFound, reloadCollector = oc.updateExistingNginxOSSReceiver(nginxConfigContext) @@ -405,10 +394,14 @@ func (oc *Collector) checkForNewNginxReceivers(nginxConfigContext *model.NginxCo } } + tcplogReceiversFound := oc.updateTcplogReceivers(nginxConfigContext) + if tcplogReceiversFound { + reloadCollector = true + } + return reloadCollector } -// Todo: consider update scenario for new syslogreceivers func (oc *Collector) updateExistingNginxPlusReceiver( nginxConfigContext *model.NginxConfigContext, ) (nginxReceiverFound, reloadCollector bool) { @@ -472,6 +465,52 @@ func (oc *Collector) updateExistingNginxOSSReceiver( return nginxReceiverFound, reloadCollector } +func (oc *Collector) updateTcplogReceivers(nginxConfigContext *model.NginxConfigContext) bool { + oc.config.Collector.Receivers.TcplogReceivers = make([]config.TcplogReceiver, 0) + + if nginxConfigContext.NAPSysLogServers != nil { + for _, napSysLogServer := range nginxConfigContext.NAPSysLogServers { + oc.config.Collector.Receivers.TcplogReceivers = append( + oc.config.Collector.Receivers.TcplogReceivers, + config.TcplogReceiver{ + ListenAddress: napSysLogServer, + Operators: []config.Operator{ + { + Type: "add", + Fields: map[string]string{ + "field": "body", + "value": timestampConversionExpression, + }, + }, + { + Type: "syslog_parser", + Fields: map[string]string{ + "protocol": "rfc3164", + }, + }, + { + Type: "key_value_parser", + Fields: map[string]string{ + "parse_from": "attributes.message", + "parse_to": "body", + "pair_delimiter": "\",\"", + }, + }, + { + Type: "remove", + Fields: map[string]string{ + "field": "attributes.message", + }, + }, + }, + }, + ) + } + } + + return len(oc.config.Collector.Receivers.TcplogReceivers) > 0 +} + // nolint: revive func (oc *Collector) updateResourceAttributes( attributesToAdd []config.ResourceAttribute, diff --git a/internal/collector/otel_collector_plugin_test.go b/internal/collector/otel_collector_plugin_test.go index 9953263605..ddc4bd91df 100644 --- a/internal/collector/otel_collector_plugin_test.go +++ b/internal/collector/otel_collector_plugin_test.go @@ -178,18 +178,13 @@ func TestCollector_ProcessNginxConfigUpdateTopic(t *testing.T) { receivers: config.Receivers{ HostMetrics: nil, OtlpReceivers: nil, - SyslogReceivers: []config.SyslogReceiver{ - { - Server: []string{"127.0.0.1:1515"}, - InstanceID: "00", - Protocol: "rfc3164"}, - }, NginxPlusReceivers: []config.NginxPlusReceiver{ { InstanceID: "123", PlusAPI: "", }, }, + TcplogReceivers: make([]config.TcplogReceiver, 0), }, }, { @@ -210,12 +205,6 @@ func TestCollector_ProcessNginxConfigUpdateTopic(t *testing.T) { receivers: config.Receivers{ HostMetrics: nil, OtlpReceivers: nil, - SyslogReceivers: []config.SyslogReceiver{ - { - Server: []string{"127.0.0.1:1515"}, - InstanceID: "00", - Protocol: "rfc3164"}, - }, NginxReceivers: []config.NginxReceiver{ { InstanceID: "123", @@ -228,6 +217,7 @@ func TestCollector_ProcessNginxConfigUpdateTopic(t *testing.T) { }, }, }, + TcplogReceivers: make([]config.TcplogReceiver, 0), }, }, } diff --git a/internal/collector/otelcol.tmpl b/internal/collector/otelcol.tmpl index 9f915eff58..bcae88d423 100644 --- a/internal/collector/otelcol.tmpl +++ b/internal/collector/otelcol.tmpl @@ -1,13 +1,4 @@ receivers: -{{- range $index, $otlpReceiver := .Receivers.SyslogReceivers }} - {{- $instanceID := .InstanceID }} - {{- range $serverIndex, $server := .Server }} - syslog/{{$index}}{{$serverIndex}}: - tcp: - listen_address: "{{ $server }}" - protocol: rfc3164 - {{- end }} -{{- end }} {{- if ne .Receivers.HostMetrics nil }} hostmetrics: collection_interval: {{ .Receivers.HostMetrics.CollectionInterval }} @@ -83,6 +74,17 @@ receivers: endpoint: "{{- .PlusAPI -}}" collection_interval: 10s {{- end }} +{{- range $index, $tcplogReceiver := .Receivers.TcplogReceivers }} + tcplog/{{$index}}: + listen_address: "{{- .ListenAddress -}}" + operators: +{{- range $index, $operator := .Operators }} + - type: {{.Type}} +{{- range $key, $value := .Fields }} + {{$key}}: {{$value}} +{{- end }} +{{- end }} +{{- end }} processors: {{- if ne .Processors.Resource nil }} @@ -244,11 +246,11 @@ service: {{- if ne .Exporters.Debug nil }} - debug {{- end }} - {{- if and (ne .Receivers.SyslogReceivers nil) (ne .Exporters.OtlpExporters nil) }} + {{- if ne .Receivers.TcplogReceivers nil }} logs: receivers: - {{- range .Receivers.SyslogReceivers }} - - syslog/{{- .InstanceID -}} + {{- range $index, $tcplogReceiver := .Receivers.TcplogReceivers }} + - tcplog/{{$index}} {{- end }} processors: {{- if ne .Processors.Batch nil }} diff --git a/internal/collector/settings_test.go b/internal/collector/settings_test.go index baf29807fe..47b764c16b 100644 --- a/internal/collector/settings_test.go +++ b/internal/collector/settings_test.go @@ -100,6 +100,27 @@ func TestTemplateWrite(t *testing.T) { }, }) + cfg.Collector.Receivers.TcplogReceivers = []config.TcplogReceiver{ + { + ListenAddress: "localhost:151", + Operators: []config.Operator{ + { + Type: "add", + Fields: map[string]string{ + "field": "body", + "value": `EXPR(split(body, ",")[0])`, + }, + }, + { + Type: "remove", + Fields: map[string]string{ + "field": "attributes.message", + }, + }, + }, + }, + } + cfg.Collector.Extensions.HeadersSetter = &config.HeadersSetter{ Headers: []config.Header{ { diff --git a/internal/config/defaults.go b/internal/config/defaults.go index b386a236b6..1716320b34 100644 --- a/internal/config/defaults.go +++ b/internal/config/defaults.go @@ -12,7 +12,6 @@ import ( ) const ( - //Todo: Add default Protocol for nap DefGracefulShutdownPeriod = 5 * time.Second DefNginxReloadMonitoringPeriod = 10 * time.Second DefTreatErrorsAsWarnings = false @@ -64,14 +63,6 @@ const ( DefCollectorExtensionsHealthTLSCAPath = "" DefCollectorExtensionsHealthTLSSkipVerify = false DefCollectorExtensionsHealthTLServerNameKey = "" - - DefCollectorPrometheusExporterServerHost = "" - DefCollectorPrometheusExporterServerPort = 0 - DefCollectorPrometheusExporterTLSCertPath = "" - DefCollectorPrometheusExporterTLSKeyPath = "" - DefCollectorPrometheusExporterTLSCAPath = "" - DefCollectorPrometheusExporterTLSSkipVerify = false - DefCollectorPrometheusExporterTLServerNameKey = "" ) func DefaultFeatures() []string { diff --git a/internal/config/types.go b/internal/config/types.go index a37797f239..3f17a06093 100644 --- a/internal/config/types.go +++ b/internal/config/types.go @@ -164,7 +164,7 @@ type ( OtlpReceivers []OtlpReceiver `yaml:"-" mapstructure:"otlp_receivers"` NginxReceivers []NginxReceiver `yaml:"-" mapstructure:"nginx_receivers"` NginxPlusReceivers []NginxPlusReceiver `yaml:"-" mapstructure:"nginx_plus_receivers"` - SyslogReceivers []SyslogReceiver `yaml:"-" mapstructure:"syslog_receiver"` + TcplogReceivers []TcplogReceiver `yaml:"-" mapstructure:"tcplog_receivers"` } OtlpReceiver struct { @@ -173,10 +173,17 @@ type ( OtlpTLSConfig *OtlpTLSConfig `yaml:"-" mapstructure:"tls"` } - SyslogReceiver struct { - InstanceID string `yaml:"-" mapstructure:"instance_id"` - Server []string `yaml:"-" mapstructure:"server"` - Protocol string `yaml:"-" mapstructure:"protocol"` + TcplogReceiver struct { + ListenAddress string `yaml:"-" mapstructure:"listen_address"` + Operators []Operator `yaml:"-" mapstructure:"operators"` + } + + // There are many types of operators with different field names so we use a generic map to store the fields. + // See here for more info: + // https://github.com/open-telemetry/opentelemetry-collector-contrib/blob/main/pkg/stanza/docs/operators/README.md + Operator struct { + Fields map[string]string `yaml:"-" mapstructure:"fields"` + Type string `yaml:"-" mapstructure:"type"` } NginxReceiver struct { diff --git a/internal/model/config.go b/internal/model/config.go index faeb3c121e..2e44c4fef0 100644 --- a/internal/model/config.go +++ b/internal/model/config.go @@ -12,13 +12,13 @@ import ( ) type NginxConfigContext struct { - StubStatus string - PlusAPI string - InstanceID string - Files []*v1.File - AccessLogs []*AccessLog - ErrorLogs []*ErrorLog - Syslog []string + StubStatus string + PlusAPI string + InstanceID string + Files []*v1.File + AccessLogs []*AccessLog + ErrorLogs []*ErrorLog + NAPSysLogServers []string } func (ncc *NginxConfigContext) Equal(otherNginxConfigContext *NginxConfigContext) bool { @@ -46,6 +46,10 @@ func (ncc *NginxConfigContext) Equal(otherNginxConfigContext *NginxConfigContext return false } + if !reflect.DeepEqual(ncc.NAPSysLogServers, otherNginxConfigContext.NAPSysLogServers) { + return false + } + return true } @@ -86,12 +90,6 @@ type ErrorLog struct { Readable bool } -type NAP struct { - Enable bool - Syslog bool - SyslogServer string -} - type ( WriteStatus int ) diff --git a/internal/watcher/instance/nginx_config_parser.go b/internal/watcher/instance/nginx_config_parser.go index 5e555ff125..bf2d812b2e 100644 --- a/internal/watcher/instance/nginx_config_parser.go +++ b/internal/watcher/instance/nginx_config_parser.go @@ -37,7 +37,6 @@ const ( stubStatusAPIDirective = "stub_status" apiFormat = "http://%s%s" locationDirective = "location" - napDirective = "app_protect_security_log" ) type ( @@ -92,6 +91,8 @@ func (ncp *NginxConfigParser) createNginxConfigContext( instance *mpi.Instance, payload *crossplane.Payload, ) (*model.NginxConfigContext, error) { + napSyslogServersFound := make(map[string]bool) + nginxConfigContext := &model.NginxConfigContext{ InstanceID: instance.GetInstanceMeta().GetInstanceId(), } @@ -100,7 +101,6 @@ func (ncp *NginxConfigParser) createNginxConfigContext( for _, conf := range payload.Config { formatMap := make(map[string]string) - syslogMap := make(map[string]bool) err := ncp.crossplaneConfigTraverse(ctx, &conf, func(ctx context.Context, parent, directive *crossplane.Directive) error { switch directive.Directive { @@ -130,14 +130,18 @@ func (ncp *NginxConfigParser) createNginxConfigContext( matches := re.FindStringSubmatch(syslogArg) if len(matches) > 1 { syslogServer := matches[1] - if !syslogMap[syslogServer] { - nginxConfigContext.Syslog = append(nginxConfigContext.Syslog, syslogServer) - syslogMap[syslogServer] = true + if !napSyslogServersFound[syslogServer] { + nginxConfigContext.NAPSysLogServers = append( + nginxConfigContext.NAPSysLogServers, + syslogServer, + ) + napSyslogServersFound[syslogServer] = true + slog.DebugContext(ctx, "Found NAP syslog server", "address", syslogServer) } - slog.InfoContext(ctx, "Captured syslog server", "syslog_server", syslogServer) } } } + return nil }, ) diff --git a/internal/watcher/instance/nginx_config_parser_test.go b/internal/watcher/instance/nginx_config_parser_test.go index adfa084eb6..2d49840aae 100644 --- a/internal/watcher/instance/nginx_config_parser_test.go +++ b/internal/watcher/instance/nginx_config_parser_test.go @@ -277,18 +277,19 @@ func TestNginxConfigParser_Parse(t *testing.T) { require.NoError(t, err) tests := []struct { - instance *mpi.Instance - name string - syslog []string + instance *mpi.Instance + name string + syslogServers []string }{ { - name: "Test 1: Valid response", - instance: protos.GetNginxOssInstance([]string{}), + name: "Test 1: Valid response", + instance: protos.GetNginxOssInstance([]string{}), + syslogServers: []string{"127.0.0.1:1515"}, }, { - name: "Test 2: Error response", - instance: protos.GetNginxPlusInstance([]string{}), - syslog: []string{"127.0.0.1:1515"}, + name: "Test 2: Error response", + instance: protos.GetNginxPlusInstance([]string{}), + syslogServers: []string{"127.0.0.1:1515"}, }, } @@ -300,7 +301,7 @@ func TestNginxConfigParser_Parse(t *testing.T) { ltsvAccessLog.Name(), errorLog.Name(), test.instance.GetInstanceMeta().GetInstanceId(), - test.syslog, + test.syslogServers, ) expectedConfigContext.Files = append(expectedConfigContext.Files, &mpi.File{ diff --git a/test/config/collector/test-opentelemetry-collector-agent.yaml b/test/config/collector/test-opentelemetry-collector-agent.yaml index cf2a0bdd1b..1ef1532443 100644 --- a/test/config/collector/test-opentelemetry-collector-agent.yaml +++ b/test/config/collector/test-opentelemetry-collector-agent.yaml @@ -1,8 +1,4 @@ receivers: - syslog/00: - tcp: - listen_address: "127.0.0.1:1515" - protocol: rfc3164 hostmetrics: collection_interval: 1m0s initial_delay: 1s @@ -26,6 +22,14 @@ receivers: access_logs: - log_format: "$remote_addr - $remote_user [$time_local] \"$request\" $status $body_bytes_sent \"$http_referer\" \"$http_user_agent\" \"$http_x_forwarded_for\"\"$upstream_cache_status\"" file_path: "/var/log/nginx/access-custom.conf" + tcplog/0: + listen_address: "localhost:151" + operators: + - type: add + field: body + value: EXPR(split(body, ",")[0]) + - type: remove + field: attributes.message processors: batch: @@ -90,7 +94,7 @@ service: - debug logs: receivers: - - syslog/00 + - tcplog/0 processors: - batch exporters: diff --git a/test/config/nginx/nginx-with-multiple-access-logs.conf b/test/config/nginx/nginx-with-multiple-access-logs.conf index 83582169a6..4d22a456e9 100644 --- a/test/config/nginx/nginx-with-multiple-access-logs.conf +++ b/test/config/nginx/nginx-with-multiple-access-logs.conf @@ -4,6 +4,7 @@ worker_processes auto; error_log %s notice; pid /var/run/nginx.pid; +load_module modules/ngx_http_app_protect_module.so; events { worker_connections 1024; @@ -33,5 +34,7 @@ http { server { access_log %s ltsv; + + app_protect_security_log "/etc/app_protect/conf/log_default.json" syslog:server=127.0.0.1:1515; } } diff --git a/test/model/config.go b/test/model/config.go index c5a242a49d..59c82395de 100644 --- a/test/model/config.go +++ b/test/model/config.go @@ -14,13 +14,14 @@ func GetConfigContext() *model.NginxConfigContext { } } +// nolint: revive func GetConfigContextWithNames( accessLogName, combinedAccessLogName, ltsvAccessLogName, errorLogName string, instanceID string, - syslog []string, + syslogServers []string, ) *model.NginxConfigContext { return &model.NginxConfigContext{ AccessLogs: []*model.AccessLog{ @@ -52,7 +53,7 @@ func GetConfigContextWithNames( Permissions: "0600", }, }, - InstanceID: instanceID, - Syslog: syslog, + InstanceID: instanceID, + NAPSysLogServers: syslogServers, } } diff --git a/test/protos/instances.go b/test/protos/instances.go index fa6a3eb6ee..6ea7c9b1eb 100644 --- a/test/protos/instances.go +++ b/test/protos/instances.go @@ -102,9 +102,6 @@ func GetNginxPlusInstance(expectedModules []string) *v1.Instance { "stream_proxy_protocol_vendor_module", "stream_realip_module", "stream_ssl_module", "stream_ssl_preread_module", }, - AppProtectDirectives: []string{ - `app_protect_security_log "/etc/app_protect/conf/log_default.json" syslog:server=127.0.0.1:514`, - }, PlusApi: "", }, }, diff --git a/test/types/config.go b/test/types/config.go index 9073b160f5..681aa0cc02 100644 --- a/test/types/config.go +++ b/test/types/config.go @@ -75,13 +75,6 @@ func AgentConfig() *config.Config { }, }, }, - SyslogReceivers: []config.SyslogReceiver{ - { - Server: []string{"127.0.0.1:1515"}, - InstanceID: "00", - Protocol: "rfc3164", - }, - }, HostMetrics: &config.HostMetrics{ CollectionInterval: time.Minute, InitialDelay: time.Second, From 1b7a3598600341dc2d40b293f6e73581e205dfed Mon Sep 17 00:00:00 2001 From: dhurley Date: Thu, 28 Nov 2024 15:22:01 +0000 Subject: [PATCH 09/12] Replace syslog receiver with tcplog receiver --- internal/collector/otel_collector_plugin_test.go | 9 +++++++-- internal/collector/settings_test.go | 4 +++- 2 files changed, 10 insertions(+), 3 deletions(-) diff --git a/internal/collector/otel_collector_plugin_test.go b/internal/collector/otel_collector_plugin_test.go index ddc4bd91df..d27c931e76 100644 --- a/internal/collector/otel_collector_plugin_test.go +++ b/internal/collector/otel_collector_plugin_test.go @@ -9,6 +9,7 @@ import ( "context" "errors" "fmt" + "path/filepath" "strings" "testing" @@ -27,6 +28,8 @@ import ( ) func TestCollector_New(t *testing.T) { + tmpDir := t.TempDir() + tests := []struct { config *config.Config expectedError error @@ -57,7 +60,7 @@ func TestCollector_New(t *testing.T) { name: "Successful initialization", config: &config.Config{ Collector: &config.Collector{ - Log: &config.Log{Path: "/tmp/test.log"}, + Log: &config.Log{Path: filepath.Join(tmpDir, "test.log")}, }, }, expectedError: nil, @@ -80,6 +83,8 @@ func TestCollector_New(t *testing.T) { } func TestCollector_Init(t *testing.T) { + tmpDir := t.TempDir() + tests := []struct { name string expectedLog string @@ -105,7 +110,7 @@ func TestCollector_Init(t *testing.T) { logBuf := &bytes.Buffer{} stub.StubLoggerWith(logBuf) - conf.Collector.Log = &config.Log{Path: "/tmp/test.log"} + conf.Collector.Log = &config.Log{Path: filepath.Join(tmpDir, "test.log")} if tt.expectedError { conf.Collector.Receivers = config.Receivers{} diff --git a/internal/collector/settings_test.go b/internal/collector/settings_test.go index 47b764c16b..f9857f31fc 100644 --- a/internal/collector/settings_test.go +++ b/internal/collector/settings_test.go @@ -49,8 +49,10 @@ func TestConfigProviderSettings(t *testing.T) { } func TestTemplateWrite(t *testing.T) { + tmpDir := t.TempDir() + cfg := types.AgentConfig() - actualConfPath := filepath.Join("/tmp/", "nginx-agent-otelcol-test.yaml") + actualConfPath := filepath.Join(tmpDir, "nginx-agent-otelcol-test.yaml") cfg.Collector.ConfigPath = actualConfPath cfg.Collector.Exporters.PrometheusExporter = &config.PrometheusExporter{ From 6f0eddeb8c845fc8c5bcc73d04c35886a821282c Mon Sep 17 00:00:00 2001 From: dhurley Date: Thu, 28 Nov 2024 15:46:40 +0000 Subject: [PATCH 10/12] Replace syslog receiver with tcplog receiver --- internal/collector/otel_collector_plugin.go | 7 +++++++ internal/collector/otelcol.tmpl | 5 +++++ internal/collector/settings_test.go | 9 +++++++++ .../collector/test-opentelemetry-collector-agent.yaml | 7 +++++++ 4 files changed, 28 insertions(+) diff --git a/internal/collector/otel_collector_plugin.go b/internal/collector/otel_collector_plugin.go index 72d74ee3f6..c6dbd2599d 100644 --- a/internal/collector/otel_collector_plugin.go +++ b/internal/collector/otel_collector_plugin.go @@ -502,6 +502,13 @@ func (oc *Collector) updateTcplogReceivers(nginxConfigContext *model.NginxConfig "field": "attributes.message", }, }, + { + Type: "add", + Fields: map[string]string{ + "field": "resource.instance.id", + "value": nginxConfigContext.InstanceID, + }, + }, }, }, ) diff --git a/internal/collector/otelcol.tmpl b/internal/collector/otelcol.tmpl index bcae88d423..52fa876cf0 100644 --- a/internal/collector/otelcol.tmpl +++ b/internal/collector/otelcol.tmpl @@ -253,6 +253,11 @@ service: - tcplog/{{$index}} {{- end }} processors: + {{- if ne .Processors.Resource nil }} + {{- if .Processors.Resource.Attributes }} + - resource + {{- end }} + {{- end }} {{- if ne .Processors.Batch nil }} - batch {{- end }} diff --git a/internal/collector/settings_test.go b/internal/collector/settings_test.go index f9857f31fc..c2bf75e31a 100644 --- a/internal/collector/settings_test.go +++ b/internal/collector/settings_test.go @@ -54,6 +54,15 @@ func TestTemplateWrite(t *testing.T) { cfg := types.AgentConfig() actualConfPath := filepath.Join(tmpDir, "nginx-agent-otelcol-test.yaml") cfg.Collector.ConfigPath = actualConfPath + cfg.Collector.Processors.Resource = &config.Resource{ + Attributes: []config.ResourceAttribute{ + { + Key: "resource.id", + Action: "add", + Value: "12345", + }, + }, + } cfg.Collector.Exporters.PrometheusExporter = &config.PrometheusExporter{ Server: &config.ServerConfig{ diff --git a/test/config/collector/test-opentelemetry-collector-agent.yaml b/test/config/collector/test-opentelemetry-collector-agent.yaml index 1ef1532443..35fec11fb4 100644 --- a/test/config/collector/test-opentelemetry-collector-agent.yaml +++ b/test/config/collector/test-opentelemetry-collector-agent.yaml @@ -32,6 +32,11 @@ receivers: field: attributes.message processors: + resource: + attributes: + - key: resource.id + action: add + value: 12345 batch: send_batch_size: 8192 timeout: 200ms @@ -87,6 +92,7 @@ service: - otlp/0 - nginx/123 processors: + - resource - batch exporters: - otlp/0 @@ -96,6 +102,7 @@ service: receivers: - tcplog/0 processors: + - resource - batch exporters: - otlp/0 From 5faae621d55fd521a17e96c3adf3ebffefc3fbae Mon Sep 17 00:00:00 2001 From: dhurley Date: Thu, 28 Nov 2024 16:52:39 +0000 Subject: [PATCH 11/12] Replace syslog receiver with tcplog receiver --- internal/collector/otel_collector_plugin.go | 22 ++++++++-- .../collector/otel_collector_plugin_test.go | 40 ++++++++++++++++++- 2 files changed, 57 insertions(+), 5 deletions(-) diff --git a/internal/collector/otel_collector_plugin.go b/internal/collector/otel_collector_plugin.go index c6dbd2599d..a614bbd9be 100644 --- a/internal/collector/otel_collector_plugin.go +++ b/internal/collector/otel_collector_plugin.go @@ -466,10 +466,14 @@ func (oc *Collector) updateExistingNginxOSSReceiver( } func (oc *Collector) updateTcplogReceivers(nginxConfigContext *model.NginxConfigContext) bool { - oc.config.Collector.Receivers.TcplogReceivers = make([]config.TcplogReceiver, 0) - + newTcplogReceiverAdded := false if nginxConfigContext.NAPSysLogServers != nil { + napLoop: for _, napSysLogServer := range nginxConfigContext.NAPSysLogServers { + if oc.doesTcplogReceiverAlreadyExist(napSysLogServer) { + continue napLoop + } + oc.config.Collector.Receivers.TcplogReceivers = append( oc.config.Collector.Receivers.TcplogReceivers, config.TcplogReceiver{ @@ -512,10 +516,22 @@ func (oc *Collector) updateTcplogReceivers(nginxConfigContext *model.NginxConfig }, }, ) + + newTcplogReceiverAdded = true + } + } + + return newTcplogReceiverAdded +} + +func (oc *Collector) doesTcplogReceiverAlreadyExist(listenAddress string) bool { + for _, tcplogReceiver := range oc.config.Collector.Receivers.TcplogReceivers { + if listenAddress == tcplogReceiver.ListenAddress { + return true } } - return len(oc.config.Collector.Receivers.TcplogReceivers) > 0 + return false } // nolint: revive diff --git a/internal/collector/otel_collector_plugin_test.go b/internal/collector/otel_collector_plugin_test.go index d27c931e76..28c76c3867 100644 --- a/internal/collector/otel_collector_plugin_test.go +++ b/internal/collector/otel_collector_plugin_test.go @@ -189,7 +189,6 @@ func TestCollector_ProcessNginxConfigUpdateTopic(t *testing.T) { PlusAPI: "", }, }, - TcplogReceivers: make([]config.TcplogReceiver, 0), }, }, { @@ -222,7 +221,6 @@ func TestCollector_ProcessNginxConfigUpdateTopic(t *testing.T) { }, }, }, - TcplogReceivers: make([]config.TcplogReceiver, 0), }, }, } @@ -652,6 +650,44 @@ func TestCollector_updateResourceAttributes(t *testing.T) { } } +func TestCollector_updateTcplogReceivers(t *testing.T) { + conf := types.OTelConfig(t) + conf.Collector.Log.Path = "" + conf.Collector.Processors.Batch = nil + conf.Collector.Processors.Attribute = nil + conf.Collector.Processors.Resource = nil + + collector, err := New(conf) + require.NoError(t, err) + + nginxConfigContext := &model.NginxConfigContext{ + NAPSysLogServers: []string{ + "localhost:151", + }, + } + + assert.Empty(t, conf.Collector.Receivers.TcplogReceivers) + + t.Run("Test 1: New TcplogReceiver added", func(tt *testing.T) { + tcplogReceiverAdded := collector.updateTcplogReceivers(nginxConfigContext) + + assert.True(tt, tcplogReceiverAdded) + assert.Len(tt, conf.Collector.Receivers.TcplogReceivers, 1) + assert.Equal(tt, "localhost:151", conf.Collector.Receivers.TcplogReceivers[0].ListenAddress) + assert.Len(tt, conf.Collector.Receivers.TcplogReceivers[0].Operators, 5) + }) + + // Calling updateTcplogReceivers shouldn't update the TcplogReceivers slice + // since there is already a receiver with the same ListenAddress + t.Run("Test 2: TcplogReceiver already exists", func(tt *testing.T) { + tcplogReceiverAdded := collector.updateTcplogReceivers(nginxConfigContext) + assert.False(t, tcplogReceiverAdded) + assert.Len(t, conf.Collector.Receivers.TcplogReceivers, 1) + assert.Equal(t, "localhost:151", conf.Collector.Receivers.TcplogReceivers[0].ListenAddress) + assert.Len(t, conf.Collector.Receivers.TcplogReceivers[0].Operators, 5) + }) +} + func createFakeCollector() *typesfakes.FakeCollectorInterface { fakeCollector := &typesfakes.FakeCollectorInterface{} fakeCollector.RunStub = func(ctx context.Context) error { return nil } From bbc3eb550e593a845f241958ea55683fc4bf677f Mon Sep 17 00:00:00 2001 From: dhurley Date: Thu, 28 Nov 2024 17:10:49 +0000 Subject: [PATCH 12/12] Replace syslog receiver with tcplog receiver --- internal/collector/otel_collector_plugin.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/internal/collector/otel_collector_plugin.go b/internal/collector/otel_collector_plugin.go index a614bbd9be..baf8255c8a 100644 --- a/internal/collector/otel_collector_plugin.go +++ b/internal/collector/otel_collector_plugin.go @@ -509,7 +509,7 @@ func (oc *Collector) updateTcplogReceivers(nginxConfigContext *model.NginxConfig { Type: "add", Fields: map[string]string{ - "field": "resource.instance.id", + "field": "resource[\"instance.id\"]", "value": nginxConfigContext.InstanceID, }, },