Merge pull request #28 from sunnyneo/master

Added Nginx support

elkserver/logstash/conf.d/40-redir-nginx.conf

@@ -0,0 +1,121 @@
+# Part of RedELK
+#
+# In this file we configure the logstash filtes for Apache logs
+#
+# Author: Outflank B.V. / Marc Smeets
+#
+
+filter {
+  if [infralogtype] == "redirtraffic" and [redirprogram] == "nginx" {
+
+    # Filebeat introduces the source field for the name of the log file path. But this collides with source object from the Elastic Common Schema.
+    # We have no need for the filebeat's field as its also stored in log.file.path. So we drop it.
+    mutate {
+      remove_field => [ "source" ]
+    }
+
+     # Let's first trim the syslog-like info from the log line
+    grok {
+      match => { "message" => [ "\[%{HTTPDATE:redirtraffic.timestamp}\] (%{SYSLOGHOST:sysloghostname}|-) %{PROG:syslogprogram}(?:\[%{POSINT:syslogpid}\]): %{GREEDYDATA:messagenosyslog}" ] }
+    }
+
+    # now matching the real Apache log lines. We have several log line formats we need to match:
+    # - Lines without X-Forwarded-For identified with "xforwardedfor:-"
+    # - Lines with X-Forwarded-For set, identified with "xforwardedfor:$SOMEIP"
+    # - any other weird sitution, i.e. cutoff lines when the log lne is larger than the redir's logbuffer size
+    #
+    # We'll walk through them one by one
+    #
+
+    if "xforwardedfor:-" in [message] {
+      # Lines without X-Forwarded-For identified with "xforwardedfor:-"
+      grok {
+        match => { "messagenosyslog" => [ "frontend:(?<redir.frontendname>([^/]*))/%{IPORHOST:redir.frontendip}:%{POSINT:redir.frontendport} backend:%{NOTSPACE:redir.backendname} client:%{IPORHOST:redirtraffic.sourceip}:%{POSINT:redirtraffic.sourceport} xforwardedfor:- headers:\{(?<redirtraffic.headersall>([^\}]*))} statuscode:%{POSINT:redirtraffic.httpstatus} request:%{GREEDYDATA:redirtraffic.httprequest}" ] }
+      }
+    } else if "request:" in [message] {
+    # Lines with X-Forwarded-For set. We already filtered out the 'xfordwardedfor:-', so anything left with a large enough log line should be good 
+      grok {
+        match => { "messagenosyslog" => [ "frontend:(?<redir.frontendname>([^/]*))/%{IPORHOST:redir.frontendip}:%{POSINT:redir.frontendport} backend:%{NOTSPACE:redir.backendname} client:%{IPORHOST:redirtraffic.sourceipcdn}:%{POSINT:redirtraffic.sourceportcdn} xforwardedfor:%{IPORHOST:redirtraffic.sourceip} headers:\{(?<redirtraffic.headersall>([^\}]*))} statuscode:%{POSINT:redirtraffic.httpstatus} request:%{GREEDYDATA:redirtraffic.httprequest}" ] }
+        add_tag => [ "redirtrafficxforwardedfor" ]
+      }
+    } else {
+    # catchall situation, i.e. cutoff lines when the log lne is larger than the redir's logbuffer size
+      grok {
+        match => { "messagenosyslog" => [ "frontend:(?<redir.frontendname>([^/]*))/%{IPORHOST:redir.frontendip}:%{POSINT:redir.frontendport} backend:%{NOTSPACE:redir.backendname}  %{GREEDYDATA:redirtraffic.catchall}" ] }
+        add_tag => [ "redirlongmessagecatchall" ]
+      }
+    }
+
+    if [messagenosyslog] {
+      mutate {
+        remove_field => [ "messagenosyslog" ]
+      }
+    }
+
+    # map header values onto dedicated fields and split the values of the headersall field into an array
+    if [redirtraffic.headersall] {
+      # map to dedicated fields
+      grok {
+        match => { "redirtraffic.headersall" => [ "(?<redirtraffic.headeruseragent>([^|]*))\|(?<redirtraffic.headerhost>([^|]*))\|(?<redirtraffic.headerxforwardedfor>([^|]*))\|(?<redirtraffic.headerxforwardedproto>([^|]*))\|(?<redirtraffic.headerxhost>([^|]*))\|(?<redirtraffic.headerforwarded>([^|]*))\|(?<redirtraffic.headervia>([^|]*))" ] }
+      }      
+      # split the values into an array
+      mutate {
+        split => { "redirtraffic.headersall" => "|" }
+      }
+    }
+
+    # Set the timestamp from the log to @timestamp, example: 15/Apr/2018:19:22:31 +0000
+    date {
+      match => [ "redirtraffic.timestamp", "dd/MMM/yyyy:HH:mm:ss Z" ]
+      target => "@timestamp"
+      timezone => "Etc/UTC"
+    }
+
+    # Add data to the redirraffic.sourceip
+    if [redirtraffic.sourceip] {
+      # duplicate field so we can replace it with reverse DNS lookup
+      mutate {
+        add_field => { "redirtraffic.sourcedns" => "%{redirtraffic.sourceip}" }
+      }
+      # do reverse DNS lookup
+      dns {
+        reverse => ["redirtraffic.sourcedns"]
+        action => "replace"
+        timeout => "2.0"
+      }
+      # add geo ip info from City DB
+      geoip {
+        source => "redirtraffic.sourceip"
+      }
+      # add geo ip info from ASN DB
+      geoip { 
+        source => "redirtraffic.sourceip"
+        default_database_type => "ASN"
+        database => "/usr/share/logstash/GeoLite2-dbs/GeoLite2-ASN.mmdb"
+      }
+    }
+    # Add data to the redirtraffic.sourceipcdn
+    if [redirtraffic.sourceipcdn] {
+      # duplicate field so we can replace it with reverse DNS lookup
+      mutate {
+        add_field => { "redirtraffic.sourcednscdn" => "%{redirtraffic.sourceipcdn}" }
+      }
+      # do reverse DNS lookup
+      dns {
+        reverse => ["redirtraffic.sourcednscdn"]
+        action => "replace"
+        timeout => "2.0"
+      }
+      # add geo ip info from City DB
+      geoip {
+        source => "redirtraffic.sourceipcdn"
+      }
+      # add geo ip info from ASN DB
+      geoip { 
+        source => "redirtraffic.sourceipcdn"
+        default_database_type => "ASN"
+        database => "/usr/share/logstash/GeoLite2-dbs/GeoLite2-ASN.mmdb"
+      }
+    }
+  }
+}

example-data-and-configs/nginx/redelk-redir-nginx.conf

@@ -0,0 +1,147 @@
+user www-data;
+worker_processes auto;
+pid /run/nginx.pid;
+include /etc/nginx/modules-enabled/*.conf;
+
+events {
+	worker_connections 768;
+	# multi_accept on;
+}
+
+http {
+
+	# Create variable, can't use $hostname as it is a known variable in nginx
+	map $redir_hostname $redir_hostname { 
+		default "www.example.com";
+	}
+	map $backend_name $backend_name { 
+		default "decoy";
+	}
+	map $frontend_name $frontend_name { 
+		default "example-www";
+	}
+
+	#To parse Forwarded header https://www.nginx.com/resources/wiki/start/topics/examples/forwarded/
+	map $http_forwarded $proxy_add_forwarded {
+                # If the incoming Forwarded header is syntactically valid, append to it
+                "~^(,[ \\t]*)*([!#$%&'*+.^_`|~0-9A-Za-z-]+=([!#$%&'*+.^_`|~0-9A-Za-z-]+|\"([\\t \\x21\\x23-\\x5B\\x5D-\\x7E\\x80-\\xFF]|\\\\[\\t \\x21-\\x7E\\x80-\\xFF])*\"))?(;([!#$%&'*+.^_`|~0-9A-Za-z-]+=([!#$%&'*+.^_`|~0-9A-Za-z-]+|\"([\\t \\x21\\x23-\\x5B\\x5D-\\x7E\\x80-\\xFF]|\\\\[\\t \\x21-\\x7E\\x80-\\xFF])*\"))?)*([ \\t]*,([ \\t]*([!#$%&'*+.^_`|~0-9A-Za-z-]+=([!#$%&'*+.^_`|~0-9A-Za-z-]+|\"([\\t \\x21\\x23-\\x5B\\x5D-\\x7E\\x80-\\xFF]|\\\\[\\t \\x21-\\x7E\\x80-\\xFF])*\"))?(;([!#$%&'*+.^_`|~0-9A-Za-z-]+=([!#$%&'*+.^_`|~0-9A-Za-z-]+|\"([\\t \\x21\\x23-\\x5B\\x5D-\\x7E\\x80-\\xFF]|\\\\[\\t \\x21-\\x7E\\x80-\\xFF])*\"))?)*)?)*$" "$http_forwarded";
+
+                # Otherwise, replace it
+                default "-";
+    }
+
+	#Nginx does not support Via header by default, omitting it in logs
+	# Based on 
+  	# https://github.com/outflanknl/RedELK/blob/master/example-data-and-configs/Apache/redelk-redir-apache.conf
+    log_format redelklog '[$time_local] $redir_hostname nginx[$pid]: frontend:$frontend_name/$server_addr:$server_port backend:$backend_name client:$remote_addr:$remote_port xforwardedfor:$http_x_forwarded_for headers:{$http_user_agent|$host|$http_x_forwarded_for|$http_x_forwarded_proto|$http_x_forwarded_host|$proxy_add_forwarded|-|} statuscode:$status request:$request';
+
+	##
+	# Basic Settings
+	##
+	sendfile on;
+	tcp_nopush on;
+	tcp_nodelay on;
+	keepalive_timeout 65;
+	types_hash_max_size 2048;
+	# server_tokens off;
+
+	# server_names_hash_bucket_size 64;
+	# server_name_in_redirect off;
+
+	include /etc/nginx/mime.types;
+	default_type application/octet-stream;
+
+	##
+	# SSL Settings
+	##
+
+	ssl_protocols TLSv1 TLSv1.1 TLSv1.2; # Dropping SSLv3, ref: POODLE
+	ssl_prefer_server_ciphers on;
+
+	##
+	# Logging Settings
+	##
+
+	access_log /var/log/nginx/access-redelk.log redelklog;
+	access_log /var/log/nginx/access.log;
+	error_log /var/log/nginx/error.log;
+
+	##
+	# Gzip Settings
+	##
+
+	gzip on;
+
+	# gzip_vary on;
+	# gzip_proxied any;
+	# gzip_comp_level 6;
+	# gzip_buffers 16 8k;
+	# gzip_http_version 1.1;
+	# gzip_types text/plain text/css application/json application/javascript text/xml application/xml application/xml+rss text/javascript;
+
+	##
+	# Virtual Host Configs
+	##
+
+	include /etc/nginx/conf.d/*.conf;
+	include /etc/nginx/sites-enabled/*;
+
+	server {
+        listen 80;
+        server_name _;
+
+        location / {
+            return 301 https://$host$request_uri;
+        }
+    }
+
+    server {
+        listen 443;
+        server_name www.random-domain.com;
+        root /var/www/html;
+
+        client_body_buffer_size 10K;
+        client_header_buffer_size 1k;
+        client_max_body_size 100m;
+
+        ssl on;
+        ssl_certificate /etc/letsencrypt/live/www.random-domain.com/fullchain.pem;
+        ssl_certificate_key /etc/letsencrypt/live/www.random-domain.com/privkey.pem;
+
+
+		# C2 autossh tunnel
+		location ~* ^/(hello\.html|bye\.html)$ {
+            proxy_pass https://127.0.0.1:4433;
+            proxy_set_header Host $host;
+            proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for;
+        }
+
+        # Decoy
+        location / {
+            proxy_pass https://www.google.com;
+            proxy_set_header Host www.google.com;
+        }
+	}
+}
+
+
+#mail {
+#	# See sample authentication script at:
+#	# http://wiki.nginx.org/ImapAuthenticateWithApachePhpScript
+# 
+#	# auth_http localhost/auth.php;
+#	# pop3_capabilities "TOP" "USER";
+#	# imap_capabilities "IMAP4rev1" "UIDPLUS";
+# 
+#	server {
+#		listen     localhost:110;
+#		protocol   pop3;
+#		proxy      on;
+#	}
+# 
+#	server {
+#		listen     localhost:143;
+#		protocol   imap;
+#		proxy      on;
+#	}
+#}

redirs/filebeat/filebeat.yml

@@ -15,6 +15,14 @@ filebeat.prospectors:
   fields:
     infralogtype: redirtraffic
     redirprogram: apache
+- type: log
+  enabled: true
+  fields_under_root: true
+  paths:
+    - /var/log/nginx/access-redelk.log
+  fields:
+    infralogtype: redirtraffic
+    redirprogram: nginx
 
 filebeat.config.modules:
   path: ${path.config}/modules.d/*.yml