Merge branch 'develop' into stable

contrib/log-courier.init

@@ -0,0 +1,96 @@
+#!/bin/sh
+#
+# log-courier Log Courier
+#
+# chkconfig: 2345 90 10
+# description: Controls the Log Courier daemon
+#
+### BEGIN INIT INFO
+# Provides:          log-courier
+# Required-Start:    $local_fs $remote_fs $syslog
+# Required-Stop:     $local_fs $remote_fs $syslog
+# Default-Start:     2 3 4 5
+# Default-Stop:      0 1 6
+# Short-Description: Log Courier
+### END INIT INFO
+
+# source function library
+. /etc/rc.d/init.d/functions
+
+DAEMON='/usr/sbin/log-courier'
+DATA_DIR='/var/lib/log-courier'
+CONFIG_FILE='/etc/log-courier/log-courier.conf'
+PID_FILE='/var/run/log-courier.pid'
+
+do_start() {
+	echo -n "Starting Log Courier: "
+	status -p $PID_FILE $DAEMON &>/dev/null
+	RC=$?
+	if [ $RC -eq 0 ]; then
+		success
+	else
+		cd $DATA_DIR
+		nohup ${DAEMON} -config="${CONFIG_FILE}" </dev/null &>/dev/null &
+		RC=$?
+		echo $! > ${PID_FILE}
+		[ $RC -eq 0 ] && success || failure
+	fi
+	echo
+	return $?
+}
+
+do_reload() {
+	echo -n "Reloading Log Courier: "
+	killproc -p $PID_FILE $DAEMON -HUP
+	RC=$?
+	echo
+}
+
+case "$1" in
+	start)
+		$0 configtest && do_start
+		RC=$?
+	;;
+	stop)
+		echo -n "Stopping Log Courier: "
+		killproc -p $PID_FILE -d 30 $DAEMON
+		RC=$?
+		echo
+	;;
+	status)
+		echo "Log Courier status:"
+		status -p $PID_FILE $DAEMON
+		RC=$?
+	;;
+	reload)
+		$0 configtest && do_reload
+	;;
+	restart)
+		$0 configtest
+		RC=$?
+		if [ $RC -eq 0 ]; then
+			$0 stop
+			do_start
+			RC=$?
+		fi
+	;;
+	configtest)
+		echo -n "Configuration test: "
+		TESTRESULT=$( ${DAEMON} -config="${CONFIG_FILE}" -config-test )
+		RC=$?
+		if [ $RC -ne 0 ]; then
+			failure
+			echo
+			echo "${TESTRESULT}"
+		else
+			success
+			echo
+		fi
+	;;
+	*)
+		echo "Usage: $0 start|stop|status|reload|restart|configtest"
+		exit 1
+	;;
+esac
+
+exit $RC

docs/AdministrationUtility.md

@@ -18,9 +18,11 @@
 The `lc-admin` command allows you to remotely (or locally) connect to a running
 Log Courier instance to monitor and control log shipping.
 
-In order to connect, the `admin enabled` general configuration entry must be set
-to `true` and an `admin port` specified. See [Configuration](Configuration.md)
-for more information on these options.
+To enable a Log Courier instance to receive administration connections, set the
+`admin enabled` general configuration entry to `true`. To specify a custom
+listen address, set the `admin listen address` entry. See
+[Configuration](Configuration.md) for more information on these options and the
+default listen address.
 
 ## Available Commands
 
@@ -60,8 +62,7 @@ Following is an example of the output this command provides.
 
 The `lc-admin` command accepts the following command line options.
 
-	-host="127.0.0.1": the Log Courier host to connect to (default 127.0.0.1)
-	-port=1234: the Log Courier monitor port (default 1234)
+	-connect="tcp:127.0.0.1:1234": the Log Courier address to connect to
 	-quiet=false: quietly execute the command line argument and output only the result
 	-version=false: display the Log Courier client version
 	-watch=false: repeat the command specified on the command line every second

docs/ChangeLog.md

@@ -4,6 +4,7 @@
 <!-- DON'T EDIT THIS SECTION, INSTEAD RE-RUN doctoc TO UPDATE -->
 **Table of Contents**  *generated with [DocToc](http://doctoc.herokuapp.com/)*
 
+- [0.14](#014)
 - [0.13](#013)
 - [0.12](#012)
 - [0.11](#011)
@@ -13,6 +14,34 @@
 
 <!-- END doctoc generated TOC please keep comment here to allow auto update -->
 
+## 0.14
+
+*18th September 2014*
+
+**Breaking Changes**
+
+* The 'file' field in generated events has been renamed to 'path'. This
+normalises the events with those generated by Logstash itself, and means the
+Logstash `multiline` filter's default `stream_identity` setting is compatible.
+
+**Changes**
+
+* Fix connection failure and retry sometimes entering an unrecoverable state
+that freezes log shipping
+* Fix ProtocolError with large log packets and on idle connections (since 0.13)
+* Provide more information when the gem encounters ProtocolError failures
+* Fix ssl_verify usage triggering error, "Either 'ssl_verify_default_ca' or
+'ssl_verify_ca' must be specified when ssl_verify is true" (#41)
+* Fix  (#45)
+* Restore message reliability and correctly perform partial ack. Since 0.9 a
+full event spool from log-courier could be lost (default 1024) instead of just
+* Significantly improve Log Courier gem performance within JRuby by switching
+JrJackson parse mode from string to raw+bigdecimal
+* Add unix domain socket support to the administration connection
+* Provide publisher connection status via the administration connection
+* Gracefully handle lines greater than 1 MiB in size by splitting and tagging
+them, and make the size configurable (#40)
+
 ## 0.13
 
 *30th August 2014*

docs/Configuration.md

@@ -13,12 +13,13 @@
   - [Fileglob](#fileglob)
 - [`"general"`](#general)
   - [`"admin enabled"`](#admin-enabled)
-  - [`"admin bind address"`](#admin-bind-address)
-  - [`"admin port"`](#admin-port)
+  - [`"admin listen address"`](#admin-listen-address)
   - [`"persist directory"`](#persist-directory)
   - [`"prospect interval"`](#prospect-interval)
   - [`"spool size"`](#spool-size)
+  - [`"spool max bytes"`](#spool-max-bytes)
   - [`"spool timeout"`](#spool-timeout)
+  - [`"max line bytes"`](#max-line-bytes)
   - [`"log level"`](#log-level)
   - [`"log stdout"`](#log-stdout)
   - [`"log syslog"`](#log-syslog)
@@ -153,26 +154,32 @@ of new log files.
 
 ### `"admin enabled"`
 
-*Boolean. Optional. Default: false*
+*Boolean. Optional. Default: false*  
 *Requires restart*
 
 Enables the administration listener that the `lc-admin` utility can connect to.
 
-### `"admin bind address"`
+### `"admin listen address"`
 
-*String. Optional. Default: 127.0.0.1*
+*String. Optional. Default: tcp:127.0.0.1:1234*
 
-The TCP address the administration listener should listen on.
+The address the administration listener should listen on in the format
+`transport:address`.
 
-### `"admin port"`
+Allowed transports are "tcp", "tcp4", "tcp6" (Windows and *nix) and "unix"
+(*nix only). For the tcp transports the address format is `host:port`. For the
+unix transport the address should specify a filename to use when creating the
+unix domain socket. If no transport name is specified, "tcp" is assumed.
 
-*Number. Required with "admin enabled" = true*
+Examples:
 
-The TCP port the administration listener should listen on.
+	127.0.0.1:1234
+	tcp:127.0.0.1:1234
+	unix:/var/run/log-courier/admin.socket
 
 ### `"persist directory"`
 
-*String. Optional. Default: "."*
+*String. Optional. Default: "."*  
 *Requires restart*
 
 The directory that Log Courier should store its persistence data in. The default
@@ -207,31 +214,53 @@ usage.
 easily cope with over 10,000 events a second on most machines and uses little
 memory. It is useful only in very specific circumstances.*
 
+### `"spool max bytes"`
+
+*Number. Optional. Default: 10485760*
+
+The maximum size of an event spool, before compression. If an incomplete spool
+does not have enough room for the next event, it will be flushed immediately.
+
+If this value is modified, the receiving end should also be configured with the
+new limit. For the Logstash plugin, this is the `max_packet_size` setting.
+
 ### `"spool timeout"`
 
 *Duration. Optional. Default: 5*
 
 The maximum amount of time to wait for a full spool. If an incomplete spool is
-not filled within this time limit, the spool will be flushed regardless.
+not filled within this time limit, the spool will be flushed immediately.
+
+### `"max line bytes"`
+
+*Number. Optional. Default: 1048576*
+
+The maxmimum line length to process. If a line exceeds this length, it will be
+split across multiple events. Each split line will have a "tag" field added
+containing the tag "splitline". The final part of the line will not have a "tag"
+field added.
+
+If the `fields` configuration already contained a "tags" entry, and it is an
+array, it will be appended to. Otherwise, the "tag" field will be left as is.
 
 ### `"log level"`
 
-*String. Optional. Default: "info".
-Available values: "critical", "error", "warning", "notice", "info", "debug"*
+*String. Optional. Default: "info".  
+Available values: "critical", "error", "warning", "notice", "info", "debug"*  
 *Requires restart*
 
 The maximum level of detail to produce in Log Courier's internal log.
 
 ### `"log stdout"`
 
-*Boolean. Optional. Default: true*
+*Boolean. Optional. Default: true*  
 *Requires restart*
 
 Enables sending of Log Courier's internal log to the console (stdout). May be used in conjunction with `"log syslog"` and `"log file"`.
 
 ### `"log syslog"`
 
-*Boolean. Optional. Default: false*
+*Boolean. Optional. Default: false*  
 *Requires restart*
 
 Enables sending of Log Courier's internal log to syslog. May be used in conjunction with `"log stdout"` and `"log file"`.
@@ -240,7 +269,7 @@ Enables sending of Log Courier's internal log to syslog. May be used in conjunct
 
 ### `"log file"`
 
-*Filepath. Optional*
+*Filepath. Optional*  
 *Requires restart*
 
 A log file to save Log Courier's internal log into. May be used in conjunction with `"log stdout"` and `"log syslog"`.

docs/LogstashIntegration.md

@@ -81,6 +81,8 @@ The following options are available for the input plugin:
 * ssl_verify_ca - Path to an SSL CA certificate to use for client certificate
 verification (tls)
 * curve_secret_key - CurveZMQ secret key for the server (zmq)
+* max_packet_size - The maximum packet size to accept (default 10485760,
+corresponds to Log Courier's `"spool max bytes"`)
 
 The following options are available for the output plugin:
 

lib/log-courier/client.rb

@@ -236,7 +236,7 @@ def run_io
         rescue => e
           # Unknown error occurred
           @logger.warn("[LogCourierClient] Unknown error: #{e}") unless @logger.nil?
-          @logger.debug("[LogCourierClient] #{e.backtrace}: #{e.message} (#{e.class})") unless @logger.nil? || !@logger.debug?
+          @logger.warn("[LogCourierClient] #{e.backtrace}: #{e.message} (#{e.class})") unless @logger.nil?
         end
 
         # Disconnect and retry payloads
@@ -311,8 +311,7 @@ def buffer_jdat_data_event(buffer, event)
     def process_pong(message)
       # Sanity
       if message.length != 0
-        # TODO: log something
-        raise ClientProtocolError
+        raise ClientProtocolError, "Unexpected data attached to pong message (#{message.length})"
       end
 
       # No longer pending a PONG
@@ -322,8 +321,7 @@ def process_pong(message)
     def process_ackn(message)
       # Sanity
       if message.length != 20
-        # TODO: log something
-        raise ClientProtocolError
+        raise ClientProtocolError, "ACKN message size invalid (#{message.length})"
       end
 
       # Grab nonce

lib/log-courier/client_tls.rb

@@ -115,7 +115,7 @@ def run_send(io_control)
       # Just shutdown
     rescue => e
       @logger.warn("[LogCourierClient] Unknown SSL write error: #{e}") unless @logger.nil?
-      @logger.debug("[LogCourierClient] #{e.backtrace}: #{e.message} (#{e.class})") unless @logger.nil? || !@logger.debug?
+      @logger.warn("[LogCourierClient] #{e.backtrace}: #{e.message} (#{e.class})") unless @logger.nil?
       io_control << ['F']
     end
 
@@ -151,7 +151,7 @@ def run_recv(io_control)
       # Just shutdown
     rescue => e
       @logger.warn("[LogCourierClient] Unknown SSL read error: #{e}") unless @logger.nil?
-      @logger.debug("[LogCourierClient] #{e.backtrace}: #{e.message} (#{e.class})") unless @logger.nil? || !@logger.debug?
+      @logger.warn("[LogCourierClient] #{e.backtrace}: #{e.message} (#{e.class})") unless @logger.nil?
       io_control << ['F']
     end
 
@@ -211,7 +211,7 @@ def tls_connect
       0
     rescue => e
       @logger.warn("[LogCourierClient] Unknown connection failure to #{@options[:addresses][0]}:#{@options[:port]}: #{e}") unless @logger.nil?
-      @logger.debug("[LogCourierClient] #{e.backtrace}: #{e.message} (#{e.class})") unless @logger.nil? || !@logger.debug?
+      @logger.warn("[LogCourierClient] #{e.backtrace}: #{e.message} (#{e.class})") unless @logger.nil?
     end
   end
 end