diff --git a/.gitignore b/.gitignore new file mode 100644 index 0000000..e3e0b40 --- /dev/null +++ b/.gitignore @@ -0,0 +1,57 @@ +Makefile.in +aclocal.m4 +compile +config.guess +config.h.in +config.sub +configure +depcomp +aclocal.m4 +ar-lib +install-sh +missing +test-driver +kids-*.tar.gz + +debug/ +src/store/.dirstamp +core.* +epm/kids.list +client/c/sample/loggen +client/c/sample/echo2kids +client/c/sample/subscribe +client/python/build/ +client/python/dist/ +*.egg-info/ +.installed.cfg +Makefile +*.a +*.heap +*.la +linux/ +autom4te.cache/ +.deps +.lib +*.log +*.pyc +*.dSYM/ +*.o +*~ +*.out +*.bak +*.scan +*core +config.h +*.status +src/kids +stamp-h1 +epm/linux-*/ + +*.deps +*.dirstamp +debug/* +callgrind.* +!client/c/sample/Makefile +\#*\# +.\#* +test/runtest* diff --git a/ChangeLog b/ChangeLog new file mode 100644 index 0000000..9903bee --- /dev/null +++ b/ChangeLog @@ -0,0 +1,6 @@ +Kids ChangeLog +============== + +Version 1.0.0 +------------- +First public version diff --git a/LICENSE b/LICENSE new file mode 100644 index 0000000..aa10541 --- /dev/null +++ b/LICENSE @@ -0,0 +1,24 @@ +Copyright (c) 2014, Zhihu Inc. +Some rights reserved. + +Redistribution and use in source and binary forms, with or without +modification, are permitted provided that the following conditions are met: + * Redistributions of source code must retain the above copyright + notice, this list of conditions and the following disclaimer. + * Redistributions in binary form must reproduce the above copyright + notice, this list of conditions and the following disclaimer in the + documentation and/or other materials provided with the distribution. + * Neither the name of the Kids nor the + names of its contributors may be used to endorse or promote products + derived from this software without specific prior written permission. + +THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND +ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED +WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE +DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE FOR ANY +DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES +(INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; +LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND +ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS +SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. diff --git a/Makefile.am b/Makefile.am new file mode 100644 index 0000000..c144892 --- /dev/null +++ b/Makefile.am @@ -0,0 +1,12 @@ +SUBDIRS = deps/ae src doc test + +all-am: + cd deps/ae && $(MAKE) + +install: + cd src && $(MAKE) install + +deb: + cd epm && epm -a amd64 -f deb kids + +test: check diff --git a/README.md b/README.md new file mode 100644 index 0000000..1bf5a01 --- /dev/null +++ b/README.md @@ -0,0 +1,84 @@ +kids +==== + +Kids is a log aggregation system. + +It aggregates messages like [Scribe](https://github.com/facebookarchive/scribe) and its pub/sub pattern is ported from [Redis](http://redis.io/). + + +Features +-------- + +* Real-time subscription +* Distributed collection +* Message persistence +* Multithreading +* Redis protocol +* No third-party dependencies + + +Installation +------------ + +You need a complier with C++11 support like GCC 4.7 (or later) or [Clang](http://clang.llvm.org). + + ./autogen.sh + ./configure # --prefix=/your/path + make + make test # optional + make install + +By default, it will be installed to `/usr/local/bin/kids`. +You can use the `--prefix` option to specify the installation location. +Run `./configure --help` for more config options. + + +Quickstart +---------- + +In the distributed mode, first start kids with the `server.conf`: + + kids -c sample/server.conf + +Next, edit `host` and `port` in `networkstore` in `sample/agent.conf` as: + + store network primary { + host kidsserver; + port 3388; + } + +Then, run kids with the modified config file: + + kids -c sample/agent.conf + +Finally, use `publish` command in Redis protocol to send log to kids agent. +All the log will be resent to your kids server and persistently stored to disk for analysis later. +You can also use `subscribe` or `psubscribe` in Redis protocol to get real-time log from kids server. + +Full explanation of config file, see [here](doc/config.md). + +You can directly run `kids -c sample/server.conf` on single-server mode without agent, but it is NOT recommended. + +Run `kids --help` for more running options. + + +License +------- + +Kids is BSD-licensed, see LICENSE for more details. + + +FAQ +--- + +Q: What is the meaning of "kids"? +A: "kids" is the recursive acronym of "__K__ids __I__s a __D__ata __S__tream". + + +Architecture +------------ + +![image](doc/image/arch.jpg) + +You can view the Chinese version README [here](README.zh_CN.md) + diff --git a/README.zh_CN.md b/README.zh_CN.md new file mode 100644 index 0000000..509f055 --- /dev/null +++ b/README.zh_CN.md @@ -0,0 +1,70 @@ +kids +==== + +Kids 是一个日志聚合系统。 + +采用 [Scribe](https://github.com/facebookarchive/scribe) 的消息聚合模型和 [Redis](http://redis.io/) 的 pub/sub 模型。 + + +特性 +---- + +* 实时订阅 +* 分布式收集,集中存储 +* 多线程模型 +* 使用 Redis 协议 +* 无第三方依赖 + + +安装 +---- + +编译 kids 需要 C++11 支持,如 GCC 4.7 或更高版本或 [Clang](http://clang.llvm.org) + + ./autogen.sh + ./configure # --prefix=/your/path + make + make test #optional + make install + +默认情况下,kids 会被安装至 `/usr/local/bin/kids`,使用 `--prefix` 选项设置指定的安装位置,运行 `./configure --help` 获取更多设置选项。 + + +快速开始 +-------- + +在分布式模式下,首先使用 `server.conf` 启动 kids server: + + kids -c sample/server.conf + +然后,修改 `networkstore` 中的 `host` 与 `port`: + + store network primary { + host kidsserver; + port 3388; + } + +然后使用修改后的 `agent.conf` 启动 kids agent: + + kids -c sample/agent.conf + +最后,将需要收集的日志以 Redis 协议的 `publish` 命令发送到 kids agent,agent 会将它们转发至 kids server,server 端会对其进行持久化存储,同时,可以在 server 端直接使用 `subscribe` 或者 `psubscribe` 命令实时订阅日志。 + +配置文件的具体选项详见 [配置](doc/config.zh_CN.md)。 + +你也可以不使用 kids agent 而直接运行 `kids -c sample/server.conf` 使用单服务器模式,但是这是不被推荐的行为。 + +开源协议 +-------- +Kids 使用 BSD 协议,具体内容详见 LICENSE 文件。 + +FAQ +--- +Q: 为什么叫「kids」? +A: 「kids」是「__K__ids __I__s a __D__ata __S__tream」的递归缩写。 + +架构图 +------ +![image](doc/image/arch.jpg) + +[English Version](README.md) diff --git a/TODO b/TODO new file mode 100644 index 0000000..e69de29 diff --git a/autogen.sh b/autogen.sh new file mode 100755 index 0000000..3361b68 --- /dev/null +++ b/autogen.sh @@ -0,0 +1,16 @@ +#! /bin/sh + +if [ -x "`which autoreconf 2>/dev/null`" ] ; then + exec autoreconf -ivf +fi + +if glibtoolize --version > /dev/null 2>&1; then + LIBTOOLIZE='glibtoolize' +else + LIBTOOLIZE='libtoolize' +fi + +$LIBTOOLIZE && \ +aclocal && \ +automake --add-missing --force-missing --include-deps && \ +autoconf diff --git a/client/c/README b/client/c/README new file mode 100644 index 0000000..447fc74 --- /dev/null +++ b/client/c/README @@ -0,0 +1 @@ +simply use hiredis as kids client diff --git a/client/c/sample/Makefile b/client/c/sample/Makefile new file mode 100644 index 0000000..a36b0a2 --- /dev/null +++ b/client/c/sample/Makefile @@ -0,0 +1,13 @@ +all: loggen subscribe echo2kids + +subscribe: subscribe.c + gcc -g -o subscribe subscribe.c -lhiredis + +loggen: loggen.c + gcc -g -o loggen loggen.c -lhiredis + +echo2kids: echo2kids.c + gcc -g -o echo2kids echo2kids.c -lhiredis + +clean: + rm -rf loggen subscribe echo2kids diff --git a/client/c/sample/echo2kids.c b/client/c/sample/echo2kids.c new file mode 100644 index 0000000..0e5179e --- /dev/null +++ b/client/c/sample/echo2kids.c @@ -0,0 +1,99 @@ +#include +#include +#include +#include +#include + +#include "hiredis/hiredis.h" + +void Usage(const char *program_name) { + printf("Usage: %s [options]\n" + "options:\n" + " -s, --socket unix socket of kids\n" + " -h, --host host of kids (at least one of ip and socket must be specified)\n" + " -p, --port port of kids (default is 3388)\n" + " -t, --topic topic of log send to kids (default is \"KID_SAMPLE\")\n" + " -h, --help print help and quit\n\n", + program_name); + exit(0); +} + +const char* const short_options = "s:h:p:t:f:n:qH"; +const struct option long_options[] = { + { "socket", required_argument, NULL, 's' }, + { "host", required_argument, NULL, 'h' }, + { "port", required_argument, NULL, 'p' }, + { "topic", required_argument, NULL, 't' }, + { "help", no_argument, NULL, 'H' }, + { NULL, 0, NULL, 0 }, +}; + +int main(int argc, char **argv) { + redisContext *c; + redisReply *reply; + const char *socket = NULL; + const char *ip = "127.0.0.1"; + int port = 3388; + const char *topic = "loggen"; + char logbuf[10240]; + int opt; + int daemon; + struct timeval timeout = { 1, 500000 }; // 1.5 seconds + + while ((opt = getopt_long(argc, argv, short_options, long_options, NULL)) > 0) { + switch (opt) { + case 's': + socket = optarg; + break; + case 'h': + ip = optarg; + break; + case 'p': + port = atoi(optarg); + break; + case 't': + topic = optarg; + break; + case 'd': + daemon = 1; + break; + case 'H': + default: + Usage(argv[0]); + } + } + + if (socket == NULL && ip == NULL) { + Usage(argv[0]); + } + + if (socket) { + c = redisConnectUnixWithTimeout(socket, timeout); + } else { + c = redisConnectWithTimeout((char*)ip, port, timeout); + } + if (c->err) { + printf("Connection error: %s\n", c->errstr); + exit(1); + } + + char *line = NULL; + size_t linecap = 0; + ssize_t linelen; + while (1) { + if ((linelen = getline(&line, &linecap, stdin)) <= 0) { + break; + } + line[linelen - 1] = '\0'; + reply = redisCommand(c, "PUBLISH %s %s", topic, line); + if (reply) { + freeReplyObject(reply); + } else { + printf("LOG %s : failed\n", logbuf); + return -1; + } + + } + + redisFree(c); +} diff --git a/client/c/sample/loggen.c b/client/c/sample/loggen.c new file mode 100644 index 0000000..9d7ed4b --- /dev/null +++ b/client/c/sample/loggen.c @@ -0,0 +1,145 @@ +#include +#include +#include +#include +#include + +#include "hiredis/hiredis.h" + +void Usage(const char *program_name) { + printf("Usage: %s [options]\n" + "options:\n" + " -s, --socket unix socket of kids\n" + " -h, --host host of kids (at least one of ip and socket must be specified)\n" + " -p, --port port of kids (default is 3388)\n" + " -t, --topic topic of log send to kids (default is \"KID_SAMPLE\")\n" + " -f, --file read logs from file\n" + " -n, --num number of logs send to kids (default is -1 means infinite)\n" + " -q, --quiet don't print logs on screen\n" + " --period