From 0197e416c34b1ff71aaf3d01745f2c4d8444752e Mon Sep 17 00:00:00 2001 From: dccmx Date: Fri, 19 Dec 2014 16:30:16 +0800 Subject: [PATCH] First public release. --- .gitignore | 57 + ChangeLog | 6 + LICENSE | 24 + Makefile.am | 12 + README.md | 84 + README.zh_CN.md | 70 + TODO | 0 autogen.sh | 16 + client/c/README | 1 + client/c/sample/Makefile | 13 + client/c/sample/echo2kids.c | 99 + client/c/sample/loggen.c | 145 + client/c/sample/subscribe.c | 127 + client/go/kids.go | 247 + client/go/kids_test.go | 83 + configure.ac | 103 + deps/ae/Makefile.am | 15 + deps/ae/ae.c | 435 + deps/ae/ae.h | 118 + deps/ae/ae_epoll.c | 130 + deps/ae/ae_evport.c | 315 + deps/ae/ae_kqueue.c | 132 + deps/ae/ae_select.c | 99 + deps/ae/anet.c | 441 + deps/ae/anet.h | 60 + deps/ae/configure.ac | 52 + deps/ae/fmacros.h | 48 + deps/ae/libae.h | 17 + deps/ae/zmalloc.c | 351 + deps/ae/zmalloc.h | 85 + doc/Makefile.am | 4 + doc/config.md | 50 + doc/config.zh_CN.md | 51 + doc/image/arch.jpg | Bin 0 -> 34941 bytes doc/image/thread.jpg | Bin 0 -> 79351 bytes doc/overview.md | 1 + doc/overview.zh_CN.md | 1 + doc/store.md | 124 + doc/store.zh_CN.md | 112 + samples/agent.conf | 41 + samples/kids | 124 + samples/kids.conf | 25 + samples/server.conf | 19 + src/Makefile.am | 33 + src/buffer.cc | 248 + src/buffer.h | 69 + src/client.cc | 601 + src/client.h | 86 + src/common.h | 66 + src/conf.cc | 161 + src/conf.h | 87 + src/constants.h | 60 + src/filesystem.cc | 243 + src/filesystem.h | 43 + src/kids.cc | 184 + src/kids.h | 62 + src/lexer.c | 484 + src/lexer.rl | 74 + src/logger.cc | 108 + src/logger.h | 74 + src/master.cc | 223 + src/master.h | 61 + src/msgqueue.cc | 96 + src/msgqueue.h | 65 + src/parser.c | 1212 ++ src/parser.h | 8 + src/parser.y | 190 + src/pubsub.cc | 135 + src/sds.c | 731 ++ src/sds.h | 107 + src/store/bufferstore.cc | 146 + src/store/bufferstore.h | 41 + src/store/filestore.cc | 171 + src/store/filestore.h | 47 + src/store/multiplestore.cc | 84 + src/store/multiplestore.h | 35 + src/store/networkstore.cc | 320 + src/store/networkstore.h | 50 + src/store/prioritystore.cc | 68 + src/store/prioritystore.h | 28 + src/store/store.cc | 53 + src/store/store.h | 68 + src/storer.cc | 131 + src/storer.h | 53 + src/util.cc | 308 + src/util.h | 20 + src/worker.cc | 236 + src/worker.h | 73 + test/Makefile.am | 62 + test/data/conf/agent.conf | 36 + test/data/conf/server.conf | 26 + test/funcational/qps.py | 79 + test/funcational/test.py | 124 + test/gtest/gtest-all.cc | 9592 ++++++++++++++++ test/gtest/gtest.h | 20061 ++++++++++++++++++++++++++++++++++ test/test_buffer.cc | 143 + test/test_conf.cc | 92 + test/test_filesystem.cc | 73 + test/test_main.cc | 16 + test/test_message.cc | 34 + test/test_pattern.cc | 24 + test/test_util.cc | 47 + tools/collect_kids.py | 43 + 103 files changed, 41832 insertions(+) create mode 100644 .gitignore create mode 100644 ChangeLog create mode 100644 LICENSE create mode 100644 Makefile.am create mode 100644 README.md create mode 100644 README.zh_CN.md create mode 100644 TODO create mode 100755 autogen.sh create mode 100644 client/c/README create mode 100644 client/c/sample/Makefile create mode 100644 client/c/sample/echo2kids.c create mode 100644 client/c/sample/loggen.c create mode 100644 client/c/sample/subscribe.c create mode 100644 client/go/kids.go create mode 100644 client/go/kids_test.go create mode 100644 configure.ac create mode 100644 deps/ae/Makefile.am create mode 100644 deps/ae/ae.c create mode 100644 deps/ae/ae.h create mode 100644 deps/ae/ae_epoll.c create mode 100644 deps/ae/ae_evport.c create mode 100644 deps/ae/ae_kqueue.c create mode 100644 deps/ae/ae_select.c create mode 100644 deps/ae/anet.c create mode 100644 deps/ae/anet.h create mode 100644 deps/ae/configure.ac create mode 100644 deps/ae/fmacros.h create mode 100644 deps/ae/libae.h create mode 100644 deps/ae/zmalloc.c create mode 100644 deps/ae/zmalloc.h create mode 100644 doc/Makefile.am create mode 100644 doc/config.md create mode 100644 doc/config.zh_CN.md create mode 100644 doc/image/arch.jpg create mode 100644 doc/image/thread.jpg create mode 100644 doc/overview.md create mode 100644 doc/overview.zh_CN.md create mode 100644 doc/store.md create mode 100644 doc/store.zh_CN.md create mode 100644 samples/agent.conf create mode 100755 samples/kids create mode 100644 samples/kids.conf create mode 100644 samples/server.conf create mode 100644 src/Makefile.am create mode 100644 src/buffer.cc create mode 100644 src/buffer.h create mode 100644 src/client.cc create mode 100644 src/client.h create mode 100644 src/common.h create mode 100644 src/conf.cc create mode 100644 src/conf.h create mode 100644 src/constants.h create mode 100644 src/filesystem.cc create mode 100644 src/filesystem.h create mode 100644 src/kids.cc create mode 100644 src/kids.h create mode 100644 src/lexer.c create mode 100644 src/lexer.rl create mode 100644 src/logger.cc create mode 100644 src/logger.h create mode 100644 src/master.cc create mode 100644 src/master.h create mode 100644 src/msgqueue.cc create mode 100644 src/msgqueue.h create mode 100644 src/parser.c create mode 100644 src/parser.h create mode 100644 src/parser.y create mode 100644 src/pubsub.cc create mode 100644 src/sds.c create mode 100644 src/sds.h create mode 100644 src/store/bufferstore.cc create mode 100644 src/store/bufferstore.h create mode 100644 src/store/filestore.cc create mode 100644 src/store/filestore.h create mode 100644 src/store/multiplestore.cc create mode 100644 src/store/multiplestore.h create mode 100644 src/store/networkstore.cc create mode 100644 src/store/networkstore.h create mode 100644 src/store/prioritystore.cc create mode 100644 src/store/prioritystore.h create mode 100644 src/store/store.cc create mode 100644 src/store/store.h create mode 100644 src/storer.cc create mode 100644 src/storer.h create mode 100644 src/util.cc create mode 100644 src/util.h create mode 100644 src/worker.cc create mode 100644 src/worker.h create mode 100644 test/Makefile.am create mode 100644 test/data/conf/agent.conf create mode 100644 test/data/conf/server.conf create mode 100755 test/funcational/qps.py create mode 100755 test/funcational/test.py create mode 100644 test/gtest/gtest-all.cc create mode 100644 test/gtest/gtest.h create mode 100644 test/test_buffer.cc create mode 100644 test/test_conf.cc create mode 100644 test/test_filesystem.cc create mode 100644 test/test_main.cc create mode 100644 test/test_message.cc create mode 100644 test/test_pattern.cc create mode 100644 test/test_util.cc create mode 100644 tools/collect_kids.py diff --git a/.gitignore b/.gitignore new file mode 100644 index 0000000..e3e0b40 --- /dev/null +++ b/.gitignore @@ -0,0 +1,57 @@ +Makefile.in +aclocal.m4 +compile +config.guess +config.h.in +config.sub +configure +depcomp +aclocal.m4 +ar-lib +install-sh +missing +test-driver +kids-*.tar.gz + +debug/ +src/store/.dirstamp +core.* +epm/kids.list +client/c/sample/loggen +client/c/sample/echo2kids +client/c/sample/subscribe +client/python/build/ +client/python/dist/ +*.egg-info/ +.installed.cfg +Makefile +*.a +*.heap +*.la +linux/ +autom4te.cache/ +.deps +.lib +*.log +*.pyc +*.dSYM/ +*.o +*~ +*.out +*.bak +*.scan +*core +config.h +*.status +src/kids +stamp-h1 +epm/linux-*/ + +*.deps +*.dirstamp +debug/* +callgrind.* +!client/c/sample/Makefile +\#*\# +.\#* +test/runtest* diff --git a/ChangeLog b/ChangeLog new file mode 100644 index 0000000..9903bee --- /dev/null +++ b/ChangeLog @@ -0,0 +1,6 @@ +Kids ChangeLog +============== + +Version 1.0.0 +------------- +First public version diff --git a/LICENSE b/LICENSE new file mode 100644 index 0000000..aa10541 --- /dev/null +++ b/LICENSE @@ -0,0 +1,24 @@ +Copyright (c) 2014, Zhihu Inc. +Some rights reserved. + +Redistribution and use in source and binary forms, with or without +modification, are permitted provided that the following conditions are met: + * Redistributions of source code must retain the above copyright + notice, this list of conditions and the following disclaimer. + * Redistributions in binary form must reproduce the above copyright + notice, this list of conditions and the following disclaimer in the + documentation and/or other materials provided with the distribution. + * Neither the name of the Kids nor the + names of its contributors may be used to endorse or promote products + derived from this software without specific prior written permission. + +THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND +ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED +WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE +DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE FOR ANY +DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES +(INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; +LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND +ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS +SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. diff --git a/Makefile.am b/Makefile.am new file mode 100644 index 0000000..c144892 --- /dev/null +++ b/Makefile.am @@ -0,0 +1,12 @@ +SUBDIRS = deps/ae src doc test + +all-am: + cd deps/ae && $(MAKE) + +install: + cd src && $(MAKE) install + +deb: + cd epm && epm -a amd64 -f deb kids + +test: check diff --git a/README.md b/README.md new file mode 100644 index 0000000..1bf5a01 --- /dev/null +++ b/README.md @@ -0,0 +1,84 @@ +kids +==== + +Kids is a log aggregation system. + +It aggregates messages like [Scribe](https://github.com/facebookarchive/scribe) and its pub/sub pattern is ported from [Redis](http://redis.io/). + + +Features +-------- + +* Real-time subscription +* Distributed collection +* Message persistence +* Multithreading +* Redis protocol +* No third-party dependencies + + +Installation +------------ + +You need a complier with C++11 support like GCC 4.7 (or later) or [Clang](http://clang.llvm.org). + + ./autogen.sh + ./configure # --prefix=/your/path + make + make test # optional + make install + +By default, it will be installed to `/usr/local/bin/kids`. +You can use the `--prefix` option to specify the installation location. +Run `./configure --help` for more config options. + + +Quickstart +---------- + +In the distributed mode, first start kids with the `server.conf`: + + kids -c sample/server.conf + +Next, edit `host` and `port` in `networkstore` in `sample/agent.conf` as: + + store network primary { + host kidsserver; + port 3388; + } + +Then, run kids with the modified config file: + + kids -c sample/agent.conf + +Finally, use `publish` command in Redis protocol to send log to kids agent. +All the log will be resent to your kids server and persistently stored to disk for analysis later. +You can also use `subscribe` or `psubscribe` in Redis protocol to get real-time log from kids server. + +Full explanation of config file, see [here](doc/config.md). + +You can directly run `kids -c sample/server.conf` on single-server mode without agent, but it is NOT recommended. + +Run `kids --help` for more running options. + + +License +------- + +Kids is BSD-licensed, see LICENSE for more details. + + +FAQ +--- + +Q: What is the meaning of "kids"? +A: "kids" is the recursive acronym of "__K__ids __I__s a __D__ata __S__tream". + + +Architecture +------------ + +![image](doc/image/arch.jpg) + +You can view the Chinese version README [here](README.zh_CN.md) + diff --git a/README.zh_CN.md b/README.zh_CN.md new file mode 100644 index 0000000..509f055 --- /dev/null +++ b/README.zh_CN.md @@ -0,0 +1,70 @@ +kids +==== + +Kids 是一个日志聚合系统。 + +采用 [Scribe](https://github.com/facebookarchive/scribe) 的消息聚合模型和 [Redis](http://redis.io/) 的 pub/sub 模型。 + + +特性 +---- + +* 实时订阅 +* 分布式收集，集中存储 +* 多线程模型 +* 使用 Redis 协议 +* 无第三方依赖 + + +安装 +---- + +编译 kids 需要 C++11 支持，如 GCC 4.7 或更高版本或 [Clang](http://clang.llvm.org) + + ./autogen.sh + ./configure # --prefix=/your/path + make + make test #optional + make install + +默认情况下，kids 会被安装至 `/usr/local/bin/kids`，使用 `--prefix` 选项设置指定的安装位置，运行 `./configure --help` 获取更多设置选项。 + + +快速开始 +-------- + +在分布式模式下，首先使用 `server.conf` 启动 kids server： + + kids -c sample/server.conf + +然后，修改 `networkstore` 中的 `host` 与 `port`： + + store network primary { + host kidsserver; + port 3388; + } + +然后使用修改后的 `agent.conf` 启动 kids agent： + + kids -c sample/agent.conf + +最后，将需要收集的日志以 Redis 协议的 `publish` 命令发送到 kids agent，agent 会将它们转发至 kids server，server 端会对其进行持久化存储，同时，可以在 server 端直接使用 `subscribe` 或者 `psubscribe` 命令实时订阅日志。 + +配置文件的具体选项详见 [配置](doc/config.zh_CN.md)。 + +你也可以不使用 kids agent 而直接运行 `kids -c sample/server.conf` 使用单服务器模式，但是这是不被推荐的行为。 + +开源协议 +-------- +Kids 使用 BSD 协议，具体内容详见 LICENSE 文件。 + +FAQ +--- +Q: 为什么叫「kids」? +A: 「kids」是「__K__ids __I__s a __D__ata __S__tream」的递归缩写。 + +架构图 +------ +![image](doc/image/arch.jpg) + +[English Version](README.md) diff --git a/TODO b/TODO new file mode 100644 index 0000000..e69de29 diff --git a/autogen.sh b/autogen.sh new file mode 100755 index 0000000..3361b68 --- /dev/null +++ b/autogen.sh @@ -0,0 +1,16 @@ +#! /bin/sh + +if [ -x "`which autoreconf 2>/dev/null`" ] ; then + exec autoreconf -ivf +fi + +if glibtoolize --version > /dev/null 2>&1; then + LIBTOOLIZE='glibtoolize' +else + LIBTOOLIZE='libtoolize' +fi + +$LIBTOOLIZE && \ +aclocal && \ +automake --add-missing --force-missing --include-deps && \ +autoconf diff --git a/client/c/README b/client/c/README new file mode 100644 index 0000000..447fc74 --- /dev/null +++ b/client/c/README @@ -0,0 +1 @@ +simply use hiredis as kids client diff --git a/client/c/sample/Makefile b/client/c/sample/Makefile new file mode 100644 index 0000000..a36b0a2 --- /dev/null +++ b/client/c/sample/Makefile @@ -0,0 +1,13 @@ +all: loggen subscribe echo2kids + +subscribe: subscribe.c + gcc -g -o subscribe subscribe.c -lhiredis + +loggen: loggen.c + gcc -g -o loggen loggen.c -lhiredis + +echo2kids: echo2kids.c + gcc -g -o echo2kids echo2kids.c -lhiredis + +clean: + rm -rf loggen subscribe echo2kids diff --git a/client/c/sample/echo2kids.c b/client/c/sample/echo2kids.c new file mode 100644 index 0000000..0e5179e --- /dev/null +++ b/client/c/sample/echo2kids.c @@ -0,0 +1,99 @@ +#include +#include +#include +#include +#include + +#include "hiredis/hiredis.h" + +void Usage(const char *program_name) { + printf("Usage: %s [options]\n" + "options:\n" + " -s, --socket unix socket of kids\n" + " -h, --host host of kids (at least one of ip and socket must be specified)\n" + " -p, --port port of kids (default is 3388)\n" + " -t, --topic topic of log send to kids (default is \"KID_SAMPLE\")\n" + " -h, --help print help and quit\n\n", + program_name); + exit(0); +} + +const char* const short_options = "s:h:p:t:f:n:qH"; +const struct option long_options[] = { + { "socket", required_argument, NULL, 's' }, + { "host", required_argument, NULL, 'h' }, + { "port", required_argument, NULL, 'p' }, + { "topic", required_argument, NULL, 't' }, + { "help", no_argument, NULL, 'H' }, + { NULL, 0, NULL, 0 }, +}; + +int main(int argc, char **argv) { + redisContext *c; + redisReply *reply; + const char *socket = NULL; + const char *ip = "127.0.0.1"; + int port = 3388; + const char *topic = "loggen"; + char logbuf[10240]; + int opt; + int daemon; + struct timeval timeout = { 1, 500000 }; // 1.5 seconds + + while ((opt = getopt_long(argc, argv, short_options, long_options, NULL)) > 0) { + switch (opt) { + case 's': + socket = optarg; + break; + case 'h': + ip = optarg; + break; + case 'p': + port = atoi(optarg); + break; + case 't': + topic = optarg; + break; + case 'd': + daemon = 1; + break; + case 'H': + default: + Usage(argv[0]); + } + } + + if (socket == NULL && ip == NULL) { + Usage(argv[0]); + } + + if (socket) { + c = redisConnectUnixWithTimeout(socket, timeout); + } else { + c = redisConnectWithTimeout((char*)ip, port, timeout); + } + if (c->err) { + printf("Connection error: %s\n", c->errstr); + exit(1); + } + + char *line = NULL; + size_t linecap = 0; + ssize_t linelen; + while (1) { + if ((linelen = getline(&line, &linecap, stdin)) <= 0) { + break; + } + line[linelen - 1] = '\0'; + reply = redisCommand(c, "PUBLISH %s %s", topic, line); + if (reply) { + freeReplyObject(reply); + } else { + printf("LOG %s : failed\n", logbuf); + return -1; + } + + } + + redisFree(c); +} diff --git a/client/c/sample/loggen.c b/client/c/sample/loggen.c new file mode 100644 index 0000000..9d7ed4b --- /dev/null +++ b/client/c/sample/loggen.c @@ -0,0 +1,145 @@ +#include +#include +#include +#include +#include + +#include "hiredis/hiredis.h" + +void Usage(const char *program_name) { + printf("Usage: %s [options]\n" + "options:\n" + " -s, --socket unix socket of kids\n" + " -h, --host host of kids (at least one of ip and socket must be specified)\n" + " -p, --port port of kids (default is 3388)\n" + " -t, --topic topic of log send to kids (default is \"KID_SAMPLE\")\n" + " -f, --file read logs from file\n" + " -n, --num number of logs send to kids (default is -1 means infinite)\n" + " -q, --quiet don't print logs on screen\n" + " --period