Merge pull request #65 from finaldie/1.5

Merge branch 1.5 into master

.Makefile.dep

@@ -1,6 +1,8 @@
+PB_CONF_ARGS ?= CXXFLAGS="-O2 -g -fPIC -std=c++11 -DNDEBUG"
+
 ifeq ($(disable_fast_proto), false)
-    CPP_PROTO = --cpp_implementation
-    PB_LIB_PATH = export LD_LIBRARY_PATH=../src/.libs
+    CPP_PROTO += --cpp_implementation
+    PB_LIB_PATH += export LD_LIBRARY_PATH=../src/.libs
 else
     CPP_PROTO =
     PB_LIB_PATH = true
@@ -17,6 +19,13 @@ metrics:
 
 skull-ft:
 
+deps/libyaml/Makefile:
+	cd deps/libyaml && ./bootstrap && ./configure --enable-shared=no \
+	    CFLAGS="-O2 -g -fPIC"
+
+libyaml: deps/libyaml/Makefile
+	cd deps/libyaml && $(MAKE) $(MAKE_FLAGS)
+
 jemalloc: deps/jemalloc/Makefile
 	cd deps/jemalloc && $(MAKE) $(MAKE_FLAGS)
 
@@ -33,9 +42,10 @@ protobuf-py: protobuf-cpp
 protobuf: protobuf-py
 
 deps/protobuf/Makefile:
-	cd deps/protobuf && ./autogen.sh && ./configure --enable-static=no
+	cd deps/protobuf && ./autogen.sh && ./configure $(PB_CONF_ARGS)
 
 .PHONY: flibs metrics skull-ft jemalloc protobuf protobuf-py protobuf-cpp
+.PHONY: libyaml
 
 # install targets
 install-core:
@@ -83,10 +93,15 @@ clean-jemalloc:
 	    $(MAKE) $(MAKE_FLAGS) -C ./deps/jemalloc clean || exit "$$?"
 	rm -f ./deps/jemalloc/Makefile
 
+clean-libyaml:
+	! test -f ./deps/libyaml/Makefile || \
+	    $(MAKE) $(MAKE_FLAGS) -C ./deps/libyaml clean || exit "$$?"
+	rm -f ./deps/libyaml/Makefile
+
 clean-protobuf:
 	! test -f ./deps/protobuf/Makefile || \
 	    $(MAKE) $(MAKE_FLAGS) -C ./deps/protobuf clean || exit "$$?"
 	rm -f ./deps/protobuf/Makefile
 	cd ./deps/protobuf/python && $(python_path) setup.py clean
 
-.PHONY: clean-flibs clean-skull-ft clean-jemalloc clean-protobuf
+.PHONY: clean-flibs clean-skull-ft clean-jemalloc clean-protobuf clean-libyaml

.gitmodules

@@ -10,3 +10,6 @@
 [submodule "deps/protobuf"]
 	path = deps/protobuf
 	url = git://github.com/google/protobuf.git
+[submodule "deps/libyaml"]
+	path = deps/libyaml
+	url = git://github.com/yaml/libyaml.git

.travis.yml

@@ -13,7 +13,7 @@ before_install:
     - cat /etc/issue
     - sudo apt-get update -qq
     - >
-      sudo apt-get install autoconf libtool valgrind expect libyaml-dev
+      sudo apt-get install autoconf libtool valgrind expect
       python3-dev python3-pip python3-setuptools python3-wheel libc-ares-dev
     - sudo pip3 install PyYAML pympler WebOb
     - sudo pip3 freeze
@@ -36,15 +36,13 @@ install: true
 before_script: true
 
 script:
-    - make dep -j4 disable_fast_proto=true
-    - sudo env PATH=$PATH make install-dep disable_fast_proto=true
+    - make dep -j4 minimal_deps=true FLIB_CFLAGS=-DFLIB_SKIP_LEGACY
+    - sudo env PATH=$PATH make install-dep minimal_deps=true
     - make -j4
     - make clean
     - make -j4 debug=true
     - make clean
     - make -j4 disable_jemalloc=true
-    - make clean
-    - make -j4 disable_jemalloc=true FLIB_CFLAGS=-DFLIB_SKIP_LEGACY
     - sudo make install
     - make check verbose=true
 
@@ -56,4 +54,4 @@ after_failure: false
 
 notifications:
     email:
-      - [email protected]
+        - [email protected]

ChangeLog.md

@@ -1,3 +1,12 @@
+- 2019-01-08 1.5.2
+  - Engine
+    - **New:** Add building option for minize the dependency
+    - **Enhancement:** Static link libyaml to reduce external dependency
+    - **Enhancement:** APIs loading can be omitted instead of exit
+    - **Enhancement:** Show API loading status in admin `info` command
+  - API
+    - **Enhancement:** Static link libprotobuf `api-cpp` to reduce dependency
+    - **Enhancement:** Refine python APIs
 - 2019-01-02 1.5.1
   - Engine
     - **Fix:** Prevent crash if ep pool run out of file descriptor resource

Makefile

@@ -1,20 +1,26 @@
 MAKE ?= make
+MAKE_FLAGS +=
+
 prefix ?= /usr/local
 disable_jemalloc ?= false
 disable_fast_proto ?= false
-export python_path ?= /usr/bin/python3
+minimal_deps ?= true
 
-MAKE_FLAGS +=
+export python_path ?= /usr/bin/python3
 
 all: api-cpp api-py
 
 api-cpp: core
 api-py: core
 
 ifeq ($(disable_jemalloc), false)
-dep: metrics flibs skull-ft protobuf jemalloc
+dep: metrics flibs skull-ft protobuf libyaml jemalloc
 else
-dep: metrics flibs skull-ft protobuf
+dep: metrics flibs skull-ft protobuf libyaml
+endif
+
+ifeq ($(minimal_deps), true)
+    disable_fast_proto := true
 endif
 
 core:
@@ -28,13 +34,14 @@ valgrind-check:
 
 install-dep: install-skull-ft install-protobuf
 
-install: install-core install-scripts install-api install-api-cpp install-others
-install: install-ft install-api-py
+install: install-core install-scripts install-ft install-others
+install: install-api install-api-cpp install-api-py
 
 clean: clean-api-cpp clean-api-py
 	cd src && $(MAKE) $@
 
 clean-dep: clean-flibs clean-skull-ft clean-jemalloc clean-protobuf
+clean-dep: clean-libyaml
 
 .PHONY: all dep core check valgrind-check install clean clean-dep
 

README.md

@@ -51,7 +51,7 @@ _Skull_ requires:
 ### Install Dependencies
 ```console
 # Install System Dependencies
-sudo apt-get install autoconf libtool valgrind expect libyaml-dev python3-dev python3-pip;
+sudo apt-get install autoconf libtool valgrind expect python3-dev python3-pip;
 sudo pip3 install PyYAML pympler WebOb;
 
 # Clone and Build Dependencies (For example: project folder is 'skull')

scripts/bin/skull-config

@@ -161,15 +161,15 @@ function _echo_common_includes() {
 }
 
 function _echo_common_libs() {
-    echo "-lskull-api-cpp -Wl,--no-as-needed -lprotobuf"
+    echo "-Wl,--no-as-needed"
 }
 
 function _echo_common_test_libs() {
     echo "-lskull-unittest-cpp -lskull-unittest-c -lprotobuf"
 }
 
 function _echo_libs() {
-    echo "-lprotobuf -lskull-api-cpp -Wl,--no-as-needed -lskull-common-cpp"
+    echo "-Wl,--no-as-needed -lskull-common-cpp"
 }
 
 function _echo_test_libs() {
@@ -227,5 +227,5 @@ function _echo_valgrind-dir() {
 # main
 get_root
 
-process_cmd $@
+process_cmd "$@"
 

src/Makefile

@@ -2,7 +2,7 @@ include Makefile.inc
 
 # dependency library list
 SKULL_LIBS = \
-    -lyaml \
+    $(LIBYAML_STATIC) \
     -lflibs \
     $(JEMALLOC_STATIC) \
     -pthread \

src/Makefile.inc

@@ -73,6 +73,8 @@ SKULL_TP_CXX   = $(CXX) $(SKULL_TP_CXXFLAGS)
 SKULL_CXX_LD   = $(CXX) $(SKULL_LDFLAGS)
 SKULL_CXX_UTLD = $(CXX) $(SKULL_UTLDFLAGS)
 
+SKULL_AR = ar -rcu
+
 # jemalloc
 disable_jemalloc ?= false
 
@@ -82,3 +84,5 @@ ifeq ($(disable_jemalloc), false)
 else
     JEMALLOC_STATIC :=
 endif
+
+LIBYAML_STATIC := ../deps/libyaml/src/.libs/libyaml.a

src/api/sk_config.h

@@ -94,8 +94,8 @@ typedef struct sk_config_t {
     int    command_port;
     const char* command_bind;
 
-    // Supportted languages, value: char*
-    flist* langs;
+    // Supported languages, value: char*
+    fhash* langs;
 
     // max open fds
     int    max_fds;

src/api/sk_core.h

@@ -87,6 +87,7 @@ typedef struct sk_core_t {
     flist*           drivers;        // emement type: sk_driver_t*
     fhash*           unique_modules; // key: module name; value: sk_module_t*
     fhash*           services;       // key: service name; value: sk_service_t*
+    fhash*           apis;           // key: api name; value: loaded or not
     const char*      working_dir;
 
     // admin module and config

src/common/sk_loader.c

@@ -13,17 +13,15 @@ typedef struct sk_mloader_tbl_t {
 static sk_mloader_tbl_t* mloader_tbl = NULL;
 
 static
-sk_mloader_tbl_t* sk_mloader_tbl_create()
-{
+sk_mloader_tbl_t* sk_mloader_tbl_create() {
     sk_mloader_tbl_t* tbl = calloc(1, sizeof(*tbl));
     tbl->tbl = fhash_str_create(0, FHASH_MASK_AUTO_REHASH);
 
     return tbl;
 }
 
 static
-void sk_mloader_tbl_destroy(sk_mloader_tbl_t* tbl)
-{
+void sk_mloader_tbl_destroy(sk_mloader_tbl_t* tbl) {
     if (!tbl) return;
 
     // Destroy module loader tbl
@@ -38,26 +36,22 @@ void sk_mloader_tbl_destroy(sk_mloader_tbl_t* tbl)
 }
 
 static
-sk_module_loader_t* sk_mloader_tbl_get(const char* type)
-{
+sk_module_loader_t* sk_mloader_tbl_get(const char* type) {
     return fhash_str_get(mloader_tbl->tbl, type);
 }
 
 static
-void sk_mloader_tbl_set(const char* type, sk_module_loader_t* loader)
-{
+void sk_mloader_tbl_set(const char* type, sk_module_loader_t* loader) {
     fhash_str_set(mloader_tbl->tbl, type, loader);
 }
 
 static
-sk_module_loader_t* sk_mloader_tbl_del(const char* type)
-{
+sk_module_loader_t* sk_mloader_tbl_del(const char* type) {
     return fhash_str_del(mloader_tbl->tbl, type);
 }
 
 sk_module_t* sk_module_load(const sk_module_cfg_t* cfg,
-                            const char* conf_name)
-{
+                            const char* conf_name) {
     const char* type = cfg->type;
     const char* name = cfg->name;
 
@@ -98,8 +92,7 @@ sk_module_t* sk_module_load(const sk_module_cfg_t* cfg,
     return module;
 }
 
-void sk_module_unload(sk_module_t* module)
-{
+void sk_module_unload(sk_module_t* module) {
     const sk_module_cfg_t* cfg = module->cfg;
     const char* type = cfg->type;
     sk_module_loader_t* loader = sk_mloader_tbl_get(type);
@@ -109,8 +102,7 @@ void sk_module_unload(sk_module_t* module)
     SK_ASSERT_MSG(!ret, "module unload failed: ret = %d\n", ret);
 }
 
-void sk_module_loader_register(const char* type, sk_module_loader_t loader)
-{
+void sk_module_loader_register(const char* type, sk_module_loader_t loader) {
     if (!mloader_tbl) {
         mloader_tbl = sk_mloader_tbl_create();
     }
@@ -124,8 +116,7 @@ void sk_module_loader_register(const char* type, sk_module_loader_t loader)
     sk_mloader_tbl_set(type, mloader);
 }
 
-sk_module_loader_t* sk_module_loader_unregister(const char* type)
-{
+sk_module_loader_t* sk_module_loader_unregister(const char* type) {
     if (!type) return NULL;
     return sk_mloader_tbl_del(type);
 }

src/common/sk_user_loader.c

@@ -16,17 +16,17 @@ typedef struct sk_user_api_t {
 
 static fhash* user_libs = NULL;
 
-int sk_userlib_load(const char* filename)
+int sk_userlib_load(const char* libname)
 {
     if (!user_libs) {
         user_libs = fhash_str_create(0, FHASH_MASK_AUTO_REHASH);
     }
 
     // Check whether lib has already loaded
-    const char* lib = fhash_str_get(user_libs, filename);
+    const char* lib = fhash_str_get(user_libs, libname);
     if (lib) {
-        sk_print("user lib %s has already loaded\n", filename);
-        printf("user lib %s has already loaded\n", filename);
+        sk_print("user lib %s has already loaded\n", libname);
+        printf("user lib %s has already loaded\n", libname);
         return 0;
     }
 
@@ -36,15 +36,16 @@ int sk_userlib_load(const char* filename)
 
     char fullname[SK_USER_LIBNAME_MAX];
     memset(fullname, 0, SK_USER_LIBNAME_MAX);
-    snprintf(fullname, SK_USER_LIBNAME_MAX, "%s/%s", SK_USER_PREFIX1, filename);
+    snprintf(fullname, SK_USER_LIBNAME_MAX, "%s/%s", SK_USER_PREFIX1, libname);
+    int flags = RTLD_NOW | RTLD_GLOBAL;
 
-    void* handler = dlopen(fullname, RTLD_NOW | RTLD_GLOBAL);
+    void* handler = dlopen(fullname, flags);
     if (!handler) {
         fprintf(stdout, "Notice: cannot open %s: %s\n", fullname, dlerror());
 
         memset(fullname, 0, SK_USER_LIBNAME_MAX);
-        snprintf(fullname, SK_USER_LIBNAME_MAX, "%s/%s", SK_USER_PREFIX2, filename);
-        handler = dlopen(fullname, RTLD_NOW);
+        snprintf(fullname, SK_USER_LIBNAME_MAX, "%s/%s", SK_USER_PREFIX2, libname);
+        handler = dlopen(fullname, flags);
         if (!handler) {
             fprintf(stderr, "Error: cannot open %s: %s\n", fullname, dlerror());
             return 1;
@@ -58,24 +59,30 @@ int sk_userlib_load(const char* filename)
     *(void**)(&user_api->load) = dlsym(handler, SK_API_LOAD_FUNCNAME);
     if ((error = dlerror()) != NULL) {
         fprintf(stderr, "Error: cannot setup user 'load' api %s:%s, reason: %s\n",
-                 filename, SK_API_LOAD_FUNCNAME, error);
+                 libname, SK_API_LOAD_FUNCNAME, error);
+        goto error;
         return 1;
     }
 
     // 3. Find api unloader
     *(void**)(&user_api->unload) = dlsym(handler, SK_API_UNLOAD_FUNCNAME);
     if ((error = dlerror()) != NULL) {
         fprintf(stderr, "Error: cannot setup user 'unload' api %s:%s, reason: %s\n",
-                 filename, SK_API_UNLOAD_FUNCNAME, error);
+                 libname, SK_API_UNLOAD_FUNCNAME, error);
+        goto error;
         return 1;
     }
 
     // 4. Add to userlibs table
-    fhash_str_set(user_libs, filename, user_api);
+    fhash_str_set(user_libs, libname, user_api);
 
     // 5. Run api loader
     user_api->load();
     return 0;
+
+error:
+    free(user_api);
+    return 1;
 }
 
 void sk_userlib_unload()