out_es: new 'replace_dots' option (#708)

On Elasticsearch 2.0-2.3, key names with dots were not allowed for hence was required to replace every dot with an underscore. This requirement is not longer true on newer versions of Elasticsearch. The following patch introduce a new option called 'replace_dot' which is disabled by default. So now the plugin will only replace the dots if the flag is enabled for use cases where old versions of Elasticsearch are in use. Signed-off-by: Eduardo Silva <[email protected]>
fluent · Aug 8, 2018 · 6764ac7 · 6764ac7 · konstantin-kornienko-epam · Aug 8, 2018
1 parent 7eb7aee
commit 6764ac7
Show file tree

Hide file tree

Showing 3 changed files with 26 additions and 8 deletions.
diff --git a/plugins/out_es/es.c b/plugins/out_es/es.c
@@ -35,7 +35,9 @@
 
 struct flb_output_plugin out_es_plugin;
 
-static inline void es_pack_map_content(msgpack_packer *tmp_pck, msgpack_object map)
+static inline void es_pack_map_content(msgpack_packer *tmp_pck,
+                                       msgpack_object map,
+                                       struct flb_elasticsearch *ctx)
 {
     int i;
     char *ptr_key = NULL;
@@ -79,11 +81,13 @@ static inline void es_pack_map_content(msgpack_packer *tmp_pck, msgpack_object m
          *
          *   https://goo.gl/R5NMTr
          */
-        char *p   = ptr_key;
-        char *end = ptr_key + key_size;
-        while (p != end) {
-            if (*p == '.') *p = '_';
-            p++;
+        if (ctx->replace_dots == FLB_TRUE) {
+            char *p   = ptr_key;
+            char *end = ptr_key + key_size;
+            while (p != end) {
+                if (*p == '.') *p = '_';
+                p++;
+            }
         }
 
         /* Append the key */
@@ -102,7 +106,7 @@ static inline void es_pack_map_content(msgpack_packer *tmp_pck, msgpack_object m
          */
         if (v->type == MSGPACK_OBJECT_MAP) {
             msgpack_pack_map(tmp_pck, v->via.map.size);
-            es_pack_map_content(tmp_pck, *v);
+            es_pack_map_content(tmp_pck, *v, ctx);
         }
         else {
             msgpack_pack_object(tmp_pck, *v);
@@ -278,7 +282,7 @@ static char *elasticsearch_format(void *data, size_t bytes,
          * Elasticsearch have a restriction that key names cannot contain
          * a dot; if some dot is found, it's replaced with an underscore.
          */
-        es_pack_map_content(&tmp_pck, map);
+        es_pack_map_content(&tmp_pck, map, ctx);
 
         if (ctx->generate_id == FLB_TRUE) {
             MurmurHash3_x64_128(tmp_sbuf.data, tmp_sbuf.size, 42, hash);

diff --git a/plugins/out_es/es.h b/plugins/out_es/es.h
@@ -42,6 +42,12 @@ struct flb_elasticsearch {
     /* HTTP Client Setup */
     size_t buffer_size;
 
+    /*
+     * If enabled, replace field name dots with underscore, required for
+     * Elasticsearch 2.0-2.3.
+     */
+    int replace_dots;
+
     /*
      * Logstash compatibility options
      * ==============================

diff --git a/plugins/out_es/es_conf.c b/plugins/out_es/es_conf.c
@@ -247,6 +247,14 @@ struct flb_elasticsearch *flb_es_conf_create(struct flb_output_instance *ins,
         ctx->generate_id = FLB_FALSE;
     }
 
+    /* Replace dots */
+    tmp = flb_output_get_property("replace_dots", ins);
+    if (tmp) {
+        ctx->replace_dots = flb_utils_bool(tmp);
+    }
+    else {
+        ctx->replace_dots = FLB_FALSE;
+    }
     return ctx;
 }