parser: add Time_System_Timezone

Signed-off-by: braydonk <[email protected]>
fluent · Apr 29, 2024 · aafe617 · aafe617
1 parent 9c7a4bf
commit aafe617
Show file tree

Hide file tree

Showing 19 changed files with 203 additions and 46 deletions.
diff --git a/include/fluent-bit/flb_parser.h b/include/fluent-bit/flb_parser.h
@@ -48,6 +48,7 @@ struct flb_parser {
     char *time_fmt_full;  /* original given time format */
     char *time_key;       /* field name that contains the time */
     int time_offset;      /* fixed UTC offset */
+    int time_system_timezone; /* use the system timezone as a fallback */
     int time_keep;        /* keep time field */
     int time_strict;      /* parse time field strictly */
     int logfmt_no_bare_keys; /* in logfmt parsers, require all keys to have values */
@@ -74,13 +75,18 @@ enum {
     FLB_PARSER_TYPE_HEX,
 };
 
-static inline time_t flb_parser_tm2time(const struct flb_tm *src)
+static inline time_t flb_parser_tm2time(const struct flb_tm *src, 
+                                        int use_system_timezone)
 {
     struct tm tmp;
     time_t res;
 
     tmp = src->tm;
-    res = timegm(&tmp) - flb_tm_gmtoff(src);
+    if (use_system_timezone) {
+        res = mktime(&tmp);
+    } else {
+        res = timegm(&tmp) - flb_tm_gmtoff(src);
+    }
     return res;
 }
 
@@ -92,6 +98,7 @@ struct flb_parser *flb_parser_create(const char *name, const char *format,
                                      const char *time_offset,
                                      int time_keep,
                                      int time_strict,
+                                     int time_system_timezone,
                                      int logfmt_no_bare_keys,
                                      struct flb_parser_types *types,
                                      int types_len,

diff --git a/plugins/in_kubernetes_events/kubernetes_events.c b/plugins/in_kubernetes_events/kubernetes_events.c
@@ -225,7 +225,7 @@ static int record_get_field_time(msgpack_object *obj, const char *fieldname, str
         return -2;
     }
 
-    val->tm.tv_sec = flb_parser_tm2time(&tm);
+    val->tm.tv_sec = flb_parser_tm2time(&tm, FLB_FALSE);
     val->tm.tv_nsec = 0;
 
     return 0;

diff --git a/src/flb_parser.c b/src/flb_parser.c
@@ -152,6 +152,7 @@ struct flb_parser *flb_parser_create(const char *name, const char *format,
                                      const char *time_offset,
                                      int time_keep,
                                      int time_strict,
+                                     int time_system_timezone,
                                      int logfmt_no_bare_keys,
                                      struct flb_parser_types *types,
                                      int types_len,
@@ -312,8 +313,17 @@ struct flb_parser *flb_parser_create(const char *name, const char *format,
             p->time_frac_secs = (tmp + 2);
         }
 
-        /* Optional fixed timezone offset */
-        if (time_offset) {
+        /* 
+         * Fall back to the system timezone 
+         * if there is no zone parsed from the log.  
+         */
+        p->time_system_timezone = time_system_timezone;
+
+        /* 
+         * Optional fixed timezone offset, only applied if 
+         * not falling back to system timezone. 
+         */
+        if (!p->time_system_timezone && time_offset) {
             diff = 0;
             len = strlen(time_offset);
             ret = flb_parser_tzone_offset(time_offset, len, &diff);
@@ -487,6 +497,7 @@ static int parser_conf_file(const char *cfg, struct flb_cf *cf,
     int skip_empty;
     int time_keep;
     int time_strict;
+    int time_system_timezone;
     int logfmt_no_bare_keys;
     int types_len;
     struct mk_list *head;
@@ -561,6 +572,13 @@ static int parser_conf_file(const char *cfg, struct flb_cf *cf,
             flb_sds_destroy(tmp_str);
         }
 
+        time_system_timezone = FLB_FALSE;
+        tmp_str = get_parser_key(config, cf, s, "time_system_timezone");
+        if (tmp_str) {
+            time_system_timezone = flb_utils_bool(tmp_str);
+            flb_sds_destroy(tmp_str);
+        }
+
         /* time_offset (UTC offset) */
         time_offset = get_parser_key(config, cf, s, "time_offset");
 
@@ -587,7 +605,8 @@ static int parser_conf_file(const char *cfg, struct flb_cf *cf,
         /* Create the parser context */
         if (!flb_parser_create(name, format, regex, skip_empty,
                                time_fmt, time_key, time_offset, time_keep, time_strict,
-                               logfmt_no_bare_keys, types, types_len, decoders, config)) {
+                               time_system_timezone, logfmt_no_bare_keys, types, types_len, 
+                               decoders, config)) {
             goto fconf_error;
         }
 

diff --git a/src/flb_parser_json.c b/src/flb_parser_json.c
@@ -207,7 +207,7 @@ int flb_parser_json_do(struct flb_parser *parser,
         skip = map_size;
     }
     else {
-        time_lookup = flb_parser_tm2time(&tm);
+        time_lookup = flb_parser_tm2time(&tm, parser->time_system_timezone);
     }
 
     /* Compose a new map without the time_key field */

diff --git a/src/flb_parser_logfmt.c b/src/flb_parser_logfmt.c
@@ -166,7 +166,7 @@ static int logfmt_parser(struct flb_parser *parser,
                                   parser->name, parser->time_fmt_full);
                         return -1;
                     }
-                    *time_lookup = flb_parser_tm2time(&tm);
+                    *time_lookup = flb_parser_tm2time(&tm, parser->time_system_timezone);
                 }
                 time_found = FLB_TRUE;
             }

diff --git a/src/flb_parser_ltsv.c b/src/flb_parser_ltsv.c
@@ -139,7 +139,7 @@ static int ltsv_parser(struct flb_parser *parser,
                                  parser->name, parser->time_fmt_full);
                        return -1;
                     }
-                    *time_lookup = flb_parser_tm2time(&tm);
+                    *time_lookup = flb_parser_tm2time(&tm, parser->time_system_timezone);
                 }
                 time_found = FLB_TRUE;
             }

diff --git a/src/flb_parser_regex.c b/src/flb_parser_regex.c
@@ -87,7 +87,7 @@ static void cb_results(const char *name, const char *value,
             }
 
             pcb->time_frac = frac;
-            pcb->time_lookup = flb_parser_tm2time(&tm);
+            pcb->time_lookup = flb_parser_tm2time(&tm, parser->time_system_timezone);
 
             if (parser->time_keep == FLB_FALSE) {
                 pcb->num_skipped++;

diff --git a/src/multiline/flb_ml_parser_cri.c b/src/multiline/flb_ml_parser_cri.c
@@ -40,6 +40,7 @@ static struct flb_parser *cri_parser_create(struct flb_config *config)
                           NULL,                    /* time offset */
                           FLB_TRUE,                /* time keep */
                           FLB_FALSE,               /* time strict */
+                          FLB_FALSE,               /* time system timezone */
                           FLB_FALSE,               /* no bare keys */
                           NULL,                    /* parser types */
                           0,                       /* types len */

diff --git a/src/multiline/flb_ml_parser_docker.c b/src/multiline/flb_ml_parser_docker.c
@@ -35,6 +35,7 @@ static struct flb_parser *docker_parser_create(struct flb_config *config)
                           NULL,                   /* time offset */
                           FLB_TRUE,               /* time keep */
                           FLB_FALSE,              /* time strict */
+                          FLB_FALSE,              /* time system timezone */
                           FLB_FALSE,              /* no bare keys */
                           NULL,                   /* parser types */
                           0,                      /* types len */

diff --git a/tests/internal/fuzzers/engine_fuzzer.c b/tests/internal/fuzzers/engine_fuzzer.c
@@ -143,7 +143,7 @@ int LLVMFuzzerInitialize(int *argc, char ***argv) {
 
     parser = flb_parser_create("timestamp", "regex", "^(?<time>.*)$", FLB_TRUE,
                                 "%s.%L", "time", NULL, MK_FALSE, 0, FLB_FALSE,
-                               NULL, 0, NULL, ctx->config);
+                               FLB_FALSE, NULL, 0, NULL, ctx->config);
     filter_ffd = flb_filter(ctx, (char *) "parser", NULL);
     int ret;
     ret = flb_filter_set(ctx, filter_ffd, "Match", "test",

diff --git a/tests/internal/fuzzers/parse_json_fuzzer.c b/tests/internal/fuzzers/parse_json_fuzzer.c
@@ -51,7 +51,7 @@ int LLVMFuzzerTestOneInput(const uint8_t *data, size_t size){
     }
 
     fuzz_parser = flb_parser_create("fuzzer", "json", NULL, FLB_TRUE, NULL,
-                                    NULL, NULL, MK_FALSE, MK_TRUE, FLB_FALSE,
+                                    NULL, NULL, MK_FALSE, MK_TRUE, FLB_FALSE, FLB_FALSE,
                                     NULL, 0, NULL, fuzz_config);
     if (fuzz_parser) {
         flb_parser_do(fuzz_parser, (char*)data, size,

diff --git a/tests/internal/fuzzers/parse_logfmt_fuzzer.c b/tests/internal/fuzzers/parse_logfmt_fuzzer.c
@@ -32,8 +32,8 @@ int LLVMFuzzerTestOneInput(const uint8_t *data, size_t size){
     }
     fuzz_parser = flb_parser_create("fuzzer", "logfmt", NULL, FLB_TRUE,
                                     NULL, NULL, NULL, MK_FALSE,
-                                    MK_TRUE, FLB_FALSE, NULL, 0, NULL,
-                                    fuzz_config);
+                                    MK_TRUE, FLB_FALSE, FLB_FALSE, NULL, 0,
+                                    NULL, fuzz_config);
     if (fuzz_parser) {
         flb_parser_do(fuzz_parser, (char*)data, size,
                       &out_buf, &out_size, &out_time);

diff --git a/tests/internal/fuzzers/parser_fuzzer.c b/tests/internal/fuzzers/parser_fuzzer.c
@@ -154,7 +154,7 @@ int LLVMFuzzerTestOneInput(const uint8_t *data, size_t size)
     /* now call into the parser */
     fuzz_parser = flb_parser_create("fuzzer", format, pregex, FLB_TRUE,
             time_fmt, time_key, time_offset, time_keep, 0, FLB_FALSE,
-            types, types_len, list, fuzz_config);
+            FLB_FALSE, types, types_len, list, fuzz_config);
 
     /* Second step is to use the random parser to parse random input */
     if (fuzz_parser != NULL) {

diff --git a/tests/internal/parser.c b/tests/internal/parser.c
@@ -231,7 +231,7 @@ void test_parser_time_lookup()
             continue;
         }
 
-        epoch = flb_parser_tm2time(&tm);
+        epoch = flb_parser_tm2time(&tm, FLB_FALSE);
         epoch -= year_diff;
         TEST_CHECK(t->epoch == epoch);
         TEST_CHECK(t->frac_seconds == ns);

diff --git a/tests/internal/parser_json.c b/tests/internal/parser_json.c
@@ -171,7 +171,7 @@ void test_basic()
     }
 
     parser = flb_parser_create("json", "json", NULL, FLB_FALSE, NULL, NULL, NULL,
-                               FLB_FALSE, FLB_FALSE, FLB_FALSE,
+                               FLB_FALSE, FLB_FALSE, FLB_FALSE, FLB_FALSE,
                                NULL, 0, NULL, config);
     if (!TEST_CHECK(parser != NULL)) {
         TEST_MSG("flb_parser_create failed");
@@ -223,7 +223,7 @@ void test_time_key()
     }
 
     parser = flb_parser_create("json", "json", NULL, FLB_FALSE, "%Y-%m-%dT%H:%M:%S.%L", "time", NULL,
-                               FLB_FALSE, FLB_FALSE, FLB_FALSE,
+                               FLB_FALSE, FLB_FALSE, FLB_FALSE, FLB_FALSE,
                                NULL, 0, NULL, config);
     if (!TEST_CHECK(parser != NULL)) {
         TEST_MSG("flb_parser_create failed");
@@ -284,7 +284,7 @@ void test_time_keep()
     }
 
     parser = flb_parser_create("json", "json", NULL, FLB_FALSE, "%Y-%m-%dT%H:%M:%S.%L", "time", NULL,
-                               FLB_TRUE /*time_keep */, FLB_FALSE, FLB_FALSE,
+                               FLB_TRUE /*time_keep */, FLB_FALSE, FLB_FALSE, FLB_FALSE,
                                NULL, 0, NULL, config);
     if (!TEST_CHECK(parser != NULL)) {
         TEST_MSG("flb_parser_create failed");
@@ -365,7 +365,7 @@ void test_types_is_not_supported()
     types->type = FLB_PARSER_TYPE_HEX;
 
     parser = flb_parser_create("json", "json", NULL, FLB_FALSE, NULL, NULL, NULL,
-                               FLB_FALSE, FLB_FALSE, FLB_FALSE,
+                               FLB_FALSE, FLB_FALSE, FLB_FALSE, FLB_FALSE,
                                types, 1, NULL, config);
     if (!TEST_CHECK(parser != NULL)) {
         TEST_MSG("flb_parser_create failed");
@@ -450,7 +450,7 @@ void test_decode_field_json()
     }
 
     parser = flb_parser_create("json", "json", NULL, FLB_FALSE, NULL, NULL, NULL,
-                               FLB_FALSE, FLB_FALSE, FLB_FALSE,
+                               FLB_FALSE, FLB_FALSE, FLB_FALSE, FLB_FALSE,
                                NULL, 0, decoder, config);
     if (!TEST_CHECK(parser != NULL)) {
         TEST_MSG("flb_parser_create failed");
@@ -505,7 +505,7 @@ void test_time_key_kept_if_parse_fails()
     }
 
     parser = flb_parser_create("json", "json", NULL, FLB_FALSE, time_format, "time", NULL,
-                               FLB_FALSE, FLB_TRUE, FLB_FALSE,
+                               FLB_FALSE, FLB_TRUE, FLB_FALSE, FLB_FALSE,
                                NULL, 0, NULL, config);
     if (!TEST_CHECK(parser != NULL)) {
         TEST_MSG("flb_parser_create failed");

diff --git a/tests/internal/parser_logfmt.c b/tests/internal/parser_logfmt.c
@@ -171,7 +171,7 @@ void test_basic()
     }
 
     parser = flb_parser_create("logfmt", "logfmt", NULL, FLB_FALSE, NULL, NULL, NULL,
-                               FLB_FALSE, FLB_FALSE, FLB_FALSE,
+                               FLB_FALSE, FLB_FALSE, FLB_FALSE, FLB_FALSE,
                                NULL, 0, NULL, config);
     if (!TEST_CHECK(parser != NULL)) {
         TEST_MSG("flb_parser_create failed");
@@ -223,7 +223,7 @@ void test_time_key()
     }
 
     parser = flb_parser_create("logfmt", "logfmt", NULL, FLB_FALSE, "%Y-%m-%dT%H:%M:%S.%L", "time", NULL,
-                               FLB_FALSE, FLB_FALSE, FLB_FALSE,
+                               FLB_FALSE, FLB_FALSE, FLB_FALSE, FLB_FALSE,
                                NULL, 0, NULL, config);
     if (!TEST_CHECK(parser != NULL)) {
         TEST_MSG("flb_parser_create failed");
@@ -284,7 +284,7 @@ void test_time_keep()
     }
 
     parser = flb_parser_create("logfmt", "logfmt", NULL, FLB_FALSE, "%Y-%m-%dT%H:%M:%S.%L", "time", NULL,
-                               FLB_TRUE /*time_keep */, FLB_FALSE, FLB_FALSE,
+                               FLB_TRUE /*time_keep */, FLB_FALSE, FLB_FALSE, FLB_FALSE,
                                NULL, 0, NULL, config);
     if (!TEST_CHECK(parser != NULL)) {
         TEST_MSG("flb_parser_create failed");
@@ -361,7 +361,7 @@ void test_types()
     types->type = FLB_PARSER_TYPE_HEX;
 
     parser = flb_parser_create("logfmt", "logfmt", NULL, FLB_FALSE, NULL, NULL, NULL,
-                               FLB_FALSE, FLB_FALSE, FLB_FALSE,
+                               FLB_FALSE, FLB_FALSE, FLB_FALSE, FLB_FALSE,
                                types, 1, NULL, config);
     if (!TEST_CHECK(parser != NULL)) {
         TEST_MSG("flb_parser_create failed");

diff --git a/tests/internal/parser_ltsv.c b/tests/internal/parser_ltsv.c
@@ -171,7 +171,7 @@ void test_basic()
     }
 
     parser = flb_parser_create("ltsv", "ltsv", NULL, FLB_FALSE, NULL, NULL, NULL,
-                               FLB_FALSE, FLB_FALSE, FLB_FALSE,
+                               FLB_FALSE, FLB_FALSE, FLB_FALSE, FLB_FALSE,
                                NULL, 0, NULL, config);
     if (!TEST_CHECK(parser != NULL)) {
         TEST_MSG("flb_parser_create failed");
@@ -223,7 +223,7 @@ void test_time_key()
     }
 
     parser = flb_parser_create("ltsv", "ltsv", NULL, FLB_FALSE, "%Y-%m-%dT%H:%M:%S.%L", "time", NULL,
-                               FLB_FALSE, FLB_FALSE, FLB_FALSE,
+                               FLB_FALSE, FLB_FALSE, FLB_FALSE, FLB_FALSE,
                                NULL, 0, NULL, config);
     if (!TEST_CHECK(parser != NULL)) {
         TEST_MSG("flb_parser_create failed");
@@ -284,7 +284,7 @@ void test_time_keep()
     }
 
     parser = flb_parser_create("ltsv", "ltsv", NULL, FLB_FALSE, "%Y-%m-%dT%H:%M:%S.%L", "time", NULL,
-                               FLB_TRUE /*time_keep */, FLB_FALSE, FLB_FALSE,
+                               FLB_TRUE /*time_keep */, FLB_FALSE, FLB_FALSE, FLB_FALSE,
                                NULL, 0, NULL, config);
     if (!TEST_CHECK(parser != NULL)) {
         TEST_MSG("flb_parser_create failed");
@@ -361,7 +361,7 @@ void test_types()
     types->type = FLB_PARSER_TYPE_HEX;
 
     parser = flb_parser_create("ltsv", "ltsv", NULL, FLB_FALSE, NULL, NULL, NULL,
-                               FLB_FALSE, FLB_FALSE, FLB_FALSE,
+                               FLB_FALSE, FLB_FALSE, FLB_FALSE, FLB_FALSE,
                                types, 1, NULL, config);
     if (!TEST_CHECK(parser != NULL)) {
         TEST_MSG("flb_parser_create failed");
@@ -446,7 +446,7 @@ void test_decode_field_json()
     }
 
     parser = flb_parser_create("ltsv", "ltsv", NULL, FLB_FALSE, NULL, NULL, NULL,
-                               FLB_FALSE, FLB_FALSE, FLB_FALSE,
+                               FLB_FALSE, FLB_FALSE, FLB_FALSE, FLB_FALSE,
                                NULL, 0, decoder, config);
     if (!TEST_CHECK(parser != NULL)) {
         TEST_MSG("flb_parser_create failed");

diff --git a/tests/internal/parser_regex.c b/tests/internal/parser_regex.c
@@ -172,7 +172,7 @@ void test_basic()
     }
 
     parser = flb_parser_create("regex", "regex", regex, FLB_FALSE, NULL, NULL, NULL,
-                               FLB_FALSE, FLB_FALSE, FLB_FALSE,
+                               FLB_FALSE, FLB_FALSE, FLB_FALSE, FLB_FALSE,
                                NULL, 0, NULL, config);
     if (!TEST_CHECK(parser != NULL)) {
         TEST_MSG("flb_parser_create failed");
@@ -225,7 +225,7 @@ void test_time_key()
     }
 
     parser = flb_parser_create("regex", "regex", regex, FLB_FALSE, "%Y-%m-%dT%H:%M:%S.%L", "time", NULL,
-                               FLB_FALSE, FLB_FALSE, FLB_FALSE,
+                               FLB_FALSE, FLB_FALSE, FLB_FALSE, FLB_FALSE,
                                NULL, 0, NULL, config);
     if (!TEST_CHECK(parser != NULL)) {
         TEST_MSG("flb_parser_create failed");
@@ -287,7 +287,7 @@ void test_time_keep()
     }
 
     parser = flb_parser_create("regex", "regex", regex, FLB_FALSE, "%Y-%m-%dT%H:%M:%S.%L", "time", NULL,
-                               FLB_TRUE /*time_keep */, FLB_FALSE, FLB_FALSE,
+                               FLB_TRUE /*time_keep */, FLB_FALSE, FLB_FALSE, FLB_FALSE,
                                NULL, 0, NULL, config);
     if (!TEST_CHECK(parser != NULL)) {
         TEST_MSG("flb_parser_create failed");
@@ -365,7 +365,7 @@ void test_types()
     types->type = FLB_PARSER_TYPE_HEX;
 
     parser = flb_parser_create("regex", "regex", regex, FLB_FALSE, NULL, NULL, NULL,
-                               FLB_FALSE, FLB_FALSE, FLB_FALSE,
+                               FLB_FALSE, FLB_FALSE, FLB_FALSE, FLB_FALSE,
                                types, 1, NULL, config);
     if (!TEST_CHECK(parser != NULL)) {
         TEST_MSG("flb_parser_create failed");
@@ -451,7 +451,7 @@ void test_decode_field_json()
     }
 
     parser = flb_parser_create("regex", "regex", regex, FLB_FALSE, NULL, NULL, NULL,
-                               FLB_FALSE, FLB_FALSE, FLB_FALSE,
+                               FLB_FALSE, FLB_FALSE, FLB_FALSE, FLB_FALSE,
                                NULL, 0, decoder, config);
     if (!TEST_CHECK(parser != NULL)) {
         TEST_MSG("flb_parser_create failed");