From 0e4dd45c807aa533d6158ad5145dcd9791a70507 Mon Sep 17 00:00:00 2001 From: vircoys Date: Thu, 28 Sep 2023 18:07:51 +0800 Subject: [PATCH] feat: Migrate all GuanceCloud pipeline functions to platypus (#45) --- go.mod | 99 ++- go.sum | 247 ++++-- internal/cmd/platypus/run/run.go | 29 +- pkg/engine/callref_test.go | 2 +- pkg/engine/op_test.go | 10 +- pkg/hash/fnv1a.go | 45 + pkg/hash/fnv1a_test.go | 110 +++ pkg/inimpl/guancecloud/funcs/all.go | 80 -- pkg/inimpl/guancecloud/funcs/fn_grok_test.go | 219 ----- pkg/inimpl/guancecloud/funcs/fn_printf.go | 66 -- .../guancecloud/funcs/fn_printf_test.go | 51 -- .../guancecloud/funcs/fn_replace_test.go | 121 --- pkg/inimpl/guancecloud/funcs/fn_use_test.go | 72 -- pkg/inimpl/guancecloud/funcs/md/fn.md | 810 ------------------ pkg/inimpl/guancecloud/input/point.go | 316 ------- pkg/inimpl/guancecloud/input/point_test.go | 38 - pkg/inimpl/guancecloud/ptinput/funcs/all.go | 146 ++++ .../guancecloud/ptinput/funcs/all_doc.go | 688 +++++++++++++++ .../guancecloud/ptinput/funcs/all_doc_en.go | 562 ++++++++++++ .../ptinput/funcs/all_doc_en_test.go | 52 ++ .../guancecloud/ptinput/funcs/all_doc_test.go | 49 ++ .../{ => ptinput}/funcs/all_test.go | 24 +- .../ptinput/funcs/dateparse_test.go | 164 ++++ .../{ => ptinput}/funcs/fn_addkey.go | 10 +- .../{ => ptinput}/funcs/fn_addkey_test.go | 15 +- .../{ => ptinput}/funcs/fn_addpattern.go | 0 .../{ => ptinput}/funcs/fn_addpattern_test.go | 14 +- .../ptinput/funcs/fn_adjust_timezone.go | 93 ++ .../ptinput/funcs/fn_adjust_timezone_test.go | 166 ++++ .../guancecloud/ptinput/funcs/fn_agg.go | 285 ++++++ .../guancecloud/ptinput/funcs/fn_agg_test.go | 130 +++ .../guancecloud/ptinput/funcs/fn_append.go | 75 ++ .../ptinput/funcs/fn_append_test.go | 106 +++ .../guancecloud/ptinput/funcs/fn_b64dec.go | 70 ++ .../ptinput/funcs/fn_b64dec_test.go | 79 ++ .../guancecloud/ptinput/funcs/fn_b64enc.go | 67 ++ .../ptinput/funcs/fn_b64enc_test.go | 78 ++ .../{ => ptinput}/funcs/fn_cast.go | 4 +- .../{ => ptinput}/funcs/fn_cast_test.go | 21 +- .../guancecloud/ptinput/funcs/fn_cidr.go | 108 +++ .../guancecloud/ptinput/funcs/fn_cidr_test.go | 76 ++ .../funcs/fn_conv_traceid_hex_to_dec.go | 84 ++ .../ptinput/funcs/fn_conv_traceid_test.go | 129 +++ .../guancecloud/ptinput/funcs/fn_cover.go | 153 ++++ .../ptinput/funcs/fn_cover_test.go | 150 ++++ .../ptinput/funcs/fn_create_point.go | 191 +++++ .../ptinput/funcs/fn_create_point_test.go | 335 ++++++++ .../{ => ptinput}/funcs/fn_datetime.go | 77 +- .../{ => ptinput}/funcs/fn_datetime_test.go | 231 ++--- .../guancecloud/ptinput/funcs/fn_decoder.go | 120 +++ .../ptinput/funcs/fn_decoder_test.go | 82 ++ .../{ => ptinput}/funcs/fn_default_time.go | 20 +- .../funcs/fn_default_time_test.go | 219 ++--- .../ptinput/funcs/fn_default_time_with_fmt.go | 115 +++ .../funcs/fn_default_time_with_fmt_test.go | 124 +++ .../guancecloud/ptinput/funcs/fn_delete.go | 150 ++++ .../ptinput/funcs/fn_delete_test.go | 125 +++ .../guancecloud/ptinput/funcs/fn_drop.go | 23 + .../ptinput/funcs/fn_drop_origin_data.go | 21 + .../ptinput/funcs/fn_drop_origin_data_test.go | 60 ++ .../guancecloud/ptinput/funcs/fn_drop_test.go | 68 ++ .../{ => ptinput}/funcs/fn_dropkey.go | 0 .../{ => ptinput}/funcs/fn_dropkey_test.go | 19 +- .../ptinput/funcs/fn_duration_precision.go | 94 ++ .../funcs/fn_duration_precision_test.go | 102 +++ .../{ => ptinput}/funcs/fn_exit.go | 0 .../{ => ptinput}/funcs/fn_exit_test.go | 28 +- .../guancecloud/ptinput/funcs/fn_expr.go | 22 + .../guancecloud/ptinput/funcs/fn_geo_ip.go | 76 ++ .../ptinput/funcs/fn_gep_ip_test.go | 149 ++++ .../{ => ptinput}/funcs/fn_getkey.go | 0 .../{ => ptinput}/funcs/fn_getkey_test.go | 12 +- .../{ => ptinput}/funcs/fn_grok.go | 4 +- .../guancecloud/ptinput/funcs/fn_grok_test.go | 223 +++++ .../guancecloud/ptinput/funcs/fn_group.go | 165 ++++ .../guancecloud/ptinput/funcs/fn_group_in.go | 108 +++ .../ptinput/funcs/fn_group_in_test.go | 110 +++ .../ptinput/funcs/fn_group_test.go | 102 +++ .../guancecloud/ptinput/funcs/fn_json.go | 310 +++++++ .../guancecloud/ptinput/funcs/fn_json_test.go | 164 ++++ .../guancecloud/ptinput/funcs/fn_jsonall.go | 22 + pkg/inimpl/guancecloud/ptinput/funcs/fn_kv.go | 328 +++++++ .../guancecloud/ptinput/funcs/fn_kv_test.go | 108 +++ .../guancecloud/{ => ptinput}/funcs/fn_len.go | 0 .../{ => ptinput}/funcs/fn_len_test.go | 22 +- .../{ => ptinput}/funcs/fn_load_json.go | 5 +- .../{ => ptinput}/funcs/fn_load_json_test.go | 21 +- .../guancecloud/ptinput/funcs/fn_lowercase.go | 56 ++ .../ptinput/funcs/fn_lowercase_test.go | 106 +++ .../guancecloud/ptinput/funcs/fn_match.go | 72 ++ .../ptinput/funcs/fn_match_test.go | 77 ++ .../guancecloud/ptinput/funcs/fn_nullif.go | 63 ++ .../ptinput/funcs/fn_nullif_test.go | 99 +++ .../ptinput/funcs/fn_parse_date.go | 158 ++++ .../ptinput/funcs/fn_parse_date_test.go | 278 ++++++ .../ptinput/funcs/fn_parse_duration.go | 69 ++ .../ptinput/funcs/fn_parse_duration_test.go | 171 ++++ .../guancecloud/ptinput/funcs/fn_parse_int.go | 110 +++ .../ptinput/funcs/fn_parse_int_test.go | 164 ++++ .../{ => ptinput}/funcs/fn_rename.go | 0 .../{ => ptinput}/funcs/fn_rename_test.go | 17 +- .../{ => ptinput}/funcs/fn_replace.go | 37 +- .../ptinput/funcs/fn_replace_test.go | 121 +++ .../guancecloud/ptinput/funcs/fn_sample.go | 80 ++ .../ptinput/funcs/fn_sample_test.go | 99 +++ .../{ => ptinput}/funcs/fn_set_measurement.go | 0 .../funcs/fn_set_mesaurement_test.go | 20 +- .../{ => ptinput}/funcs/fn_set_tag.go | 6 +- .../{ => ptinput}/funcs/fn_set_tag_test.go | 27 +- .../{ => ptinput}/funcs/fn_sql_cover.go | 4 +- .../{ => ptinput}/funcs/fn_sql_cover_test.go | 52 +- .../{ => ptinput}/funcs/fn_strfmt.go | 4 +- .../{ => ptinput}/funcs/fn_strfmt_test.go | 82 +- .../guancecloud/ptinput/funcs/fn_timestamp.go | 48 ++ .../ptinput/funcs/fn_timstamp_test.go | 100 +++ .../{ => ptinput}/funcs/fn_trim.go | 4 +- .../{ => ptinput}/funcs/fn_trim_test.go | 19 +- .../{ => ptinput}/funcs/fn_uppercase.go | 4 +- .../{ => ptinput}/funcs/fn_uppercase_test.go | 51 +- .../guancecloud/ptinput/funcs/fn_url_parse.go | 69 ++ .../ptinput/funcs/fn_url_parse_test.go | 150 ++++ .../{ => ptinput}/funcs/fn_urldecode.go | 4 +- .../{ => ptinput}/funcs/fn_urldecode_test.go | 25 +- .../guancecloud/{ => ptinput}/funcs/fn_use.go | 5 +- .../guancecloud/ptinput/funcs/fn_use_test.go | 62 ++ .../guancecloud/ptinput/funcs/fn_useragent.go | 54 ++ .../ptinput/funcs/fn_useragent_test.go | 140 +++ .../ptinput/funcs/fn_valid_json.go | 41 + .../ptinput/funcs/fn_valid_json_test.go | 100 +++ .../ptinput/funcs/fn_value_type.go | 47 + .../ptinput/funcs/fn_value_type_test.go | 181 ++++ .../guancecloud/{ => ptinput}/funcs/fn_xml.go | 4 +- .../{ => ptinput}/funcs/fn_xml_test.go | 25 +- .../funcs/func_query_refer_table_test.go | 268 ++++++ .../{ => ptinput}/funcs/funcs_b_test.go | 14 +- .../funcs/funcs_benchmark_test.go | 73 +- .../ptinput/funcs/funcs_mquery_refer_table.go | 191 +++++ .../ptinput/funcs/funcs_query_refer_table.go | 158 ++++ .../{ => ptinput}/funcs/funcs_test.go | 8 +- .../guancecloud/{ => ptinput}/funcs/handle.go | 67 +- .../guancecloud/ptinput/funcs/handle_test.go | 27 + .../{ => ptinput}/funcs/ifelse_test.go | 19 +- .../ptinput/funcs/md/add_key.en.md | 27 + .../guancecloud/ptinput/funcs/md/add_key.md | 27 + .../ptinput/funcs/md/add_pattern.en.md | 54 ++ .../ptinput/funcs/md/add_pattern.md | 54 ++ .../ptinput/funcs/md/adjust_timezone.en.md | 59 ++ .../ptinput/funcs/md/adjust_timezone.md | 58 ++ .../ptinput/funcs/md/agg_create.en.md | 20 + .../ptinput/funcs/md/agg_create.md | 20 + .../ptinput/funcs/md/agg_metric.en.md | 57 ++ .../ptinput/funcs/md/agg_metric.md | 56 ++ .../guancecloud/ptinput/funcs/md/append.en.md | 25 + .../guancecloud/ptinput/funcs/md/append.md | 25 + .../guancecloud/ptinput/funcs/md/b64dec.en.md | 22 + .../guancecloud/ptinput/funcs/md/b64dec.md | 22 + .../guancecloud/ptinput/funcs/md/b64enc.en.md | 22 + .../guancecloud/ptinput/funcs/md/b64enc.md | 22 + .../guancecloud/ptinput/funcs/md/cast.en.md | 25 + .../guancecloud/ptinput/funcs/md/cast.md | 25 + .../guancecloud/ptinput/funcs/md/cidr.en.md | 25 + .../guancecloud/ptinput/funcs/md/cidr.md | 28 + .../funcs/md/conv_traceid_w3c_to_dd.en.md | 31 + .../funcs/md/conv_traceid_w3c_to_dd.md | 31 + .../guancecloud/ptinput/funcs/md/cover.en.md | 22 + .../guancecloud/ptinput/funcs/md/cover.md | 22 + .../ptinput/funcs/md/create_point.en.md | 25 + .../ptinput/funcs/md/create_point.md | 26 + .../ptinput/funcs/md/datetime.en.md | 103 +++ .../guancecloud/ptinput/funcs/md/datetime.md | 101 +++ .../guancecloud/ptinput/funcs/md/decode.en.md | 16 + .../guancecloud/ptinput/funcs/md/decode.md | 14 + .../ptinput/funcs/md/default_time.en.md | 79 ++ .../ptinput/funcs/md/default_time.md | 80 ++ .../guancecloud/ptinput/funcs/md/delete.en.md | 25 + .../guancecloud/ptinput/funcs/md/delete.md | 25 + .../guancecloud/ptinput/funcs/md/drop.en.md | 23 + .../guancecloud/ptinput/funcs/md/drop.md | 22 + .../ptinput/funcs/md/drop_key.en.md | 26 + .../guancecloud/ptinput/funcs/md/drop_key.md | 28 + .../ptinput/funcs/md/drop_origin_data.en.md | 14 + .../ptinput/funcs/md/drop_origin_data.md | 15 + .../ptinput/funcs/md/duration_precision.en.md | 20 + .../ptinput/funcs/md/duration_precision.md | 18 + .../guancecloud/ptinput/funcs/md/exit.en.md | 20 + .../guancecloud/ptinput/funcs/md/exit.md | 21 + .../ptinput/funcs/md/format_int.en.md | 46 + .../ptinput/funcs/md/format_int.md | 46 + .../guancecloud/ptinput/funcs/md/geoip.en.md | 34 + .../guancecloud/ptinput/funcs/md/geoip.md | 34 + .../ptinput/funcs/md/get_key.en.md | 40 + .../guancecloud/ptinput/funcs/md/get_key.md | 40 + .../guancecloud/ptinput/funcs/md/grok.en.md | 51 ++ .../guancecloud/ptinput/funcs/md/grok.md | 52 ++ .../ptinput/funcs/md/group_between.en.md | 38 + .../ptinput/funcs/md/group_between.md | 38 + .../ptinput/funcs/md/group_in.en.md | 15 + .../guancecloud/ptinput/funcs/md/group_in.md | 15 + .../guancecloud/ptinput/funcs/md/json.en.md | 110 +++ .../guancecloud/ptinput/funcs/md/json.md | 109 +++ .../ptinput/funcs/md/kv_split.en.md | 95 ++ .../guancecloud/ptinput/funcs/md/kv_split.md | 95 ++ .../guancecloud/ptinput/funcs/md/len.en.md | 27 + .../guancecloud/ptinput/funcs/md/len.md | 27 + .../ptinput/funcs/md/load_json.en.md | 27 + .../guancecloud/ptinput/funcs/md/load_json.md | 27 + .../ptinput/funcs/md/lowercase.en.md | 23 + .../guancecloud/ptinput/funcs/md/lowercase.md | 24 + .../guancecloud/ptinput/funcs/md/match.en.md | 28 + .../guancecloud/ptinput/funcs/md/match.md | 28 + .../ptinput/funcs/md/mquery_refer_table.en.md | 38 + .../ptinput/funcs/md/mquery_refer_table.md | 40 + .../guancecloud/ptinput/funcs/md/nullif.en.md | 34 + .../guancecloud/ptinput/funcs/md/nullif.md | 33 + .../ptinput/funcs/md/parse_date.en.md | 31 + .../ptinput/funcs/md/parse_date.md | 31 + .../ptinput/funcs/md/parse_duration.en.md | 31 + .../ptinput/funcs/md/parse_duration.md | 33 + .../ptinput/funcs/md/parse_int.en.md | 69 ++ .../guancecloud/ptinput/funcs/md/parse_int.md | 69 ++ .../ptinput/funcs/md/query_refer_table.en.md | 41 + .../ptinput/funcs/md/query_refer_table.md | 41 + .../guancecloud/ptinput/funcs/md/rename.en.md | 38 + .../guancecloud/ptinput/funcs/md/rename.md | 39 + .../ptinput/funcs/md/replace.en.md | 31 + .../guancecloud/ptinput/funcs/md/replace.md | 32 + .../guancecloud/ptinput/funcs/md/sample.en.md | 18 + .../guancecloud/ptinput/funcs/md/sample.md | 19 + .../ptinput/funcs/md/set_measurement.en.md | 25 + .../ptinput/funcs/md/set_measurement.md | 24 + .../ptinput/funcs/md/set_tag.en.md | 44 + .../guancecloud/ptinput/funcs/md/set_tag.md | 44 + .../ptinput/funcs/md/sql_cover.en.md | 17 + .../guancecloud/ptinput/funcs/md/sql_cover.md | 15 + .../guancecloud/ptinput/funcs/md/strfmt.en.md | 24 + .../guancecloud/ptinput/funcs/md/strfmt.md | 24 + .../ptinput/funcs/md/timestamp.en.md | 54 ++ .../guancecloud/ptinput/funcs/md/timestamp.md | 51 ++ .../guancecloud/ptinput/funcs/md/trim.en.md | 25 + .../guancecloud/ptinput/funcs/md/trim.md | 25 + .../ptinput/funcs/md/uppercase.en.md | 23 + .../guancecloud/ptinput/funcs/md/uppercase.md | 24 + .../ptinput/funcs/md/url_decode.en.md | 24 + .../ptinput/funcs/md/url_decode.md | 24 + .../ptinput/funcs/md/url_parse.en.md | 47 + .../guancecloud/ptinput/funcs/md/url_parse.md | 47 + .../guancecloud/ptinput/funcs/md/use.en.md | 32 + .../guancecloud/ptinput/funcs/md/use.md | 31 + .../ptinput/funcs/md/user_agent.en.md | 28 + .../ptinput/funcs/md/user_agent.md | 28 + .../ptinput/funcs/md/valid_json.en.md | 48 ++ .../ptinput/funcs/md/valid_json.md | 48 ++ .../ptinput/funcs/md/value_type.en.md | 38 + .../ptinput/funcs/md/value_type.md | 37 + .../guancecloud/ptinput/funcs/md/xml.en.md | 60 ++ .../guancecloud/ptinput/funcs/md/xml.md | 60 ++ .../guancecloud/{ => ptinput}/funcs/utils.go | 289 +++++-- .../{ => ptinput}/funcs/utilts_test.go | 2 +- pkg/inimpl/guancecloud/ptinput/ip2isp/bit.go | 265 ++++++ pkg/inimpl/guancecloud/ptinput/ip2isp/isp.go | 137 +++ pkg/inimpl/guancecloud/ptinput/ipdb/bit.go | 265 ++++++ .../ptinput/ipdb/geoip/geolite2.go | 114 +++ pkg/inimpl/guancecloud/ptinput/ipdb/ipdb.go | 83 ++ .../guancecloud/ptinput/ipdb/ipdb_test.go | 67 ++ .../guancecloud/ptinput/ipdb/iploc/iploc.go | 190 ++++ .../ptinput/ipdb/iploc/iploc_test.go | 79 ++ .../guancecloud/ptinput/plmap/aggmap.go | 456 ++++++++++ .../guancecloud/ptinput/plmap/aggmap_test.go | 146 ++++ .../guancecloud/ptinput/plmap/manager.go | 19 + pkg/inimpl/guancecloud/ptinput/point.go | 319 +++++++ pkg/inimpl/guancecloud/ptinput/point_test.go | 120 +++ .../guancecloud/ptinput/refertable/cli.go | 51 ++ .../guancecloud/ptinput/refertable/query.go | 111 +++ .../ptinput/refertable/query_test.go | 196 +++++ .../guancecloud/ptinput/refertable/runner.go | 232 +++++ .../ptinput/refertable/runner_test.go | 118 +++ .../guancecloud/ptinput/refertable/table.go | 236 +++++ .../ptinput/refertable/table_sqlite_other.go | 227 +++++ .../ptinput/refertable/table_sqlite_test.go | 211 +++++ .../refertable/table_sqlite_windows_386.go | 34 + .../ptinput/refertable/table_test.go | 122 +++ pkg/inimpl/guancecloud/ptinput/utils/utils.go | 34 + 282 files changed, 19772 insertions(+), 2537 deletions(-) create mode 100644 pkg/hash/fnv1a.go create mode 100644 pkg/hash/fnv1a_test.go delete mode 100644 pkg/inimpl/guancecloud/funcs/all.go delete mode 100644 pkg/inimpl/guancecloud/funcs/fn_grok_test.go delete mode 100644 pkg/inimpl/guancecloud/funcs/fn_printf.go delete mode 100644 pkg/inimpl/guancecloud/funcs/fn_printf_test.go delete mode 100644 pkg/inimpl/guancecloud/funcs/fn_replace_test.go delete mode 100644 pkg/inimpl/guancecloud/funcs/fn_use_test.go delete mode 100644 pkg/inimpl/guancecloud/funcs/md/fn.md delete mode 100644 pkg/inimpl/guancecloud/input/point.go delete mode 100644 pkg/inimpl/guancecloud/input/point_test.go create mode 100644 pkg/inimpl/guancecloud/ptinput/funcs/all.go create mode 100644 pkg/inimpl/guancecloud/ptinput/funcs/all_doc.go create mode 100644 pkg/inimpl/guancecloud/ptinput/funcs/all_doc_en.go create mode 100644 pkg/inimpl/guancecloud/ptinput/funcs/all_doc_en_test.go create mode 100644 pkg/inimpl/guancecloud/ptinput/funcs/all_doc_test.go rename pkg/inimpl/guancecloud/{ => ptinput}/funcs/all_test.go (58%) create mode 100644 pkg/inimpl/guancecloud/ptinput/funcs/dateparse_test.go rename pkg/inimpl/guancecloud/{ => ptinput}/funcs/fn_addkey.go (80%) rename pkg/inimpl/guancecloud/{ => ptinput}/funcs/fn_addkey_test.go (90%) rename pkg/inimpl/guancecloud/{ => ptinput}/funcs/fn_addpattern.go (100%) rename pkg/inimpl/guancecloud/{ => ptinput}/funcs/fn_addpattern_test.go (94%) create mode 100644 pkg/inimpl/guancecloud/ptinput/funcs/fn_adjust_timezone.go create mode 100644 pkg/inimpl/guancecloud/ptinput/funcs/fn_adjust_timezone_test.go create mode 100644 pkg/inimpl/guancecloud/ptinput/funcs/fn_agg.go create mode 100644 pkg/inimpl/guancecloud/ptinput/funcs/fn_agg_test.go create mode 100644 pkg/inimpl/guancecloud/ptinput/funcs/fn_append.go create mode 100644 pkg/inimpl/guancecloud/ptinput/funcs/fn_append_test.go create mode 100644 pkg/inimpl/guancecloud/ptinput/funcs/fn_b64dec.go create mode 100644 pkg/inimpl/guancecloud/ptinput/funcs/fn_b64dec_test.go create mode 100644 pkg/inimpl/guancecloud/ptinput/funcs/fn_b64enc.go create mode 100644 pkg/inimpl/guancecloud/ptinput/funcs/fn_b64enc_test.go rename pkg/inimpl/guancecloud/{ => ptinput}/funcs/fn_cast.go (95%) rename pkg/inimpl/guancecloud/{ => ptinput}/funcs/fn_cast_test.go (91%) create mode 100644 pkg/inimpl/guancecloud/ptinput/funcs/fn_cidr.go create mode 100644 pkg/inimpl/guancecloud/ptinput/funcs/fn_cidr_test.go create mode 100644 pkg/inimpl/guancecloud/ptinput/funcs/fn_conv_traceid_hex_to_dec.go create mode 100644 pkg/inimpl/guancecloud/ptinput/funcs/fn_conv_traceid_test.go create mode 100644 pkg/inimpl/guancecloud/ptinput/funcs/fn_cover.go create mode 100644 pkg/inimpl/guancecloud/ptinput/funcs/fn_cover_test.go create mode 100644 pkg/inimpl/guancecloud/ptinput/funcs/fn_create_point.go create mode 100644 pkg/inimpl/guancecloud/ptinput/funcs/fn_create_point_test.go rename pkg/inimpl/guancecloud/{ => ptinput}/funcs/fn_datetime.go (55%) rename pkg/inimpl/guancecloud/{ => ptinput}/funcs/fn_datetime_test.go (53%) create mode 100644 pkg/inimpl/guancecloud/ptinput/funcs/fn_decoder.go create mode 100644 pkg/inimpl/guancecloud/ptinput/funcs/fn_decoder_test.go rename pkg/inimpl/guancecloud/{ => ptinput}/funcs/fn_default_time.go (84%) rename pkg/inimpl/guancecloud/{ => ptinput}/funcs/fn_default_time_test.go (81%) create mode 100644 pkg/inimpl/guancecloud/ptinput/funcs/fn_default_time_with_fmt.go create mode 100644 pkg/inimpl/guancecloud/ptinput/funcs/fn_default_time_with_fmt_test.go create mode 100644 pkg/inimpl/guancecloud/ptinput/funcs/fn_delete.go create mode 100644 pkg/inimpl/guancecloud/ptinput/funcs/fn_delete_test.go create mode 100644 pkg/inimpl/guancecloud/ptinput/funcs/fn_drop.go create mode 100644 pkg/inimpl/guancecloud/ptinput/funcs/fn_drop_origin_data.go create mode 100644 pkg/inimpl/guancecloud/ptinput/funcs/fn_drop_origin_data_test.go create mode 100644 pkg/inimpl/guancecloud/ptinput/funcs/fn_drop_test.go rename pkg/inimpl/guancecloud/{ => ptinput}/funcs/fn_dropkey.go (100%) rename pkg/inimpl/guancecloud/{ => ptinput}/funcs/fn_dropkey_test.go (78%) create mode 100644 pkg/inimpl/guancecloud/ptinput/funcs/fn_duration_precision.go create mode 100644 pkg/inimpl/guancecloud/ptinput/funcs/fn_duration_precision_test.go rename pkg/inimpl/guancecloud/{ => ptinput}/funcs/fn_exit.go (100%) rename pkg/inimpl/guancecloud/{ => ptinput}/funcs/fn_exit_test.go (71%) create mode 100644 pkg/inimpl/guancecloud/ptinput/funcs/fn_expr.go create mode 100644 pkg/inimpl/guancecloud/ptinput/funcs/fn_geo_ip.go create mode 100644 pkg/inimpl/guancecloud/ptinput/funcs/fn_gep_ip_test.go rename pkg/inimpl/guancecloud/{ => ptinput}/funcs/fn_getkey.go (100%) rename pkg/inimpl/guancecloud/{ => ptinput}/funcs/fn_getkey_test.go (81%) rename pkg/inimpl/guancecloud/{ => ptinput}/funcs/fn_grok.go (95%) create mode 100644 pkg/inimpl/guancecloud/ptinput/funcs/fn_grok_test.go create mode 100644 pkg/inimpl/guancecloud/ptinput/funcs/fn_group.go create mode 100644 pkg/inimpl/guancecloud/ptinput/funcs/fn_group_in.go create mode 100644 pkg/inimpl/guancecloud/ptinput/funcs/fn_group_in_test.go create mode 100644 pkg/inimpl/guancecloud/ptinput/funcs/fn_group_test.go create mode 100644 pkg/inimpl/guancecloud/ptinput/funcs/fn_json.go create mode 100644 pkg/inimpl/guancecloud/ptinput/funcs/fn_json_test.go create mode 100644 pkg/inimpl/guancecloud/ptinput/funcs/fn_jsonall.go create mode 100644 pkg/inimpl/guancecloud/ptinput/funcs/fn_kv.go create mode 100644 pkg/inimpl/guancecloud/ptinput/funcs/fn_kv_test.go rename pkg/inimpl/guancecloud/{ => ptinput}/funcs/fn_len.go (100%) rename pkg/inimpl/guancecloud/{ => ptinput}/funcs/fn_len_test.go (73%) rename pkg/inimpl/guancecloud/{ => ptinput}/funcs/fn_load_json.go (88%) rename pkg/inimpl/guancecloud/{ => ptinput}/funcs/fn_load_json_test.go (83%) create mode 100644 pkg/inimpl/guancecloud/ptinput/funcs/fn_lowercase.go create mode 100644 pkg/inimpl/guancecloud/ptinput/funcs/fn_lowercase_test.go create mode 100644 pkg/inimpl/guancecloud/ptinput/funcs/fn_match.go create mode 100644 pkg/inimpl/guancecloud/ptinput/funcs/fn_match_test.go create mode 100644 pkg/inimpl/guancecloud/ptinput/funcs/fn_nullif.go create mode 100644 pkg/inimpl/guancecloud/ptinput/funcs/fn_nullif_test.go create mode 100644 pkg/inimpl/guancecloud/ptinput/funcs/fn_parse_date.go create mode 100644 pkg/inimpl/guancecloud/ptinput/funcs/fn_parse_date_test.go create mode 100644 pkg/inimpl/guancecloud/ptinput/funcs/fn_parse_duration.go create mode 100644 pkg/inimpl/guancecloud/ptinput/funcs/fn_parse_duration_test.go create mode 100644 pkg/inimpl/guancecloud/ptinput/funcs/fn_parse_int.go create mode 100644 pkg/inimpl/guancecloud/ptinput/funcs/fn_parse_int_test.go rename pkg/inimpl/guancecloud/{ => ptinput}/funcs/fn_rename.go (100%) rename pkg/inimpl/guancecloud/{ => ptinput}/funcs/fn_rename_test.go (86%) rename pkg/inimpl/guancecloud/{ => ptinput}/funcs/fn_replace.go (84%) create mode 100644 pkg/inimpl/guancecloud/ptinput/funcs/fn_replace_test.go create mode 100644 pkg/inimpl/guancecloud/ptinput/funcs/fn_sample.go create mode 100644 pkg/inimpl/guancecloud/ptinput/funcs/fn_sample_test.go rename pkg/inimpl/guancecloud/{ => ptinput}/funcs/fn_set_measurement.go (100%) rename pkg/inimpl/guancecloud/{ => ptinput}/funcs/fn_set_mesaurement_test.go (83%) rename pkg/inimpl/guancecloud/{ => ptinput}/funcs/fn_set_tag.go (90%) rename pkg/inimpl/guancecloud/{ => ptinput}/funcs/fn_set_tag_test.go (84%) rename pkg/inimpl/guancecloud/{ => ptinput}/funcs/fn_sql_cover.go (94%) rename pkg/inimpl/guancecloud/{ => ptinput}/funcs/fn_sql_cover_test.go (66%) rename pkg/inimpl/guancecloud/{ => ptinput}/funcs/fn_strfmt.go (95%) rename pkg/inimpl/guancecloud/{ => ptinput}/funcs/fn_strfmt_test.go (64%) create mode 100644 pkg/inimpl/guancecloud/ptinput/funcs/fn_timestamp.go create mode 100644 pkg/inimpl/guancecloud/ptinput/funcs/fn_timstamp_test.go rename pkg/inimpl/guancecloud/{ => ptinput}/funcs/fn_trim.go (95%) rename pkg/inimpl/guancecloud/{ => ptinput}/funcs/fn_trim_test.go (76%) rename pkg/inimpl/guancecloud/{ => ptinput}/funcs/fn_uppercase.go (93%) rename pkg/inimpl/guancecloud/{ => ptinput}/funcs/fn_uppercase_test.go (69%) create mode 100644 pkg/inimpl/guancecloud/ptinput/funcs/fn_url_parse.go create mode 100644 pkg/inimpl/guancecloud/ptinput/funcs/fn_url_parse_test.go rename pkg/inimpl/guancecloud/{ => ptinput}/funcs/fn_urldecode.go (93%) rename pkg/inimpl/guancecloud/{ => ptinput}/funcs/fn_urldecode_test.go (79%) rename pkg/inimpl/guancecloud/{ => ptinput}/funcs/fn_use.go (93%) create mode 100644 pkg/inimpl/guancecloud/ptinput/funcs/fn_use_test.go create mode 100644 pkg/inimpl/guancecloud/ptinput/funcs/fn_useragent.go create mode 100644 pkg/inimpl/guancecloud/ptinput/funcs/fn_useragent_test.go create mode 100644 pkg/inimpl/guancecloud/ptinput/funcs/fn_valid_json.go create mode 100644 pkg/inimpl/guancecloud/ptinput/funcs/fn_valid_json_test.go create mode 100644 pkg/inimpl/guancecloud/ptinput/funcs/fn_value_type.go create mode 100644 pkg/inimpl/guancecloud/ptinput/funcs/fn_value_type_test.go rename pkg/inimpl/guancecloud/{ => ptinput}/funcs/fn_xml.go (96%) rename pkg/inimpl/guancecloud/{ => ptinput}/funcs/fn_xml_test.go (85%) create mode 100644 pkg/inimpl/guancecloud/ptinput/funcs/func_query_refer_table_test.go rename pkg/inimpl/guancecloud/{ => ptinput}/funcs/funcs_b_test.go (98%) rename pkg/inimpl/guancecloud/{ => ptinput}/funcs/funcs_benchmark_test.go (88%) create mode 100644 pkg/inimpl/guancecloud/ptinput/funcs/funcs_mquery_refer_table.go create mode 100644 pkg/inimpl/guancecloud/ptinput/funcs/funcs_query_refer_table.go rename pkg/inimpl/guancecloud/{ => ptinput}/funcs/funcs_test.go (96%) rename pkg/inimpl/guancecloud/{ => ptinput}/funcs/handle.go (86%) create mode 100644 pkg/inimpl/guancecloud/ptinput/funcs/handle_test.go rename pkg/inimpl/guancecloud/{ => ptinput}/funcs/ifelse_test.go (86%) create mode 100644 pkg/inimpl/guancecloud/ptinput/funcs/md/add_key.en.md create mode 100644 pkg/inimpl/guancecloud/ptinput/funcs/md/add_key.md create mode 100644 pkg/inimpl/guancecloud/ptinput/funcs/md/add_pattern.en.md create mode 100644 pkg/inimpl/guancecloud/ptinput/funcs/md/add_pattern.md create mode 100644 pkg/inimpl/guancecloud/ptinput/funcs/md/adjust_timezone.en.md create mode 100644 pkg/inimpl/guancecloud/ptinput/funcs/md/adjust_timezone.md create mode 100644 pkg/inimpl/guancecloud/ptinput/funcs/md/agg_create.en.md create mode 100644 pkg/inimpl/guancecloud/ptinput/funcs/md/agg_create.md create mode 100644 pkg/inimpl/guancecloud/ptinput/funcs/md/agg_metric.en.md create mode 100644 pkg/inimpl/guancecloud/ptinput/funcs/md/agg_metric.md create mode 100644 pkg/inimpl/guancecloud/ptinput/funcs/md/append.en.md create mode 100644 pkg/inimpl/guancecloud/ptinput/funcs/md/append.md create mode 100644 pkg/inimpl/guancecloud/ptinput/funcs/md/b64dec.en.md create mode 100644 pkg/inimpl/guancecloud/ptinput/funcs/md/b64dec.md create mode 100644 pkg/inimpl/guancecloud/ptinput/funcs/md/b64enc.en.md create mode 100644 pkg/inimpl/guancecloud/ptinput/funcs/md/b64enc.md create mode 100644 pkg/inimpl/guancecloud/ptinput/funcs/md/cast.en.md create mode 100644 pkg/inimpl/guancecloud/ptinput/funcs/md/cast.md create mode 100644 pkg/inimpl/guancecloud/ptinput/funcs/md/cidr.en.md create mode 100644 pkg/inimpl/guancecloud/ptinput/funcs/md/cidr.md create mode 100644 pkg/inimpl/guancecloud/ptinput/funcs/md/conv_traceid_w3c_to_dd.en.md create mode 100644 pkg/inimpl/guancecloud/ptinput/funcs/md/conv_traceid_w3c_to_dd.md create mode 100644 pkg/inimpl/guancecloud/ptinput/funcs/md/cover.en.md create mode 100644 pkg/inimpl/guancecloud/ptinput/funcs/md/cover.md create mode 100644 pkg/inimpl/guancecloud/ptinput/funcs/md/create_point.en.md create mode 100644 pkg/inimpl/guancecloud/ptinput/funcs/md/create_point.md create mode 100644 pkg/inimpl/guancecloud/ptinput/funcs/md/datetime.en.md create mode 100644 pkg/inimpl/guancecloud/ptinput/funcs/md/datetime.md create mode 100644 pkg/inimpl/guancecloud/ptinput/funcs/md/decode.en.md create mode 100644 pkg/inimpl/guancecloud/ptinput/funcs/md/decode.md create mode 100644 pkg/inimpl/guancecloud/ptinput/funcs/md/default_time.en.md create mode 100644 pkg/inimpl/guancecloud/ptinput/funcs/md/default_time.md create mode 100644 pkg/inimpl/guancecloud/ptinput/funcs/md/delete.en.md create mode 100644 pkg/inimpl/guancecloud/ptinput/funcs/md/delete.md create mode 100644 pkg/inimpl/guancecloud/ptinput/funcs/md/drop.en.md create mode 100644 pkg/inimpl/guancecloud/ptinput/funcs/md/drop.md create mode 100644 pkg/inimpl/guancecloud/ptinput/funcs/md/drop_key.en.md create mode 100644 pkg/inimpl/guancecloud/ptinput/funcs/md/drop_key.md create mode 100644 pkg/inimpl/guancecloud/ptinput/funcs/md/drop_origin_data.en.md create mode 100644 pkg/inimpl/guancecloud/ptinput/funcs/md/drop_origin_data.md create mode 100644 pkg/inimpl/guancecloud/ptinput/funcs/md/duration_precision.en.md create mode 100644 pkg/inimpl/guancecloud/ptinput/funcs/md/duration_precision.md create mode 100644 pkg/inimpl/guancecloud/ptinput/funcs/md/exit.en.md create mode 100644 pkg/inimpl/guancecloud/ptinput/funcs/md/exit.md create mode 100644 pkg/inimpl/guancecloud/ptinput/funcs/md/format_int.en.md create mode 100644 pkg/inimpl/guancecloud/ptinput/funcs/md/format_int.md create mode 100644 pkg/inimpl/guancecloud/ptinput/funcs/md/geoip.en.md create mode 100644 pkg/inimpl/guancecloud/ptinput/funcs/md/geoip.md create mode 100644 pkg/inimpl/guancecloud/ptinput/funcs/md/get_key.en.md create mode 100644 pkg/inimpl/guancecloud/ptinput/funcs/md/get_key.md create mode 100644 pkg/inimpl/guancecloud/ptinput/funcs/md/grok.en.md create mode 100644 pkg/inimpl/guancecloud/ptinput/funcs/md/grok.md create mode 100644 pkg/inimpl/guancecloud/ptinput/funcs/md/group_between.en.md create mode 100644 pkg/inimpl/guancecloud/ptinput/funcs/md/group_between.md create mode 100644 pkg/inimpl/guancecloud/ptinput/funcs/md/group_in.en.md create mode 100644 pkg/inimpl/guancecloud/ptinput/funcs/md/group_in.md create mode 100644 pkg/inimpl/guancecloud/ptinput/funcs/md/json.en.md create mode 100644 pkg/inimpl/guancecloud/ptinput/funcs/md/json.md create mode 100644 pkg/inimpl/guancecloud/ptinput/funcs/md/kv_split.en.md create mode 100644 pkg/inimpl/guancecloud/ptinput/funcs/md/kv_split.md create mode 100644 pkg/inimpl/guancecloud/ptinput/funcs/md/len.en.md create mode 100644 pkg/inimpl/guancecloud/ptinput/funcs/md/len.md create mode 100644 pkg/inimpl/guancecloud/ptinput/funcs/md/load_json.en.md create mode 100644 pkg/inimpl/guancecloud/ptinput/funcs/md/load_json.md create mode 100644 pkg/inimpl/guancecloud/ptinput/funcs/md/lowercase.en.md create mode 100644 pkg/inimpl/guancecloud/ptinput/funcs/md/lowercase.md create mode 100644 pkg/inimpl/guancecloud/ptinput/funcs/md/match.en.md create mode 100644 pkg/inimpl/guancecloud/ptinput/funcs/md/match.md create mode 100644 pkg/inimpl/guancecloud/ptinput/funcs/md/mquery_refer_table.en.md create mode 100644 pkg/inimpl/guancecloud/ptinput/funcs/md/mquery_refer_table.md create mode 100644 pkg/inimpl/guancecloud/ptinput/funcs/md/nullif.en.md create mode 100644 pkg/inimpl/guancecloud/ptinput/funcs/md/nullif.md create mode 100644 pkg/inimpl/guancecloud/ptinput/funcs/md/parse_date.en.md create mode 100644 pkg/inimpl/guancecloud/ptinput/funcs/md/parse_date.md create mode 100644 pkg/inimpl/guancecloud/ptinput/funcs/md/parse_duration.en.md create mode 100644 pkg/inimpl/guancecloud/ptinput/funcs/md/parse_duration.md create mode 100644 pkg/inimpl/guancecloud/ptinput/funcs/md/parse_int.en.md create mode 100644 pkg/inimpl/guancecloud/ptinput/funcs/md/parse_int.md create mode 100644 pkg/inimpl/guancecloud/ptinput/funcs/md/query_refer_table.en.md create mode 100644 pkg/inimpl/guancecloud/ptinput/funcs/md/query_refer_table.md create mode 100644 pkg/inimpl/guancecloud/ptinput/funcs/md/rename.en.md create mode 100644 pkg/inimpl/guancecloud/ptinput/funcs/md/rename.md create mode 100644 pkg/inimpl/guancecloud/ptinput/funcs/md/replace.en.md create mode 100644 pkg/inimpl/guancecloud/ptinput/funcs/md/replace.md create mode 100644 pkg/inimpl/guancecloud/ptinput/funcs/md/sample.en.md create mode 100644 pkg/inimpl/guancecloud/ptinput/funcs/md/sample.md create mode 100644 pkg/inimpl/guancecloud/ptinput/funcs/md/set_measurement.en.md create mode 100644 pkg/inimpl/guancecloud/ptinput/funcs/md/set_measurement.md create mode 100644 pkg/inimpl/guancecloud/ptinput/funcs/md/set_tag.en.md create mode 100644 pkg/inimpl/guancecloud/ptinput/funcs/md/set_tag.md create mode 100644 pkg/inimpl/guancecloud/ptinput/funcs/md/sql_cover.en.md create mode 100644 pkg/inimpl/guancecloud/ptinput/funcs/md/sql_cover.md create mode 100644 pkg/inimpl/guancecloud/ptinput/funcs/md/strfmt.en.md create mode 100644 pkg/inimpl/guancecloud/ptinput/funcs/md/strfmt.md create mode 100644 pkg/inimpl/guancecloud/ptinput/funcs/md/timestamp.en.md create mode 100644 pkg/inimpl/guancecloud/ptinput/funcs/md/timestamp.md create mode 100644 pkg/inimpl/guancecloud/ptinput/funcs/md/trim.en.md create mode 100644 pkg/inimpl/guancecloud/ptinput/funcs/md/trim.md create mode 100644 pkg/inimpl/guancecloud/ptinput/funcs/md/uppercase.en.md create mode 100644 pkg/inimpl/guancecloud/ptinput/funcs/md/uppercase.md create mode 100644 pkg/inimpl/guancecloud/ptinput/funcs/md/url_decode.en.md create mode 100644 pkg/inimpl/guancecloud/ptinput/funcs/md/url_decode.md create mode 100644 pkg/inimpl/guancecloud/ptinput/funcs/md/url_parse.en.md create mode 100644 pkg/inimpl/guancecloud/ptinput/funcs/md/url_parse.md create mode 100644 pkg/inimpl/guancecloud/ptinput/funcs/md/use.en.md create mode 100644 pkg/inimpl/guancecloud/ptinput/funcs/md/use.md create mode 100644 pkg/inimpl/guancecloud/ptinput/funcs/md/user_agent.en.md create mode 100644 pkg/inimpl/guancecloud/ptinput/funcs/md/user_agent.md create mode 100644 pkg/inimpl/guancecloud/ptinput/funcs/md/valid_json.en.md create mode 100644 pkg/inimpl/guancecloud/ptinput/funcs/md/valid_json.md create mode 100644 pkg/inimpl/guancecloud/ptinput/funcs/md/value_type.en.md create mode 100644 pkg/inimpl/guancecloud/ptinput/funcs/md/value_type.md create mode 100644 pkg/inimpl/guancecloud/ptinput/funcs/md/xml.en.md create mode 100644 pkg/inimpl/guancecloud/ptinput/funcs/md/xml.md rename pkg/inimpl/guancecloud/{ => ptinput}/funcs/utils.go (55%) rename pkg/inimpl/guancecloud/{ => ptinput}/funcs/utilts_test.go (98%) create mode 100644 pkg/inimpl/guancecloud/ptinput/ip2isp/bit.go create mode 100644 pkg/inimpl/guancecloud/ptinput/ip2isp/isp.go create mode 100644 pkg/inimpl/guancecloud/ptinput/ipdb/bit.go create mode 100644 pkg/inimpl/guancecloud/ptinput/ipdb/geoip/geolite2.go create mode 100644 pkg/inimpl/guancecloud/ptinput/ipdb/ipdb.go create mode 100644 pkg/inimpl/guancecloud/ptinput/ipdb/ipdb_test.go create mode 100644 pkg/inimpl/guancecloud/ptinput/ipdb/iploc/iploc.go create mode 100644 pkg/inimpl/guancecloud/ptinput/ipdb/iploc/iploc_test.go create mode 100644 pkg/inimpl/guancecloud/ptinput/plmap/aggmap.go create mode 100644 pkg/inimpl/guancecloud/ptinput/plmap/aggmap_test.go create mode 100644 pkg/inimpl/guancecloud/ptinput/plmap/manager.go create mode 100644 pkg/inimpl/guancecloud/ptinput/point.go create mode 100644 pkg/inimpl/guancecloud/ptinput/point_test.go create mode 100644 pkg/inimpl/guancecloud/ptinput/refertable/cli.go create mode 100644 pkg/inimpl/guancecloud/ptinput/refertable/query.go create mode 100644 pkg/inimpl/guancecloud/ptinput/refertable/query_test.go create mode 100644 pkg/inimpl/guancecloud/ptinput/refertable/runner.go create mode 100644 pkg/inimpl/guancecloud/ptinput/refertable/runner_test.go create mode 100644 pkg/inimpl/guancecloud/ptinput/refertable/table.go create mode 100644 pkg/inimpl/guancecloud/ptinput/refertable/table_sqlite_other.go create mode 100644 pkg/inimpl/guancecloud/ptinput/refertable/table_sqlite_test.go create mode 100644 pkg/inimpl/guancecloud/ptinput/refertable/table_sqlite_windows_386.go create mode 100644 pkg/inimpl/guancecloud/ptinput/refertable/table_test.go create mode 100644 pkg/inimpl/guancecloud/ptinput/utils/utils.go diff --git a/go.mod b/go.mod index 58cf29b..648768e 100644 --- a/go.mod +++ b/go.mod @@ -3,42 +3,93 @@ module github.com/GuanceCloud/platypus go 1.19 require ( - github.com/DataDog/datadog-agent/pkg/obfuscate v0.39.0 - github.com/GuanceCloud/grok v1.1.2 - github.com/antchfx/xmlquery v1.3.12 + github.com/DataDog/datadog-agent/pkg/obfuscate v0.47.1 + github.com/GuanceCloud/cliutils v0.1.5-0.20230926031706-4c70e9a29d6a + github.com/GuanceCloud/grok v1.1.3 + github.com/antchfx/xmlquery v1.3.18 github.com/araddon/dateparse v0.0.0-20201001162425-8aadafed4dc4 + github.com/hashicorp/go-retryablehttp v0.7.4 github.com/influxdata/influxdb1-client v0.0.0-20220302092344-a9ab5670611c - github.com/mssola/user_agent v0.5.3 - github.com/spf13/cast v1.5.0 - github.com/spf13/cobra v1.5.0 - github.com/stretchr/testify v1.8.0 - github.com/tidwall/gjson v1.14.3 + github.com/ip2location/ip2location-go v8.3.0+incompatible + github.com/itchyny/timefmt-go v0.1.5 + github.com/mssola/user_agent v0.6.0 + github.com/oschwald/geoip2-golang v1.9.0 + github.com/spf13/cast v1.5.1 + github.com/spf13/cobra v1.7.0 + github.com/stretchr/testify v1.8.4 + github.com/tidwall/gjson v1.17.0 go.uber.org/zap v1.23.0 + golang.org/x/text v0.11.0 + gopkg.in/yaml.v2 v2.4.0 + modernc.org/sqlite v1.25.0 ) require ( - github.com/DataDog/datadog-go/v5 v5.1.0 // indirect - github.com/Microsoft/go-winio v0.5.1 // indirect - github.com/antchfx/xpath v1.2.1 // indirect - github.com/cespare/xxhash/v2 v2.1.1 // indirect + github.com/BurntSushi/toml v1.3.2 // indirect + github.com/DataDog/datadog-go/v5 v5.1.1 // indirect + github.com/Microsoft/go-winio v0.6.1 // indirect + github.com/aliyun/aliyun-oss-go-sdk v2.1.2+incompatible // indirect + github.com/antchfx/xpath v1.2.4 // indirect + github.com/bytedance/sonic v1.8.0 // indirect + github.com/cespare/xxhash/v2 v2.2.0 // indirect + github.com/chenzhuoyu/base64x v0.0.0-20221115062448-fe3a3abad311 // indirect github.com/cpuguy83/go-md2man/v2 v2.0.2 // indirect - github.com/davecgh/go-spew v1.1.1 // indirect - github.com/dgraph-io/ristretto v0.1.0 // indirect - github.com/dustin/go-humanize v1.0.0 // indirect - github.com/golang/glog v0.0.0-20160126235308-23def4e6c14b // indirect - github.com/golang/groupcache v0.0.0-20200121045136-8c9f03a8e57e // indirect - github.com/inconshreveable/mousetrap v1.0.0 // indirect + github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc // indirect + github.com/dustin/go-humanize v1.0.1 // indirect + github.com/gin-contrib/sse v0.1.0 // indirect + github.com/gin-gonic/gin v1.9.0 // indirect + github.com/go-playground/locales v0.14.1 // indirect + github.com/go-playground/universal-translator v0.18.1 // indirect + github.com/go-playground/validator/v10 v10.11.2 // indirect + github.com/goccy/go-json v0.10.0 // indirect + github.com/golang/groupcache v0.0.0-20210331224755-41bb18bfe9da // indirect + github.com/golang/protobuf v1.5.3 // indirect + github.com/google/uuid v1.3.0 // indirect + github.com/hashicorp/go-cleanhttp v0.5.2 // indirect + github.com/inconshreveable/mousetrap v1.1.0 // indirect + github.com/json-iterator/go v1.1.12 // indirect + github.com/kballard/go-shellquote v0.0.0-20180428030007-95032a82bc51 // indirect + github.com/klauspost/cpuid/v2 v2.2.3 // indirect + github.com/leodido/go-urn v1.2.1 // indirect + github.com/mattn/go-isatty v0.0.19 // indirect + github.com/matttproud/golang_protobuf_extensions v1.0.4 // indirect + github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd // indirect + github.com/modern-go/reflect2 v1.0.2 // indirect + github.com/oschwald/maxminddb-golang v1.11.0 // indirect + github.com/outcaste-io/ristretto v0.2.1 // indirect + github.com/pelletier/go-toml/v2 v2.0.6 // indirect github.com/pkg/errors v0.9.1 // indirect - github.com/pmezard/go-difflib v1.0.0 // indirect + github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2 // indirect + github.com/prometheus/client_model v0.4.0 // indirect + github.com/prometheus/common v0.44.0 // indirect + github.com/remyoudompheng/bigfft v0.0.0-20230129092748-24d4a6f8daec // indirect + github.com/rs/xid v1.2.1 // indirect github.com/russross/blackfriday/v2 v2.1.0 // indirect + github.com/satori/go.uuid v1.2.0 // indirect github.com/spf13/pflag v1.0.5 // indirect github.com/tidwall/match v1.1.1 // indirect github.com/tidwall/pretty v1.2.0 // indirect - go.uber.org/atomic v1.9.0 // indirect + github.com/twitchyliquid64/golang-asm v0.15.1 // indirect + github.com/ugorji/go/codec v1.2.9 // indirect + go.uber.org/atomic v1.11.0 // indirect go.uber.org/multierr v1.6.0 // indirect - golang.org/x/net v0.0.0-20220127200216-cd36cc0744dd // indirect - golang.org/x/sys v0.0.0-20220804214406-8e32c043e418 // indirect - golang.org/x/text v0.3.7 // indirect - gopkg.in/yaml.v2 v2.4.0 // indirect + golang.org/x/arch v0.0.0-20210923205945-b76863e36670 // indirect + golang.org/x/crypto v0.11.0 // indirect + golang.org/x/mod v0.12.0 // indirect + golang.org/x/net v0.12.0 // indirect + golang.org/x/sys v0.10.0 // indirect + golang.org/x/time v0.3.0 // indirect + golang.org/x/tools v0.11.0 // indirect + google.golang.org/protobuf v1.31.0 // indirect + gopkg.in/natefinch/lumberjack.v2 v2.0.0 // indirect gopkg.in/yaml.v3 v3.0.1 // indirect + lukechampine.com/uint128 v1.2.0 // indirect + modernc.org/cc/v3 v3.40.0 // indirect + modernc.org/ccgo/v3 v3.16.13 // indirect + modernc.org/libc v1.24.1 // indirect + modernc.org/mathutil v1.5.0 // indirect + modernc.org/memory v1.6.0 // indirect + modernc.org/opt v0.1.3 // indirect + modernc.org/strutil v1.1.3 // indirect + modernc.org/token v1.0.1 // indirect ) diff --git a/go.sum b/go.sum index 4174c65..f65df02 100644 --- a/go.sum +++ b/go.sum @@ -1,129 +1,278 @@ -github.com/DataDog/datadog-agent/pkg/obfuscate v0.39.0 h1:kXdXGOYeH2pKa3tdNGdrixT2tnXrJ4xLE9WypL4X7HE= -github.com/DataDog/datadog-agent/pkg/obfuscate v0.39.0/go.mod h1:MxVcCIC42tBIjPm93BHdh9/vw2LivRiptj3HygI+GGQ= -github.com/DataDog/datadog-go/v5 v5.1.0 h1:Zmq3tCk9+Tdq8Du73M71Zo6Dyx+cEo9QkCSCqQlHFaQ= -github.com/DataDog/datadog-go/v5 v5.1.0/go.mod h1:KhiYb2Badlv9/rofz+OznKoEF5XKTonWyhx5K83AP8E= -github.com/GuanceCloud/grok v1.1.2 h1:xscdb39kbPjiO8HfhORt/wDZuQqhi8N/WyjNcQnWujA= -github.com/GuanceCloud/grok v1.1.2/go.mod h1:AHkJZYf7Qbo1FTZT6htdyScpICpgnkQ5+Hc0EmA88vM= +github.com/BurntSushi/toml v1.3.2 h1:o7IhLm0Msx3BaB+n3Ag7L8EVlByGnpq14C4YWiu/gL8= +github.com/BurntSushi/toml v1.3.2/go.mod h1:CxXYINrC8qIiEnFrOxCa7Jy5BFHlXnUU2pbicEuybxQ= +github.com/DataDog/datadog-agent/pkg/obfuscate v0.47.1 h1:KILbqSN3VXtfzQAzAMemgE/bR6nJYEVnw3il2Tk4YjA= +github.com/DataDog/datadog-agent/pkg/obfuscate v0.47.1/go.mod h1:e933RWa4kAWuHi5jpzEuOiULlv21HcCFEVIYegmaB5c= +github.com/DataDog/datadog-go/v5 v5.1.1 h1:JLZ6s2K1pG2h9GkvEvMdEGqMDyVLEAccdX5TltWcLMU= +github.com/DataDog/datadog-go/v5 v5.1.1/go.mod h1:KhiYb2Badlv9/rofz+OznKoEF5XKTonWyhx5K83AP8E= +github.com/GuanceCloud/cliutils v0.1.5-0.20230926031706-4c70e9a29d6a h1:5KxYeDSCen6CqfIA4EqNVFTqxBpkvunAyZ+5luWHops= +github.com/GuanceCloud/cliutils v0.1.5-0.20230926031706-4c70e9a29d6a/go.mod h1:F3RrCmZ8YLeHgaRPIMkPgEiO5s6/GHHfE9QLzzLBeMI= +github.com/GuanceCloud/grok v1.1.3 h1:0Phqza1ChCro6r9YG6AfpBB7GM0aYSXFRREFmbjF5AI= +github.com/GuanceCloud/grok v1.1.3/go.mod h1:AHkJZYf7Qbo1FTZT6htdyScpICpgnkQ5+Hc0EmA88vM= github.com/Microsoft/go-winio v0.5.0/go.mod h1:JPGBdM1cNvN/6ISo+n8V5iA4v8pBzdOpzfwIujj1a84= -github.com/Microsoft/go-winio v0.5.1 h1:aPJp2QD7OOrhO5tQXqQoGSJc+DjDtWTGLOmNyAm6FgY= github.com/Microsoft/go-winio v0.5.1/go.mod h1:JPGBdM1cNvN/6ISo+n8V5iA4v8pBzdOpzfwIujj1a84= -github.com/antchfx/xmlquery v1.3.12 h1:6TMGpdjpO/P8VhjnaYPXuqT3qyJ/VsqoyNTmJzNBTQ4= -github.com/antchfx/xmlquery v1.3.12/go.mod h1:3w2RvQvTz+DaT5fSgsELkSJcdNgkmg6vuXDEuhdwsPQ= -github.com/antchfx/xpath v1.2.1 h1:qhp4EW6aCOVr5XIkT+l6LJ9ck/JsUH/yyauNgTQkBF8= -github.com/antchfx/xpath v1.2.1/go.mod h1:i54GszH55fYfBmoZXapTHN8T8tkcHfRgLyVwwqzXNcs= +github.com/Microsoft/go-winio v0.6.1 h1:9/kr64B9VUZrLm5YYwbGtUJnMgqWVOdUAXu6Migciow= +github.com/Microsoft/go-winio v0.6.1/go.mod h1:LRdKpFKfdobln8UmuiYcKPot9D2v6svN5+sAH+4kjUM= +github.com/aliyun/aliyun-oss-go-sdk v2.1.2+incompatible h1:kB3yKV/bBZLX5Rm8aOaFsM6M7ZmyOiUgAElhfn/R27c= +github.com/aliyun/aliyun-oss-go-sdk v2.1.2+incompatible/go.mod h1:T/Aws4fEfogEE9v+HPhhw+CntffsBHJ8nXQCwKr0/g8= +github.com/antchfx/xmlquery v1.3.18 h1:FSQ3wMuphnPPGJOFhvc+cRQ2CT/rUj4cyQXkJcjOwz0= +github.com/antchfx/xmlquery v1.3.18/go.mod h1:Afkq4JIeXut75taLSuI31ISJ/zeq+3jG7TunF7noreA= +github.com/antchfx/xpath v1.2.4 h1:dW1HB/JxKvGtJ9WyVGJ0sIoEcqftV3SqIstujI+B9XY= +github.com/antchfx/xpath v1.2.4/go.mod h1:i54GszH55fYfBmoZXapTHN8T8tkcHfRgLyVwwqzXNcs= github.com/araddon/dateparse v0.0.0-20201001162425-8aadafed4dc4 h1:OkS1BqB3CzLtGRznRyvriSY8jeaVk2CrDn2ZiRQgMUI= github.com/araddon/dateparse v0.0.0-20201001162425-8aadafed4dc4/go.mod h1:hMAUZFIkk4B1FouGxqlogyMyU6BwY/UiVmmbbzz9Up8= +github.com/baiyubin/aliyun-sts-go-sdk v0.0.0-20180326062324-cfa1a18b161f h1:ZNv7On9kyUzm7fvRZumSyy/IUiSC7AzL0I1jKKtwooA= github.com/benbjohnson/clock v1.1.0 h1:Q92kusRqC1XV2MjkWETPvjJVqKetz1OzxZB7mHJLju8= -github.com/cespare/xxhash/v2 v2.1.1 h1:6MnRN8NT7+YBpUIWxHtefFZOKTAPgGjpQSxqLNn0+qY= +github.com/bytedance/sonic v1.5.0/go.mod h1:ED5hyg4y6t3/9Ku1R6dU/4KyJ48DZ4jPhfY1O2AihPM= +github.com/bytedance/sonic v1.8.0 h1:ea0Xadu+sHlu7x5O3gKhRpQ1IKiMrSiHttPF0ybECuA= +github.com/bytedance/sonic v1.8.0/go.mod h1:i736AoUSYt75HyZLoJW9ERYxcy6eaN6h4BZXU064P/U= github.com/cespare/xxhash/v2 v2.1.1/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs= +github.com/cespare/xxhash/v2 v2.2.0 h1:DC2CZ1Ep5Y4k3ZQ899DldepgrayRUGE6BBZ/cd9Cj44= +github.com/cespare/xxhash/v2 v2.2.0/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs= +github.com/chenzhuoyu/base64x v0.0.0-20211019084208-fb5309c8db06/go.mod h1:DH46F32mSOjUmXrMHnKwZdA8wcEefY7UVqBKYGjpdQY= +github.com/chenzhuoyu/base64x v0.0.0-20221115062448-fe3a3abad311 h1:qSGYFH7+jGhDF8vLC+iwCD4WpbV1EBDSzWkJODFLams= +github.com/chenzhuoyu/base64x v0.0.0-20221115062448-fe3a3abad311/go.mod h1:b583jCggY9gE99b6G5LEC39OIiVsWj+R97kbl5odCEk= github.com/cpuguy83/go-md2man/v2 v2.0.2 h1:p1EgwI/C7NhT0JmVkwCD2ZBK8j4aeHQX2pMHHBfMQ6w= github.com/cpuguy83/go-md2man/v2 v2.0.2/go.mod h1:tgQtvFlXSQOSOSIRvRPT7W67SCa46tRHOmNcaadrF8o= github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= -github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= -github.com/dgraph-io/ristretto v0.1.0 h1:Jv3CGQHp9OjuMBSne1485aDpUkTKEcUqF+jm/LuerPI= -github.com/dgraph-io/ristretto v0.1.0/go.mod h1:fux0lOrBhrVCJd3lcTHsIJhq1T2rokOu6v9Vcb3Q9ug= +github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc h1:U9qPSI2PIWSS1VwoXQT9A3Wy9MM3WgvqSxFWenqJduM= +github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/dgryski/go-farm v0.0.0-20190423205320-6a90982ecee2 h1:tdlZCpZ/P9DhczCTSixgIKmwPv6+wP5DGjqLYw5SUiA= github.com/dgryski/go-farm v0.0.0-20190423205320-6a90982ecee2/go.mod h1:SqUrOPUnsFjfmXRMNPybcSiG0BgUW2AuFH8PAnS2iTw= -github.com/dustin/go-humanize v1.0.0 h1:VSnTsYCnlFHaM2/igO1h6X3HA71jcobQuxemgkq4zYo= github.com/dustin/go-humanize v1.0.0/go.mod h1:HtrtbFcZ19U5GC7JDqmcUSB87Iq5E25KnS6fMYU6eOk= -github.com/frankban/quicktest v1.14.3 h1:FJKSZTDHjyhriyC81FLQ0LY93eSai0ZyR/ZIkd3ZUKE= -github.com/golang/glog v0.0.0-20160126235308-23def4e6c14b h1:VKtxabqXZkF25pY9ekfRL6a582T4P37/31XEstQ5p58= -github.com/golang/glog v0.0.0-20160126235308-23def4e6c14b/go.mod h1:SBH7ygxi8pfUlaOkMMuAQtPIUF8ecWP5IEl/CR7VP2Q= -github.com/golang/groupcache v0.0.0-20200121045136-8c9f03a8e57e h1:1r7pUrabqp18hOBcwBwiTsbnFeTZHV9eER/QT5JVZxY= -github.com/golang/groupcache v0.0.0-20200121045136-8c9f03a8e57e/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc= +github.com/dustin/go-humanize v1.0.1 h1:GzkhY7T5VNhEkwH0PVJgjz+fX1rhBrR7pRT3mDkpeCY= +github.com/dustin/go-humanize v1.0.1/go.mod h1:Mu1zIs6XwVuF/gI1OepvI0qD18qycQx+mFykh5fBlto= +github.com/frankban/quicktest v1.14.4 h1:g2rn0vABPOOXmZUj+vbmUp0lPoXEMuhTpIluN0XL9UY= +github.com/gin-contrib/sse v0.1.0 h1:Y/yl/+YNO8GZSjAhjMsSuLt29uWRFHdHYUb5lYOV9qE= +github.com/gin-contrib/sse v0.1.0/go.mod h1:RHrZQHXnP2xjPF+u1gW/2HnVO7nvIa9PG3Gm+fLHvGI= +github.com/gin-gonic/gin v1.9.0 h1:OjyFBKICoexlu99ctXNR2gg+c5pKrKMuyjgARg9qeY8= +github.com/gin-gonic/gin v1.9.0/go.mod h1:W1Me9+hsUSyj3CePGrd1/QrKJMSJ1Tu/0hFEH89961k= +github.com/go-playground/assert/v2 v2.2.0 h1:JvknZsQTYeFEAhQwI4qEt9cyV5ONwRHC+lYKSsYSR8s= +github.com/go-playground/locales v0.14.1 h1:EWaQ/wswjilfKLTECiXz7Rh+3BjFhfDFKv/oXslEjJA= +github.com/go-playground/locales v0.14.1/go.mod h1:hxrqLVvrK65+Rwrd5Fc6F2O76J/NuW9t0sjnWqG1slY= +github.com/go-playground/universal-translator v0.18.1 h1:Bcnm0ZwsGyWbCzImXv+pAJnYK9S473LQFuzCbDbfSFY= +github.com/go-playground/universal-translator v0.18.1/go.mod h1:xekY+UJKNuX9WP91TpwSH2VMlDf28Uj24BCp08ZFTUY= +github.com/go-playground/validator/v10 v10.11.2 h1:q3SHpufmypg+erIExEKUmsgmhDTyhcJ38oeKGACXohU= +github.com/go-playground/validator/v10 v10.11.2/go.mod h1:NieE624vt4SCTJtD87arVLvdmjPAeV8BQlHtMnw9D7s= +github.com/goccy/go-json v0.10.0 h1:mXKd9Qw4NuzShiRlOXKews24ufknHO7gx30lsDyokKA= +github.com/goccy/go-json v0.10.0/go.mod h1:6MelG93GURQebXPDq3khkgXZkazVtN9CRI+MGFi0w8I= +github.com/golang/groupcache v0.0.0-20210331224755-41bb18bfe9da h1:oI5xCqsCo564l8iNU+DwB5epxmsaqB+rhGL0m5jtYqE= +github.com/golang/groupcache v0.0.0-20210331224755-41bb18bfe9da/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc= github.com/golang/mock v1.6.0/go.mod h1:p6yTPP+5HYm5mzsMV8JkE6ZKdX+/wYM6Hr+LicevLPs= -github.com/google/go-cmp v0.5.7 h1:81/ik6ipDQS2aGcBfIN5dHDB36BwrStyeAQquSYCV4o= -github.com/inconshreveable/mousetrap v1.0.0 h1:Z8tu5sraLXCXIcARxBp/8cbvlwVa7Z1NHg9XEKhtSvM= -github.com/inconshreveable/mousetrap v1.0.0/go.mod h1:PxqpIevigyE2G7u3NXJIT2ANytuPF1OarO4DADm73n8= +github.com/golang/protobuf v1.2.0/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= +github.com/golang/protobuf v1.5.0/go.mod h1:FsONVRAS9T7sI+LIUmWTfcYkHO4aIWwzhcaSAoJOfIk= +github.com/golang/protobuf v1.5.3 h1:KhyjKVUg7Usr/dYsdSqoFveMYd5ko72D+zANwlG1mmg= +github.com/golang/protobuf v1.5.3/go.mod h1:XVQd3VNwM+JqD3oG2Ue2ip4fOMUkwXdXDdiuN0vRsmY= +github.com/google/go-cmp v0.5.5/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= +github.com/google/go-cmp v0.5.9 h1:O2Tfq5qg4qc4AmwVlvv0oLiVAGB7enBSJ2x2DqQFi38= +github.com/google/gofuzz v1.0.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/M65Eg= +github.com/google/pprof v0.0.0-20221118152302-e6195bd50e26 h1:Xim43kblpZXfIBQsbuBVKCudVG457BR2GZFIz3uw3hQ= +github.com/google/uuid v1.3.0 h1:t6JiXgmwXMjEs8VusXIJk2BXHsn+wx8BZdTaoZ5fu7I= +github.com/google/uuid v1.3.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= +github.com/hashicorp/go-cleanhttp v0.5.2 h1:035FKYIWjmULyFRBKPs8TBQoi0x6d9G4xc9neXJWAZQ= +github.com/hashicorp/go-cleanhttp v0.5.2/go.mod h1:kO/YDlP8L1346E6Sodw+PrpBSV4/SoxCXGY6BqNFT48= +github.com/hashicorp/go-hclog v0.9.2 h1:CG6TE5H9/JXsFWJCfoIVpKFIkFe6ysEuHirp4DxCsHI= +github.com/hashicorp/go-hclog v0.9.2/go.mod h1:5CU+agLiy3J7N7QjHK5d05KxGsuXiQLrjA0H7acj2lQ= +github.com/hashicorp/go-retryablehttp v0.7.4 h1:ZQgVdpTdAL7WpMIwLzCfbalOcSUdkDZnpUv3/+BxzFA= +github.com/hashicorp/go-retryablehttp v0.7.4/go.mod h1:Jy/gPYAdjqffZ/yFGCFV2doI5wjtH1ewM9u8iYVjtX8= +github.com/inconshreveable/mousetrap v1.1.0 h1:wN+x4NVGpMsO7ErUn/mUI3vEoE6Jt13X2s0bqwp9tc8= +github.com/inconshreveable/mousetrap v1.1.0/go.mod h1:vpF70FUmC8bwa3OWnCshd2FqLfsEA9PFc4w1p2J65bw= github.com/influxdata/influxdb1-client v0.0.0-20220302092344-a9ab5670611c h1:qSHzRbhzK8RdXOsAdfDgO49TtqC1oZ+acxPrkfTxcCs= github.com/influxdata/influxdb1-client v0.0.0-20220302092344-a9ab5670611c/go.mod h1:qj24IKcXYK6Iy9ceXlo3Tc+vtHo9lIhSX5JddghvEPo= -github.com/kr/pretty v0.3.0 h1:WgNl7dwNpEZ6jJ9k1snq4pZsg7DOEN8hP9Xw0Tsjwk0= +github.com/ip2location/ip2location-go v8.3.0+incompatible h1:QwUE+FlSbo6bjOWZpv2Grb57vJhWYFNPyBj2KCvfWaM= +github.com/ip2location/ip2location-go v8.3.0+incompatible/go.mod h1:3JUY1TBjTx1GdA7oRT7Zeqfc0bg3lMMuU5lXmzdpuME= +github.com/itchyny/timefmt-go v0.1.5 h1:G0INE2la8S6ru/ZI5JecgyzbbJNs5lG1RcBqa7Jm6GE= +github.com/itchyny/timefmt-go v0.1.5/go.mod h1:nEP7L+2YmAbT2kZ2HfSs1d8Xtw9LY8D2stDBckWakZ8= +github.com/json-iterator/go v1.1.12 h1:PV8peI4a0ysnczrg+LtxykD8LfKY9ML6u2jnxaEnrnM= +github.com/json-iterator/go v1.1.12/go.mod h1:e30LSqwooZae/UwlEbR2852Gd8hjQvJoHmT4TnhNGBo= +github.com/kballard/go-shellquote v0.0.0-20180428030007-95032a82bc51 h1:Z9n2FFNUXsshfwJMBgNA0RU6/i7WVaAegv3PtuIHPMs= +github.com/kballard/go-shellquote v0.0.0-20180428030007-95032a82bc51/go.mod h1:CzGEWj7cYgsdH8dAjBGEr58BoE7ScuLd+fwFZ44+/x8= +github.com/klauspost/cpuid/v2 v2.0.9/go.mod h1:FInQzS24/EEf25PyTYn52gqo7WaD8xa0213Md/qVLRg= +github.com/klauspost/cpuid/v2 v2.2.3 h1:sxCkb+qR91z4vsqw4vGGZlDgPz3G7gjaLyK3V8y70BU= +github.com/klauspost/cpuid/v2 v2.2.3/go.mod h1:RVVoqg1df56z8g3pUjL/3lE5UfnlrJX8tyFgg4nqhuY= +github.com/kr/pretty v0.3.1 h1:flRD4NNwYAUpkphVc1HcthR4KEIFJ65n8Mw5qdRn3LE= github.com/kr/text v0.2.0 h1:5Nx0Ya0ZqY2ygV366QzturHI13Jq95ApcVaJBhpS+AY= +github.com/leodido/go-urn v1.2.1 h1:BqpAaACuzVSgi/VLzGZIobT2z4v53pjosyNd9Yv6n/w= +github.com/leodido/go-urn v1.2.1/go.mod h1:zt4jvISO2HfUBqxjfIshjdMTYS56ZS/qv49ictyFfxY= +github.com/mattn/go-isatty v0.0.19 h1:JITubQf0MOLdlGRuRq+jtsDlekdYPia9ZFsB8h/APPA= +github.com/mattn/go-isatty v0.0.19/go.mod h1:W+V8PltTTMOvKvAeJH7IuucS94S2C6jfK/D7dTCTo3Y= github.com/mattn/go-runewidth v0.0.9/go.mod h1:H031xJmbD/WCDINGzjvQ9THkh0rPKHF+m2gUSrubnMI= -github.com/mssola/user_agent v0.5.3 h1:lBRPML9mdFuIZgI2cmlQ+atbpJdLdeVl2IDodjBR578= -github.com/mssola/user_agent v0.5.3/go.mod h1:TTPno8LPY3wAIEKRpAtkdMT0f8SE24pLRGPahjCH4uw= +github.com/mattn/go-sqlite3 v1.14.16 h1:yOQRA0RpS5PFz/oikGwBEqvAWhWg5ufRz4ETLjwpU1Y= +github.com/matttproud/golang_protobuf_extensions v1.0.4 h1:mmDVorXM7PCGKw94cs5zkfA9PSy5pEvNWRP0ET0TIVo= +github.com/matttproud/golang_protobuf_extensions v1.0.4/go.mod h1:BSXmuO+STAnVfrANrmjBb36TMTDstsz7MSK+HVaYKv4= +github.com/modern-go/concurrent v0.0.0-20180228061459-e0a39a4cb421/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q= +github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd h1:TRLaZ9cD/w8PVh93nsPXa1VrQ6jlwL5oN8l14QlcNfg= +github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q= +github.com/modern-go/reflect2 v1.0.2 h1:xBagoLtFs94CBntxluKeaWgTMpvLxC4ur3nMaC9Gz0M= +github.com/modern-go/reflect2 v1.0.2/go.mod h1:yWuevngMOJpCy52FWWMvUC8ws7m/LJsjYzDa0/r8luk= +github.com/mssola/user_agent v0.6.0 h1:uwPR4rtWlCHRFyyP9u2KOV0u8iQXmS7Z7feTrstQwk4= +github.com/mssola/user_agent v0.6.0/go.mod h1:TTPno8LPY3wAIEKRpAtkdMT0f8SE24pLRGPahjCH4uw= +github.com/oschwald/geoip2-golang v1.9.0 h1:uvD3O6fXAXs+usU+UGExshpdP13GAqp4GBrzN7IgKZc= +github.com/oschwald/geoip2-golang v1.9.0/go.mod h1:BHK6TvDyATVQhKNbQBdrj9eAvuwOMi2zSFXizL3K81Y= +github.com/oschwald/maxminddb-golang v1.11.0 h1:aSXMqYR/EPNjGE8epgqwDay+P30hCBZIveY0WZbAWh0= +github.com/oschwald/maxminddb-golang v1.11.0/go.mod h1:YmVI+H0zh3ySFR3w+oz8PCfglAFj3PuCmui13+P9zDg= +github.com/outcaste-io/ristretto v0.2.1 h1:KCItuNIGJZcursqHr3ghO7fc5ddZLEHspL9UR0cQM64= +github.com/outcaste-io/ristretto v0.2.1/go.mod h1:W8HywhmtlopSB1jeMg3JtdIhf+DYkLAr0VN/s4+MHac= +github.com/pelletier/go-toml/v2 v2.0.6 h1:nrzqCb7j9cDFj2coyLNLaZuJTLjWjlaz6nvTvIwycIU= +github.com/pelletier/go-toml/v2 v2.0.6/go.mod h1:eumQOmlWiOPt5WriQQqoM5y18pDHwha2N+QD+EUNTek= github.com/pkg/errors v0.9.1 h1:FEBLx1zS214owpjy7qsBeixbURkuhQAwrK5UwLGTwt4= github.com/pkg/errors v0.9.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= -github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= -github.com/rogpeppe/go-internal v1.6.1 h1:/FiVV8dS/e+YqF2JvO3yXRFbBLTIuSDkuC7aBOAvL+k= +github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2 h1:Jamvg5psRIccs7FGNTlIRMkT8wgtp5eCXdBlqhYGL6U= +github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= +github.com/prometheus/client_model v0.4.0 h1:5lQXD3cAg1OXBf4Wq03gTrXHeaV0TQvGfUooCfx1yqY= +github.com/prometheus/client_model v0.4.0/go.mod h1:oMQmHW1/JoDwqLtg57MGgP/Fb1CJEYF2imWWhWtMkYU= +github.com/prometheus/common v0.44.0 h1:+5BrQJwiBB9xsMygAB3TNvpQKOwlkc25LbISbrdOOfY= +github.com/prometheus/common v0.44.0/go.mod h1:ofAIvZbQ1e/nugmZGz4/qCb9Ap1VoSTIO7x0VV9VvuY= +github.com/remyoudompheng/bigfft v0.0.0-20200410134404-eec4a21b6bb0/go.mod h1:qqbHyh8v60DhA7CoWK5oRCqLrMHRGoxYCSS9EjAz6Eo= +github.com/remyoudompheng/bigfft v0.0.0-20230129092748-24d4a6f8daec h1:W09IVJc94icq4NjY3clb7Lk8O1qJ8BdBEF8z0ibU0rE= +github.com/remyoudompheng/bigfft v0.0.0-20230129092748-24d4a6f8daec/go.mod h1:qqbHyh8v60DhA7CoWK5oRCqLrMHRGoxYCSS9EjAz6Eo= +github.com/rogpeppe/go-internal v1.10.0 h1:TMyTOH3F/DB16zRVcYyreMH6GnZZrwQVAoYjRBZyWFQ= +github.com/rs/xid v1.2.1 h1:mhH9Nq+C1fY2l1XIpgxIiUOfNpRBYH1kKcr+qfKgjRc= +github.com/rs/xid v1.2.1/go.mod h1:+uKXf+4Djp6Md1KODXJxgGQPKngRmWyn10oCKFzNHOQ= github.com/russross/blackfriday/v2 v2.1.0 h1:JIOH55/0cWyOuilr9/qlrm0BSXldqnqwMsf35Ld67mk= github.com/russross/blackfriday/v2 v2.1.0/go.mod h1:+Rmxgy9KzJVeS9/2gXHxylqXiyQDYRxCVz55jmeOWTM= +github.com/satori/go.uuid v1.2.0 h1:0uYX9dsZ2yD7q2RtLRtPSdGDWzjeM3TbMJP9utgA0ww= +github.com/satori/go.uuid v1.2.0/go.mod h1:dA0hQrYB0VpLJoorglMZABFdXlWrHn1NEOzdhQKdks0= github.com/scylladb/termtables v0.0.0-20191203121021-c4c0b6d42ff4/go.mod h1:C1a7PQSMz9NShzorzCiG2fk9+xuCgLkPeCvMHYR2OWg= github.com/sirupsen/logrus v1.7.0/go.mod h1:yWOB1SBYBC5VeMP7gHvWumXLIWorT60ONWic61uBYv0= -github.com/spf13/cast v1.5.0 h1:rj3WzYc11XZaIZMPKmwP96zkFEnnAmV8s6XbB2aY32w= -github.com/spf13/cast v1.5.0/go.mod h1:SpXXQ5YoyJw6s3/6cMTQuxvgRl3PCJiyaX9p6b155UU= -github.com/spf13/cobra v1.5.0 h1:X+jTBEBqF0bHN+9cSMgmfuvv2VHJ9ezmFNf9Y/XstYU= -github.com/spf13/cobra v1.5.0/go.mod h1:dWXEIy2H428czQCjInthrTRUg7yKbok+2Qi/yBIJoUM= +github.com/spf13/cast v1.5.1 h1:R+kOtfhWQE6TVQzY+4D7wJLBgkdVasCEFxSUBYBYIlA= +github.com/spf13/cast v1.5.1/go.mod h1:b9PdjNptOpzXr7Rq1q9gJML/2cdGQAo69NKzQ10KN48= +github.com/spf13/cobra v1.7.0 h1:hyqWnYt1ZQShIddO5kBpj3vu05/++x6tJ6dg8EC572I= +github.com/spf13/cobra v1.7.0/go.mod h1:uLxZILRyS/50WlhOIKD7W6V5bgeIt+4sICxh6uRMrb0= github.com/spf13/pflag v1.0.5 h1:iy+VFUOCP1a+8yFto/drg2CJ5u0yRoB7fZw3DKv/JXA= github.com/spf13/pflag v1.0.5/go.mod h1:McXfInJRrz4CZXVZOBLb0bTZqETkiAhM9Iw0y3An2Bg= github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= -github.com/stretchr/objx v0.4.0 h1:M2gUjqZET1qApGOWNSnZ49BAIMX4F/1plDv3+l31EJ4= github.com/stretchr/objx v0.4.0/go.mod h1:YvHI0jy2hoMjB+UWwv71VJQ9isScKT/TqJzVSSt89Yw= +github.com/stretchr/objx v0.5.0 h1:1zr/of2m5FGMsad5YfcqgdqdWrIhu+EBEJRhR1U7z/c= +github.com/stretchr/objx v0.5.0/go.mod h1:Yh+to48EsGEfYuaHDzXPcE3xhTkx73EhmCGUpEOglKo= github.com/stretchr/testify v1.2.2/go.mod h1:a8OnRcib4nhh0OaRAV+Yts87kKdq0PP7pXfy6kDkUVs= github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI= -github.com/stretchr/testify v1.4.0/go.mod h1:j7eGeouHqKxXV5pUuKE4zz7dFj8WfuZ+81PSLYec5m4= github.com/stretchr/testify v1.6.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= github.com/stretchr/testify v1.7.0/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= github.com/stretchr/testify v1.7.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= -github.com/stretchr/testify v1.8.0 h1:pSgiaMZlXftHpm5L7V1+rVB+AZJydKsMxsQBIJw4PKk= github.com/stretchr/testify v1.8.0/go.mod h1:yNjHg4UonilssWZ8iaSj1OCr/vHnekPRkoO+kdMU+MU= -github.com/tidwall/gjson v1.14.3 h1:9jvXn7olKEHU1S9vwoMGliaT8jq1vJ7IH/n9zD9Dnlw= -github.com/tidwall/gjson v1.14.3/go.mod h1:/wbyibRr2FHMks5tjHJ5F8dMZh3AcwJEMf5vlfC0lxk= +github.com/stretchr/testify v1.8.1/go.mod h1:w2LPCIKwWwSfY2zedu0+kehJoqGctiVI29o6fzry7u4= +github.com/stretchr/testify v1.8.4 h1:CcVxjf3Q8PM0mHUKJCdn+eZZtm5yQwehR5yeSVQQcUk= +github.com/stretchr/testify v1.8.4/go.mod h1:sz/lmYIOXD/1dqDmKjjqLyZ2RngseejIcXlSw2iwfAo= +github.com/tidwall/gjson v1.17.0 h1:/Jocvlh98kcTfpN2+JzGQWQcqrPQwDrVEMApx/M5ZwM= +github.com/tidwall/gjson v1.17.0/go.mod h1:/wbyibRr2FHMks5tjHJ5F8dMZh3AcwJEMf5vlfC0lxk= github.com/tidwall/match v1.1.1 h1:+Ho715JplO36QYgwN9PGYNhgZvoUSc9X2c80KVTi+GA= github.com/tidwall/match v1.1.1/go.mod h1:eRSPERbgtNPcGhD8UCthc6PmLEQXEWd3PRB5JTxsfmM= github.com/tidwall/pretty v1.2.0 h1:RWIZEg2iJ8/g6fDDYzMpobmaoGh5OLl4AXtGUGPcqCs= github.com/tidwall/pretty v1.2.0/go.mod h1:ITEVvHYasfjBbM0u2Pg8T2nJnzm8xPwvNhhsoaGGjNU= +github.com/twitchyliquid64/golang-asm v0.15.1 h1:SU5vSMR7hnwNxj24w34ZyCi/FmDZTkS4MhqMhdFk5YI= +github.com/twitchyliquid64/golang-asm v0.15.1/go.mod h1:a1lVb/DtPvCB8fslRZhAngC2+aY1QWCk3Cedj/Gdt08= +github.com/ugorji/go/codec v1.2.9 h1:rmenucSohSTiyL09Y+l2OCk+FrMxGMzho2+tjr5ticU= +github.com/ugorji/go/codec v1.2.9/go.mod h1:UNopzCgEMSXjBc6AOMqYvWC1ktqTAfzJZUZgYf6w6lg= github.com/yuin/goldmark v1.3.5/go.mod h1:mwnBkeHKe2W/ZEtQ+71ViKU8L12m81fl3OWwC1Zlc8k= +github.com/yuin/goldmark v1.4.13/go.mod h1:6yULJ656Px+3vBD8DxQVa3kxgyrAnzto9xy5taEt/CY= go.uber.org/atomic v1.7.0/go.mod h1:fEN4uk6kAWBTFdckzkM89CLk9XfWZrxpCo0nPH17wJc= -go.uber.org/atomic v1.9.0 h1:ECmE8Bn/WFTYwEW/bpKD3M8VtR/zQVbavAoalC1PYyE= go.uber.org/atomic v1.9.0/go.mod h1:fEN4uk6kAWBTFdckzkM89CLk9XfWZrxpCo0nPH17wJc= +go.uber.org/atomic v1.10.0/go.mod h1:LUxbIzbOniOlMKjJjyPfpl4v+PKK2cNJn91OQbhoJI0= +go.uber.org/atomic v1.11.0 h1:ZvwS0R+56ePWxUNi+Atn9dWONBPp/AUETXlHW0DxSjE= +go.uber.org/atomic v1.11.0/go.mod h1:LUxbIzbOniOlMKjJjyPfpl4v+PKK2cNJn91OQbhoJI0= go.uber.org/goleak v1.1.11 h1:wy28qYRKZgnJTxGxvye5/wgWr1EKjmUDGYox5mGlRlI= go.uber.org/multierr v1.6.0 h1:y6IPFStTAIT5Ytl7/XYmHvzXQ7S3g/IeZW9hyZ5thw4= go.uber.org/multierr v1.6.0/go.mod h1:cdWPpRnG4AhwMwsgIHip0KRBQjJy5kYEpYjJxpXp9iU= go.uber.org/zap v1.23.0 h1:OjGQ5KQDEUawVHxNwQgPpiypGHOxo2mNZsOqTak4fFY= go.uber.org/zap v1.23.0/go.mod h1:D+nX8jyLsMHMYrln8A0rJjFt/T/9/bGgIhAqxv5URuY= +golang.org/x/arch v0.0.0-20210923205945-b76863e36670 h1:18EFjUmQOcUvxNYSkA6jO9VAiXCnxFY6NyDX0bHDmkU= +golang.org/x/arch v0.0.0-20210923205945-b76863e36670/go.mod h1:5om86z9Hs0C8fWVUuoMHwpExlXzs5Tkyp9hOrfG7pp8= golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w= golang.org/x/crypto v0.0.0-20191011191535-87dc89f01550/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= +golang.org/x/crypto v0.0.0-20210921155107-089bfa567519/go.mod h1:GvvjBRRGRdwPK5ydBHafDWAxML/pGHZbMvKqRZ5+Abc= +golang.org/x/crypto v0.11.0 h1:6Ewdq3tDic1mg5xRO4milcWCfMVQhI4NkqWWvqejpuA= +golang.org/x/crypto v0.11.0/go.mod h1:xgJhtzW8F9jGdVFWZESrid1U1bjeNy4zgy5cRr/CIio= golang.org/x/mod v0.4.2/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= +golang.org/x/mod v0.6.0-dev.0.20220419223038-86c51ed26bb4/go.mod h1:jJ57K6gSWd91VN4djpZkiMVwK6gcyfeH4XE8wZrZaV4= +golang.org/x/mod v0.12.0 h1:rmsUpXtvNzj340zd98LZ4KntptpfRHwpFOHG188oHXc= +golang.org/x/mod v0.12.0/go.mod h1:iBbtSCu2XBx23ZKBPSOrRkjjQPZFPuis4dIYUhu/chs= golang.org/x/net v0.0.0-20190404232315-eb5bcb51f2a3/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= golang.org/x/net v0.0.0-20190620200207-3b0461eec859/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= +golang.org/x/net v0.0.0-20210226172049-e18ecbb05110/go.mod h1:m0MpNAwzfU5UDzcl9v0D8zg8gWTRqZa9RBIspLL5mdg= golang.org/x/net v0.0.0-20210405180319-a5a99cb37ef4/go.mod h1:p54w0d4576C0XHj96bSt6lcn1PtDYWL6XObtHCRCNQM= -golang.org/x/net v0.0.0-20220127200216-cd36cc0744dd h1:O7DYs+zxREGLKzKoMQrtrEacpb0ZVXA5rIwylE2Xchk= -golang.org/x/net v0.0.0-20220127200216-cd36cc0744dd/go.mod h1:CfG3xpIq0wQ8r1q4Su4UZFWDARRcnwPjda9FqA0JpMk= +golang.org/x/net v0.0.0-20220722155237-a158d28d115b/go.mod h1:XRhObCWvk6IyKnWLug+ECip1KBveYUHfp+8e9klMJ9c= +golang.org/x/net v0.7.0/go.mod h1:2Tu9+aMcznHK/AK1HMvgo6xiTLG5rD5rZLDS+rp2Bjs= +golang.org/x/net v0.12.0 h1:cfawfvKITfUsFCeJIHJrbSxpeu/E81khclypR0GVT50= +golang.org/x/net v0.12.0/go.mod h1:zEVYFnQC7m/vmpQFELhcD1EWkZlX69l4oqgmer6hfKA= +golang.org/x/sync v0.0.0-20181221193216-37e7f081c4d4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20210220032951-036812b2e83c/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.0.0-20220722155255-886fb9371eb4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.3.0 h1:ftCYgMx6zT/asHUrPw8BLLscYtGznsLAnjq5RH9P66E= golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20190412213103-97732733099d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20191026070338-33540a1f6037/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20200930185726-fdedc70b468f/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20201119102817-f84b799fce68/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20210124154548-22da62e12c0c/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20210330210617-4fbd30eecc44/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20210510120138-977fb7262007/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20210615035016-665e8c7367d1/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.0.0-20211216021012-1d35b9e2eb4e/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.0.0-20220804214406-8e32c043e418 h1:9vYwv7OjYaky/tlAeD7C4oC9EsPTlaFl1H2jS++V+ME= -golang.org/x/sys v0.0.0-20220804214406-8e32c043e418/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.0.0-20220520151302-bc2c85ada10a/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.0.0-20220627191245-f75cf1eec38b/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.0.0-20220704084225-05e143d24a9e/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.0.0-20220722155257-8c9f86f7a55f/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.5.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.6.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.10.0 h1:SqMFp9UcQJZa+pmYuAKjd9xq1f0j5rLcDIk0mj4qAsA= +golang.org/x/sys v0.10.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo= golang.org/x/term v0.0.0-20210927222741-03fcf44c2211/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8= +golang.org/x/term v0.5.0/go.mod h1:jMB1sMXY+tzblOD4FWmEbocvup2/aLOaQEp7JmGp78k= golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= -golang.org/x/text v0.3.7 h1:olpwvP2KacW1ZWvsR7uQhoyTYvKAupfQrRGBFM352Gk= golang.org/x/text v0.3.7/go.mod h1:u+2+/6zg+i71rQMx5EYifcz6MCKuco9NR6JIITiCfzQ= +golang.org/x/text v0.7.0/go.mod h1:mrYo+phRRbMaCq/xk9113O4dZlRixOauAjOtrjsXDZ8= +golang.org/x/text v0.11.0 h1:LAntKIrcmeSKERyiOh0XMV39LXS8IE9UL2yP7+f5ij4= +golang.org/x/text v0.11.0/go.mod h1:TvPlkZtksWOMsz7fbANvkp4WM8x/WCo/om8BMLbz+aE= +golang.org/x/time v0.3.0 h1:rg5rLMjNzMS1RkNLzCG38eapWhnYLFYXDXj2gOlr8j4= +golang.org/x/time v0.3.0/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= golang.org/x/tools v0.0.0-20191119224855-298f0cb1881e/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= golang.org/x/tools v0.1.1/go.mod h1:o0xws9oXOQQZyjljx8fwUC0k7L1pTE6eaCbjGeHmOkk= +golang.org/x/tools v0.1.12/go.mod h1:hNGJHUnrk76NpqgfD5Aqm5Crs+Hm0VOH/i9J2+nxYbc= +golang.org/x/tools v0.11.0 h1:EMCa6U9S2LtZXLAMoWiR/R8dAQFRqbAitmbJ2UKhoi8= +golang.org/x/tools v0.11.0/go.mod h1:anzJrxPjNtfgiYQYirP2CPGzGLxrH2u2QBhn6Bf3qY8= golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20191011141410-1b5146add898/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= -golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1 h1:go1bK/D/BFZV2I8cIQd1NKEZ+0owSTG1fDTci4IqFcE= +golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= -gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405 h1:yhCVgyC4o1eVCa2tZl7eS0r+SDo693bJlVdllGtEeKM= +google.golang.org/protobuf v1.26.0-rc.1/go.mod h1:jlhhOSvTdKEhbULTjvd4ARK9grFBp09yW+WbY/TyQbw= +google.golang.org/protobuf v1.26.0/go.mod h1:9q0QmTI4eRPtz6boOQmLYwt+qCgq0jsYwAQnmE0givc= +google.golang.org/protobuf v1.31.0 h1:g0LDEJHgrBl9N9r17Ru3sqWhkIx2NB67okBHPwC7hs8= +google.golang.org/protobuf v1.31.0/go.mod h1:HV8QOd/L58Z+nl8r43ehVNZIU/HEI6OcFqwMG9pJV4I= gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= -gopkg.in/yaml.v2 v2.2.2/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= +gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c h1:Hei/4ADfdWqJk1ZMxUNpqntNwaWcugrBjAiHlqqRiVk= +gopkg.in/natefinch/lumberjack.v2 v2.0.0 h1:1Lc07Kr7qY4U2YPouBjpCLxpiyxIVoxqXgkXLknAOE8= +gopkg.in/natefinch/lumberjack.v2 v2.0.0/go.mod h1:l0ndWWf7gzL7RNwBG7wST/UCcT4T24xpD6X8LsfU/+k= gopkg.in/yaml.v2 v2.4.0 h1:D8xgwECY7CYvx+Y2n4sBz93Jn9JRvxdiyyo8CTfuKaY= gopkg.in/yaml.v2 v2.4.0/go.mod h1:RDklbk79AGWmwhnvt/jBztapEOGDOx6ZbXqjP6csGnQ= gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA= gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= +lukechampine.com/uint128 v1.2.0 h1:mBi/5l91vocEN8otkC5bDLhi2KdCticRiwbdB0O+rjI= +lukechampine.com/uint128 v1.2.0/go.mod h1:c4eWIwlEGaxC/+H1VguhU4PHXNWDCDMUlWdIWl2j1gk= +modernc.org/cc/v3 v3.40.0 h1:P3g79IUS/93SYhtoeaHW+kRCIrYaxJ27MFPv+7kaTOw= +modernc.org/cc/v3 v3.40.0/go.mod h1:/bTg4dnWkSXowUO6ssQKnOV0yMVxDYNIsIrzqTFDGH0= +modernc.org/ccgo/v3 v3.16.13 h1:Mkgdzl46i5F/CNR/Kj80Ri59hC8TKAhZrYSaqvkwzUw= +modernc.org/ccgo/v3 v3.16.13/go.mod h1:2Quk+5YgpImhPjv2Qsob1DnZ/4som1lJTodubIcoUkY= +modernc.org/ccorpus v1.11.6 h1:J16RXiiqiCgua6+ZvQot4yUuUy8zxgqbqEEUuGPlISk= +modernc.org/httpfs v1.0.6 h1:AAgIpFZRXuYnkjftxTAZwMIiwEqAfk8aVB2/oA6nAeM= +modernc.org/libc v1.24.1 h1:uvJSeCKL/AgzBo2yYIPPTy82v21KgGnizcGYfBHaNuM= +modernc.org/libc v1.24.1/go.mod h1:FmfO1RLrU3MHJfyi9eYYmZBfi/R+tqZ6+hQ3yQQUkak= +modernc.org/mathutil v1.5.0 h1:rV0Ko/6SfM+8G+yKiyI830l3Wuz1zRutdslNoQ0kfiQ= +modernc.org/mathutil v1.5.0/go.mod h1:mZW8CKdRPY1v87qxC/wUdX5O1qDzXMP5TH3wjfpga6E= +modernc.org/memory v1.6.0 h1:i6mzavxrE9a30whzMfwf7XWVODx2r5OYXvU46cirX7o= +modernc.org/memory v1.6.0/go.mod h1:PkUhL0Mugw21sHPeskwZW4D6VscE/GQJOnIpCnW6pSU= +modernc.org/opt v0.1.3 h1:3XOZf2yznlhC+ibLltsDGzABUGVx8J6pnFMS3E4dcq4= +modernc.org/opt v0.1.3/go.mod h1:WdSiB5evDcignE70guQKxYUl14mgWtbClRi5wmkkTX0= +modernc.org/sqlite v1.25.0 h1:AFweiwPNd/b3BoKnBOfFm+Y260guGMF+0UFk0savqeA= +modernc.org/sqlite v1.25.0/go.mod h1:FL3pVXie73rg3Rii6V/u5BoHlSoyeZeIgKZEgHARyCU= +modernc.org/strutil v1.1.3 h1:fNMm+oJklMGYfU9Ylcywl0CO5O6nTfaowNsh2wpPjzY= +modernc.org/strutil v1.1.3/go.mod h1:MEHNA7PdEnEwLvspRMtWTNnp2nnyvMfkimT1NKNAGbw= +modernc.org/tcl v1.15.2 h1:C4ybAYCGJw968e+Me18oW55kD/FexcHbqH2xak1ROSY= +modernc.org/token v1.0.1 h1:A3qvTqOwexpfZZeyI0FeGPDlSWX5pjZu9hF4lU+EKWg= +modernc.org/token v1.0.1/go.mod h1:UGzOrNV1mAFSEB63lOFHIpNRUVMvYTc6yu1SMY/XTDM= +modernc.org/z v1.7.3 h1:zDJf6iHjrnB+WRD88stbXokugjyc0/pB91ri1gO6LZY= +rsc.io/pdf v0.1.1/go.mod h1:n8OzWcQ6Sp37PL01nO98y4iUCRdTGarVfzxY20ICaU4= diff --git a/internal/cmd/platypus/run/run.go b/internal/cmd/platypus/run/run.go index 3d4f3d6..8081ea7 100644 --- a/internal/cmd/platypus/run/run.go +++ b/internal/cmd/platypus/run/run.go @@ -9,11 +9,12 @@ import ( "os" "time" + "github.com/GuanceCloud/cliutils/point" "github.com/GuanceCloud/platypus/internal/logger" "github.com/GuanceCloud/platypus/pkg/engine" plruntime "github.com/GuanceCloud/platypus/pkg/engine/runtime" - "github.com/GuanceCloud/platypus/pkg/inimpl/guancecloud/funcs" - "github.com/GuanceCloud/platypus/pkg/inimpl/guancecloud/input" + "github.com/GuanceCloud/platypus/pkg/inimpl/guancecloud/ptinput" + "github.com/GuanceCloud/platypus/pkg/inimpl/guancecloud/ptinput/funcs" "github.com/influxdata/influxdb1-client/models" influxdb "github.com/influxdata/influxdb1-client/v2" "go.uber.org/zap/zapcore" @@ -84,7 +85,7 @@ func runScript(ctx context.Context, options *Options, script *plruntime.Script) return fmt.Errorf("read input file error: %w", err) } - var measurement string + var name string var tags map[string]string var fields map[string]any tn := time.Now() @@ -103,25 +104,23 @@ func runScript(ctx context.Context, options *Options, script *plruntime.Script) } fields = f tags = pt.Tags() - measurement = pt.Name() + name = pt.Name() tn = pt.Time() case TypeText: - measurement = "default_name" + name = "default_name" fields = map[string]any{"message": string(data)} default: return fmt.Errorf("unsupported input type: %s", options.Type) } var dropped bool - pt := input.GetPoint() - defer input.PutPoint(pt) - input.InitPt(pt, measurement, tags, fields, tn) + pt := ptinput.NewPlPoint(point.UnknownCategory, name, tags, fields, tn) - fields = pt.Fields - tags = pt.Tags - dropped = pt.Drop - tn = pt.Time - measurement = pt.Measurement + fields = pt.Fields() + tags = pt.Tags() + dropped = pt.Dropped() + tn = pt.PtTime() + name = pt.GetPtName() errR := engine.RunScriptWithRMapIn(script, pt, nil) if errR != nil { @@ -139,7 +138,7 @@ func runScript(ctx context.Context, options *Options, script *plruntime.Script) encoder.SetEscapeHTML(false) encoder.SetIndent("", defaultJSONIndent) if err := encoder.Encode(map[string]any{ - "measurement": measurement, + "measurement": name, "tags": tags, "fields": fields, "time": tn, @@ -148,7 +147,7 @@ func runScript(ctx context.Context, options *Options, script *plruntime.Script) } l.Infof("Platypus Output Data:\n%s", buf.String()) case OutTypeLineProtocol: - pt, err := influxdb.NewPoint(measurement, tags, fields, tn) + pt, err := influxdb.NewPoint(name, tags, fields, tn) if err != nil { return fmt.Errorf("encode point output error: %w", err) } diff --git a/pkg/engine/callref_test.go b/pkg/engine/callref_test.go index 3a66492..845711c 100644 --- a/pkg/engine/callref_test.go +++ b/pkg/engine/callref_test.go @@ -8,7 +8,7 @@ package engine import ( "testing" - "github.com/GuanceCloud/platypus/pkg/inimpl/guancecloud/funcs" + "github.com/GuanceCloud/platypus/pkg/inimpl/guancecloud/ptinput/funcs" "github.com/stretchr/testify/assert" ) diff --git a/pkg/engine/op_test.go b/pkg/engine/op_test.go index 0ae8f77..db339ab 100644 --- a/pkg/engine/op_test.go +++ b/pkg/engine/op_test.go @@ -9,9 +9,10 @@ import ( "testing" "time" + "github.com/GuanceCloud/cliutils/point" "github.com/GuanceCloud/platypus/pkg/engine/runtime" - "github.com/GuanceCloud/platypus/pkg/inimpl/guancecloud/funcs" - "github.com/GuanceCloud/platypus/pkg/inimpl/guancecloud/input" + "github.com/GuanceCloud/platypus/pkg/inimpl/guancecloud/ptinput" + "github.com/GuanceCloud/platypus/pkg/inimpl/guancecloud/ptinput/funcs" "github.com/stretchr/testify/assert" ) @@ -102,8 +103,7 @@ for a = 0; a < 12; a = a + 1 { script := scripts["abc.ppl"] - pt := input.GetPoint() - pt = input.InitPt(pt, "test", nil, nil, time.Now()) + pt := ptinput.NewPlPoint(point.Logging, "test", nil, nil, time.Now()) err := runtime.RunScriptWithRMapIn(script, pt, nil) if err != nil { t.Fatal(err.Error()) @@ -119,5 +119,5 @@ for a = 0; a < 12; a = a + 1 { "len1": int64(2), "len2": int64(3), "list_": "[1,2,{\"a\":[1,2,2]},{}]", - }, pt.Fields) + }, pt.Fields()) } diff --git a/pkg/hash/fnv1a.go b/pkg/hash/fnv1a.go new file mode 100644 index 0000000..b141fbd --- /dev/null +++ b/pkg/hash/fnv1a.go @@ -0,0 +1,45 @@ +// Unless explicitly stated otherwise all files in this repository are licensed +// under the MIT License. +// This product includes software developed at Guance Cloud (https://www.guance.com/). +// Copyright 2021-present Guance, Inc. + +// Package hash used to calculate hash +package hash + +const ( + // pkg "hash/fnv". + offset64 = 14695981039346656037 + prime64 = 1099511628211 +) + +func Fnv1aNew() uint64 { + return offset64 +} + +func Fnv1aHashAdd(h uint64, s string) uint64 { + for i := 0; i < len(s); i++ { + h ^= uint64(s[i]) + h *= prime64 + } + return h +} + +func Fnv1aStrHash(s string) uint64 { + var h uint64 = offset64 + for i := 0; i < len(s); i++ { + h ^= uint64(s[i]) + h *= prime64 + } + return h +} + +func Fnv1aHash(v []string) uint64 { + var h uint64 = offset64 + for _, s := range v { + for j := 0; j < len(s); j++ { + h ^= uint64(s[j]) + h *= prime64 + } + } + return h +} diff --git a/pkg/hash/fnv1a_test.go b/pkg/hash/fnv1a_test.go new file mode 100644 index 0000000..1000bce --- /dev/null +++ b/pkg/hash/fnv1a_test.go @@ -0,0 +1,110 @@ +// Unless explicitly stated otherwise all files in this repository are licensed +// under the MIT License. +// This product includes software developed at Guance Cloud (https://www.guance.com/). +// Copyright 2021-present Guance, Inc. + +package hash + +import ( + "hash/fnv" + "testing" + + "github.com/stretchr/testify/assert" +) + +func TestHash(t *testing.T) { + d := []string{ + `k8s_pod_namespace=d["k8s_pod_namespace"]`, + `k8s_container_name=d["k8s_container_name"]`, + `k8s_c ontainer_name=1`, `k8s_node_name=1, + fields=1`, + } + + h1 := Fnv1aNew() + for _, s := range d { + h1 = Fnv1aHashAdd(h1, s) + } + + h2 := Fnv1aHash(d) + + h3 := hash4ts(d) + + var s string + for _, v := range d { + s += v + } + h4 := Fnv1aStrHash(s) + + assert.NotEqual(t, Fnv1aNew(), h1) + assert.Equal(t, h1, h2) + assert.Equal(t, h1, h3) + assert.Equal(t, h1, h4) +} + +func BenchmarkIHashAdd(b *testing.B) { + d := []string{ + `k8s_pod_namespace=d["k8s_pod_namespace"]`, + `k8s_container_name=d["k8s_container_name"]`, + `k8s_c ontainer_name=1`, `k8s_node_name=1, + fields=1`, + } + for n := 0; n < b.N; n++ { + h := Fnv1aNew() + for _, s := range d { + h = Fnv1aHashAdd(h, s) + } + } +} + +func BenchmarkIHash(b *testing.B) { + d := []string{ + `k8s_pod_namespace=d["k8s_pod_namespace"]`, + `k8s_container_name=d["k8s_container_name"]`, + `k8s_c ontainer_name=1`, `k8s_node_name=1, + fields=1`, + } + for n := 0; n < b.N; n++ { + Fnv1aHash(d) + // b.Error(Fnv1aHash2(d)) + } +} + +func BenchmarkHash(b *testing.B) { + d := []string{ + `k8s_pod_namespace=d["k8s_pod_namespace"]`, + `k8s_container_name=d["k8s_container_name"]`, + `k8s_c ontainer_name=1`, `k8s_node_name=1, + fields=1`, + } + for n := 0; n < b.N; n++ { + hash4ts(d) + } +} + +func BenchmarkHash2(b *testing.B) { + d := [][]byte{ + []byte(`k8s_pod_namespace=d["k8s_pod_namespace"]`), + []byte(`k8s_container_name=d["k8s_container_name"]`), + []byte(`k8s_c ontainer_name=1`), []byte(`k8s_node_name=1, + fields=1`), + } + for n := 0; n < b.N; n++ { + hash4tb(d) + } +} + +func hash4ts(v []string) uint64 { + h := fnv.New64a() + for _, v := range v { + h.Write([]byte(v)) + } + return h.Sum64() +} + +func hash4tb(v [][]byte) uint64 { + h := fnv.New64a() + for _, v := range v { + h.Write(v) + } + return h.Sum64() +} diff --git a/pkg/inimpl/guancecloud/funcs/all.go b/pkg/inimpl/guancecloud/funcs/all.go deleted file mode 100644 index 29d45db..0000000 --- a/pkg/inimpl/guancecloud/funcs/all.go +++ /dev/null @@ -1,80 +0,0 @@ -// Unless explicitly stated otherwise all files in this repository are licensed -// under the MIT License. -// This product includes software developed at Guance Cloud (https://www.guance.com/). -// Copyright 2021-present Guance, Inc. - -// Package funcs for pipeline -package funcs - -import ( - "github.com/GuanceCloud/platypus/internal/logger" - "github.com/GuanceCloud/platypus/pkg/engine/runtime" - "go.uber.org/zap" -) - -var l logger.Logger = logger.NewStdoutLogger("funcs", zap.DebugLevel) - -func InitLog(logger *zap.SugaredLogger) { - l = logger -} - -var FuncsMap = map[string]runtime.FuncCall{ - "grok": Grok, - "add_key": AddKey, - "add_pattern": AddPattern, - "cast": Cast, - "datetime": DateTime, - "default_time": DefaultTime, - - "drop_key": Dropkey, - "exit": Exit, - "get_key": Getkey, - - "len": Len, - "load_json": LoadJSON, - "rename": Rename, - "set_tag": SetTag, - "set_measurement": SetMeasurement, - "strfmt": Strfmt, - "trim": Trim, - "uppercase": Uppercase, - "use": Use, - "url_decode": URLDecode, - - "printf": Printf, - "replace": Replace, - "xml": XML, - // "match": Match, - "sql_cover": SQLCover, -} - -var FuncsCheckMap = map[string]runtime.FuncCheck{ - "grok": GrokChecking, - "add_key": AddkeyChecking, - "add_pattern": AddPatternChecking, - "cast": CastChecking, - "datetime": DateTimeChecking, - "default_time": DefaultTimeChecking, - - "drop_key": DropkeyChecking, - "exit": ExitChecking, - "get_key": GetkeyChecking, - - "len": LenChecking, - "load_json": LoadJSONChecking, - - "rename": RenameChecking, - "set_measurement": SetMeasurementChecking, - "set_tag": SetTagChecking, - "strfmt": StrfmtChecking, - "trim": TrimChecking, - "uppercase": UppercaseChecking, - "use": UseChecking, - "url_decode": URLDecodeChecking, - - "printf": PrintfChecking, - "replace": ReplaceChecking, - "sql_cover": SQLCoverChecking, - "xml": XMLChecking, - // "match": MatchChecking, -} diff --git a/pkg/inimpl/guancecloud/funcs/fn_grok_test.go b/pkg/inimpl/guancecloud/funcs/fn_grok_test.go deleted file mode 100644 index 5fdab3d..0000000 --- a/pkg/inimpl/guancecloud/funcs/fn_grok_test.go +++ /dev/null @@ -1,219 +0,0 @@ -// Unless explicitly stated otherwise all files in this repository are licensed -// under the MIT License. -// This product includes software developed at Guance Cloud (https://www.guance.com/). -// Copyright 2021-present Guance, Inc. - -package funcs - -import ( - "testing" - "time" - - "github.com/stretchr/testify/assert" -) - -func TestGrok(t *testing.T) { - cases := []struct { - name, pl, in string - expected interface{} - fail bool - outkey string - }{ - { - name: "normal_return_t", - pl: ` -add_pattern("_second", "(?:(?:[0-5]?[0-9]|60)(?:[:.,][0-9]+)?)") -add_pattern("_minute", "(?:[0-5][0-9])") -add_pattern("_hour", "(?:2[0123]|[01]?[0-9])") -add_pattern("time", "([^0-9]?)%{_hour:hour}:%{_minute:minute}(?::%{_second:second})([^0-9]?)") -add_key(grok_match_ok, grok(_, "%{time}")) ; printf("%v", grok_match_ok)`, - in: "12:13:14.123", - expected: true, - outkey: "grok_match_ok", - }, - { - name: "normal_return_f", - pl: ` - add_pattern("_second", "(?:(?:[0-5]?[0-9]|60)(?:[:.,][0-9]+)?)") - add_pattern("_minute", "(?:[0-5][0-9])") - add_pattern("_hour", "(?:2[0123]|[01]?[0-9])") - add_pattern("time", "([^0-9]?)%{_hour:hour}:%{_minute:minute}(?::%{_second:second})([^0-9]?)") - add_key(grok_match_ok, grok(_, "%{time}"))`, - in: "12 :13:14.123", - expected: false, - outkey: "grok_match_ok", - }, - { - name: "normal_return_sample_t", - pl: ` - add_key(grok_match_ok, grok(_, "12 :13:14.123"))`, - in: "12 :13:14.123", - expected: true, - outkey: "grok_match_ok", - }, - { - name: "normal_return_sample_f", - pl: ` - add_key(grok_match_ok, grok(_, "12 :13:14.123"))`, - in: "12:13:14.123", - expected: false, - outkey: "grok_match_ok", - }, - { - name: "normal", - pl: ` - add_pattern("_second", "(?:(?:[0-5]?[0-9]|60)(?:[:.,][0-9]+)?)") - add_pattern("_minute", "(?:[0-5][0-9])") - add_pattern("_hour", "(?:2[0123]|[01]?[0-9])") - add_pattern("time", "([^0-9]?)%{_hour:hour}:%{_minute:minute}(?::%{_second:second})([^0-9]?)") - grok(_, "%{time}")`, - in: "12:13:14.123", - expected: "14.123", - outkey: "second", - }, - { - name: "normal", - pl: ` - add_pattern("_second", "(?:(?:[0-5]?[0-9]|60)(?:[:.,][0-9]+)?)") - add_pattern("_minute", "(?:[0-5][0-9])") - add_pattern("_hour", "(?:2[0123]|[01]?[0-9])") - add_pattern("time", "([^0-9]?)%{_hour:hour}:%{_minute:minute}(?::%{_second:second})([^0-9]?)") - grok(_, "%{time}")`, - in: "12:13:14", - expected: "13", - outkey: "minute", - }, - { - name: "normal", - pl: ` - add_pattern("_second", "(?:(?:[0-5]?[0-9]|60)(?:[:.,][0-9]+)?)") - add_pattern("_minute", "(?:[0-5][0-9])") - add_pattern("_hour", "(?:2[0123]|[01]?[0-9])") - add_pattern("time", "([^0-9]?)%{_hour:hour}:%{_minute:minute}(?::%{_second:second})([^0-9]?)") - grok(_, "%{time}")`, - in: "12:13:14", - expected: "12", - outkey: "hour", - }, - { - name: "normal", - pl: ` - add_pattern("_second", "(?:(?:[0-5]?[0-9]|60)(?:[:.,][0-9]+)?)") - add_pattern("_minute", "(?:[0-5][0-9])") - add_pattern("_hour", "(?:2[0123]|[01]?[0-9])") - add_pattern("time", "([^0-9]?)%{_hour:hour}:%{_minute:minute}(?::%{_second:second})([^0-9]?)") - grok(_, "%{time}")`, - in: "12:13:14", - expected: "14", - outkey: "second", - }, - { - name: "normal", - pl: ` - add_pattern("time", "%{NUMBER:time:float}") - grok(_, '''%{time} - %{WORD:word:string} - %{WORD:code:int} - %{WORD:w1}''')`, - in: `1.1 - s - 123cvf - aa222`, - expected: int64(0), - outkey: "code", - }, - { - name: "normal", - pl: ` - add_pattern("time", "%{NUMBER:time:float}") - grok(_, '''%{time} - %{WORD:word:string} - %{WORD:code:int} - %{WORD:w1}''')`, - in: `1.1 - s - 123 - aa222`, - expected: int64(123), - outkey: "code", - }, - { - name: "normal", - pl: ` - add_pattern("time", "%{NUMBER:time:float}") - grok(_, '''%{time} - %{WORD:word:str} - %{WORD:code:int} - %{WORD:w1}''')`, - in: `1.1 - s - 123 - aa222`, - expected: int64(123), - outkey: "code", - }, - { - name: "normal", - pl: ` - add_pattern("_second", "(?:(?:[0-5]?[0-9]|60)(?:[:.,][0-9]+)?)") - add_pattern("_minute", "(?:[0-5][0-9])") - add_pattern("_hour", "(?:2[0123]|[01]?[0-9])") - add_pattern("time", "([^0-9]?)%{_hour:hour:string}:%{_minute:minute:int}(?::%{_second:second:float})([^0-9]?)") - grok(_, "%{WORD:date} %{time}")`, - in: "2021/1/11 2:13:14.123", - expected: float64(14.123), - outkey: "second", - }, - { - name: "trim_space", - in: " not_space ", - pl: `add_pattern("d", "[\\s\\S]*") - grok(_, "%{d:item}")`, - expected: "not_space", - outkey: "item", - }, - { - name: "trim_space, enable", - in: " not_space ", - pl: `add_pattern("d", "[\\s\\S]*") - grok(_, "%{d:item}", true)`, - expected: "not_space", - outkey: "item", - }, - { - name: "trim_space, disable", - in: " not_space ", - pl: `add_pattern("d", "[\\s\\S]*") - grok(_, "%{d:item}", false)`, - expected: " not_space ", - outkey: "item", - }, - } - - for idx, tc := range cases { - t.Run(tc.name, func(t *testing.T) { - runner, err := NewTestingRunner(tc.pl) - if err != nil { - if tc.fail { - t.Logf("[%d]expect error: %s", idx, err) - } else { - t.Errorf("[%d] failed: %s", idx, err) - } - return - } - - _, _, f, _, _, err := runScript(runner, "test", nil, map[string]interface{}{ - "message": tc.in, - }, time.Now()) - - assert.Equal(t, err, nil) - - t.Log(f) - - v, ok := f[tc.outkey] - assert.Equal(t, true, ok) - assert.Equal(t, tc.expected, v) - t.Logf("[%d] PASS", idx) - }) - } -} diff --git a/pkg/inimpl/guancecloud/funcs/fn_printf.go b/pkg/inimpl/guancecloud/funcs/fn_printf.go deleted file mode 100644 index 3f9c372..0000000 --- a/pkg/inimpl/guancecloud/funcs/fn_printf.go +++ /dev/null @@ -1,66 +0,0 @@ -// Unless explicitly stated otherwise all files in this repository are licensed -// under the MIT License. -// This product includes software developed at Guance Cloud (https://www.guance.com/). -// Copyright 2021-present Guance, Inc. - -package funcs - -import ( - "fmt" - - "github.com/GuanceCloud/platypus/pkg/ast" - "github.com/GuanceCloud/platypus/pkg/engine/runtime" - "github.com/GuanceCloud/platypus/pkg/errchain" - "github.com/spf13/cast" -) - -func PrintfChecking(ctx *runtime.Context, funcExpr *ast.CallExpr) *errchain.PlError { - if len(funcExpr.Param) < 1 { - return runtime.NewRunError(ctx, "function `%s' requires at least one argument", funcExpr.NamePos) - } - if _, err := getKeyName(funcExpr.Param[0]); err != nil { - return runtime.NewRunError(ctx, err.Error(), funcExpr.Param[0].StartPos()) - } - return nil -} - -func Printf(ctx *runtime.Context, funcExpr *ast.CallExpr) *errchain.PlError { - outdata := make([]interface{}, 0) - - if len(funcExpr.Param) < 1 { - return runtime.NewRunError(ctx, - "function `%s' requires at least one argument", funcExpr.NamePos) - } - - fmtStr := getArgStr(ctx, funcExpr.Param[0]) - - if fmtStr == "" { - return nil - } - - for i := 1; i < len(funcExpr.Param); i++ { - if v, _, err := runtime.RunStmt(ctx, funcExpr.Param[i]); err != nil { - return err - } else { - outdata = append(outdata, v) - } - } - - if _, err := fmt.Printf(fmtStr, outdata...); err != nil { - l.Debug(err) - } - return nil -} - -func getArgStr(ctx *runtime.Context, node *ast.Node) string { - if node == nil { - return "" - } - - if v, dtype, err := runtime.RunStmt(ctx, node); err == nil { - if dtype == ast.String { - return cast.ToString(v) - } - } - return "" -} diff --git a/pkg/inimpl/guancecloud/funcs/fn_printf_test.go b/pkg/inimpl/guancecloud/funcs/fn_printf_test.go deleted file mode 100644 index 5d59bbe..0000000 --- a/pkg/inimpl/guancecloud/funcs/fn_printf_test.go +++ /dev/null @@ -1,51 +0,0 @@ -// Unless explicitly stated otherwise all files in this repository are licensed -// under the MIT License. -// This product includes software developed at Guance Cloud (https://www.guance.com/). -// Copyright 2021-present Guance, Inc. - -package funcs - -import ( - "testing" - "time" -) - -func TestPrintf(t *testing.T) { - cases := []struct { - name, pl, in string - expected interface{} - fail bool - outkey string - }{ - { - name: "normal", - pl: ` - count = 10 - a= "count %d\n" - printf(a, count) - `, - expected: nil, - outkey: "", - }, - } - for idx, tc := range cases { - t.Run(tc.name, func(t *testing.T) { - runner, err := NewTestingRunner(tc.pl) - if err != nil { - if tc.fail { - t.Logf("[%d]expect error: %s", idx, err) - } else { - t.Errorf("[%d] failed: %s", idx, err) - } - return - } - _, _, _, _, _, err = runScript(runner, "test", nil, map[string]interface{}{ - "message": tc.in, - }, time.Now()) - - if err != nil { - t.Error(err) - } - }) - } -} diff --git a/pkg/inimpl/guancecloud/funcs/fn_replace_test.go b/pkg/inimpl/guancecloud/funcs/fn_replace_test.go deleted file mode 100644 index 91532da..0000000 --- a/pkg/inimpl/guancecloud/funcs/fn_replace_test.go +++ /dev/null @@ -1,121 +0,0 @@ -// Unless explicitly stated otherwise all files in this repository are licensed -// under the MIT License. -// This product includes software developed at Guance Cloud (https://www.guance.com/). -// Copyright 2021-present Guance, Inc. - -package funcs - -import ( - "testing" - "time" - - "github.com/stretchr/testify/assert" -) - -func TestReplace(t *testing.T) { - cases := []struct { - name string - pl, in string - outKey string - expected interface{} - fail bool - }{ - { - name: `normal1`, - pl: `input = load_json(_); str1xxx = input["str1xxx"] - replace(str1xxx, "(1[0-9]{2})[0-9]{4}([0-9]{4})", "$1****$2")`, - in: `{"str1xxx": "13789123014"}`, - outKey: "str1xxx", - fail: false, - expected: "137****3014", - }, - - { - name: `normal2`, - pl: `input = load_json(_); str1xxx = input["str1xxx"] - replace(str1xxx, "([a-z]*) \\w*", "$1 ***")`, - in: `{"str1xxx": "zhang san"}`, - outKey: "str1xxx", - expected: "zhang ***", - fail: false, - }, - - { - name: `normal3`, - pl: `input = load_json(_); str1xxx = input["str1xxx"] - replace(str1xxx, "([1-9]{4})[0-9]{10}([0-9]{4})", "$1**********$2")`, - in: `{"str1xxx": "362201200005302565"}`, - outKey: "str1xxx", - expected: "3622**********2565", - fail: false, - }, - - { - name: `normal4`, - pl: `input = load_json(_); str1xxx = input["str1xxx"] - replace(str1xxx, '([\u4e00-\u9fa5])[\u4e00-\u9fa5]([\u4e00-\u9fa5])', "$1＊$2")`, - in: `{"str1xxx": "小阿卡"}`, - outKey: "str1xxx", - expected: "小＊卡", - fail: false, - }, - - { - name: `normal5`, - pl: `input = load_json(_); str1xxx = input["str1xxx"]; add_key(str1xxx); printf("%v", get_key(str1xxx)) - replace(str1, '([\u4e00-\u9fa5])[\u4e00-\u9fa5]([\u4e00-\u9fa5])', "$1＊$2")`, - in: `{"str1xxx": "小阿卡"}`, - outKey: "str1xxx", - expected: "小阿卡", - fail: false, - }, - - { - name: `not enough args`, - pl: `input = load_json(_); str1xxx = input["str1xxx"] - replace(str1xxx, '([\u4e00-\u9fa5])[\u4e00-\u9fa5]([\u4e00-\u9fa5])')`, - in: `{"str1xxx": "小阿卡"}`, - fail: true, - }, - - { - name: `invalid arg type`, - pl: `input = load_json(_); str1xxx = input["str1xxx"] - replace(str1xxx, 2, "$1＊$2")`, - in: `{"str1xxx": "小阿卡"}`, - fail: true, - }, - } - - for idx, tc := range cases { - t.Run(tc.name, func(t *testing.T) { - runner, err := NewTestingRunner(tc.pl) - if err != nil { - if tc.fail { - t.Logf("[%d]expect error: %s", idx, err) - } else { - t.Errorf("[%d] failed: %s", idx, err) - } - return - } - - _, _, f, _, _, err := runScript(runner, "test", nil, map[string]interface{}{ - "message": tc.in, - }, time.Now()) - if err != nil { - t.Fatal(err) - } - - if v, ok := f[tc.outKey]; !ok { - if !tc.fail { - t.Errorf("[%d]expect error: %s", idx, err) - } - } else { - assert.Equal(t, tc.expected, v) - t.Logf("[%d] PASS", idx) - } - - t.Logf("[%d] PASS", idx) - }) - } -} diff --git a/pkg/inimpl/guancecloud/funcs/fn_use_test.go b/pkg/inimpl/guancecloud/funcs/fn_use_test.go deleted file mode 100644 index 566f7a4..0000000 --- a/pkg/inimpl/guancecloud/funcs/fn_use_test.go +++ /dev/null @@ -1,72 +0,0 @@ -// Unless explicitly stated otherwise all files in this repository are licensed -// under the MIT License. -// This product includes software developed at Guance Cloud (https://www.guance.com/). -// Copyright 2021-present Guance, Inc. - -package funcs - -import ( - "testing" - "time" - - "github.com/GuanceCloud/platypus/pkg/inimpl/guancecloud/input" - "github.com/stretchr/testify/assert" -) - -func TestUse(t *testing.T) { - case1 := map[string]string{ - "a.ppl": "if true {use(\"b.ppl\")}", - "b.ppl": "add_key(b,1)", - "d.ppl": "use(\"c.ppl\")", - "c.ppl": "use(\"a.ppl\") use(\"d.ppl\") use(\"fcName.ppl\")", - } - - ret := [2][]string{ - {"a.ppl", "b.ppl"}, - {"d.ppl", "c.ppl"}, - } - - timenow := time.Now() - - retCheck := input.Point{ - Measurement: "default", - Tags: map[string]string{ - "ax": "1", - }, - Fields: map[string]interface{}{ - "b": int64(1), - }, - Time: timenow, - Drop: false, - } - - ret1, ret2 := NewTestingRunner2(case1) - assert.Equal(t, len(ret[0]), len(ret1)) - assert.Equal(t, len(ret[1]), len(ret2)) - - for _, v := range ret[0] { - if _, ok := ret1[v]; !ok { - t.Error(v) - } - } - - for _, v := range ret[1] { - if _, ok := ret2[v]; !ok { - t.Error(v) - } - } - - for _, name := range ret[0] { - m, tags, f, _, drop, err := runScript(ret1[name], "default", map[string]string{ - "ax": "1", - }, nil, timenow) - if err != nil { - t.Error(err) - continue - } - assert.Equal(t, retCheck.Tags, tags) - assert.Equal(t, retCheck.Fields, f) - assert.Equal(t, retCheck.Drop, drop) - assert.Equal(t, retCheck.Measurement, m) - } -} diff --git a/pkg/inimpl/guancecloud/funcs/md/fn.md b/pkg/inimpl/guancecloud/funcs/md/fn.md deleted file mode 100644 index 5460c27..0000000 --- a/pkg/inimpl/guancecloud/funcs/md/fn.md +++ /dev/null @@ -1,810 +0,0 @@ -# GuanceCloud Pipeline Functions - -## `printf()` - -函数原型：`printf(fmt=required, key1=optional, key2, ...)` - -函数说明：对已提取 `key1,key2...` 指定的字段内容根据 `fmt` 进行格式化，并把格式化后的内容输出到标准输出 - -函数参数 - -- `fmt`: 格式化字符串模板 -- `key1，key2`: 占位符对应的参数 - -示例: - -```python -count = 10 -printf("count: %d", count) -``` - -## `load_json()` - -函数原型：`fn load_json(val: str) nil|bool|float|map|list` - -函数说明：将 json 字符串转换成 map、list、nil、bool、float 的其中一种，可通过 index 表达式取值及修改值。 - -参数: - -- `val`: 要求是 string 类型的数据 - -示例: - -```python -# _: {"a":{"first": [2.2, 1.1], "ff": "[2.2, 1.1]","second":2,"third":"aBC","forth":true},"age":47} -abc = load_json(_) - -add_key(abc, abc["a"]["first"][-1]) - -abc["a"]["first"][-1] = 11 - -# 需要将堆栈上的数据同步到 point 中 -add_key(abc, abc["a"]["first"][-1]) - -add_key(len_abc, len(abc)) - -add_key(len_abc, len(load_json(abc["a"]["ff"]))) -``` - -## `len()` - -函数原型：`fn len(val: str|map|list) int` - -函数说明：计算 string 字节数，map 和 list 的元素个数。 - -参数: - -- `val`: 可以是 map、list 或 string - -示例: - -```python -# 例 1 -add_key(abc, len("abc")) -# 输出 -{ - "abc": 3, -} - -# 例 2 -add_key(abc, len(["abc"])) -#处理结果 -{ - "abc": 1, -} -``` - -## `add_pattern()` - -函数原型：`fn add_pattern(name: str, pattern: str)` - -函数说明：创建自定义 grok 模式。grok 模式有作用域限制, 如在 if else 语句内将产生新的作用域, 该 pattern 仅在此作用域内有效。该函数不可覆盖同一作用域或者上一作用域已经存在的 grok 模式 - -参数: - -- `name`：模式命名 -- `pattern`: 自定义模式内容 - -示例: - -```python -# 待处理数据: "11,abc,end1", "22,abc,end1", "33,abc,end3" - -# pipline脚本 -add_pattern("aa", "\\d{2}") -grok(_, "%{aa:aa}") -if false { - -} else { - add_pattern("bb", "[a-z]{3}") - if aa == "11" { - add_pattern("cc", "end1") - grok(_, "%{aa:aa},%{bb:bb},%{cc:cc}") - } elif aa == "22" { - # 此处使用 pattern cc 将导致编译失败: no pattern found for %{cc} - grok(_, "%{aa:aa},%{bb:bb},%{INT:cc}") - } elif aa == "33" { - add_pattern("bb", "[\\d]{5}") # 此处覆盖 bb 失败 - add_pattern("cc", "end3") - grok(_, "%{aa:aa},%{bb:bb},%{cc:cc}") - } -} - -# 处理结果 -{ - "aa": "11" - "bb": "abc" - "cc": "end1" - "message": "11,abc,end1" -} - -{ - "aa": "22", - "message": "22,abc,end1" -} - -{ - "aa": "33" - "bb": "abc" - "cc": "end3" - "message": "33,abc,end3" -} -``` - -## `grok()` - -函数原型：`fn grok(input: str, pattern: str, trim_space: bool = true) bool` - -函数说明：通过 `pattern` 提取文本串 `input` 中的内容，当 pattern 匹配 input 成功时返回 true 否则返回 false。 - -参数: - -- `input`：待提取文本，可以是原始文本（`_`）或经过初次提取之后的某个 `key` -- `pattern`: grok 表达式，表达式中支持指定 key 的数据类型：bool, float, int, string(对应 ppl 的 str，亦可写为 str)，默认为 string -- `trim_space`: 删除提取出的字符中的空白首尾字符，默认值为 true - -```python -grok(_, pattern) # 直接使用输入的文本作为原始数据 -grok(key, pattern) # 对之前已经提取出来的某个 key，做再次 grok -``` - -示例: - -```python -# 待处理数据: "12/01/2021 21:13:14.123" - -# pipline脚本 -add_pattern("_second", "(?:(?:[0-5]?[0-9]|60)(?:[:.,][0-9]+)?)") -add_pattern("_minute", "(?:[0-5][0-9])") -add_pattern("_hour", "(?:2[0123]|[01]?[0-9])") -add_pattern("time", "([^0-9]?)%{_hour:hour:string}:%{_minute:minute:int}(?::%{_second:second:float})([^0-9]?)") - -grok_match_ok = grok(_, "%{DATE_US:date} %{time}") - -add_key(grok_match_ok) - -# 处理结果 -{ - "date": "12/01/2021", - "hour": "21", - "message": "12/01/2021 21:13:14.123", - "minute": 13, - "second": 14.123 -} - -{ - "date": "12/01/2021", - "grok_match_ok": true, - "hour": "21", - "message": "12/01/2021 21:13:14.123", - "minute": 13, - "second": 14.123, - "status": "unknown", - "time": 1665994187473917724 -} -``` - -## `add_key()` - -##函数原型：`fn add_key(key, value)` - -函数说明：往 point 中增加一个字段 - -函数参数 - -- `key`: 新增的 key 名称 -- `value`：作为 key 的值 - -示例: - -```python -# 待处理数据: {"age": 17, "name": "zhangsan", "height": 180} - -# 处理脚本 -add_key(city, "shanghai") - -# 处理结果 -{ - "age": 17, - "height": 180, - "name": "zhangsan", - "city": "shanghai" -} -``` - -## `get_key()` - -函数原型：`fn get_key(key)` - -函数说明：从 point 中读取 key 的值，而不是堆栈上的变量的值 - -函数参数 - -- `key_name`: key 的名称 - -示例一: - -```python -# scipt 1 -key = "shanghai" -add_key(key) -key = "tokyo" -add_key(add_new_key, key) - -# 处理结果 -{ - "add_new_key": "tokyo", - "key": "shanghai", -} - -``` - -## `set_tag()` - -函数原型：`fn set_tag(key, value: str)` - -函数说明：对指定字段标记为 tag 输出，设置为 tag 后，其他函数仍可对该变量操作。如果被置为 tag 的 key 是已经切割出来的 field，那么它将不会在 field 中出现，这样可以避免切割出来的 field key 跟已有数据上的 tag key 重名 - -函数参数 - -- `key`: 待标记为 tag 的字段 -- `value`: 可以为字符串字面量或者变量 - -```python -# in << {"str": "13789123014"} -set_tag(str) -json(_, str) # str == "13789123014" -replace(str, "(1[0-9]{2})[0-9]{4}([0-9]{4})", "$1****$2") -# Extracted data(drop: false, cost: 49.248µs): -# { -# "message": "{\"str\": \"13789123014\", \"str_b\": \"3\"}", -# "str#": "137****3014" -# } -# * 字符 `#` 仅为 datakit --pl --txt 输出展示时字段为 tag 的标记 - -# in << {"str_a": "2", "str_b": "3"} -json(_, str_a) -set_tag(str_a, "3") # str_a == 3 -# Extracted data(drop: false, cost: 30.069µs): -# { -# "message": "{\"str_a\": \"2\", \"str_b\": \"3\"}", -# "str_a#": "3" -# } - - -# in << {"str_a": "2", "str_b": "3"} -json(_, str_a) -json(_, str_b) -set_tag(str_a, str_b) # str_a == str_b == "3" -# Extracted data(drop: false, cost: 32.903µs): -# { -# "message": "{\"str_a\": \"2\", \"str_b\": \"3\"}", -# "str_a#": "3", -# "str_b": "3" -# } -``` - -## `rename()` - -函数原型：`fn rename(new_key, old_key)` - -函数说明：将已提取的字段重新命名 - -参数: - -- `new_key`: 新字段名 -- `old_key`: 已提取的字段名 - -```python -# 把已提取的 abc 字段重新命名为 abc1 -rename('abc1', abc) - -# or - -rename(abc1, abc) -``` - -示例： - -```python -# 待处理数据: {"info": {"age": 17, "name": "zhangsan", "height": 180}} - -# 处理脚本 -json(_, info.name, "姓名") - -# 处理结果 -{ - "message": "{\"info\": {\"age\": 17, \"name\": \"zhangsan\", \"height\": 180}}", - "zhangsan": { - "age": 17, - "height": 180, - "姓名": "zhangsan" - } -} -``` - -## `set_measurement()` - -函数原型：`fn set_measurement(name: str, delete_key: bool = false)` - -函数说明：改变行协议的 name - -函数参数 - -- `name`: 值作为 mesaurement name，可传入字符串常量或变量 -- `delete_key`: 如果在 point 中存在与变量同名的 tag 或 field 则删除它 - -## `cast()` - -函数原型：`fn cast(key, dst_type: str)` - -函数说明：将 key 值转换成指定类型 - -函数参数 - -- `key`: 已提取的某字段 -- `type`：转换的目标类型，支持 `\"str\", \"float\", \"int\", \"bool\"` 这几种，目标类型需要用英文状态双引号括起来 - -示例: - -```python -# 待处理数据: {"first": 1,"second":2,"third":"aBC","forth":true} - -# 处理脚本 -json(_, first) -cast(first, "str") - -# 处理结果 -{ - "first": "1" -} -``` - -## `default_time()` - -函数原型：`fn default_time(key: str, timezone: str = "")` - -函数说明：以提取的某个字段作为最终数据的时间戳 - -函数参数 - -- `key`: 指定的 key， key 的数据类型需要为字符串类型 -- `timezone`: 指定待格式化的时间文本所使用的时区，可选参数，默认为当前系统时区，时区示例 `+8/-8/+8:30` - -待处理数据支持以下格式化时间 - -| 日期格式 | 日期格式 | 日期格式 | 日期格式 | -| ----- | ---- | ---- | ---- | -| `2014-04-26 17:24:37.3186369` | `May 8, 2009 5:57:51 PM` | `2012-08-03 18:31:59.257000000` | `oct 7, 1970` | -| `2014-04-26 17:24:37.123` | `oct 7, '70` | `2013-04-01 22:43` | `oct. 7, 1970` | -| `2013-04-01 22:43:22` | `oct. 7, 70` | `2014-12-16 06:20:00 UTC` | `Mon Jan 2 15:04:05 2006` | -| `2014-12-16 06:20:00 GMT` | `Mon Jan 2 15:04:05 MST 2006` | `2014-04-26 05:24:37 PM` | `Mon Jan 02 15:04:05 -0700 2006` | -| `2014-04-26 13:13:43 +0800` | `Monday, 02-Jan-06 15:04:05 MST` | `2014-04-26 13:13:43 +0800 +08` | `Mon, 02 Jan 2006 15:04:05 MST` | -| `2014-04-26 13:13:44 +09:00` | `Tue, 11 Jul 2017 16:28:13 +0200 (CEST)` | `2012-08-03 18:31:59.257000000 +0000 UTC` | `Mon, 02 Jan 2006 15:04:05 -0700` | -| `2015-09-30 18:48:56.35272715 +0000 UTC` | `Thu, 4 Jan 2018 17:53:36 +0000` | `2015-02-18 00:12:00 +0000 GMT` | `Mon 30 Sep 2018 09:09:09 PM UTC` | -| `2015-02-18 00:12:00 +0000 UTC` | `Mon Aug 10 15:44:11 UTC+0100 2015` | `2015-02-08 03:02:00 +0300 MSK m=+0.000000001` | `Thu, 4 Jan 2018 17:53:36 +0000` | -| `2015-02-08 03:02:00.001 +0300 MSK m=+0.000000001` | `Fri Jul 03 2015 18:04:07 GMT+0100 (GMT Daylight Time)` | `2017-07-19 03:21:51+00:00` | `September 17, 2012 10:09am` | -| `2014-04-26` | `September 17, 2012 at 10:09am PST-08` | `2014-04` | `September 17, 2012, 10:10:09` | -| `2014` | `2014:3:31` | `2014-05-11 08:20:13,787` | `2014:03:31` | -| `3.31.2014` | `2014:4:8 22:05` | `03.31.2014` | `2014:04:08 22:05` | -| `08.21.71` | `2014:04:2 03:00:51` | `2014.03` | `2014:4:02 03:00:51` | -| `2014.03.30` | `2012:03:19 10:11:59` | `20140601` | `2012:03:19 10:11:59.3186369` | -| `20140722105203` | `2014年04月08日` | `1332151919` | `2006-01-02T15:04:05+0000` | -| `1384216367189` | `2009-08-12T22:15:09-07:00` | `1384216367111222` | `2009-08-12T22:15:09` | -| `1384216367111222333` | `2009-08-12T22:15:09Z` | - -JSON 提取示例: - -```python -# 原始 json -{ - "time":"06/Jan/2017:16:16:37 +0000", - "second":2, - "third":"abc", - "forth":true -} - -# pipeline 脚本 -json(_, time) # 提取 time 字段 -default_time(time) # 将提取到的 time 字段转换成时间戳 - -# 处理结果 -{ - "time": 1483719397000000000, -} -``` - -文本提取示例: - -```python -# 原始日志文本 -2021-01-11T17:43:51.887+0800 DEBUG io io/io.go:458 post cost 6.87021ms - -# pipeline 脚本 -grok(_, '%{TIMESTAMP_ISO8601:log_time}') # 提取日志时间，并将字段命名为 log_time -default_time(log_time) # 将提取到的 log_time 字段转换成时间戳 - -# 处理结果 -{ - "log_time": 1610358231887000000, -} - -# 对于 logging 采集的数据，最好将时间字段命名为 time，否则 logging 采集器会以当前时间填充 -rename("time", log_time) - -# 处理结果 -{ - "time": 1610358231887000000, -} -``` - -## `drop_key()` - -函数原型：`fn drop_key(key)` - -函数说明：删除已提取字段 - -函数参数 - -- `key`: 待删除字段名 - -示例: - -```python -data = `{\"age\": 17, \"name\": \"zhangsan\", \"height\": 180}` - -# 处理脚本 -json(_, age,) -json(_, name) -json(_, height) -drop_key(height) - -# 处理结果 -{ - "age": 17, - "name": "zhangsan" -} -``` - -## `use()` - -函数原型：`fn use(name: str)` - -参数: - -- `name`: 脚本名，如 abp.p - -函数说明：调用其他脚本，可在被调用的脚本访问当前的所有数据 -示例： - -```python -# 待处理数据: {"ip":"1.2.3.4"} - -# 处理脚本 a.p -use(\"b.p\") - -# 处理脚本 b.p -json(_, ip) -geoip(ip) - -# 执行脚本 a.p 的处理结果 -{ - "city" : "Brisbane", - "country" : "AU", - "ip" : "1.2.3.4", - "province" : "Queensland", - "isp" : "unknown" - "message" : "{\"ip\": \"1.2.3.4\"}", -} -``` - -## `exit()` - -函数原型：`fn exit()` - -函数说明：结束当前一条日志的解析，若未调用函数 drop() 仍会输出已经解析的部分 - -```python -# in << {"str_a": "2", "str_b": "3"} -json(_, str_a) -if str_a == "2"{ - exit() -} -json(_, str_b) - -# Extracted data(drop: false, cost: 48.233µs): -# { -# "message": "{\"str_a\": \"2\", \"str_b\": \"3\"}", -# "str_a": "2" -# } -``` - -示例二: - -```python -# scipt 2 -key = "shanghai" -add_key(key) -key = "tokyo" -add_key(add_new_key, get_key(key)) - -#处理结果 -{ - "add_new_key": "shanghai", - "key": "shanghai", -} -``` - -## `cover()` {#fn-cover} - -函数原型：`fn cover(key: str, range: list)` - -函数说明：对指定字段上获取的字符串数据，按范围进行数据脱敏处理 - -函数参数 - -- `key`: 待提取字段 -- `range`: 脱敏字符串的索引范围（`[start,end]`） start和end均支持负数下标，用来表达从尾部往前追溯的语义。区间合理即可，end如果大于字符串最大长度会默认成最大长度 - -示例: - -```python -# 待处理数据 {"str": "13789123014"} -json(_, `str`) -cover(`str`, [8, 9]) - -# 待处理数据 {"abc": "13789123014"} -json(_, abc) -cover(abc, [2, 4]) -``` - -## `datetime()` {#fn-datetime} - -函数原型：`fn datetime(key, precision: str, fmt: str)` - -函数说明：将时间戳转成指定日期格式 - -函数参数 - -- `key`: 已经提取的时间戳 (必选参数) -- `precision`：输入的时间戳精度(s, ms) -- `fmt`：日期格式，时间格式, 支持以下模版 - -```python -ANSIC = "Mon Jan _2 15:04:05 2006" -UnixDate = "Mon Jan _2 15:04:05 MST 2006" -RubyDate = "Mon Jan 02 15:04:05 -0700 2006" -RFC822 = "02 Jan 06 15:04 MST" -RFC822Z = "02 Jan 06 15:04 -0700" // RFC822 with numeric zone -RFC850 = "Monday, 02-Jan-06 15:04:05 MST" -RFC1123 = "Mon, 02 Jan 2006 15:04:05 MST" -RFC1123Z = "Mon, 02 Jan 2006 15:04:05 -0700" // RFC1123 with numeric zone -RFC3339 = "2006-01-02T15:04:05Z07:00" -RFC3339Nano = "2006-01-02T15:04:05.999999999Z07:00" -Kitchen = "3:04PM" -``` - -示例: - -```python -# 待处理数据: -# { -# "a":{ -# "timestamp": "1610960605000", -# "second":2 -# }, -# "age":47 -# } - -# 处理脚本 -json(_, a.timestamp) datetime(a.timestamp, 'ms', 'RFC3339') -``` - -## `replace()` - -函数原型：`fn replace(key: str, regex: str, replace_str: str)` - -函数说明：对指定字段上获取的字符串数据按正则进行替换 - -函数参数 - -- `key`: 待提取字段 -- `regex`: 正则表达式 -- `replace_str`: 替换的字符串 - -示例: - -```python -# 电话号码：{"str": "13789123014"} -json(_, str) -replace(str, "(1[0-9]{2})[0-9]{4}([0-9]{4})", "$1****$2") - -# 英文名 {"str": "zhang san"} -json(_, str) -replace(str, "([a-z]*) \\w*", "$1 ***") - -# 身份证号 {"str": "362201200005302565"} -json(_, str) -replace(str, "([1-9]{4})[0-9]{10}([0-9]{4})", "$1**********$2") - -# 中文名 {"str": "小阿卡"} -json(_, str) -replace(str, '([\u4e00-\u9fa5])[\u4e00-\u9fa5]([\u4e00-\u9fa5])', "$1＊$2") -``` - -## `sql_cover()` - -函数原型：`fn sql_cover(sql_test: str)` - -函数说明：脱敏sql语句 - -```python -# in << {"select abc from def where x > 3 and y < 5"} -sql_cover(_) - -# Extracted data(drop: false, cost: 33.279µs): -# { -# "message": "select abc from def where x > ? and y < ?" -# } -``` - -## `strfmt()` - -函数原型：`fn strfmt(key, fmt: str, args ...: int|float|bool|str|list|map|nil)` - -函数说明：对已提取 `arg1, arg2, ...` 指定的字段内容根据 `fmt` 进行格式化，并把格式化后的内容写入 `key` 字段中 - -函数参数 - -- `key`: 指定格式化后数据写入字段名 -- `fmt`: 格式化字符串模板 -- `args`:可变参数，可以是多个已提取的待格式化字段名 - -示例: - -```python -# 待处理数据: {"a":{"first":2.3,"second":2,"third":"abc","forth":true},"age":47} - -# 处理脚本 -json(_, a.second) -json(_, a.thrid) -cast(a.second, "int") -json(_, a.forth) -strfmt(bb, "%v %s %v", a.second, a.thrid, a.forth) -``` - -## `trim()` - -函数原型：`fn trim(key, cutset: str = "")` - -函数说明：删除 key 中首尾中指定的字符，cutset 为空字符串时默认删除所有空白符 - -函数参数： - -- `key`: 已提取的某字段，字符串类型 -- `cutset`: 删除 key 中出现在 cutset 字符串的中首尾字符 - -示例: - -```python -# 待处理数据: "trim(key, cutset)" - -# 处理脚本 -add_key(test_data, "ACCAA_test_DataA_ACBA") -trim(test_data, "ABC_") - -# 处理结果 -{ - "test_data": "test_Data" -} -``` - -## `uppercase()` - -函数原型：`fn uppercase(key: str)` - -函数说明：将已提取 key 中内容转换成大写 - -函数参数 - -- `key`: 指定已提取的待转换字段名，将 `key` 内容转成大写 - -示例: - -```python -# 待处理数据: {"first": "hello","second":2,"third":"aBC","forth":true} - -# 处理脚本 -json(_, first) uppercase(first) - -# 处理结果 -{ - "first": "HELLO" -} -``` - -## `url_decode()` - -函数原型：`fn url_decode(key: str)` - -函数说明：将已提取 `key` 中的 URL 解析成明文 - -参数: - -- `key`: 已经提取的某个 `key` - -示例： - -```python -# 待处理数据: {"url":"http%3a%2f%2fwww.baidu.com%2fs%3fwd%3d%e6%b5%8b%e8%af%95"} - -# 处理脚本 -json(_, url) url_decode(url) - -# 处理结果 -{ - "message": "{"url":"http%3a%2f%2fwww.baidu.com%2fs%3fwd%3d%e6%b5%8b%e8%af%95"}", - "url": "http://www.baidu.com/s?wd=测试" -} -``` - -## `xml()` - -函数原型：`fn xml(input: str, xpath_expr: str, key_name)` - -函数说明：通过 xpath 表达式，从 XML 中提取字段。 - -参数： - -- input: 待提取的 XML -- xpath_expr: xpath 表达式 -- key_name: 提取后数据写入新 key - -示例一： - -```python -# 待处理数据 - - valuex - ... - ... - - element_a_1 - element_a_2 - - - -# 处理脚本 -xml(_, '/entry/fieldarray//fielda[1]/text()', field_a_1) - -# 处理结果 -{ - "field_a_1": "element_a_1", # 提取了 element_a_1 - "message": "\t\t\u003centry\u003e\n \u003cfieldx\u003evaluex\u003c/fieldx\u003e\n \u003cfieldy\u003e...\u003c/fieldy\u003e\n \u003cfieldz\u003e...\u003c/fieldz\u003e\n \u003cfieldarray\u003e\n \u003cfielda\u003eelement_a_1\u003c/fielda\u003e\n \u003cfielda\u003eelement_a_2\u003c/fielda\u003e\n \u003c/fieldarray\u003e\n \u003c/entry\u003e", - "status": "unknown", - "time": 1655522989104916000 -} -``` - -示例二： - -```python -# 待处理数据 - - ORD12345 - V11111 - - -# 处理脚本 -xml(_, '/OrderEvent/@actionCode', action_code) -xml(_, '/OrderEvent/OrderNumber/text()', OrderNumber) - -# 处理结果 -{ - "OrderNumber": "ORD12345", - "action_code": "5", - "message": "\u003cOrderEvent actionCode = \"5\"\u003e\n \u003cOrderNumber\u003eORD12345\u003c/OrderNumber\u003e\n \u003cVendorNumber\u003eV11111\u003c/VendorNumber\u003e\n\u003c/OrderEvent\u003e", - "status": "unknown", - "time": 1655523193632471000 -} -``` diff --git a/pkg/inimpl/guancecloud/input/point.go b/pkg/inimpl/guancecloud/input/point.go deleted file mode 100644 index cc94d95..0000000 --- a/pkg/inimpl/guancecloud/input/point.go +++ /dev/null @@ -1,316 +0,0 @@ -// Unless explicitly stated otherwise all files in this repository are licensed -// under the MIT License. -// This product includes software developed at Guance Cloud (https://www.guance.com/). -// Copyright 2021-present Guance, Inc. - -package input - -import ( - "fmt" - "sync" - "time" - - "github.com/GuanceCloud/platypus/pkg/ast" - plruntime "github.com/GuanceCloud/platypus/pkg/engine/runtime" - - "github.com/spf13/cast" -) - -type ( - KeyKind uint - PtFlag uint -) - -const ( - PtMeasurement PtFlag = iota - PtTag - PtField - PtTFDefaulutOrKeep - PtTime -) - -const ( - KindPtDefault KeyKind = iota - KindPtTag -) - -const Originkey = "message" - -type TFMeta struct { - DType ast.DType - PtFlag PtFlag -} - -type InputWithVarbMapRW interface { - Get(key string) (any, ast.DType, bool) - Set(key string, value any, dtype ast.DType) bool - Delete(key string) bool -} - -type InputWithVarbMapR interface { - Get(key string) (any, ast.DType, bool) -} - -type InputWithoutVarbMap interface{} - -type Point struct { - Measurement string - Tags map[string]string - Fields map[string]any // int, float, bool, string, map, slice, array - Time time.Time - - Drop bool - Meta map[string]*TFMeta // [DType, PtFlag] -} - -func InitPt(pt *Point, m string, t map[string]string, f map[string]any, tn time.Time) *Point { - if f == nil { - f = map[string]any{} - } - - if t == nil { - t = map[string]string{} - } - - pt.Measurement = m - pt.Tags = t - pt.Fields = f - pt.Time = tn - pt.Drop = false - pt.Meta = map[string]*TFMeta{} - - for k, v := range f { - if v == nil { - pt.Meta[k] = GetMeta(ast.Nil, PtField) - continue - } - switch v.(type) { - case int32, int8, int16, int, - uint, uint16, uint32, uint64, uint8: - pt.Meta[k] = GetMeta(ast.Int, PtField) - f[k] = cast.ToInt64(v) - case int64: - pt.Meta[k] = GetMeta(ast.Int, PtField) - case float32: - pt.Meta[k] = GetMeta(ast.Float, PtField) - f[k] = cast.ToFloat64(v) - case float64: - pt.Meta[k] = GetMeta(ast.Float, PtField) - case bool: - pt.Meta[k] = GetMeta(ast.Bool, PtField) - case string: - pt.Meta[k] = GetMeta(ast.String, PtField) - } - } - - for k := range t { - pt.Meta[k] = GetMeta(ast.String, PtTag) - } - return pt -} - -func (pt *Point) Get(key string) (any, ast.DType, error) { - m, ok := pt.Meta[key] - - if !ok { - return nil, ast.Invalid, fmt.Errorf("not found") - } - - if m.DType == ast.Void || m.DType == ast.Nil { - return nil, ast.Nil, nil - } - - switch m.PtFlag { //nolint:exhaustive - case PtField: - if v, ok := pt.Fields[key]; ok { - return v, m.DType, nil - } - return nil, ast.Nil, nil - case PtTag: - if v, ok := pt.Tags[key]; ok { - return v, ast.String, nil - } - return nil, ast.Nil, nil - default: - return nil, ast.Invalid, fmt.Errorf("unsupported pt key type") - } -} - -func (pt *Point) Delete(key string) { - m, ok := pt.Meta[key] - if !ok { - return - } - - if m.PtFlag == PtField { - delete(pt.Fields, key) - } else { - delete(pt.Tags, key) - } - - delete(pt.Meta, key) - PutMeta(m) -} - -func (pt *Point) Set(key string, value any, dtype ast.DType) error { - m, ok := pt.Meta[key] - - if !ok { - m = GetMeta(dtype, PtField) - pt.Meta[key] = m - } - - // key 值为 nil 只记录 meta 中 - switch m.PtFlag { //nolint:exhaustive - case PtField: - switch dtype { //nolint:exhaustive - case ast.Nil, ast.Void, ast.Invalid: - m.DType = ast.Nil - pt.Fields[key] = nil - return nil - case ast.List, ast.Map: - if v, err := plruntime.Conv2String(value, dtype); err == nil { - pt.Fields[key] = v - m.DType = ast.String - } else { - m.DType = ast.Nil - pt.Fields[key] = nil - return nil - } - default: - pt.Fields[key] = value - m.DType = dtype - } - case PtTag: - if dtype == ast.Void || dtype == ast.Invalid { - delete(pt.Tags, key) - return nil - } - - if v, err := plruntime.Conv2String(value, dtype); err == nil { - pt.Tags[key] = v - } else { - return err - } - } - return nil -} - -func (pt *Point) SetTag(key string, value any, dtype ast.DType) error { - m, ok := pt.Meta[key] - - if !ok { - m = GetMeta(ast.String, PtTag) - pt.Meta[key] = m - } - - if m.PtFlag == PtField { - delete(pt.Fields, key) - m.DType, m.PtFlag = ast.String, PtTag - } - - if str, err := plruntime.Conv2String(value, dtype); err == nil { - pt.Tags[key] = str - return nil - } else { - pt.Tags[key] = "" - return err - } -} - -func (pt *Point) Mv2Tag(key string) error { - m, ok := pt.Meta[key] - if !ok { - pt.Meta[key] = GetMeta(ast.String, PtTag) - pt.Tags[key] = "" - return nil - } - - if m.PtFlag == PtField { - dType := m.DType - - m.DType, m.PtFlag = ast.String, PtTag - v, ok := pt.Fields[key] - if !ok { - return nil - } - delete(pt.Fields, key) - if str, err := plruntime.Conv2String(v, dType); err == nil { - pt.Tags[key] = str - } else { - pt.Tags[key] = "" - return err - } - } - - return nil -} - -func (pt *Point) SetMeasurement(m string) { - pt.Measurement = m -} - -func (pt *Point) GetMeasurement() string { - return pt.Measurement -} - -func (pt *Point) KeyTime2Time() { - if v, _, err := pt.Get("time"); err == nil { - if nanots, ok := v.(int64); ok { - t := time.Unix(nanots/int64(time.Second), - nanots%int64(time.Second)) - if !t.IsZero() { - pt.Time = t - } - pt.Delete("time") - } - } -} - -var pointPool = sync.Pool{ - New: func() any { - return &Point{} - }, -} - -var metaPool = sync.Pool{ - New: func() any { - return &TFMeta{} - }, -} - -func GetMeta(dtype ast.DType, ptflag PtFlag) *TFMeta { - meta, _ := metaPool.Get().(*TFMeta) - meta.DType = dtype - meta.PtFlag = ptflag - return meta -} - -func PutMeta(meta *TFMeta) { - metaPool.Put(meta) -} - -func GetPoint() *Point { - pt, _ := pointPool.Get().(*Point) - return pt -} - -func PutPoint(pt *Point) { - if pt == nil { - return - } - - pt.Measurement = "" - - pt.Fields = nil - pt.Tags = nil - - for k, v := range pt.Meta { - delete(pt.Meta, k) - PutMeta(v) - } - - pt.Drop = false - - pointPool.Put(pt) -} diff --git a/pkg/inimpl/guancecloud/input/point_test.go b/pkg/inimpl/guancecloud/input/point_test.go deleted file mode 100644 index c922489..0000000 --- a/pkg/inimpl/guancecloud/input/point_test.go +++ /dev/null @@ -1,38 +0,0 @@ -package input - -import ( - "testing" - "time" - - "github.com/GuanceCloud/platypus/pkg/ast" - "github.com/stretchr/testify/assert" -) - -func TestInitPoint(t *testing.T) { - pt := &Point{ - Tags: map[string]string{ - "a1": "1", - }, - Fields: map[string]any{ - "a": int32(1), - "b": int64(2), - "c": float32(3), - "d": float64(4), - "e": "5", - "f": true, - "g": nil, - }, - } - pt2 := &Point{} - InitPt(pt2, "", pt.Tags, pt.Fields, time.Now()) - assert.Equal(t, &TFMeta{DType: ast.Int, PtFlag: PtField}, pt2.Meta["a"]) - assert.Equal(t, int64(1), pt2.Fields["a"]) - assert.Equal(t, &TFMeta{DType: ast.Int, PtFlag: PtField}, pt2.Meta["b"]) - assert.Equal(t, float64(3), pt2.Fields["c"]) - assert.Equal(t, &TFMeta{DType: ast.Float, PtFlag: PtField}, pt2.Meta["c"]) - assert.Equal(t, &TFMeta{DType: ast.Float, PtFlag: PtField}, pt2.Meta["d"]) - assert.Equal(t, &TFMeta{DType: ast.String, PtFlag: PtField}, pt2.Meta["e"]) - assert.Equal(t, &TFMeta{DType: ast.Bool, PtFlag: PtField}, pt2.Meta["f"]) - assert.Equal(t, &TFMeta{DType: ast.Nil, PtFlag: PtField}, pt2.Meta["g"]) - assert.Equal(t, &TFMeta{DType: ast.String, PtFlag: PtTag}, pt2.Meta["a1"]) -} diff --git a/pkg/inimpl/guancecloud/ptinput/funcs/all.go b/pkg/inimpl/guancecloud/ptinput/funcs/all.go new file mode 100644 index 0000000..76b58b2 --- /dev/null +++ b/pkg/inimpl/guancecloud/ptinput/funcs/all.go @@ -0,0 +1,146 @@ +// Unless explicitly stated otherwise all files in this repository are licensed +// under the MIT License. +// This product includes software developed at Guance Cloud (https://www.guance.com/). +// Copyright 2021-present Guance, Inc. + +// Package funcs for pipeline +package funcs + +import ( + "github.com/GuanceCloud/cliutils/logger" + "github.com/GuanceCloud/platypus/pkg/engine/runtime" +) + +var l = logger.DefaultSLogger("funcs") + +func InitLog() { + l = logger.SLogger("funcs") +} + +var FuncsMap = map[string]runtime.FuncCall{ + "agg_create": AggCreate, + "agg_metric": AggAddMetric, + "append": Append, + "cidr": CIDR, + "grok": Grok, + "add_key": AddKey, + "delete": DeleteMapItem, + "adjust_timezone": AdjustTimezone, + "json": JSON, + "add_pattern": AddPattern, + "b64dec": B64dec, + "b64enc": B64enc, + "cast": Cast, + "datetime": DateTime, + "default_time": DefaultTime, + "default_time_with_fmt": DefaultTimeWithFmt, + "drop": Drop, + "drop_key": Dropkey, + "drop_origin_data": DropOriginData, + "exit": Exit, + "geoip": GeoIP, + "get_key": Getkey, + "group_between": Group, + "group_in": GroupIn, + "kv_split": KVSplit, + "lowercase": Lowercase, + "len": Len, + "load_json": LoadJSON, + "nullif": NullIf, + "rename": Rename, + "set_tag": SetTag, + "set_measurement": SetMeasurement, + "strfmt": Strfmt, + "trim": Trim, + "timestamp": Timestamp, + "uppercase": Uppercase, + "use": Use, + "url_decode": URLDecode, + "user_agent": UserAgent, + "parse_duration": ParseDuration, + "parse_date": ParseDate, + "cover": Cover, + "query_refer_table": QueryReferTable, + "mquery_refer_table": MQueryReferTableMulti, + "replace": Replace, + "duration_precision": DurationPrecision, + "xml": XML, + "match": Match, + "sql_cover": SQLCover, + "decode": Decode, + "sample": Sample, + "url_parse": URLParse, + "value_type": ValueType, + "vaild_json": ValidJSON, + "valid_json": ValidJSON, + "conv_traceid_w3c_to_dd": ConvTraceIDW3C2DD, + "create_point": CreatePoint, + "parse_int": ParseInt, + "format_int": FormatInt, + // disable + "json_all": JSONAll, +} + +var FuncsCheckMap = map[string]runtime.FuncCheck{ + "agg_create": AggCreateChecking, + "agg_metric": AggAddMetricChecking, + "append": AppendChecking, + "cidr": CIDRChecking, + "grok": GrokChecking, + "add_key": AddkeyChecking, + "delete": DeleteMapItemChecking, + "adjust_timezone": AdjustTimezoneChecking, + "json": JSONChecking, + "add_pattern": AddPatternChecking, + "b64dec": B64decChecking, + "b64enc": B64encChecking, + "cast": CastChecking, + "datetime": DateTimeChecking, + "default_time": DefaultTimeChecking, + "default_time_with_fmt": DefaultTimeWithFmtChecking, + "drop": DropChecking, + "drop_key": DropkeyChecking, + "drop_origin_data": DropOriginDataChecking, + "exit": ExitChecking, + "geoip": GeoIPChecking, + "get_key": GetkeyChecking, + "group_between": GroupChecking, + "group_in": GroupInChecking, + "kv_split": KVSplitChecking, + "len": LenChecking, + "load_json": LoadJSONChecking, + "lowercase": LowercaseChecking, + "nullif": NullIfChecking, + "rename": RenameChecking, + "set_measurement": SetMeasurementChecking, + "set_tag": SetTagChecking, + "strfmt": StrfmtChecking, + "trim": TrimChecking, + "timestamp": TimestampChecking, + "uppercase": UppercaseChecking, + "use": UseChecking, + "url_decode": URLDecodeChecking, + "user_agent": UserAgentChecking, + "parse_duration": ParseDurationChecking, + "parse_date": ParseDateChecking, + "cover": CoverChecking, + "query_refer_table": QueryReferTableChecking, + "mquery_refer_table": MQueryReferTableChecking, + "replace": ReplaceChecking, + "duration_precision": DurationPrecisionChecking, + "sql_cover": SQLCoverChecking, + "xml": XMLChecking, + "match": MatchChecking, + "decode": DecodeChecking, + "url_parse": URLParseChecking, + "sample": SampleChecking, + "value_type": ValueTypeChecking, + "vaild_json": ValidJSONChecking, + "valid_json": ValidJSONChecking, + "conv_traceid_w3c_to_dd": ConvTraceIDW3C2DDChecking, + "create_point": CreatePointChecking, + "parse_int": ParseIntChecking, + "format_int": FormatIntChecking, + // disable + "json_all": JSONAllChecking, +} diff --git a/pkg/inimpl/guancecloud/ptinput/funcs/all_doc.go b/pkg/inimpl/guancecloud/ptinput/funcs/all_doc.go new file mode 100644 index 0000000..4d48b49 --- /dev/null +++ b/pkg/inimpl/guancecloud/ptinput/funcs/all_doc.go @@ -0,0 +1,688 @@ +// Unless explicitly stated otherwise all files in this repository are licensed +// under the MIT License. +// This product includes software developed at Guance Cloud (https://www.guance.com/). +// Copyright 2021-present Guance, Inc. + +package funcs + +import ( + _ "embed" +) + +type PLDoc struct { + Doc string `json:"doc"` + Prototype string `json:"prototype"` + Description string `json:"description"` + Deprecated bool `json:"deprecated"` + RequiredVersion string `json:"required_version"` + FnCategory map[string][]string `json:"fn_category"` +} + +var PipelineFunctionDocs = map[string]*PLDoc{ + "add_key()": &addKeyMarkdown, + "add_pattern()": &addPatternMarkdown, + "adjust_timezone()": &adjustTimezoneMarkdown, + "agg_create()": &aggCreateMarkdown, + "agg_metric()": &aggMetricMarkdown, + "append()": &appendMarkdown, + "b64dec()": &b64decMarkdown, + "b64enc()": &b64encMarkdown, + "cast()": &castMarkdown, + "delete()": &deleteMarkdown, + "cidr()": &cidrMarkdown, + "cover()": &coverMarkdown, + "datetime()": &datetimeMarkdown, + "decode": &decodeMarkdown, + "default_time()": &defaultTimeMarkdown, + "drop()": &dropMarkdown, + "drop_key()": &dropKeyMarkdown, + "drop_origin_data()": &dropOriginDataMarkdown, + "duration_precision()": &durationPrecisionMarkdown, + "exit()": &exitMarkdown, + "geoip()": &geoIPMarkdown, + "get_key()": &getKeyMarkdown, + "grok()": &grokMarkdown, + "group_between()": &groupBetweenMarkdown, + "group_in()": &groupInMarkdown, + "kv_split()": &kvSplitMarkdown, + "json()": &jsonMarkdown, + "len()": &lenMarkdown, + "load_json()": &loadJSONMarkdown, + "lowercase()": &lowercaseMarkdown, + "nullif()": &nullIfMarkdown, + "parse_date()": &parseDateMarkdown, + "parse_duration()": &parseDurationMarkdown, + "query_refer_table()": &queryReferTableMarkdown, + "match()": &matchMarkdown, + "mquery_refer_table()": &mQueryReferTableMarkdown, + "rename()": &renameMarkdown, + "replace()": &replaceMarkdown, + "set_measurement()": &setMeasurementMarkdown, + "set_tag()": &setTagMarkdown, + "sql_cover": &sqlCoverMarkdown, + "strfmt()": &strfmtMarkdown, + "trim()": &trimMarkdown, + "uppercase()": &uppercaseMarkdown, + "url_decode()": &URLDecodeMarkdown, + "use()": &useMarkdown, + "user_agent()": &userAgentMarkdown, + "xml()": &xmlMarkdown, + "sample()": &sampleMarkdown, + "url_parse()": &urlParseMarkdown, + "timestamp()": ×tampMarkdown, + "value_type()": &valueTypeMarkdown, + "valid_json()": &validJSONMarkdown, + "conv_traceid_w3c_to_dd()": &convTraceID128MD, + "create_point()": &createPointMarkdown, + "parse_int()": &parseIntMarkdown, + "format_int()": &formatIntMarkdown, +} + +var PipelineFunctionDocsEN = map[string]*PLDoc{ + "add_key()": &addKeyMarkdownEN, + "add_pattern()": &addPatternMarkdownEN, + "adjust_timezone()": &adjustTimezoneMarkdownEN, + "agg_create()": &aggCreateMarkdownEN, + "agg_metric()": &aggMetricMarkdownEN, + "append()": &appendMarkdownEN, + "b64dec()": &b64decMarkdownEN, + "b64enc()": &b64encMarkdownEN, + "cast()": &castMarkdownEN, + "delete()": &deleteMarkdownEN, + "cidr()": &cidrMarkdownEN, + "cover()": &coverMarkdownEN, + "datetime()": &datetimeMarkdownEN, + "decode": &decodeMarkdownEN, + "default_time()": &defaultTimeMarkdownEN, + "drop()": &dropMarkdownEN, + "drop_key()": &dropKeyMarkdownEN, + "drop_origin_data()": &dropOriginDataMarkdownEN, + "duration_precision()": &durationPrecisionMarkdownEN, + "exit()": &exitMarkdownEN, + "geoip()": &geoIPMarkdownEN, + "get_key()": &getKeyMarkdownEN, + "grok()": &grokMarkdownEN, + "group_between()": &groupBetweenMarkdownEN, + "group_in()": &groupInMarkdownEN, + "kv_split()": &kvSplitMarkdownEN, + "json()": &jsonMarkdownEN, + "len()": &lenMarkdownEN, + "load_json()": &loadJSONMarkdownEN, + "lowercase()": &lowercaseMarkdownEN, + "nullif()": &nullIfMarkdownEN, + "parse_date()": &parseDateMarkdownEN, + "parse_duration()": &parseDurationMarkdownEN, + "query_refer_table()": &queryReferTableMarkdownEN, + "match()": &matchMarkdownEN, + "mquery_refer_table()": &mQueryReferTableMarkdownEN, + "rename()": &renameMarkdownEN, + "replace()": &replaceMarkdownEN, + "set_measurement()": &setMeasurementMarkdownEN, + "set_tag()": &setTagMarkdownEN, + "sql_cover": &sqlCoverMarkdownEN, + "strfmt()": &strfmtMarkdownEN, + "trim()": &trimMarkdownEN, + "uppercase()": &uppercaseMarkdownEN, + "url_decode()": &URLDecodeMarkdownEN, + "use()": &useMarkdownEN, + "user_agent()": &userAgentMarkdownEN, + "xml()": &xmlMarkdownEN, + "sample()": &sampleMarkdownEN, + "url_parse()": &urlParseMarkdownEN, + "timestamp()": ×tampMarkdownEN, + "value_type()": &valueTypeMarkdownEN, + "valid_json()": &validJSONMarkdownEN, + "conv_traceid_w3c_to_dd()": &convTraceID128MDEN, + "create_point()": &createPointMarkdownEN, + "parse_int()": &parseIntMarkdownEN, + "format_int()": &formatIntMarkdownEN, +} + +// embed docs. +var ( + + //go:embed md/agg_create.md + docAggCreate string + + //go:embed md/agg_metric.md + docAggMetric string + + //go:embed md/add_pattern.md + docAddPattern string + + //go:embed md/append.md + docAppend string + + //go:embed md/b64dec.md + docB64dec string + + //go:embed md/b64enc.md + docB64enc string + + //go:embed md/cidr.md + docCIDR string + + //go:embed md/grok.md + docGrok string + + //go:embed md/json.md + docJSON string + + //go:embed md/query_refer_table.md + docQueryReferTable string + + //go:embed md/mquery_refer_table.md + docMQueryReferTable string + + //go:embed md/match.md + docMatch string + + //go:embed md/rename.md + docRename string + + //go:embed md/url_decode.md + docURLDecode string + + //go:embed md/geoip.md + docGeoIP string + + //go:embed md/datetime.md + docDatetime string + + //go:embed md/cast.md + docCast string + + //go:embed md/delete.md + docDelete string + + //go:embed md/get_key.md + docGetKey string + + //go:embed md/group_between.md + docGroupBetreen string + + //go:embed md/group_in.md + docGroupIn string + + //go:embed md/uppercase.md + docUppercase string + + //go:embed md/len.md + docLen string + + //go:embed md/load_json.md + docLoadJSON string + + //go:embed md/lowercase.md + docLowercase string + + //go:embed md/nullif.md + docNullif string + + //go:embed md/strfmt.md + docStrfmt string + + //go:embed md/drop_origin_data.md + docDropOriginData string + + //go:embed md/add_key.md + docAddKey string + + //go:embed md/default_time.md + docDefaultTime string + + //go:embed md/drop_key.md + docDropKey string + + //go:embed md/trim.md + docTrim string + + //go:embed md/user_agent.md + docUserAgent string + + //go:embed md/parse_duration.md + docParseDuration string + + //go:embed md/parse_date.md + docParseDate string + + //go:embed md/cover.md + docCover string + + //go:embed md/replace.md + docReplace string + + //go:embed md/set_measurement.md + docSetMeasurement string + + //go:embed md/set_tag.md + docSetTag string + + //go:embed md/sample.md + docSample string + + //go:embed md/drop.md + docDrop string + + //go:embed md/exit.md + docExit string + + //go:embed md/duration_precision.md + docDurationPresicion string + + //go:embed md/decode.md + docDecode string + + //go:embed md/sql_cover.md + docSQLCover string + + //go:embed md/adjust_timezone.md + docAdjustTimezone string + + //go:embed md/xml.md + docXML string + + //go:embed md/use.md + docUse string + + //go:embed md/url_parse.md + docURLParse string + + //go:embed md/timestamp.md + docTimestamp string + + //go:embed md/kv_split.md + docKVSplit string + + //go:embed md/value_type.md + docValueType string + + //go:embed md/valid_json.md + docValidJSON string + + //go:embed md/conv_traceid_w3c_to_dd.md + docConvTraceID string + + //go:embed md/create_point.md + docCreatePoint string + + //go:embed md/parse_int.md + docParseInt string + + //go:embed md/format_int.md + docFormatInt string +) + +const ( + langTagZhCN = "zh-CN" +) + +const ( + cEncodeDecode = "编解码" + cMeasurementOp = "行协议操作" + cRegExp = "RegExp" + cGrok = "Grok" + cJSON = "JSON" + cXML = "XML" + cTimeOp = "时间操作" + cTypeCast = "类型转换" + cNetwork = "网络" + cStringOp = "字符串操作" + cDesensitization = "脱敏" + cSample = "采样" + ea = "聚合" + cOther = "其他" +) + +var ( + URLDecodeMarkdown = PLDoc{ + Doc: docURLDecode, Deprecated: false, + FnCategory: map[string][]string{ + langTagZhCN: {cEncodeDecode, cNetwork}, + }, + } + addKeyMarkdown = PLDoc{ + Doc: docAddKey, Deprecated: false, + FnCategory: map[string][]string{ + langTagZhCN: {cMeasurementOp}, + }, + } + aggCreateMarkdown = PLDoc{ + Doc: docAggCreate, Deprecated: false, + FnCategory: map[string][]string{ + langTagZhCN: {ea}, + }, + } + aggMetricMarkdown = PLDoc{ + Doc: docAggMetric, Deprecated: false, + FnCategory: map[string][]string{ + langTagZhCN: {ea}, + }, + } + + addPatternMarkdown = PLDoc{ + Doc: docAddPattern, Deprecated: false, + FnCategory: map[string][]string{ + langTagZhCN: {cGrok}, + }, + } + adjustTimezoneMarkdown = PLDoc{ + Doc: docAdjustTimezone, Deprecated: false, + FnCategory: map[string][]string{ + langTagZhCN: {cTimeOp}, + }, + } + appendMarkdown = PLDoc{ + Doc: docAppend, Deprecated: false, + FnCategory: map[string][]string{ + langTagZhCN: {cOther}, + }, + } + b64decMarkdown = PLDoc{ + Doc: docB64dec, Deprecated: false, + FnCategory: map[string][]string{ + langTagZhCN: {cEncodeDecode}, + }, + } + b64encMarkdown = PLDoc{ + Doc: docB64enc, Deprecated: false, + FnCategory: map[string][]string{ + langTagZhCN: {cEncodeDecode}, + }, + } + castMarkdown = PLDoc{ + Doc: docCast, Deprecated: false, + FnCategory: map[string][]string{ + langTagZhCN: {cTypeCast}, + }, + } + deleteMarkdown = PLDoc{ + Doc: docDelete, Deprecated: false, + FnCategory: map[string][]string{ + langTagZhCN: {cJSON, cOther}, + }, + } + cidrMarkdown = PLDoc{ + Doc: docCIDR, Deprecated: false, + FnCategory: map[string][]string{ + langTagZhCN: {cNetwork}, + }, + } + coverMarkdown = PLDoc{ + Doc: docCover, Deprecated: false, + FnCategory: map[string][]string{ + langTagZhCN: {cStringOp, cDesensitization}, + }, + } + datetimeMarkdown = PLDoc{ + Doc: docDatetime, Deprecated: false, + FnCategory: map[string][]string{ + langTagZhCN: {cTimeOp}, + }, + } + decodeMarkdown = PLDoc{ + Doc: docDecode, Deprecated: false, + FnCategory: map[string][]string{ + langTagZhCN: {cEncodeDecode}, + }, + } + defaultTimeMarkdown = PLDoc{ + Doc: docDefaultTime, Deprecated: false, + FnCategory: map[string][]string{ + langTagZhCN: {cTimeOp, cMeasurementOp}, + }, + } + getKeyMarkdown = PLDoc{ + Doc: docGetKey, Deprecated: false, + FnCategory: map[string][]string{ + langTagZhCN: {cMeasurementOp}, + }, + } + dropKeyMarkdown = PLDoc{ + Doc: docDropKey, Deprecated: false, + FnCategory: map[string][]string{ + langTagZhCN: {cMeasurementOp}, + }, + } + dropMarkdown = PLDoc{ + Doc: docDrop, Deprecated: false, + FnCategory: map[string][]string{ + langTagZhCN: {cMeasurementOp}, + }, + } + dropOriginDataMarkdown = PLDoc{ + Doc: docDropOriginData, Deprecated: false, + FnCategory: map[string][]string{ + langTagZhCN: {cMeasurementOp}, + }, + } + durationPrecisionMarkdown = PLDoc{ + Doc: docDurationPresicion, Deprecated: false, + FnCategory: map[string][]string{ + langTagZhCN: {cTimeOp}, + }, + } + exitMarkdown = PLDoc{ + Doc: docExit, Deprecated: false, + FnCategory: map[string][]string{ + langTagZhCN: {cOther}, + }, + } + geoIPMarkdown = PLDoc{ + Doc: docGeoIP, Deprecated: false, + FnCategory: map[string][]string{ + langTagZhCN: {cNetwork}, + }, + } + grokMarkdown = PLDoc{ + Doc: docGrok, Deprecated: false, + FnCategory: map[string][]string{ + langTagZhCN: {cGrok, cRegExp}, + }, + } + groupBetweenMarkdown = PLDoc{ + Doc: docGroupBetreen, Deprecated: false, + FnCategory: map[string][]string{ + langTagZhCN: {cOther}, + }, + } + groupInMarkdown = PLDoc{ + Doc: docGroupIn, Deprecated: false, + FnCategory: map[string][]string{ + langTagZhCN: {cOther}, + }, + } + jsonMarkdown = PLDoc{ + Doc: docJSON, Deprecated: false, + FnCategory: map[string][]string{ + langTagZhCN: {cJSON}, + }, + } + lenMarkdown = PLDoc{ + Doc: docLen, Deprecated: false, + FnCategory: map[string][]string{ + langTagZhCN: {cOther}, + }, + } + loadJSONMarkdown = PLDoc{ + Doc: docLoadJSON, Deprecated: false, + FnCategory: map[string][]string{ + langTagZhCN: {cJSON}, + }, + } + lowercaseMarkdown = PLDoc{ + Doc: docLowercase, Deprecated: false, + FnCategory: map[string][]string{ + langTagZhCN: {cStringOp}, + }, + } + nullIfMarkdown = PLDoc{ + Doc: docNullif, Deprecated: false, + FnCategory: map[string][]string{ + langTagZhCN: {cOther}, + }, + } + parseDateMarkdown = PLDoc{ + Doc: docParseDate, Deprecated: false, + FnCategory: map[string][]string{ + langTagZhCN: {cTimeOp}, + }, + } + parseDurationMarkdown = PLDoc{ + Doc: docParseDuration, Deprecated: false, + FnCategory: map[string][]string{ + langTagZhCN: {cTimeOp}, + }, + } + queryReferTableMarkdown = PLDoc{ + Doc: docQueryReferTable, Deprecated: false, + FnCategory: map[string][]string{ + langTagZhCN: {cOther}, + }, + } + matchMarkdown = PLDoc{ + Doc: docMatch, Deprecated: false, + FnCategory: map[string][]string{ + langTagZhCN: {cRegExp}, + }, + } + mQueryReferTableMarkdown = PLDoc{ + Doc: docMQueryReferTable, Deprecated: false, + FnCategory: map[string][]string{ + langTagZhCN: {cOther}, + }, + } + renameMarkdown = PLDoc{ + Doc: docRename, Deprecated: false, + FnCategory: map[string][]string{ + langTagZhCN: {cMeasurementOp}, + }, + } + replaceMarkdown = PLDoc{ + Doc: docReplace, Deprecated: false, + FnCategory: map[string][]string{ + langTagZhCN: {cRegExp}, + }, + } + sampleMarkdown = PLDoc{ + Doc: docSample, Deprecated: false, + FnCategory: map[string][]string{ + langTagZhCN: {cSample}, + }, + } + + setMeasurementMarkdown = PLDoc{ + Doc: docSetMeasurement, Deprecated: false, + FnCategory: map[string][]string{ + langTagZhCN: {cMeasurementOp}, + }, + } + setTagMarkdown = PLDoc{ + Doc: docSetTag, Deprecated: false, + FnCategory: map[string][]string{ + langTagZhCN: {cMeasurementOp}, + }, + } + sqlCoverMarkdown = PLDoc{ + Doc: docSQLCover, Deprecated: false, + FnCategory: map[string][]string{ + langTagZhCN: {cDesensitization}, + }, + } + strfmtMarkdown = PLDoc{ + Doc: docStrfmt, Deprecated: false, + FnCategory: map[string][]string{ + langTagZhCN: {cStringOp}, + }, + } + trimMarkdown = PLDoc{ + Doc: docTrim, Deprecated: false, + FnCategory: map[string][]string{ + langTagZhCN: {cStringOp}, + }, + } + uppercaseMarkdown = PLDoc{ + Doc: docUppercase, Deprecated: false, + FnCategory: map[string][]string{ + langTagZhCN: {cStringOp}, + }, + } + userAgentMarkdown = PLDoc{ + Doc: docUserAgent, Deprecated: false, + FnCategory: map[string][]string{ + langTagZhCN: {cOther}, + }, + } + useMarkdown = PLDoc{ + Doc: docUse, Deprecated: false, + FnCategory: map[string][]string{ + langTagZhCN: {cOther}, + }, + } + xmlMarkdown = PLDoc{ + Doc: docXML, Deprecated: false, + FnCategory: map[string][]string{ + langTagZhCN: {cXML}, + }, + } + urlParseMarkdown = PLDoc{ + Doc: docURLParse, Deprecated: false, + FnCategory: map[string][]string{ + langTagZhCN: {cNetwork, cEncodeDecode}, + }, + } + timestampMarkdown = PLDoc{ + Doc: docTimestamp, Deprecated: false, + FnCategory: map[string][]string{ + langTagZhCN: {cTimeOp}, + }, + } + + kvSplitMarkdown = PLDoc{ + Doc: docKVSplit, Deprecated: false, + FnCategory: map[string][]string{ + langTagZhCN: {cRegExp}, + }, + } + + valueTypeMarkdown = PLDoc{ + Doc: docValueType, Deprecated: false, + FnCategory: map[string][]string{ + langTagZhCN: {cJSON}, + }, + } + + validJSONMarkdown = PLDoc{ + Doc: docValidJSON, Deprecated: false, + FnCategory: map[string][]string{ + langTagZhCN: {cJSON}, + }, + } + + convTraceID128MD = PLDoc{ + Doc: docConvTraceID, + FnCategory: map[string][]string{ + langTagZhCN: {cStringOp}, + }, + } + + createPointMarkdown = PLDoc{ + Doc: docCreatePoint, + FnCategory: map[string][]string{ + langTagZhCN: {cOther}, + }, + } + parseIntMarkdown = PLDoc{ + Doc: docParseInt, + FnCategory: map[string][]string{ + langTagZhCN: {cStringOp}, + }, + } + + formatIntMarkdown = PLDoc{ + Doc: docFormatInt, + FnCategory: map[string][]string{ + langTagZhCN: {cStringOp}, + }, + } +) diff --git a/pkg/inimpl/guancecloud/ptinput/funcs/all_doc_en.go b/pkg/inimpl/guancecloud/ptinput/funcs/all_doc_en.go new file mode 100644 index 0000000..b4152e9 --- /dev/null +++ b/pkg/inimpl/guancecloud/ptinput/funcs/all_doc_en.go @@ -0,0 +1,562 @@ +// Unless explicitly stated otherwise all files in this repository are licensed +// under the MIT License. +// This product includes software developed at Guance Cloud (https://www.guance.com/). +// Copyright 2021-present Guance, Inc. + +package funcs + +import ( + _ "embed" +) + +// embed docs. +var ( + + //go:embed md/agg_create.en.md + docAggCreateEN string + + //go:embed md/agg_metric.en.md + docAggMetricEN string + + //go:embed md/add_pattern.en.md + docAddPatternEN string + + //go:embed md/append.en.md + docAppendEN string + + //go:embed md/b64dec.en.md + docB64decEN string + + //go:embed md/b64enc.en.md + docB64encEN string + + //go:embed md/cidr.en.md + docCIDREN string + + //go:embed md/grok.en.md + docGrokEN string + + //go:embed md/json.en.md + docJSONEN string + + //go:embed md/query_refer_table.en.md + docQueryReferTableEN string + + //go:embed md/mquery_refer_table.en.md + docMQueryReferTableEN string + + //go:embed md/match.en.md + docMatchEN string + + //go:embed md/rename.en.md + docRenameEN string + + //go:embed md/url_decode.en.md + docURLDecodeEN string + + //go:embed md/geoip.en.md + docGeoIPEN string + + //go:embed md/datetime.en.md + docDatetimeEN string + + //go:embed md/cast.en.md + docCastEN string + + //go:embed md/delete.en.md + docDeleteEN string + + //go:embed md/get_key.en.md + docGetKeyEN string + + //go:embed md/group_between.en.md + docGroupBetreenEN string + + //go:embed md/group_in.en.md + docGroupInEN string + + //go:embed md/uppercase.en.md + docUppercaseEN string + + //go:embed md/len.en.md + docLenEN string + + //go:embed md/load_json.en.md + docLoadJSONEN string + + //go:embed md/lowercase.en.md + docLowercaseEN string + + //go:embed md/nullif.en.md + docNullifEN string + + //go:embed md/strfmt.en.md + docStrfmtEN string + + //go:embed md/drop_origin_data.en.md + docDropOriginDataEN string + + //go:embed md/add_key.en.md + docAddKeyEN string + + //go:embed md/default_time.en.md + docDefaultTimeEN string + + //go:embed md/drop_key.en.md + docDropKeyEN string + + //go:embed md/trim.en.md + docTrimEN string + + //go:embed md/user_agent.en.md + docUserAgentEN string + + //go:embed md/parse_duration.en.md + docParseDurationEN string + + //go:embed md/parse_date.en.md + docParseDateEN string + + //go:embed md/cover.en.md + docCoverEN string + + //go:embed md/replace.en.md + docReplaceEN string + + //go:embed md/set_measurement.en.md + docSetMeasurementEN string + + //go:embed md/set_tag.en.md + docSetTagEN string + + //go:embed md/sample.en.md + docSampleEN string + + //go:embed md/drop.en.md + docDropEN string + + //go:embed md/exit.en.md + docExitEN string + + //go:embed md/duration_precision.en.md + docDurationPresicionEN string + + //go:embed md/decode.en.md + docDecodeEN string + + //go:embed md/sql_cover.en.md + docSQLCoverEN string + + //go:embed md/adjust_timezone.en.md + docAdjustTimezoneEN string + + //go:embed md/xml.en.md + docXMLEN string + + //go:embed md/use.en.md + docUseEN string + + //go:embed md/url_parse.en.md + docURLParseEN string + + //go:embed md/timestamp.en.md + docTimestampEN string + + //go:embed md/kv_split.en.md + docKVSplitEN string + + //go:embed md/value_type.en.md + docValueTypeEN string + + //go:embed md/valid_json.en.md + docValidJSONEN string + + //go:embed md/conv_traceid_w3c_to_dd.en.md + docConvTraceIDEN string + + //go:embed md/create_point.en.md + docCreatePointEN string + + //go:embed md/parse_int.en.md + docParseIntEN string + + //go:embed md/format_int.en.md + docFormatIntEN string +) + +const ( + langTagEnUS = "en-US" +) + +const ( + eEncodeDecode = "Encode/Decode" + eMeasurementOp = "Point" + eRegExp = "RegExp" + eGrok = "Grok" + eJSON = "JSON" + eXML = "XML" + eTimeOp = "Time" + eTypeCast = "Type" + eNetwork = "Network" + eStringOp = "String" + eDesensitization = "Desensitization" + eSample = "Sample" + eAgg = "Aggregation" + eOther = "Other" +) + +var ( + URLDecodeMarkdownEN = PLDoc{ + Doc: docURLDecodeEN, Deprecated: false, + FnCategory: map[string][]string{ + langTagEnUS: {eEncodeDecode, eNetwork}, + }, + } + addKeyMarkdownEN = PLDoc{ + Doc: docAddKeyEN, Deprecated: false, + FnCategory: map[string][]string{ + langTagEnUS: {eMeasurementOp}, + }, + } + addPatternMarkdownEN = PLDoc{ + Doc: docAddPatternEN, Deprecated: false, + FnCategory: map[string][]string{ + langTagEnUS: {eGrok}, + }, + } + adjustTimezoneMarkdownEN = PLDoc{ + Doc: docAdjustTimezoneEN, Deprecated: false, + FnCategory: map[string][]string{ + langTagEnUS: {eTimeOp}, + }, + } + + aggCreateMarkdownEN = PLDoc{ + Doc: docAggCreateEN, Deprecated: false, + FnCategory: map[string][]string{ + langTagEnUS: {eAgg}, + }, + } + aggMetricMarkdownEN = PLDoc{ + Doc: docAggMetricEN, Deprecated: false, + FnCategory: map[string][]string{ + langTagEnUS: {eAgg}, + }, + } + + appendMarkdownEN = PLDoc{ + Doc: docAppendEN, Deprecated: false, + FnCategory: map[string][]string{ + langTagEnUS: {eOther}, + }, + } + b64decMarkdownEN = PLDoc{ + Doc: docB64decEN, Deprecated: false, + FnCategory: map[string][]string{ + langTagEnUS: {eEncodeDecode}, + }, + } + b64encMarkdownEN = PLDoc{ + Doc: docB64encEN, Deprecated: false, + FnCategory: map[string][]string{ + langTagEnUS: {eEncodeDecode}, + }, + } + castMarkdownEN = PLDoc{ + Doc: docCastEN, Deprecated: false, + FnCategory: map[string][]string{ + langTagEnUS: {eTypeCast}, + }, + } + deleteMarkdownEN = PLDoc{ + Doc: docDeleteEN, Deprecated: false, + FnCategory: map[string][]string{ + langTagEnUS: {eJSON, eOther}, + }, + } + + cidrMarkdownEN = PLDoc{ + Doc: docCIDREN, Deprecated: false, + FnCategory: map[string][]string{ + langTagEnUS: {eNetwork}, + }, + } + coverMarkdownEN = PLDoc{ + Doc: docCoverEN, Deprecated: false, + FnCategory: map[string][]string{ + langTagEnUS: {eStringOp, eDesensitization}, + }, + } + datetimeMarkdownEN = PLDoc{ + Doc: docDatetimeEN, Deprecated: false, + FnCategory: map[string][]string{ + langTagEnUS: {eTimeOp}, + }, + } + decodeMarkdownEN = PLDoc{ + Doc: docDecodeEN, Deprecated: false, + FnCategory: map[string][]string{ + langTagEnUS: {eEncodeDecode}, + }, + } + defaultTimeMarkdownEN = PLDoc{ + Doc: docDefaultTimeEN, Deprecated: false, + FnCategory: map[string][]string{ + langTagEnUS: {eTimeOp, eMeasurementOp}, + }, + } + getKeyMarkdownEN = PLDoc{ + Doc: docGetKeyEN, Deprecated: false, + FnCategory: map[string][]string{ + langTagEnUS: {eMeasurementOp}, + }, + } + dropKeyMarkdownEN = PLDoc{ + Doc: docDropKeyEN, Deprecated: false, + FnCategory: map[string][]string{ + langTagEnUS: {eMeasurementOp}, + }, + } + dropMarkdownEN = PLDoc{ + Doc: docDropEN, Deprecated: false, + FnCategory: map[string][]string{ + langTagEnUS: {eMeasurementOp}, + }, + } + dropOriginDataMarkdownEN = PLDoc{ + Doc: docDropOriginDataEN, Deprecated: false, + FnCategory: map[string][]string{ + langTagEnUS: {eMeasurementOp}, + }, + } + durationPrecisionMarkdownEN = PLDoc{ + Doc: docDurationPresicionEN, Deprecated: false, + FnCategory: map[string][]string{ + langTagEnUS: {eTimeOp}, + }, + } + exitMarkdownEN = PLDoc{ + Doc: docExitEN, Deprecated: false, + FnCategory: map[string][]string{ + langTagEnUS: {eOther}, + }, + } + geoIPMarkdownEN = PLDoc{ + Doc: docGeoIPEN, Deprecated: false, + FnCategory: map[string][]string{ + langTagEnUS: {eNetwork}, + }, + } + grokMarkdownEN = PLDoc{ + Doc: docGrokEN, Deprecated: false, + FnCategory: map[string][]string{ + langTagEnUS: {eGrok, eRegExp}, + }, + } + groupBetweenMarkdownEN = PLDoc{ + Doc: docGroupBetreenEN, Deprecated: false, + FnCategory: map[string][]string{ + langTagEnUS: {eOther}, + }, + } + groupInMarkdownEN = PLDoc{ + Doc: docGroupInEN, Deprecated: false, + FnCategory: map[string][]string{ + langTagEnUS: {eOther}, + }, + } + jsonMarkdownEN = PLDoc{ + Doc: docJSONEN, Deprecated: false, + FnCategory: map[string][]string{ + langTagEnUS: {eJSON}, + }, + } + lenMarkdownEN = PLDoc{ + Doc: docLenEN, Deprecated: false, + FnCategory: map[string][]string{ + langTagEnUS: {eOther}, + }, + } + loadJSONMarkdownEN = PLDoc{ + Doc: docLoadJSONEN, Deprecated: false, + FnCategory: map[string][]string{ + langTagEnUS: {eJSON}, + }, + } + lowercaseMarkdownEN = PLDoc{ + Doc: docLowercaseEN, Deprecated: false, + FnCategory: map[string][]string{ + langTagEnUS: {eStringOp}, + }, + } + nullIfMarkdownEN = PLDoc{ + Doc: docNullifEN, Deprecated: false, + FnCategory: map[string][]string{ + langTagEnUS: {eOther}, + }, + } + parseDateMarkdownEN = PLDoc{ + Doc: docParseDateEN, Deprecated: false, + FnCategory: map[string][]string{ + langTagEnUS: {eTimeOp}, + }, + } + parseDurationMarkdownEN = PLDoc{ + Doc: docParseDurationEN, Deprecated: false, + FnCategory: map[string][]string{ + langTagEnUS: {eTimeOp}, + }, + } + queryReferTableMarkdownEN = PLDoc{ + Doc: docQueryReferTableEN, Deprecated: false, + FnCategory: map[string][]string{ + langTagEnUS: {eOther}, + }, + } + matchMarkdownEN = PLDoc{ + Doc: docMatchEN, Deprecated: false, + FnCategory: map[string][]string{ + langTagEnUS: {eRegExp}, + }, + } + mQueryReferTableMarkdownEN = PLDoc{ + Doc: docMQueryReferTableEN, Deprecated: false, + FnCategory: map[string][]string{ + langTagEnUS: {eOther}, + }, + } + renameMarkdownEN = PLDoc{ + Doc: docRenameEN, Deprecated: false, + FnCategory: map[string][]string{ + langTagEnUS: {eMeasurementOp}, + }, + } + replaceMarkdownEN = PLDoc{ + Doc: docReplaceEN, Deprecated: false, + FnCategory: map[string][]string{ + langTagEnUS: {eRegExp}, + }, + } + sampleMarkdownEN = PLDoc{ + Doc: docSampleEN, Deprecated: false, + FnCategory: map[string][]string{ + langTagEnUS: {eSample}, + }, + } + + setMeasurementMarkdownEN = PLDoc{ + Doc: docSetMeasurementEN, Deprecated: false, + FnCategory: map[string][]string{ + langTagEnUS: {eMeasurementOp}, + }, + } + setTagMarkdownEN = PLDoc{ + Doc: docSetTagEN, Deprecated: false, + FnCategory: map[string][]string{ + langTagEnUS: {eMeasurementOp}, + }, + } + sqlCoverMarkdownEN = PLDoc{ + Doc: docSQLCoverEN, Deprecated: false, + FnCategory: map[string][]string{ + langTagEnUS: {eDesensitization}, + }, + } + strfmtMarkdownEN = PLDoc{ + Doc: docStrfmtEN, Deprecated: false, + FnCategory: map[string][]string{ + langTagEnUS: {eStringOp}, + }, + } + trimMarkdownEN = PLDoc{ + Doc: docTrimEN, Deprecated: false, + FnCategory: map[string][]string{ + langTagEnUS: {eStringOp}, + }, + } + uppercaseMarkdownEN = PLDoc{ + Doc: docUppercaseEN, Deprecated: false, + FnCategory: map[string][]string{ + langTagEnUS: {eStringOp}, + }, + } + userAgentMarkdownEN = PLDoc{ + Doc: docUserAgentEN, Deprecated: false, + FnCategory: map[string][]string{ + langTagEnUS: {eOther}, + }, + } + useMarkdownEN = PLDoc{ + Doc: docUseEN, Deprecated: false, + FnCategory: map[string][]string{ + langTagEnUS: {eOther}, + }, + } + xmlMarkdownEN = PLDoc{ + Doc: docXMLEN, Deprecated: false, + FnCategory: map[string][]string{ + langTagEnUS: {eXML}, + }, + } + urlParseMarkdownEN = PLDoc{ + Doc: docURLParseEN, Deprecated: false, + FnCategory: map[string][]string{ + langTagEnUS: {eNetwork, eEncodeDecode}, + }, + } + timestampMarkdownEN = PLDoc{ + Doc: docTimestampEN, Deprecated: false, + FnCategory: map[string][]string{ + langTagEnUS: {eTimeOp}, + }, + } + + kvSplitMarkdownEN = PLDoc{ + Doc: docKVSplitEN, Deprecated: false, + FnCategory: map[string][]string{ + langTagEnUS: {eRegExp}, + }, + } + + valueTypeMarkdownEN = PLDoc{ + Doc: docValueTypeEN, Deprecated: false, + FnCategory: map[string][]string{ + langTagEnUS: {eJSON}, + }, + } + + validJSONMarkdownEN = PLDoc{ + Doc: docValidJSONEN, Deprecated: false, + FnCategory: map[string][]string{ + langTagEnUS: {eJSON}, + }, + } + + convTraceID128MDEN = PLDoc{ + Doc: docConvTraceIDEN, + FnCategory: map[string][]string{ + langTagEnUS: {eStringOp}, + }, + } + + createPointMarkdownEN = PLDoc{ + Doc: docCreatePointEN, + FnCategory: map[string][]string{ + langTagEnUS: {eOther}, + }, + } + + parseIntMarkdownEN = PLDoc{ + Doc: docParseIntEN, + FnCategory: map[string][]string{ + langTagEnUS: {eStringOp}, + }, + } + + formatIntMarkdownEN = PLDoc{ + Doc: docFormatIntEN, + FnCategory: map[string][]string{ + langTagEnUS: {eStringOp}, + }, + } +) diff --git a/pkg/inimpl/guancecloud/ptinput/funcs/all_doc_en_test.go b/pkg/inimpl/guancecloud/ptinput/funcs/all_doc_en_test.go new file mode 100644 index 0000000..1250599 --- /dev/null +++ b/pkg/inimpl/guancecloud/ptinput/funcs/all_doc_en_test.go @@ -0,0 +1,52 @@ +// Unless explicitly stated otherwise all files in this repository are licensed +// under the MIT License. +// This product includes software developed at Guance Cloud (https://www.guance.com/). +// Copyright 2021-present Guance, Inc. + +package funcs + +import ( + "strings" + "testing" +) + +func TestAllDocEn(t *testing.T) { + if len(PipelineFunctionDocs) != len(PipelineFunctionDocsEN) { + t.Fatal("len(PipelineFunctionDocs) != len(PipelineFunctionDocsEN)") + } + protoPrefix, descPrefix := "Function prototype: ", "Function description: " + funcNameMap := map[string]bool{} + for name := range PipelineFunctionDocsEN { + funcNameMap[strings.TrimSuffix(name, "()")] = true + } + for fn := range FuncsMap { + if fn == "json_all" || + fn == "default_time_with_fmt" || + fn == "expr" || + fn == "vaild_json" { + continue + } + if _, has := funcNameMap[fn]; !has { + t.Errorf("func %s exists in FuncsMap but not in PipelineFunctionDocs", fn) + } + } + for fn, plDoc := range PipelineFunctionDocsEN { + lines := strings.Split(plDoc.Doc, "\n") + var hasProto, hasDesc bool + for _, line := range lines { + if strings.HasPrefix(line, protoPrefix) { + hasProto = true + } + if strings.HasPrefix(line, descPrefix) { + hasDesc = true + } + } + // These fields are needed by front-end. + if !hasDesc { + t.Errorf("%s does not contain '%s'", fn, protoPrefix) + } + if !hasProto { + t.Errorf("%s does not contain '%s'", fn, descPrefix) + } + } +} diff --git a/pkg/inimpl/guancecloud/ptinput/funcs/all_doc_test.go b/pkg/inimpl/guancecloud/ptinput/funcs/all_doc_test.go new file mode 100644 index 0000000..f84cd09 --- /dev/null +++ b/pkg/inimpl/guancecloud/ptinput/funcs/all_doc_test.go @@ -0,0 +1,49 @@ +// Unless explicitly stated otherwise all files in this repository are licensed +// under the MIT License. +// This product includes software developed at Guance Cloud (https://www.guance.com/). +// Copyright 2021-present Guance, Inc. + +package funcs + +import ( + "strings" + "testing" +) + +func TestAllDoc(t *testing.T) { + protoPrefix, descPrefix := "函数原型：", "函数说明：" + funcNameMap := map[string]bool{} + for name := range PipelineFunctionDocs { + funcNameMap[strings.TrimSuffix(name, "()")] = true + } + for fn := range FuncsMap { + if fn == "json_all" || + fn == "default_time_with_fmt" || + fn == "expr" || + fn == "vaild_json" { + continue + } + if _, has := funcNameMap[fn]; !has { + t.Errorf("func %s exists in FuncsMap but not in PipelineFunctionDocs", fn) + } + } + for fn, plDoc := range PipelineFunctionDocs { + lines := strings.Split(plDoc.Doc, "\n") + var hasProto, hasDesc bool + for _, line := range lines { + if strings.HasPrefix(line, protoPrefix) { + hasProto = true + } + if strings.HasPrefix(line, descPrefix) { + hasDesc = true + } + } + // These fields are needed by front-end. + if !hasDesc { + t.Errorf("%s does not contain '%s'", fn, protoPrefix) + } + if !hasProto { + t.Errorf("%s does not contain '%s'", fn, descPrefix) + } + } +} diff --git a/pkg/inimpl/guancecloud/funcs/all_test.go b/pkg/inimpl/guancecloud/ptinput/funcs/all_test.go similarity index 58% rename from pkg/inimpl/guancecloud/funcs/all_test.go rename to pkg/inimpl/guancecloud/ptinput/funcs/all_test.go index f3ae833..babcf63 100644 --- a/pkg/inimpl/guancecloud/funcs/all_test.go +++ b/pkg/inimpl/guancecloud/ptinput/funcs/all_test.go @@ -7,18 +7,19 @@ package funcs import ( "fmt" - "time" "github.com/GuanceCloud/platypus/pkg/engine" "github.com/GuanceCloud/platypus/pkg/engine/runtime" - "github.com/GuanceCloud/platypus/pkg/inimpl/guancecloud/input" + "github.com/GuanceCloud/platypus/pkg/errchain" + "github.com/GuanceCloud/platypus/pkg/inimpl/guancecloud/ptinput" ) func NewTestingRunner(script string) (*runtime.Script, error) { - name := "default.ppl" + name := "default.p" ret1, ret2 := engine.ParseScript(map[string]string{ - "default.ppl": script, - }, FuncsMap, FuncsCheckMap, + "default.p": script, + }, + FuncsMap, FuncsCheckMap, ) if len(ret1) > 0 { return ret1[name], nil @@ -33,15 +34,6 @@ func NewTestingRunner2(scripts map[string]string) (map[string]*runtime.Script, m return engine.ParseScript(scripts, FuncsMap, FuncsCheckMap) } -func runScript(proc *runtime.Script, measurement string, - tags map[string]string, fields map[string]any, tn time.Time) ( - string, map[string]string, map[string]any, time.Time, bool, error, -) { - pt := input.GetPoint() - defer input.PutPoint(pt) - - input.InitPt(pt, measurement, tags, fields, tn) - - _ = engine.RunScriptWithRMapIn(proc, pt, nil) - return pt.Measurement, pt.Tags, pt.Fields, pt.Time, pt.Drop, nil +func runScript(proc *runtime.Script, pt ptinput.PlInputPt) *errchain.PlError { + return engine.RunScriptWithRMapIn(proc, pt, nil) } diff --git a/pkg/inimpl/guancecloud/ptinput/funcs/dateparse_test.go b/pkg/inimpl/guancecloud/ptinput/funcs/dateparse_test.go new file mode 100644 index 0000000..5e18c91 --- /dev/null +++ b/pkg/inimpl/guancecloud/ptinput/funcs/dateparse_test.go @@ -0,0 +1,164 @@ +// Unless explicitly stated otherwise all files in this repository are licensed +// under the MIT License. +// This product includes software developed at Guance Cloud (https://www.guance.com/). +// Copyright 2021-present Guance, Inc. + +package funcs + +/* +import ( + "testing" + "time" + + "github.com/araddon/dateparse" +) + +func TestDateParse(t *testing.T) { + cases := []string{ + "May 8, 2009 5:57:51 PM", + "oct 7, 1970", + "oct 7, '70", + "oct. 7, 1970", + "oct. 7, 70", + "Mon Jan 2 15:04:05 2006", + "Mon Jan 2 15:04:05 MST 2006", + "Mon Jan 02 15:04:05 -0700 2006", + "Monday, 02-Jan-06 15:04:05 MST", + "Mon, 02 Jan 2006 15:04:05 MST", + "Tue, 11 Jul 2017 16:28:13 +0200 (CEST)", + "Mon, 02 Jan 2006 15:04:05 -0700", + "Mon 30 Sep 2018 09:09:09 PM UTC", + "Mon Aug 10 15:44:11 UTC+0100 2015", + "Thu, 4 Jan 2018 17:53:36 +0000", + "Fri Jul 03 2015 18:04:07 GMT+0100 (GMT Daylight Time)", + "Sun, 3 Jan 2021 00:12:23 +0800 (GMT+08:00)", + "September 17, 2012 10:09am", + "September 17, 2012 at 10:09am PST-08", + "September 17, 2012, 10:10:09", + "October 7, 1970", + "October 7th, 1970", + "12 Feb 2006, 19:17", + "12 Feb 2006 19:17", + "14 May 2019 19:11:40.164", + "7 oct 70", + "7 oct 1970", + "03 February 2013", + "1 July 2013", + "2013-Feb-03", + // dd/Mon/yyy alpha Months + "06/Jan/2008:15:04:05 -0700", + "06/Jan/2008 15:04:05 -0700", + // mm/dd/yy + "3/31/2014", + "03/31/2014", + "08/21/71", + "8/1/71", + "4/8/2014 22:05", + "04/08/2014 22:05", + "4/8/14 22:05", + "04/2/2014 03:00:51", + "8/8/1965 12:00:00 AM", + "8/8/1965 01:00:01 PM", + "8/8/1965 01:00 PM", + "8/8/1965 1:00 PM", + "8/8/1965 12:00 AM", + "4/02/2014 03:00:51", + "03/19/2012 10:11:59", + "03/19/2012 10:11:59.3186369", + // yyyy/mm/dd + "2014/3/31", + "2014/03/31", + "2014/4/8 22:05", + "2014/04/08 22:05", + "2014/04/2 03:00:51", + "2014/4/02 03:00:51", + "2012/03/19 10:11:59", + "2012/03/19 10:11:59.3186369", + // yyyy:mm:dd + "2014:3:31", + "2014:03:31", + "2014:4:8 22:05", + "2014:04:08 22:05", + "2014:04:2 03:00:51", + "2014:4:02 03:00:51", + "2012:03:19 10:11:59", + "2012:03:19 10:11:59.3186369", + // Chinese + "2014年04月08日", + // yyyy-mm-ddThh + "2006-01-02T15:04:05+0000", + "2009-08-12T22:15:09-07:00", + "2009-08-12T22:15:09", + "2009-08-12T22:15:09.988", + "2009-08-12T22:15:09Z", + "2017-07-19T03:21:51:897+0100", + "2019-05-29T08:41-04", // no seconds, 2 digit TZ offset + // yyyy-mm-dd hh:mm:ss + "2014-04-26 17:24:37.3186369", + "2012-08-03 18:31:59.257000000", + "2014-04-26 17:24:37.123", + "2013-04-01 22:43", + "2013-04-01 22:43:22", + "2014-12-16 06:20:00 UTC", + "2014-12-16 06:20:00 GMT", + "2014-04-26 05:24:37 PM", + "2014-04-26 13:13:43 +0800", + "2014-04-26 13:13:43 +0800 +08", + "2014-04-26 13:13:44 +09:00", + "2012-08-03 18:31:59.257000000 +0000 UTC", + "2015-09-30 18:48:56.35272715 +0000 UTC", + "2015-02-18 00:12:00 +0000 GMT", + "2015-02-18 00:12:00 +0000 UTC", + "2015-02-08 03:02:00 +0300 MSK m=+0.000000001", + "2015-02-08 03:02:00.001 +0300 MSK m=+0.000000001", + "2017-07-19 03:21:51+00:00", + "2014-04-26", + "2014-04", + "2014", + "2014-05-11 08:20:13,787", + // yyyy-mm-dd-07:00 + "2020-07-20+08:00", + // mm.dd.yy + "3.31.2014", + "03.31.2014", + "08.21.71", + "2014.03", + "2014.03.30", + // yyyymmdd and similar + "20140601", + "20140722105203", + // yymmdd hh:mm:yy mysql log + // 080313 05:21:55 mysqld started + "171113 14:14:20", + // unix seconds, ms, micro, nano + "1332151919", + "1384216367189", + "1384216367111222", + "1384216367111222333", + + // output of `date' command + "Mon Mar 15 15:12:26 +0800 2021", + "Mon Mar 15 07:12:26 +0000 2021", + + "Mon Mar 15 15:12:26 CST 2021", + "Mon Mar 15 07:12:26 UTC 2021", + } + + loc, err := time.LoadLocation("UTC") + if err != nil { + panic(err.Error()) + } + _ = loc + // time.Local = loc + + for _, dateExample := range cases { + tm, err := dateparse.ParseLocal(dateExample) + if err != nil { + t.Logf("parse %s failed: %s", dateExample, err) + continue + } + + t.Logf("% 32s : %v(%d)", dateExample, tm, tm.UnixNano()) + } +} +*/ diff --git a/pkg/inimpl/guancecloud/funcs/fn_addkey.go b/pkg/inimpl/guancecloud/ptinput/funcs/fn_addkey.go similarity index 80% rename from pkg/inimpl/guancecloud/funcs/fn_addkey.go rename to pkg/inimpl/guancecloud/ptinput/funcs/fn_addkey.go index 453a797..eb49cbf 100644 --- a/pkg/inimpl/guancecloud/funcs/fn_addkey.go +++ b/pkg/inimpl/guancecloud/ptinput/funcs/fn_addkey.go @@ -11,8 +11,7 @@ import ( "github.com/GuanceCloud/platypus/pkg/ast" "github.com/GuanceCloud/platypus/pkg/engine/runtime" "github.com/GuanceCloud/platypus/pkg/errchain" - "github.com/GuanceCloud/platypus/pkg/inimpl/guancecloud/input" - "github.com/GuanceCloud/platypus/pkg/token" + "github.com/GuanceCloud/platypus/pkg/inimpl/guancecloud/ptinput" ) func AddkeyChecking(ctx *runtime.Context, funcExpr *ast.CallExpr) *errchain.PlError { @@ -29,9 +28,6 @@ func AddkeyChecking(ctx *runtime.Context, funcExpr *ast.CallExpr) *errchain.PlEr } func AddKey(ctx *runtime.Context, funcExpr *ast.CallExpr) *errchain.PlError { - if funcExpr == nil { - return runtime.NewRunError(ctx, "unreachable", token.InvalidLnColPos) - } if len(funcExpr.Param) != 2 && len(funcExpr.Param) != 1 { return runtime.NewRunError(ctx, fmt.Sprintf( "func %s expected 1 or 2 args", funcExpr.Name), funcExpr.NamePos) @@ -48,7 +44,7 @@ func AddKey(ctx *runtime.Context, funcExpr *ast.CallExpr) *errchain.PlError { l.Debug(err) return nil } - _ = addKey2PtWithVal(ctx.InData(), key, v.Value, v.DType, input.KindPtDefault) + _ = addKey2PtWithVal(ctx.InData(), key, v.Value, v.DType, ptinput.KindPtDefault) return nil } @@ -59,7 +55,7 @@ func AddKey(ctx *runtime.Context, funcExpr *ast.CallExpr) *errchain.PlError { if errRun != nil { return errRun.ChainAppend(ctx.Name(), funcExpr.NamePos) } - if err := addKey2PtWithVal(ctx.InData(), key, val, dtype, input.KindPtDefault); err != nil { + if err := addKey2PtWithVal(ctx.InData(), key, val, dtype, ptinput.KindPtDefault); err != nil { l.Debug(err) return nil } diff --git a/pkg/inimpl/guancecloud/funcs/fn_addkey_test.go b/pkg/inimpl/guancecloud/ptinput/funcs/fn_addkey_test.go similarity index 90% rename from pkg/inimpl/guancecloud/funcs/fn_addkey_test.go rename to pkg/inimpl/guancecloud/ptinput/funcs/fn_addkey_test.go index 654d6cc..2174989 100644 --- a/pkg/inimpl/guancecloud/funcs/fn_addkey_test.go +++ b/pkg/inimpl/guancecloud/ptinput/funcs/fn_addkey_test.go @@ -9,6 +9,8 @@ import ( "testing" "time" + "github.com/GuanceCloud/cliutils/point" + "github.com/GuanceCloud/platypus/pkg/inimpl/guancecloud/ptinput" "github.com/stretchr/testify/assert" ) @@ -110,15 +112,18 @@ add_key(add_new_key, "shanghai") } return } - _, _, f, _, _, err := runScript(runner, - "test", nil, map[string]any{"message": tc.in}, time.Now()) + pt := ptinput.NewPlPoint(point.Logging, "test", nil, map[string]any{"message": tc.in}, time.Now()) - if err == nil { - v := f["add_new_key"] + errR := runScript(runner, pt) + + if errR == nil { + v, isTag, ok := pt.GetWithIsTag("add_new_key") + assert.Equal(t, true, ok) + assert.Equal(t, false, isTag) assert.Equal(t, tc.expect, v) t.Logf("[%d] PASS", idx) } else { - t.Error(err) + t.Error(errR) } }) } diff --git a/pkg/inimpl/guancecloud/funcs/fn_addpattern.go b/pkg/inimpl/guancecloud/ptinput/funcs/fn_addpattern.go similarity index 100% rename from pkg/inimpl/guancecloud/funcs/fn_addpattern.go rename to pkg/inimpl/guancecloud/ptinput/funcs/fn_addpattern.go diff --git a/pkg/inimpl/guancecloud/funcs/fn_addpattern_test.go b/pkg/inimpl/guancecloud/ptinput/funcs/fn_addpattern_test.go similarity index 94% rename from pkg/inimpl/guancecloud/funcs/fn_addpattern_test.go rename to pkg/inimpl/guancecloud/ptinput/funcs/fn_addpattern_test.go index 252e2a5..4ea4690 100644 --- a/pkg/inimpl/guancecloud/funcs/fn_addpattern_test.go +++ b/pkg/inimpl/guancecloud/ptinput/funcs/fn_addpattern_test.go @@ -9,6 +9,8 @@ import ( "testing" "time" + "github.com/GuanceCloud/cliutils/point" + "github.com/GuanceCloud/platypus/pkg/inimpl/guancecloud/ptinput" "github.com/stretchr/testify/assert" ) @@ -162,11 +164,15 @@ func TestAddPattern(t *testing.T) { } return } - _, _, f, _, _, err := runScript(runner, "test", nil, map[string]any{"message": tc.in}, time.Now()) - if err != nil { - t.Fatal(err) + + pt := ptinput.NewPlPoint(point.Logging, "test", nil, map[string]any{"message": tc.in}, time.Now()) + errR := runScript(runner, pt) + if errR != nil { + t.Fatal(errR) } - v := f[tc.outkey] + v, isTag, ok := pt.GetWithIsTag(tc.outkey) + assert.Equal(t, true, ok) + assert.Equal(t, false, isTag) if assert.Equal(t, tc.expect, v) { t.Logf("[%d] PASS", idx) } diff --git a/pkg/inimpl/guancecloud/ptinput/funcs/fn_adjust_timezone.go b/pkg/inimpl/guancecloud/ptinput/funcs/fn_adjust_timezone.go new file mode 100644 index 0000000..29c5b2f --- /dev/null +++ b/pkg/inimpl/guancecloud/ptinput/funcs/fn_adjust_timezone.go @@ -0,0 +1,93 @@ +// Unless explicitly stated otherwise all files in this repository are licensed +// under the MIT License. +// This product includes software developed at Guance Cloud (https://www.guance.com/). +// Copyright 2021-present Guance, Inc. + +package funcs + +import ( + "fmt" + "reflect" + "time" + + "github.com/GuanceCloud/platypus/pkg/ast" + "github.com/GuanceCloud/platypus/pkg/engine/runtime" + "github.com/GuanceCloud/platypus/pkg/errchain" + "github.com/GuanceCloud/platypus/pkg/inimpl/guancecloud/ptinput" +) + +const defaultMinuteDelta = int64(2) + +func AdjustTimezoneChecking(ctx *runtime.Context, funcExpr *ast.CallExpr) *errchain.PlError { + if len(funcExpr.Param) != 1 { + return runtime.NewRunError(ctx, fmt.Sprintf( + "func `%s' expected 1 arg", funcExpr.Name), + funcExpr.NamePos) + } + + if _, err := getKeyName(funcExpr.Param[0]); err != nil { + return runtime.NewRunError(ctx, err.Error(), funcExpr.Param[0].StartPos()) + } + + return nil +} + +func AdjustTimezone(ctx *runtime.Context, funcExpr *ast.CallExpr) *errchain.PlError { + key, err := getKeyName(funcExpr.Param[0]) + if err != nil { + return runtime.NewRunError(ctx, err.Error(), funcExpr.Param[0].StartPos()) + } + + // 默认允许 +2 分钟误差 + minuteAllow := defaultMinuteDelta + if len(funcExpr.Param) == 2 { + switch funcExpr.Param[1].NodeType { //nolint:exhaustive + case ast.TypeFloatLiteral: + minuteAllow = int64(funcExpr.Param[1].FloatLiteral.Val) + case ast.TypeIntegerLiteral: + minuteAllow = funcExpr.Param[1].IntegerLiteral.Val + default: + } + } + + if minuteAllow > 15 { + minuteAllow = 15 + } else if minuteAllow < 0 { + minuteAllow = 0 + } + + minuteAllow *= int64(time.Minute) + + logTS, err := ctx.GetKey(key) + if err != nil { + l.Debug(err) + return nil + } + + switch cont := logTS.Value.(type) { + case int64: + cont = detectTimezone(cont, time.Now().UnixNano(), minuteAllow) + if err := addKey2PtWithVal(ctx.InData(), key, cont, ast.Int, ptinput.KindPtDefault); err != nil { + l.Debug(err) + return nil + } + default: + return runtime.NewRunError(ctx, fmt.Sprintf( + "param value expect int64, got `%s`", reflect.TypeOf(cont)), + funcExpr.Param[0].StartPos()) + } + + return nil +} + +func detectTimezone(logTS, nowTS, minuteDuration int64) int64 { + logTS -= (logTS/int64(time.Hour) - nowTS/int64(time.Hour)) * int64(time.Hour) + + if logTS-nowTS > minuteDuration { + logTS -= int64(time.Hour) + } else if logTS-nowTS <= -int64(time.Hour)+minuteDuration { + logTS += int64(time.Hour) + } + + return logTS +} diff --git a/pkg/inimpl/guancecloud/ptinput/funcs/fn_adjust_timezone_test.go b/pkg/inimpl/guancecloud/ptinput/funcs/fn_adjust_timezone_test.go new file mode 100644 index 0000000..d694e41 --- /dev/null +++ b/pkg/inimpl/guancecloud/ptinput/funcs/fn_adjust_timezone_test.go @@ -0,0 +1,166 @@ +// Unless explicitly stated otherwise all files in this repository are licensed +// under the MIT License. +// This product includes software developed at Guance Cloud (https://www.guance.com/). +// Copyright 2021-present Guance, Inc. + +package funcs + +import ( + "fmt" + "testing" + "time" + + "github.com/GuanceCloud/cliutils/point" + "github.com/GuanceCloud/platypus/pkg/inimpl/guancecloud/ptinput" + "github.com/stretchr/testify/assert" +) + +const Hour8 = int64(8 * time.Hour) + +func TestAdjustTimezone(t *testing.T) { + // local timezone: utc+0800 + cst := time.FixedZone("CST", 8*3600) + time.Local = cst + + tn := time.Now().Add(time.Minute * 10) + cases := []struct { + name, pl, in string + outkey string + expect time.Time + fail bool + }{ + // { + // name: "time fmt: ANSIC", + // in: fmt.Sprintf(`{"time":"%s"}`, tn.UTC().Format(time.ANSIC)), + // pl: ` + // json(_, time) + // default_time(time) + // `, + // outkey: "time", + // expect: time.Unix(0, tn.UnixNano()-int64(tn.Nanosecond())-Hour8), + // fail: false, + // }, + { + name: "time fmt: ANSIC", + in: fmt.Sprintf(`{"time":"%s"}`, tn.UTC().Format(time.ANSIC)), + pl: ` + json(_, time) + default_time(time) + adjust_timezone(time) + `, + outkey: "time", + expect: time.Unix(0, tn.UnixNano()-int64(tn.Nanosecond())-int64(time.Hour)), + fail: false, + }, + } + + for idx, tc := range cases { + t.Run(tc.name, func(t *testing.T) { + runner, err := NewTestingRunner(tc.pl) + if err != nil { + if tc.fail { + t.Logf("[%d]expect error: %s", idx, err) + } else { + t.Errorf("[%d] failed: %s", idx, err) + } + return + } + { + pt := ptinput.NewPlPoint( + point.Logging, "test", nil, map[string]any{"message": tc.in}, time.Now()) + errR := runScript(runner, pt) + if errR != nil { + t.Fatal(errR) + } + + pt.KeyTime2Time() + var v interface{} + if tc.outkey != "time" { + v, _, _ = pt.GetWithIsTag(tc.outkey) + } else { + v = pt.PtTime() + } + assert.Equal(t, tc.expect, v) + t.Logf("[%d] PASS", idx) + } + + { + pt := ptinput.NewPlPoint(point.Logging, + "test", nil, map[string]any{"message": tc.in}, time.Now()) + errR := runScript(runner, pt) + if errR != nil { + t.Fatal(errR) + } + + pt.KeyTime2Time() + var v interface{} + if tc.outkey != "time" { + v, _, _ = pt.GetWithIsTag(tc.outkey) + } else { + v = pt.PtTime() + } + assert.Equal(t, tc.expect, v) + t.Logf("[%d] PASS", idx) + } + }) + } +} + +func TestDetectTimezone(t *testing.T) { + // logTS, nowTS, minuteAllow, expTS + tn := time.Date(2022, 6, 1, 10, 11, 12, int(time.Millisecond), time.Local) + + cases := []struct { + name string + logTS int64 + nowTS int64 + durationAllow int64 + expTS int64 + neq int64 + }{ + { + name: "id 1, hour -2, minute -2, sec -2, nanosec -15", + // log: 2022-6-1 10:11:12.001 + // now: 2022-6-1 12:11:14.014 + // exp: 2022-6-1 12:11:12.001 + logTS: time.Date(2022, 6, 1, 10, 11, 12, int(time.Millisecond), time.Local).UnixNano(), + nowTS: time.Date(2022, 6, 1, 12, 11, 14, int(time.Millisecond)*15, time.Local).UnixNano(), + expTS: time.Date(2022, 6, 1, 12, 11, 12, int(time.Millisecond), time.Local).UnixNano(), + durationAllow: defaultMinuteDelta * int64(time.Minute), + }, + { + name: "id 2, hour -2, minute 0, sec 0, nanosec +5", + // log: 2022-6-1 10:11:14.02 + // now: 2022-6-1 12:11:14.014 + // exp: 2022-7-5 12:11:14.02 + logTS: time.Date(2022, 6, 1, 10, 11, 14, int(time.Millisecond)*20, time.Local).UnixNano(), + nowTS: time.Date(2022, 6, 1, 12, 11, 14, int(time.Millisecond)*15, time.Local).UnixNano(), + expTS: time.Date(2022, 6, 1, 12, 11, 14, int(time.Millisecond)*20, time.Local).UnixNano(), + durationAllow: defaultMinuteDelta * int64(time.Minute), + }, + { + name: "id 3, hour -2, minute +2, sec +1", + logTS: tn.Add(time.Minute*2 + time.Second - 2*time.Hour).UnixNano(), + nowTS: tn.UnixNano(), + expTS: tn.Add(time.Minute*2 + time.Second - time.Hour).UnixNano(), + durationAllow: defaultMinuteDelta * int64(time.Minute), + }, + { + name: "id 3, hour -1, minute -58, sec 0", + logTS: time.Date(2022, 6, 1, 11, 1, 1, 0, time.Local).UnixNano(), + nowTS: time.Date(2022, 6, 1, 12, 59, 1, 0, time.Local).UnixNano(), + expTS: time.Date(2022, 6, 1, 13, 1, 1, 0, time.Local).UnixNano(), + durationAllow: defaultMinuteDelta * int64(time.Minute), + }, + } + for _, v := range cases { + tsAct := detectTimezone(v.logTS, v.nowTS, v.durationAllow) + + if v.neq != 0 { + tsAct += v.neq + } + assert.Equal(t, time.Unix(0, v.expTS), + time.Unix(0, tsAct), + fmt.Sprintf(v.name)) + } +} diff --git a/pkg/inimpl/guancecloud/ptinput/funcs/fn_agg.go b/pkg/inimpl/guancecloud/ptinput/funcs/fn_agg.go new file mode 100644 index 0000000..1a08154 --- /dev/null +++ b/pkg/inimpl/guancecloud/ptinput/funcs/fn_agg.go @@ -0,0 +1,285 @@ +// Unless explicitly stated otherwise all files in this repository are licensed +// under the MIT License. +// This product includes software developed at Guance Cloud (https://www.guance.com/). +// Copyright 2021-present Guance, Inc. + +package funcs + +import ( + "fmt" + "reflect" + "time" + + "github.com/GuanceCloud/cliutils/point" + "github.com/GuanceCloud/platypus/pkg/ast" + "github.com/GuanceCloud/platypus/pkg/engine/runtime" + "github.com/GuanceCloud/platypus/pkg/errchain" +) + +func AggCreateChecking(ctx *runtime.Context, funcExpr *ast.CallExpr) *errchain.PlError { + if err := reindexFuncArgs(funcExpr, []string{ + "bucket", "on_interval", "on_count", + "keep_value", "const_tags", "category", + }, 1); err != nil { + return runtime.NewRunError(ctx, err.Error(), funcExpr.NamePos) + } + + arg := funcExpr.Param[0] + switch arg.NodeType { //nolint:exhaustive + case ast.TypeStringLiteral: + default: + return runtime.NewRunError(ctx, fmt.Sprintf("param `bucket` expect StringLiteral, got %s", + arg.NodeType), arg.StartPos()) + } + + interval := time.Minute + if arg := funcExpr.Param[1]; arg != nil { + switch arg.NodeType { //nolint:exhaustive + case ast.TypeStringLiteral: + ts := arg.StringLiteral.Val + if v, err := time.ParseDuration(ts); err != nil { + return runtime.NewRunError(ctx, fmt.Sprintf("parse on_interval: %s", err.Error()), + arg.StartPos()) + } else { + interval = v + } + default: + return runtime.NewRunError(ctx, fmt.Sprintf("param `on_interval` expect StringLiteral, got %s", + arg.NodeType), arg.StartPos()) + } + } + + count := 0 + if arg := funcExpr.Param[2]; arg != nil { + switch arg.NodeType { //nolint:exhaustive + case ast.TypeIntegerLiteral: + count = int(arg.IntegerLiteral.Val) + default: + return runtime.NewRunError(ctx, fmt.Sprintf("param `on_count` expect IntegerLiteral, got %s", + arg.NodeType), arg.StartPos()) + } + } + + if interval <= 0 && count <= 0 { + return runtime.NewRunError(ctx, + "param `on_interval` and `on_count` cannot be less than or equal to 0 at the same time", arg.StartPos()) + } + + if arg := funcExpr.Param[3]; arg != nil { + switch arg.NodeType { //nolint:exhaustive + case ast.TypeBoolLiteral: + default: + return runtime.NewRunError(ctx, fmt.Sprintf("param `keep_value` expect BoolLiteral, got %s", + arg.NodeType), arg.StartPos()) + } + } + + return nil +} + +func AggCreate(ctx *runtime.Context, funcExpr *ast.CallExpr) *errchain.PlError { + pt, err := getPoint(ctx.InData()) + if err != nil { + return nil + } + + buks := pt.GetAggBuckets() + if buks == nil { + return nil + } + + ptCat := point.Metric + if arg := funcExpr.Param[5]; arg != nil { + if catName, _, err := runtime.RunStmt(ctx, arg); err != nil { + return nil + } else if catName != nil { + if catN, ok := catName.(string); ok { + ptCat = ptCategory(catN) + if ptCat == point.UnknownCategory { + return nil + } + } else { + return runtime.NewRunError(ctx, fmt.Sprintf( + "type of parameter expected to be int64, got %s", + reflect.TypeOf(catName)), arg.StartPos()) + } + } + } + + bukName := funcExpr.Param[0].StringLiteral.Val + if _, ok := buks.GetBucket(ptCat, bukName); ok { + return nil + } + + interval := time.Minute + count := 0 + keepValue := false + + if arg := funcExpr.Param[1]; arg != nil { + interval, _ = time.ParseDuration(arg.StringLiteral.Val) + } + + if arg := funcExpr.Param[2]; arg != nil { + count = int(arg.IntegerLiteral.Val) + } + + if arg := funcExpr.Param[3]; arg != nil { + keepValue = arg.BoolLiteral.Val + } + + constTags := map[string]string{} + if arg := funcExpr.Param[4]; arg != nil { + if v, _, err := runtime.RunStmt(ctx, arg); err == nil { + if v, ok := v.(map[string]any); ok { + for k, v := range v { + if v, ok := v.(string); ok { + constTags[k] = v + } + } + } + } + } + + buks.CreateBucket(ptCat, bukName, interval, count, keepValue, constTags) + + return nil +} + +func AggAddMetricChecking(ctx *runtime.Context, funcExpr *ast.CallExpr) *errchain.PlError { + if err := reindexFuncArgs(funcExpr, []string{ + "bucket", "new_field", "agg_fn", + "agg_by", "agg_field", "category", + }, 5); err != nil { + return runtime.NewRunError(ctx, err.Error(), funcExpr.NamePos) + } + + arg1 := funcExpr.Param[0] + switch arg1.NodeType { //nolint:exhaustive + case ast.TypeStringLiteral: + default: + return runtime.NewRunError(ctx, fmt.Sprintf("param `bucket` expect StringLiteral, got %s", + arg1.NodeType), arg1.StartPos()) + } + + arg2 := funcExpr.Param[1] + switch arg2.NodeType { //nolint:exhaustive + case ast.TypeStringLiteral: + default: + return runtime.NewRunError(ctx, fmt.Sprintf("param `new_field` expect StringLiteral, got %s", + arg2.NodeType), arg2.StartPos()) + } + + arg3 := funcExpr.Param[2] + switch arg3.NodeType { //nolint:exhaustive + case ast.TypeStringLiteral: + default: + return runtime.NewRunError(ctx, fmt.Sprintf("param `agg_fn` expect StringLiteral, got %s", + arg3.NodeType), arg3.StartPos()) + } + + var tags []string + arg4 := funcExpr.Param[3] + switch arg4.NodeType { //nolint:exhaustive + case ast.TypeListInitExpr: + for _, v := range arg4.ListInitExpr.List { + switch v.NodeType { //nolint:exhaustive + case ast.TypeStringLiteral: + tags = append(tags, v.StringLiteral.Val) + default: + return runtime.NewRunError(ctx, fmt.Sprintf("agg_by elem expect StringLiteral, got %s", + arg4.NodeType), arg4.StartPos()) + } + } + default: + return runtime.NewRunError(ctx, fmt.Sprintf("param `agg_by` expect StringLiteral, got %s", + arg4.NodeType), arg4.StartPos()) + } + + if len(tags) == 0 { + return runtime.NewRunError(ctx, "size of param `agg_by` is 0", arg4.StartPos()) + } + + funcExpr.PrivateData = tags + + arg5 := funcExpr.Param[4] + switch arg5.NodeType { //nolint:exhaustive + case ast.TypeStringLiteral: + default: + return runtime.NewRunError(ctx, fmt.Sprintf("param `agg_field` expect StringLiteral, got %s", + arg5.NodeType), arg5.StartPos()) + } + + return nil +} + +func AggAddMetric(ctx *runtime.Context, funcExpr *ast.CallExpr) *errchain.PlError { + pt, err := getPoint(ctx.InData()) + if err != nil { + return nil + } + + buks := pt.GetAggBuckets() + if buks == nil { + return nil + } + + bukName := funcExpr.Param[0].StringLiteral.Val + newField := funcExpr.Param[1].StringLiteral.Val + aggFn := funcExpr.Param[2].StringLiteral.Val + + aggBy, ok := funcExpr.PrivateData.([]string) + if !ok { + return nil + } + + ptCat := point.Metric + if arg := funcExpr.Param[5]; arg != nil { + if catName, _, err := runtime.RunStmt(ctx, arg); err != nil { + return nil + } else if catName != nil { + if catN, ok := catName.(string); ok { + ptCat = ptCategory(catN) + if ptCat == point.UnknownCategory { + return nil + } + } else { + return runtime.NewRunError(ctx, fmt.Sprintf( + "type of parameter expected to be int64, got %s", + reflect.TypeOf(catName)), arg.StartPos()) + } + } + } + + byValue := []string{} + for _, by := range aggBy { + if v, err := ctx.GetKey(by); err != nil { + return nil + } else { + if v, ok := v.Value.(string); ok { + byValue = append(byValue, v) + } else { + return nil + } + } + } + + if len(aggBy) != len(byValue) { + return nil + } + + aggField := funcExpr.Param[4].StringLiteral.Val + + fieldValue, err := ctx.GetKey(aggField) + if err != nil { + return nil + } + + buk, ok := buks.GetBucket(ptCat, bukName) + if !ok { + return nil + } + + buk.AddMetric(newField, aggFn, aggBy, byValue, fieldValue.Value) + + return nil +} diff --git a/pkg/inimpl/guancecloud/ptinput/funcs/fn_agg_test.go b/pkg/inimpl/guancecloud/ptinput/funcs/fn_agg_test.go new file mode 100644 index 0000000..06280dc --- /dev/null +++ b/pkg/inimpl/guancecloud/ptinput/funcs/fn_agg_test.go @@ -0,0 +1,130 @@ +// Unless explicitly stated otherwise all files in this repository are licensed +// under the MIT License. +// This product includes software developed at Guance Cloud (https://www.guance.com/). +// Copyright 2021-present Guance, Inc. + +package funcs + +import ( + "testing" + "time" + + "github.com/GuanceCloud/platypus/pkg/inimpl/guancecloud/ptinput" + "github.com/GuanceCloud/platypus/pkg/inimpl/guancecloud/ptinput/plmap" + "github.com/stretchr/testify/assert" + + "github.com/GuanceCloud/cliutils/point" +) + +func TestAgg(t *testing.T) { + cases := []struct { + name, pl string + in []string + fail bool + out map[point.Category]map[string]map[string]any + }{ + { + name: "", + pl: ` + set_tag("t0", "t1111") + set_tag("t1", "t1111") + set_tag("t2", "t2__") + f0 = _ + cast(f0, "int") + agg_create("abc", on_interval="5s",on_count=0, const_tags={"a":"b"}) + agg_create("def") + + agg_metric("abc", "f1", "avg", ["t2","t0"], "f0") + agg_metric("def", "f1", "avg", ["t1", "t2"], "f0") + + agg_create("abc", on_interval="5s",on_count=0, const_tags={"a":"b"}, + category="logging") + agg_create("def", category="logging") + + agg_metric("abc", "f1", "avg", ["t2","t0"], "f0", category="logging") + agg_metric("def", "f1", "avg", ["t1", "t2"], "f0", "logging") + `, + in: []string{`1`, `2`}, + out: map[point.Category]map[string]map[string]any{ + point.Metric: { + "abc": { + "f1": float64(1.5), + "t0": "t1111", + }, + "def": { + "f1": float64(1.5), + "t1": "t1111", + }, + }, + point.Logging: { + "abc": { + "f1": float64(1.5), + "t0": "t1111", + }, + "def": { + "f1": float64(1.5), + "t1": "t1111", + }, + }, + }, + }, + } + + for idx, tc := range cases { + t.Run(tc.name, func(t *testing.T) { + runner, err := NewTestingRunner(tc.pl) + if err != nil { + if tc.fail { + t.Logf("[%d]expect error: %s", idx, err) + } else { + t.Errorf("[%d] failed: %s", idx, err) + } + return + } + + ptsLi := map[point.Category]map[string][]*point.Point{} + fn := func(cat point.Category, n string, d any) error { + catBuk, ok := ptsLi[cat] + if !ok { + catBuk = map[string][]*point.Point{} + ptsLi[cat] = catBuk + } + catBuk[n] = append(catBuk[n], d.([]*point.Point)...) + return nil + } + + buks := plmap.NewAggBuks(fn, nil) + for _, tcIn := range tc.in { + pt := ptinput.NewPlPoint( + point.Logging, "test", nil, map[string]any{"message": tcIn}, time.Now()) + pt.SetAggBuckets(buks) + errR := runScript(runner, pt) + if errR != nil { + t.Fatal(*errR) + } + } + + buks.StopAllBukScanner() + + for cat, catBuk := range tc.out { + for bukName, kv := range catBuk { + for k, v := range kv { + if len(ptsLi[cat][bukName]) == 0 { + t.Fatal("no data") + } + pt := ptsLi[cat][bukName][0] + f := pt.InfluxFields() + tags := pt.InfluxTags() + if _, ok := f[k]; ok { + assert.Equal(t, v, f[k]) + } else if _, ok := tags[k]; ok { + assert.Equal(t, v, tags[k]) + } else { + t.Error(k) + } + } + } + } + }) + } +} diff --git a/pkg/inimpl/guancecloud/ptinput/funcs/fn_append.go b/pkg/inimpl/guancecloud/ptinput/funcs/fn_append.go new file mode 100644 index 0000000..95f5517 --- /dev/null +++ b/pkg/inimpl/guancecloud/ptinput/funcs/fn_append.go @@ -0,0 +1,75 @@ +// Unless explicitly stated otherwise all files in this repository are licensed +// under the MIT License. +// This product includes software developed at Guance Cloud (https://www.guance.com/). +// Copyright 2021-present Guance, Inc. + +package funcs + +import ( + "fmt" + + "github.com/GuanceCloud/platypus/pkg/ast" + "github.com/GuanceCloud/platypus/pkg/engine/runtime" + "github.com/GuanceCloud/platypus/pkg/errchain" +) + +func doAppendChecking(ctx *runtime.Context, funcExpr *ast.CallExpr) *errchain.PlError { + if len(funcExpr.Param) != 2 { + return runtime.NewRunError(ctx, "func %s expects 2 args", funcExpr.NamePos) + } + if funcExpr.Param[0].NodeType != ast.TypeIdentifier { + return runtime.NewRunError(ctx, fmt.Sprintf( + "the first param expects Identifier, got %s", funcExpr.Param[0].NodeType), + funcExpr.Param[0].StartPos()) + } + switch funcExpr.Param[1].NodeType { //nolint:exhaustive + case ast.TypeIdentifier, ast.TypeAttrExpr, ast.TypeBoolLiteral, + ast.TypeIntegerLiteral, ast.TypeFloatLiteral, ast.TypeStringLiteral: + default: + return runtime.NewRunError(ctx, fmt.Sprintf( + "the second param expects Identifier, AttrExpr, BoolLiteral, NumberLiteral or StringLiteral, got %s", + funcExpr.Param[1].NodeType), funcExpr.Param[1].StartPos(), + ) + } + return nil +} + +func AppendChecking(ctx *runtime.Context, funcExpr *ast.CallExpr) *errchain.PlError { + return doAppendChecking(ctx, funcExpr) +} + +func Append(ctx *runtime.Context, funcExpr *ast.CallExpr) *errchain.PlError { + if errR := doAppendChecking(ctx, funcExpr); errR != nil { + return errR + } + + elem, _, errR := runtime.RunStmt(ctx, funcExpr.Param[1]) + if errR != nil { + return errR + } + + key, err := getKeyName(funcExpr.Param[0]) + if err != nil { + return runtime.NewRunError(ctx, err.Error(), funcExpr.Param[0].StartPos()) + } + val, err := ctx.GetKey(key) + if err != nil { + l.Debugf("key `%v` does not exist, ignored", key) + return nil //nolint:nilerr + } + if val.DType != ast.List { + l.Debugf("cannot append to a %s", val.DType.String()) + return nil + } + var arr []any + switch v := val.Value.(type) { + case []any: + arr = v + default: + l.Debugf("expect []any, got %T", v) + return nil + } + arr = append(arr, elem) + ctx.Regs.ReturnAppend(arr, ast.List) + return nil +} diff --git a/pkg/inimpl/guancecloud/ptinput/funcs/fn_append_test.go b/pkg/inimpl/guancecloud/ptinput/funcs/fn_append_test.go new file mode 100644 index 0000000..47d24f1 --- /dev/null +++ b/pkg/inimpl/guancecloud/ptinput/funcs/fn_append_test.go @@ -0,0 +1,106 @@ +// Unless explicitly stated otherwise all files in this repository are licensed +// under the MIT License. +// This product includes software developed at Guance Cloud (https://www.guance.com/). +// Copyright 2021-present Guance, Inc. + +package funcs + +import ( + "testing" + "time" + + "github.com/GuanceCloud/cliutils/point" + "github.com/GuanceCloud/platypus/pkg/inimpl/guancecloud/ptinput" + "github.com/stretchr/testify/assert" +) + +func TestAppend(t *testing.T) { + cases := []struct { + name, pl, in string + expected interface{} + fail bool + outkey string + }{ + { + name: "append a float number", + pl: `abc = ["1", "2"] + abc = append(abc, 5.1) + add_key(arr, abc)`, + in: `test`, + expected: "[\"1\",\"2\",5.1]", + outkey: "arr", + }, + { + name: "append a string", + pl: `abc = ["hello"] + abc = append(abc, "world") + add_key(arr, abc)`, + in: `test`, + expected: "[\"hello\",\"world\"]", + outkey: "arr", + }, + { + name: "append a string", + pl: `abc = [1, 2] + abc = append(abc, "3") + add_key(arr, abc)`, + in: `test`, + expected: "[1,2,\"3\"]", + outkey: "arr", + }, + { + name: "append by Identifier", + pl: `a = [1, 2] + b = append(a, 3) + add_key(arr, b)`, + in: `test`, + expected: "[1,2,3]", + outkey: "arr", + }, + { + name: "append an array", + pl: `a = [1, 2] + b = [3, 4] + c = append(a, b) + add_key(arr, c)`, + in: `test`, + expected: "[1,2,[3,4]]", + outkey: "arr", + }, + { + name: "append but not assign", + pl: `a = [1, 2] + b = 3 + append(a, b) + add_key(arr, a)`, + in: `test`, + expected: "[1,2]", + outkey: "arr", + }, + } + + for idx, tc := range cases { + t.Run(tc.name, func(t *testing.T) { + runner, err := NewTestingRunner(tc.pl) + if err != nil { + if tc.fail { + t.Logf("[%d]expect error: %s", idx, err) + } else { + t.Errorf("[%d] failed: %s", idx, err) + } + return + } + pt := ptinput.NewPlPoint( + point.Logging, "test", nil, map[string]any{"message": tc.in}, time.Now()) + errR := runScript(runner, pt) + if errR != nil { + t.Fatal(*errR) + } + v, isTag, ok := pt.GetWithIsTag(tc.outkey) + assert.Equal(t, true, ok) + assert.Equal(t, tc.expected, v) + assert.Equal(t, false, isTag) + t.Logf("[%d] PASS", idx) + }) + } +} diff --git a/pkg/inimpl/guancecloud/ptinput/funcs/fn_b64dec.go b/pkg/inimpl/guancecloud/ptinput/funcs/fn_b64dec.go new file mode 100644 index 0000000..12346cf --- /dev/null +++ b/pkg/inimpl/guancecloud/ptinput/funcs/fn_b64dec.go @@ -0,0 +1,70 @@ +// Unless explicitly stated otherwise all files in this repository are licensed +// under the MIT License. +// This product includes software developed at Guance Cloud (https://www.guance.com/). +// Copyright 2021-present Guance, Inc. + +package funcs + +import ( + "encoding/base64" + "fmt" + + "github.com/GuanceCloud/platypus/pkg/ast" + "github.com/GuanceCloud/platypus/pkg/engine/runtime" + "github.com/GuanceCloud/platypus/pkg/errchain" + "github.com/GuanceCloud/platypus/pkg/inimpl/guancecloud/ptinput" +) + +func B64decChecking(ctx *runtime.Context, funcExpr *ast.CallExpr) *errchain.PlError { + if len(funcExpr.Param) != 1 { + return runtime.NewRunError(ctx, fmt.Sprintf( + "func %s expects 1 arg", funcExpr.Name), funcExpr.NamePos) + } + if funcExpr.Param[0].NodeType != ast.TypeIdentifier { + return runtime.NewRunError(ctx, fmt.Sprintf( + "expect Identifier, got %s", funcExpr.Param[0].NodeType), + funcExpr.Param[0].StartPos()) + } + return nil +} + +func B64dec(ctx *runtime.Context, funcExpr *ast.CallExpr) *errchain.PlError { + if len(funcExpr.Param) != 1 { + return runtime.NewRunError(ctx, fmt.Sprintf( + "func %s expects 1 arg", funcExpr.Name), funcExpr.NamePos) + } + if funcExpr.Param[0].NodeType != ast.TypeIdentifier { + return runtime.NewRunError(ctx, fmt.Sprintf( + "expect Identifier, got %s", funcExpr.Param[0].NodeType), + funcExpr.Param[0].StartPos()) + } + + key, err := getKeyName(funcExpr.Param[0]) + if err != nil { + return runtime.NewRunError(ctx, err.Error(), funcExpr.Param[0].StartPos()) + } + val, err := ctx.GetKey(key) + if err != nil { + l.Debugf("key `%v` does not exist, ignored", key) + return nil //nolint:nilerr + } + var cont string + + switch v := val.Value.(type) { + case string: + cont = v + default: + l.Debugf("key `%s` has value not of type string, ignored", key) + return nil + } + res, err := base64.StdEncoding.DecodeString(cont) + if err != nil { + return runtime.NewRunError(ctx, err.Error(), funcExpr.NamePos) + } + if err := addKey2PtWithVal(ctx.InData(), key, string(res), ast.String, + ptinput.KindPtDefault); err != nil { + l.Debug(err) + return nil + } + return nil +} diff --git a/pkg/inimpl/guancecloud/ptinput/funcs/fn_b64dec_test.go b/pkg/inimpl/guancecloud/ptinput/funcs/fn_b64dec_test.go new file mode 100644 index 0000000..cf17013 --- /dev/null +++ b/pkg/inimpl/guancecloud/ptinput/funcs/fn_b64dec_test.go @@ -0,0 +1,79 @@ +// Unless explicitly stated otherwise all files in this repository are licensed +// under the MIT License. +// This product includes software developed at Guance Cloud (https://www.guance.com/). +// Copyright 2021-present Guance, Inc. + +package funcs + +import ( + "testing" + "time" + + "github.com/GuanceCloud/cliutils/point" + tu "github.com/GuanceCloud/cliutils/testutil" + "github.com/GuanceCloud/platypus/pkg/inimpl/guancecloud/ptinput" +) + +func TestB64dec(t *testing.T) { + cases := []struct { + name string + outKey string + pl, in string + expected interface{} + fail bool + }{ + { + name: "normal", + pl: "json(_, `str`); b64dec(`str`)", + in: `{"str": "MTM4MzgxMzA1MTc="}`, + outKey: "str", + expected: "13838130517", + fail: false, + }, + { + name: "normal", + pl: "json(_, `str`); b64dec(`str`)", + in: `{"str": "aGVsbG8sIHdvcmxk"}`, + outKey: "str", + expected: "hello, world", + fail: false, + }, + { + name: "normal", + pl: "json(_, `str`); b64dec(`str`)", + in: `{"str": "5L2g5aW9"}`, + outKey: "str", + expected: "你好", + fail: false, + }, + } + + for idx, tc := range cases { + t.Run(tc.name, func(t *testing.T) { + runner, err := NewTestingRunner(tc.pl) + if err != nil { + if tc.fail { + t.Logf("[%d]expect error: %s", idx, err) + } else { + t.Errorf("[%d] failed: %s", idx, err) + } + return + } + + pt := ptinput.NewPlPoint( + point.Logging, "test", nil, map[string]any{"message": tc.in}, time.Now()) + errR := runScript(runner, pt) + + if errR != nil { + t.Fatal(errR) + } + + if tc.fail { + t.Logf("[%d]expect error: %s", idx, err) + } + v, _, _ := pt.Get(tc.outKey) + tu.Equals(t, tc.expected, v) + t.Logf("[%d] PASS", idx) + }) + } +} diff --git a/pkg/inimpl/guancecloud/ptinput/funcs/fn_b64enc.go b/pkg/inimpl/guancecloud/ptinput/funcs/fn_b64enc.go new file mode 100644 index 0000000..43a7a81 --- /dev/null +++ b/pkg/inimpl/guancecloud/ptinput/funcs/fn_b64enc.go @@ -0,0 +1,67 @@ +// Unless explicitly stated otherwise all files in this repository are licensed +// under the MIT License. +// This product includes software developed at Guance Cloud (https://www.guance.com/). +// Copyright 2021-present Guance, Inc. + +package funcs + +import ( + "encoding/base64" + "fmt" + + "github.com/GuanceCloud/platypus/pkg/ast" + "github.com/GuanceCloud/platypus/pkg/engine/runtime" + "github.com/GuanceCloud/platypus/pkg/errchain" + "github.com/GuanceCloud/platypus/pkg/inimpl/guancecloud/ptinput" +) + +func B64encChecking(ctx *runtime.Context, funcExpr *ast.CallExpr) *errchain.PlError { + if len(funcExpr.Param) != 1 { + return runtime.NewRunError(ctx, fmt.Sprintf( + "func %s expects 1 arg", funcExpr.Name), funcExpr.NamePos) + } + if funcExpr.Param[0].NodeType != ast.TypeIdentifier { + return runtime.NewRunError(ctx, fmt.Sprintf( + "expect Identifier, got %s", funcExpr.Param[0].NodeType), + funcExpr.Param[0].StartPos()) + } + return nil +} + +func B64enc(ctx *runtime.Context, funcExpr *ast.CallExpr) *errchain.PlError { + if len(funcExpr.Param) != 1 { + return runtime.NewRunError(ctx, fmt.Sprintf( + "func %s expects 1 arg", funcExpr.Name), funcExpr.NamePos) + } + if funcExpr.Param[0].NodeType != ast.TypeIdentifier { + return runtime.NewRunError(ctx, fmt.Sprintf( + "expect Identifier, got %s", funcExpr.Param[0].NodeType), + funcExpr.Param[0].StartPos()) + } + + key, err := getKeyName(funcExpr.Param[0]) + if err != nil { + return runtime.NewRunError(ctx, err.Error(), funcExpr.Param[0].StartPos()) + } + val, err := ctx.GetKey(key) + if err != nil { + l.Debugf("key `%v` does not exist, ignored", key) + return nil //nolint:nilerr + } + var cont string + + switch v := val.Value.(type) { + case string: + cont = v + default: + l.Debugf("key `%s` has value not of type string, ignored", key) + return nil + } + res := base64.StdEncoding.EncodeToString([]byte(cont)) + if err := addKey2PtWithVal(ctx.InData(), key, res, ast.String, + ptinput.KindPtDefault); err != nil { + l.Debug(err) + return nil + } + return nil +} diff --git a/pkg/inimpl/guancecloud/ptinput/funcs/fn_b64enc_test.go b/pkg/inimpl/guancecloud/ptinput/funcs/fn_b64enc_test.go new file mode 100644 index 0000000..4d3841a --- /dev/null +++ b/pkg/inimpl/guancecloud/ptinput/funcs/fn_b64enc_test.go @@ -0,0 +1,78 @@ +// Unless explicitly stated otherwise all files in this repository are licensed +// under the MIT License. +// This product includes software developed at Guance Cloud (https://www.guance.com/). +// Copyright 2021-present Guance, Inc. + +package funcs + +import ( + "testing" + "time" + + "github.com/GuanceCloud/cliutils/point" + tu "github.com/GuanceCloud/cliutils/testutil" + "github.com/GuanceCloud/platypus/pkg/inimpl/guancecloud/ptinput" +) + +func TestB64enc(t *testing.T) { + cases := []struct { + name string + outKey string + pl, in string + expected interface{} + fail bool + }{ + { + name: "normal", + pl: "json(_, `str`); b64enc(`str`)", + in: `{"str": "13838130517"}`, + outKey: "str", + expected: "MTM4MzgxMzA1MTc=", + fail: false, + }, + { + name: "normal", + pl: "json(_, `str`); b64enc(`str`)", + in: `{"str": "hello, world"}`, + outKey: "str", + expected: "aGVsbG8sIHdvcmxk", + fail: false, + }, + { + name: "normal", + pl: "json(_, `str`); b64enc(`str`)", + in: `{"str": "你好"}`, + outKey: "str", + expected: "5L2g5aW9", + fail: false, + }, + } + + for idx, tc := range cases { + t.Run(tc.name, func(t *testing.T) { + runner, err := NewTestingRunner(tc.pl) + if err != nil { + if tc.fail { + t.Logf("[%d]expect error: %s", idx, err) + } else { + t.Errorf("[%d] failed: %s", idx, err) + } + return + } + + pt := ptinput.NewPlPoint( + point.Logging, "test", nil, map[string]any{"message": tc.in}, time.Now()) + errR := runScript(runner, pt) + if errR != nil { + t.Fatal(errR) + } + + if tc.fail { + t.Logf("[%d]expect error: %s", idx, err) + } + v, _, _ := pt.Get(tc.outKey) + tu.Equals(t, tc.expected, v) + t.Logf("[%d] PASS", idx) + }) + } +} diff --git a/pkg/inimpl/guancecloud/funcs/fn_cast.go b/pkg/inimpl/guancecloud/ptinput/funcs/fn_cast.go similarity index 95% rename from pkg/inimpl/guancecloud/funcs/fn_cast.go rename to pkg/inimpl/guancecloud/ptinput/funcs/fn_cast.go index dc288b3..3b3fa5d 100644 --- a/pkg/inimpl/guancecloud/funcs/fn_cast.go +++ b/pkg/inimpl/guancecloud/ptinput/funcs/fn_cast.go @@ -11,7 +11,7 @@ import ( "github.com/GuanceCloud/platypus/pkg/ast" "github.com/GuanceCloud/platypus/pkg/engine/runtime" "github.com/GuanceCloud/platypus/pkg/errchain" - "github.com/GuanceCloud/platypus/pkg/inimpl/guancecloud/input" + "github.com/GuanceCloud/platypus/pkg/inimpl/guancecloud/ptinput" ) func CastChecking(ctx *runtime.Context, funcExpr *ast.CallExpr) *errchain.PlError { @@ -67,7 +67,7 @@ func Cast(ctx *runtime.Context, funcExpr *ast.CallExpr) *errchain.PlError { val, dtype := doCast(v.Value, castType) if err = addKey2PtWithVal(ctx.InData(), key, val, dtype, - input.KindPtDefault); err != nil { + ptinput.KindPtDefault); err != nil { l.Debug(err) return nil } diff --git a/pkg/inimpl/guancecloud/funcs/fn_cast_test.go b/pkg/inimpl/guancecloud/ptinput/funcs/fn_cast_test.go similarity index 91% rename from pkg/inimpl/guancecloud/funcs/fn_cast_test.go rename to pkg/inimpl/guancecloud/ptinput/funcs/fn_cast_test.go index 92f1ccd..10e3c53 100644 --- a/pkg/inimpl/guancecloud/funcs/fn_cast_test.go +++ b/pkg/inimpl/guancecloud/ptinput/funcs/fn_cast_test.go @@ -9,7 +9,9 @@ import ( "testing" "time" - "github.com/stretchr/testify/assert" + "github.com/GuanceCloud/cliutils/point" + tu "github.com/GuanceCloud/cliutils/testutil" + "github.com/GuanceCloud/platypus/pkg/inimpl/guancecloud/ptinput" ) func TestCast(t *testing.T) { @@ -131,15 +133,16 @@ func TestCast(t *testing.T) { } return } - _, _, f, _, _, err := runScript(runner, "test", nil, map[string]interface{}{ - "message": tc.in, - }, time.Now()) - if err != nil { - t.Error(err) + + pt := ptinput.NewPlPoint( + point.Logging, "test", nil, map[string]any{"message": tc.in}, time.Now()) + errR := runScript(runner, pt) + + if errR != nil { + t.Fatal(errR) } else { - t.Log(f) - v := f[tc.outkey] - assert.Equal(t, tc.expect, v) + v, _, _ := pt.Get(tc.outkey) + tu.Equals(t, tc.expect, v) t.Logf("[%d] PASS", idx) } }) diff --git a/pkg/inimpl/guancecloud/ptinput/funcs/fn_cidr.go b/pkg/inimpl/guancecloud/ptinput/funcs/fn_cidr.go new file mode 100644 index 0000000..8430c30 --- /dev/null +++ b/pkg/inimpl/guancecloud/ptinput/funcs/fn_cidr.go @@ -0,0 +1,108 @@ +// Unless explicitly stated otherwise all files in this repository are licensed +// under the MIT License. +// This product includes software developed at Guance Cloud (https://www.guance.com/). +// Copyright 2021-present Guance, Inc. + +package funcs + +import ( + "fmt" + "net/netip" + "reflect" + + "github.com/GuanceCloud/platypus/pkg/ast" + "github.com/GuanceCloud/platypus/pkg/engine/runtime" + "github.com/GuanceCloud/platypus/pkg/errchain" +) + +func CIDRChecking(ctx *runtime.Context, funcExpr *ast.CallExpr) *errchain.PlError { + if len(funcExpr.Param) != 2 { + return runtime.NewRunError(ctx, fmt.Sprintf( + "func %s expects 2 args", funcExpr.Name), funcExpr.NamePos) + } + + param0 := funcExpr.Param[0] + if !isPlVarbOrFunc(param0) && param0.NodeType != ast.TypeStringLiteral { + return runtime.NewRunError(ctx, fmt.Sprintf( + "expect StringLiteral, Identifier or AttrExpr, got %s", + param0.NodeType), param0.StartPos()) + } + + param1 := funcExpr.Param[1] + if !isPlVarbOrFunc(param1) && param1.NodeType != ast.TypeStringLiteral { + return runtime.NewRunError(ctx, fmt.Sprintf( + "expect StringLiteral, Identifier or AttrExpr, got %s", + param1.NodeType), param1.StartPos()) + } + + return nil +} + +func CIDR(ctx *runtime.Context, funcExpr *ast.CallExpr) *errchain.PlError { + if len(funcExpr.Param) != 2 { + err := fmt.Errorf("func %s expects 2 args", funcExpr.Name) + l.Debug(err) + ctx.Regs.ReturnAppend(false, ast.Bool) + return nil + } + + ipAddr, err := getStr(ctx, funcExpr.Param[0]) + if err != nil { + l.Debug(err) + ctx.Regs.ReturnAppend(false, ast.Bool) + return nil + } + + prefix, err := getStr(ctx, funcExpr.Param[1]) + if err != nil { + l.Debug(err) + ctx.Regs.ReturnAppend(false, ast.Bool) + return nil + } + + ok, err := CIDRContains(ipAddr, prefix) + if err != nil { + l.Debug(err) + } + + ctx.Regs.ReturnAppend(ok, ast.Bool) + return nil +} + +func CIDRContains(ipAddr, prefix string) (bool, error) { + network, err := netip.ParsePrefix(prefix) + if err != nil { + return false, err + } + + ip, err := netip.ParseAddr(ipAddr) + if err != nil { + return false, err + } + + return network.Contains(ip), nil +} + +func getStr(ctx *runtime.Context, node *ast.Node) (string, error) { + if node.NodeType == ast.TypeStringLiteral { + return node.StringLiteral.Val, nil + } + + keyName, err := getKeyName(node) + if err != nil { + return "", err + } + v, err := ctx.GetKey(keyName) + if err != nil { + return "", err + } + if v.DType != ast.String { + return "", err + } + + if addr, ok := v.Value.(string); ok { + return addr, nil + } else { + return "", fmt.Errorf("type: %s", reflect.TypeOf(v.Value).String()) + } +} diff --git a/pkg/inimpl/guancecloud/ptinput/funcs/fn_cidr_test.go b/pkg/inimpl/guancecloud/ptinput/funcs/fn_cidr_test.go new file mode 100644 index 0000000..4628c9b --- /dev/null +++ b/pkg/inimpl/guancecloud/ptinput/funcs/fn_cidr_test.go @@ -0,0 +1,76 @@ +// Unless explicitly stated otherwise all files in this repository are licensed +// under the MIT License. +// This product includes software developed at Guance Cloud (https://www.guance.com/). +// Copyright 2021-present Guance, Inc. + +package funcs + +import ( + "testing" + "time" + + "github.com/GuanceCloud/cliutils/point" + tu "github.com/GuanceCloud/cliutils/testutil" + "github.com/GuanceCloud/platypus/pkg/inimpl/guancecloud/ptinput" +) + +func TestCIDR(t *testing.T) { + cases := []struct { + name string + outKey string + pl, in string + expected interface{} + fail bool + }{ + { + name: "ipv4_contains", + pl: `ip = "192.0.2.233" + if cidr(ip, "192.0.2.1/24") { + add_key(ip_prefix, "192.0.2.1/24") + }`, + in: ``, + outKey: "ip_prefix", + expected: "192.0.2.1/24", + fail: false, + }, + { + name: "ipv4_not_contains", + pl: `ip = "192.0.2.233" + if cidr(ip, "192.0.1.1/24") { + add_key(ip_prefix, "192.0.1.1/24") + }`, + in: ``, + outKey: "ip_prefix", + fail: false, + }, + } + + for idx, tc := range cases { + t.Run(tc.name, func(t *testing.T) { + runner, err := NewTestingRunner(tc.pl) + if err != nil { + if tc.fail { + t.Logf("[%d]expect error: %s", idx, err) + } else { + t.Errorf("[%d] failed: %s", idx, err) + } + return + } + + pt := ptinput.NewPlPoint( + point.Logging, "test", nil, map[string]any{"message": tc.in}, time.Now()) + errR := runScript(runner, pt) + + if errR != nil { + t.Fatal(errR.Error()) + } + + if tc.fail { + t.Logf("[%d]expect error: %s", idx, errR.Error()) + } + v, _, _ := pt.Get(tc.outKey) + tu.Equals(t, tc.expected, v) + t.Logf("[%d] PASS", idx) + }) + } +} diff --git a/pkg/inimpl/guancecloud/ptinput/funcs/fn_conv_traceid_hex_to_dec.go b/pkg/inimpl/guancecloud/ptinput/funcs/fn_conv_traceid_hex_to_dec.go new file mode 100644 index 0000000..60e8ffd --- /dev/null +++ b/pkg/inimpl/guancecloud/ptinput/funcs/fn_conv_traceid_hex_to_dec.go @@ -0,0 +1,84 @@ +// Unless explicitly stated otherwise all files in this repository are licensed +// under the MIT License. +// This product includes software developed at Guance Cloud (https://www.guance.com/). +// Copyright 2021-present Guance, Inc. + +package funcs + +import ( + "encoding/binary" + "encoding/hex" + "fmt" + "strconv" + + "github.com/GuanceCloud/platypus/pkg/ast" + "github.com/GuanceCloud/platypus/pkg/engine/runtime" + "github.com/GuanceCloud/platypus/pkg/errchain" + "github.com/GuanceCloud/platypus/pkg/inimpl/guancecloud/ptinput" +) + +func ConvTraceIDW3C2DDChecking(ctx *runtime.Context, funcExpr *ast.CallExpr) *errchain.PlError { + if len(funcExpr.Param) != 1 { + return runtime.NewRunError(ctx, fmt.Sprintf( + "func %s expected 1 args", funcExpr.Name), funcExpr.NamePos) + } + + if _, err := getKeyName(funcExpr.Param[0]); err != nil { + return runtime.NewRunError(ctx, err.Error(), funcExpr.Param[0].StartPos()) + } + + return nil +} + +func ConvTraceIDW3C2DD(ctx *runtime.Context, funcExpr *ast.CallExpr) *errchain.PlError { + if len(funcExpr.Param) != 1 { + return runtime.NewRunError(ctx, fmt.Sprintf( + "func %s expected 1 args", funcExpr.Name), funcExpr.NamePos) + } + + key, err := getKeyName(funcExpr.Param[0]) + if err != nil { + return runtime.NewRunError(ctx, err.Error(), funcExpr.Param[0].StartPos()) + } + + k, err := ctx.GetKey(key) + if err != nil { + l.Debug(err) + return nil + } + + w3cTraceID, ok := k.Value.(string) + if !ok { + return nil + } + + if ddTraceID, err := convTraceW3CToDD(w3cTraceID); err != nil { + l.Debug(err) + return nil + } else { + if err := addKey2PtWithVal(ctx.InData(), key, ddTraceID, ast.String, + ptinput.KindPtDefault); err != nil { + l.Debug(err) + return nil + } + } + return nil +} + +func convTraceW3CToDD(hexStr string) (string, error) { + switch len(hexStr) { + case 32: + hexStr = hexStr[16:] + case 16: + default: + return "", fmt.Errorf("not trace/span id") + } + + b, err := hex.DecodeString(hexStr) + if err != nil { + return "", err + } + + i := binary.BigEndian.Uint64(b) + return strconv.FormatUint(i, 10), nil +} diff --git a/pkg/inimpl/guancecloud/ptinput/funcs/fn_conv_traceid_test.go b/pkg/inimpl/guancecloud/ptinput/funcs/fn_conv_traceid_test.go new file mode 100644 index 0000000..f15c390 --- /dev/null +++ b/pkg/inimpl/guancecloud/ptinput/funcs/fn_conv_traceid_test.go @@ -0,0 +1,129 @@ +// Unless explicitly stated otherwise all files in this repository are licensed +// under the MIT License. +// This product includes software developed at Guance Cloud (https://www.guance.com/). +// Copyright 2021-present Guance, Inc. + +package funcs + +import ( + "testing" + "time" + + "github.com/GuanceCloud/cliutils/point" + "github.com/GuanceCloud/platypus/pkg/inimpl/guancecloud/ptinput" + "github.com/stretchr/testify/assert" +) + +func TestConvW3CTraceID(t *testing.T) { + cases := []struct { + name, pl, in string + key string + fail bool + expect any + }{ + { + name: "value type: string", + in: `18962fdd9eea517f2ae0771ea69d6e16`, + pl: ` + grok(_, "%{NOTSPACE:trace_id}") + + conv_traceid_w3c_to_dd(trace_id) +`, + key: "trace_id", + expect: "3089600317904219670", + fail: false, + }, + + { + name: "value type: string", + in: `2f7cdb2b45447bcb43820ddf56d9a654`, + pl: ` + grok(_, "%{NOTSPACE:trace_id}") + + conv_traceid_w3c_to_dd(trace_id) +`, + key: "trace_id", + expect: "4864465800399529556", + fail: false, + }, + + { + name: "value type: string", + in: `43820ddf56d9a654`, + pl: ` + grok(_, "%{NOTSPACE:trace_id}") + + conv_traceid_w3c_to_dd(trace_id) +`, + key: "trace_id", + expect: "4864465800399529556", + fail: false, + }, + + { + name: "value type: string", + in: `03820ddf56d9a654`, + pl: ` + grok(_, "%{NOTSPACE:trace_id}") + + conv_traceid_w3c_to_dd(trace_id) +`, + key: "trace_id", + expect: "252779781972141652", + fail: false, + }, + + { + name: "value type: string", + in: `0f7cdb2b45447bcb43820ddf56d9a654`, + pl: ` + grok(_, "%{NOTSPACE:trace_id}") + + conv_traceid_w3c_to_dd(trace_id) +`, + key: "trace_id", + expect: "4864465800399529556", + fail: false, + }, + + { + name: "value type: string", + in: `10f7cdb2b45447bcb43820ddf56d9a654`, + pl: ` + grok(_, "%{NOTSPACE:trace_id}") + + conv_traceid_w3c_to_dd(trace_id) +`, + key: "trace_id", + // 原样返回 + expect: "10f7cdb2b45447bcb43820ddf56d9a654", + fail: false, + }, + } + + for idx, tc := range cases { + t.Run(tc.name, func(t *testing.T) { + runner, err := NewTestingRunner(tc.pl) + if err != nil { + if tc.fail { + t.Logf("[%d]expect error: %s", idx, err) + } else { + t.Errorf("[%d] failed: %s", idx, err) + } + return + } + pt := ptinput.NewPlPoint(point.Logging, "test", nil, map[string]any{"message": tc.in}, time.Now()) + + errR := runScript(runner, pt) + + if errR == nil { + v, _, ok := pt.Get(tc.key) + assert.Equal(t, nil, ok) + assert.Equal(t, tc.expect, v) + t.Logf("[%d] PASS", idx) + } else { + t.Error(errR) + } + }) + } +} diff --git a/pkg/inimpl/guancecloud/ptinput/funcs/fn_cover.go b/pkg/inimpl/guancecloud/ptinput/funcs/fn_cover.go new file mode 100644 index 0000000..b58bf50 --- /dev/null +++ b/pkg/inimpl/guancecloud/ptinput/funcs/fn_cover.go @@ -0,0 +1,153 @@ +// Unless explicitly stated otherwise all files in this repository are licensed +// under the MIT License. +// This product includes software developed at Guance Cloud (https://www.guance.com/). +// Copyright 2021-present Guance, Inc. + +package funcs + +import ( + "fmt" + "unicode" + "unicode/utf8" + + "github.com/GuanceCloud/platypus/pkg/ast" + "github.com/GuanceCloud/platypus/pkg/engine/runtime" + "github.com/GuanceCloud/platypus/pkg/errchain" + "github.com/GuanceCloud/platypus/pkg/inimpl/guancecloud/ptinput" +) + +func CoverChecking(ctx *runtime.Context, funcExpr *ast.CallExpr) *errchain.PlError { + if len(funcExpr.Param) != 2 { + return runtime.NewRunError(ctx, fmt.Sprintf( + "func %s expects 2 args", funcExpr.Name), funcExpr.NamePos) + } + + if _, err := getKeyName(funcExpr.Param[0]); err != nil { + return runtime.NewRunError(ctx, err.Error(), funcExpr.Param[0].StartPos()) + } + + if funcExpr.Param[1].NodeType != ast.TypeListInitExpr { + return runtime.NewRunError(ctx, fmt.Sprintf( + "param range expects ListInitExpr, got %s", funcExpr.Param[1].NodeType), + funcExpr.Param[1].StartPos()) + } + + set := funcExpr.Param[1].ListInitExpr + + if len(set.List) != 2 { + return runtime.NewRunError(ctx, fmt.Sprintf( + "param between range value `%v' is not expected", funcExpr.Param[1]), + funcExpr.Param[1].StartPos()) + } + + if (set.List[0].NodeType != ast.TypeFloatLiteral && set.List[0].NodeType != ast.TypeIntegerLiteral) || + (set.List[1].NodeType != ast.TypeFloatLiteral && set.List[1].NodeType != ast.TypeIntegerLiteral) { + return runtime.NewRunError(ctx, fmt.Sprintf( + "range value `%v' is not expected", set), funcExpr.Param[1].StartPos()) + } + + return nil +} + +func Cover(ctx *runtime.Context, funcExpr *ast.CallExpr) *errchain.PlError { + if len(funcExpr.Param) != 2 { + return runtime.NewRunError(ctx, fmt.Sprintf( + "func %s expects 2 args", funcExpr.Name), funcExpr.NamePos) + } + + key, err := getKeyName(funcExpr.Param[0]) + if err != nil { + return runtime.NewRunError(ctx, err.Error(), funcExpr.Param[0].StartPos()) + } + + if funcExpr.Param[1].NodeType != ast.TypeListInitExpr { + return nil + } + + set := funcExpr.Param[1].ListInitExpr + + var start, end int + + if len(set.List) != 2 { + return runtime.NewRunError(ctx, fmt.Sprintf( + "param between range value `%v' is not expected", set), + funcExpr.Param[1].StartPos()) + } + + switch set.List[0].NodeType { //nolint:exhaustive + case ast.TypeIntegerLiteral: + start = int(set.List[0].IntegerLiteral.Val) + case ast.TypeFloatLiteral: + start = int(set.List[0].FloatLiteral.Val) + default: + return runtime.NewRunError(ctx, fmt.Sprintf( + "range value `%v' is not expected", set), + funcExpr.Param[1].StartPos()) + } + + switch set.List[1].NodeType { //nolint:exhaustive + case ast.TypeIntegerLiteral: + end = int(set.List[1].IntegerLiteral.Val) + case ast.TypeFloatLiteral: + end = int(set.List[1].FloatLiteral.Val) + default: + return runtime.NewRunError(ctx, fmt.Sprintf( + "range value `%v' is not expected", set), + funcExpr.Param[1].StartPos()) + } + + cont1, err := ctx.GetKey(key) + if err != nil { + l.Debugf("key `%v' not exist, ignored", key) + return nil //nolint:nilerr + } + + var cont string + + switch v := cont1.Value.(type) { + case string: + cont = v + default: + return nil + } + + if end > utf8.RuneCountInString(cont) { + end = utf8.RuneCountInString(cont) + } + + // end less than 0 become greater than 0 + if end < 0 { + end += utf8.RuneCountInString(cont) + 1 + } + // start less than 0 become greater than 0 + if start <= 0 { + start += utf8.RuneCountInString(cont) + 1 + } + + // unreasonable subscript + if start > end { + l.Debug("invalid cover range") + return nil + // return runtime.NewRunError(ctx, "invalid cover range", funcExpr.Param[1].StartPos()) + } + + arrCont := []rune(cont) + + for i := 0; i < len(arrCont); i++ { + if i+1 >= start && i < end { + if unicode.Is(unicode.Han, arrCont[i]) { + arrCont[i] = rune('＊') + } else { + arrCont[i] = rune('*') + } + } + } + + if err := addKey2PtWithVal(ctx.InData(), key, string(arrCont), ast.String, + ptinput.KindPtDefault); err != nil { + l.Debug(err) + return nil + } + + return nil +} diff --git a/pkg/inimpl/guancecloud/ptinput/funcs/fn_cover_test.go b/pkg/inimpl/guancecloud/ptinput/funcs/fn_cover_test.go new file mode 100644 index 0000000..26102d2 --- /dev/null +++ b/pkg/inimpl/guancecloud/ptinput/funcs/fn_cover_test.go @@ -0,0 +1,150 @@ +// Unless explicitly stated otherwise all files in this repository are licensed +// under the MIT License. +// This product includes software developed at Guance Cloud (https://www.guance.com/). +// Copyright 2021-present Guance, Inc. + +package funcs + +import ( + "testing" + "time" + + "github.com/GuanceCloud/cliutils/point" + tu "github.com/GuanceCloud/cliutils/testutil" + "github.com/GuanceCloud/platypus/pkg/inimpl/guancecloud/ptinput" +) + +func TestDz(t *testing.T) { + cases := []struct { + name string + outKey string + pl, in string + expected interface{} + fail bool + }{ + { + name: "normal", + pl: "json(_, `str`); cover(`str`, [8, 13])", + in: `{"str": "13838130517"}`, + outKey: "str", + expected: "1383813****", + fail: false, + }, + + { + name: "normal", + pl: "json(_, `str`) ;cover(`str`, [8, 11])", + in: `{"str": "13838130517"}`, + outKey: "str", + expected: "1383813****", + fail: false, + }, + + { + name: "normal", + pl: "json(_, `str`); cover(`str`, [2, 4])", + in: `{"str": "13838130517"}`, + outKey: "str", + expected: "1***8130517", + fail: false, + }, + + { + name: "normal", + pl: "json(_, `str`) ;cover(`str`, [1, 1])", + in: `{"str": "13838130517"}`, + outKey: "str", + expected: "*3838130517", + fail: false, + }, + + { + name: "odd range", + pl: "json(_, `str`); cover(`str`, [1, 100])", + in: `{"str": "刘少波"}`, + outKey: "str", + expected: "＊＊＊", + fail: false, + }, + + { + name: "invalid range", + pl: "json(_, `str`); cover(`str`, [3, 2])", + in: `{"str": "刘少波"}`, + fail: true, + }, + + { + name: "not enough args", + pl: "json(_, `str`) ;cover(`str`)", + in: `{"str": "刘少波"}`, + fail: true, + }, + + { + name: "invalid range", + pl: "json(_, `str`); cover(`str`, [\"刘\", \"波\"])", + in: `{"str": "刘少波"}`, + fail: true, + }, + + { + name: "normal", + pl: "json(_, `str`) ;cover(`str`, [1, 2])", + in: `{"str": 123456}`, + fail: true, + }, + + { + name: "normal", + pl: "json(_, `str`) ;cover(`str`, [-1, -2])", + in: `{"str": 123456}`, + fail: true, + }, + + { + name: "normal", + pl: "json(_, `str`); cover(`str`, [-1, -2])", + in: `{"str": 123456}`, + fail: true, + }, + + { + name: "normal", + pl: "json(_, `str`) ; cast(`str`, \"int\"); cover(`str`, [-2, 10000])", + in: `{"str": 123456}`, + outKey: "str", + expected: int64(123456), + fail: false, + }, + } + + for idx, tc := range cases { + t.Run(tc.name, func(t *testing.T) { + runner, err := NewTestingRunner(tc.pl) + if err != nil { + if tc.fail { + t.Logf("[%d]expect error: %s", idx, err) + } else { + t.Errorf("[%d] failed: %s", idx, err) + } + return + } + + pt := ptinput.NewPlPoint( + point.Logging, "test", nil, map[string]any{"message": tc.in}, time.Now()) + errR := runScript(runner, pt) + + if errR != nil { + t.Fatal(errR) + } + + if tc.fail { + t.Logf("[%d]expect error: %s", idx, err) + } + v, _, _ := pt.Get(tc.outKey) + tu.Equals(t, tc.expected, v) + t.Logf("[%d] PASS", idx) + }) + } +} diff --git a/pkg/inimpl/guancecloud/ptinput/funcs/fn_create_point.go b/pkg/inimpl/guancecloud/ptinput/funcs/fn_create_point.go new file mode 100644 index 0000000..ba748b3 --- /dev/null +++ b/pkg/inimpl/guancecloud/ptinput/funcs/fn_create_point.go @@ -0,0 +1,191 @@ +// Unless explicitly stated otherwise all files in this repository are licensed +// under the MIT License. +// This product includes software developed at Guance Cloud (https://www.guance.com/). +// Copyright 2021-present Guance, Inc. + +package funcs + +import ( + "fmt" + "reflect" + "time" + + "github.com/GuanceCloud/cliutils/point" + "github.com/GuanceCloud/platypus/pkg/ast" + "github.com/GuanceCloud/platypus/pkg/engine/runtime" + "github.com/GuanceCloud/platypus/pkg/errchain" + "github.com/GuanceCloud/platypus/pkg/inimpl/guancecloud/ptinput" + "github.com/spf13/cast" +) + +func CreatePointChecking(ctx *runtime.Context, funcExpr *ast.CallExpr) *errchain.PlError { + if err := reindexFuncArgs(funcExpr, []string{ + "name", "tags", "fields", + "ts", "category", "after_use", + }, 3); err != nil { + return runtime.NewRunError(ctx, err.Error(), funcExpr.NamePos) + } + + if arg := funcExpr.Param[5]; arg != nil { + switch arg.NodeType { //nolint:exhaustive + case ast.TypeStringLiteral: + usecall := &ast.CallExpr{ + Name: "use", + NamePos: arg.StartPos(), + Param: []*ast.Node{funcExpr.Param[5]}, + } + funcExpr.PrivateData = usecall + return UseChecking(ctx, usecall) + default: + return runtime.NewRunError(ctx, fmt.Sprintf("param after_use expects StringLiteral, got %s", + arg.NodeType), arg.StartPos()) + } + } + + return nil +} + +func CreatePoint(ctx *runtime.Context, funcExpr *ast.CallExpr) *errchain.PlError { + var ptName string + var ptTags map[string]string + ptFields := map[string]any{} + ptTime := time.Now() + ptCat := point.Metric + + name, _, err := runtime.RunStmt(ctx, funcExpr.Param[0]) + if err != nil { + return err + } + + if name == nil { + return runtime.NewRunError(ctx, + "type of parameter expected to be string, got nil", + funcExpr.Param[0].StartPos()) + } + + if v, ok := name.(string); ok { + ptName = v + } else { + return runtime.NewRunError(ctx, fmt.Sprintf( + "type of parameter expected to be string, got %s", + reflect.TypeOf(v)), funcExpr.Param[0].StartPos()) + } + + if tags, _, err := runtime.RunStmt(ctx, funcExpr.Param[1]); err != nil { + return nil + } else if tags != nil { + if tKV, ok := tags.(map[string]any); ok { + if len(tKV) > 0 { + ptTags = map[string]string{} + for tagK, tagV := range tKV { + if val, ok := tagV.(string); ok { + ptTags[tagK] = val + } + } + } + } else { + return runtime.NewRunError(ctx, fmt.Sprintf( + "type of parameter expected to be map, got %s", + reflect.TypeOf(tags)), funcExpr.Param[1].StartPos()) + } + } + + if fields, _, err := runtime.RunStmt(ctx, funcExpr.Param[2]); err != nil { + return nil + } else if fields != nil { + if fKV, ok := fields.(map[string]any); ok { + if len(fKV) > 0 { + ptFields = map[string]any{} + for fK, fV := range fKV { + switch fV := fV.(type) { + case int32, int8, int16, int, + uint, uint16, uint32, uint64, uint8: + ptFields[fK] = cast.ToInt64(fV) + case float32: + ptFields[fK] = cast.ToFloat64(fV) + case []byte: + ptFields[fK] = string(fV) + case string, bool, float64, int64: + ptFields[fK] = fV + } + } + } + } else { + return runtime.NewRunError(ctx, fmt.Sprintf( + "type of parameter expected to be map, got %s", + reflect.TypeOf(fields)), funcExpr.Param[2].StartPos()) + } + } + + if arg := funcExpr.Param[3]; arg != nil { + if pTS, _, err := runtime.RunStmt(ctx, arg); err != nil { + return nil + } else if pTS != nil { + if ts, ok := pTS.(int64); ok { + if ts > 0 { + ptTime = time.Unix(0, ts) + } + } else { + return runtime.NewRunError(ctx, fmt.Sprintf( + "type of parameter expected to be int64, got %s", + reflect.TypeOf(pTS)), arg.StartPos()) + } + } + } + + if arg := funcExpr.Param[4]; arg != nil { + if catName, _, err := runtime.RunStmt(ctx, arg); err != nil { + return nil + } else if catName != nil { + if catN, ok := catName.(string); ok { + ptCat = ptCategory(catN) + if ptCat == point.UnknownCategory { + return nil + } + } else { + return runtime.NewRunError(ctx, fmt.Sprintf( + "type of parameter expected to be str, got %s", + reflect.TypeOf(catName)), arg.StartPos()) + } + } + } + + plpt := ptinput.NewPlPoint(ptCat, ptName, ptTags, ptFields, ptTime) + if arg := funcExpr.Param[5]; arg != nil { + if refCall, ok := funcExpr.PrivateData.(*ast.CallExpr); ok { + if srcipt, ok := refCall.PrivateData.(*runtime.Script); ok { + if err := runtime.RunScriptWithRMapIn(srcipt, plpt, ctx.Signal()); err != nil { + return err.ChainAppend(ctx.Name(), funcExpr.NamePos) + } + } + } + } + + if ptIn, err := getPoint(ctx.InData()); err == nil { + ptIn.AppendSubPoint(plpt) + } + + return nil +} + +func ptCategory(cat string) point.Category { + switch cat { + case point.SLogging, point.CL: + return point.Logging + case point.SMetric, point.CM: + return point.Metric + case point.STracing, point.CT: + return point.Tracing + case point.SRUM, point.CR: + return point.RUM + case point.SNetwork, point.CN: + return point.Network + case point.SObject, point.CO: + return point.Object + case point.SCustomObject, point.CCO: + return point.CustomObject + case point.SSecurity, point.CS: + return point.Security + } + return point.UnknownCategory +} diff --git a/pkg/inimpl/guancecloud/ptinput/funcs/fn_create_point_test.go b/pkg/inimpl/guancecloud/ptinput/funcs/fn_create_point_test.go new file mode 100644 index 0000000..477f173 --- /dev/null +++ b/pkg/inimpl/guancecloud/ptinput/funcs/fn_create_point_test.go @@ -0,0 +1,335 @@ +// Unless explicitly stated otherwise all files in this repository are licensed +// under the MIT License. +// This product includes software developed at Guance Cloud (https://www.guance.com/). +// Copyright 2021-present Guance, Inc. + +package funcs + +import ( + "testing" + "time" + + "github.com/GuanceCloud/cliutils/point" + "github.com/GuanceCloud/platypus/pkg/inimpl/guancecloud/ptinput" + "github.com/stretchr/testify/assert" +) + +func TestCreatePoint(t *testing.T) { + cases := []struct { + name, in string + allPl map[string]string + outkey string + expect []ptinput.PlInputPt + fail bool + }{ + { + name: "default", + in: `{"a": "1", "b": 2, "c": {"d" : "x1"}}`, + allPl: map[string]string{ + "main.p": ` + d = load_json(_) + r = {} + for x in d["c"] { + r[x] = d["c"][x] + } + r["b"] = d["b"] + create_point("n1", {"a": d["a"]}, r) + `, + }, + outkey: "abc", + expect: []ptinput.PlInputPt{ + ptinput.NewPlPoint(point.Metric, "n1", map[string]string{ + "a": "1", + }, map[string]any{ + "d": "x1", + "b": float64(2.0), + }, time.Time{}), + }, + }, + { + name: "default-use", + in: `{"a": "1", "b": 2, "c": {"d" : "x1"}}`, + allPl: map[string]string{ + "main.p": ` + d = load_json(_) + r = {} + for x in d["c"] { + r[x] = d["c"][x] + } + r["b"] = d["b"] + create_point("n1", {"a": d["a"]}, r, category="M",after_use="abc.p") + `, + "abc.p": `add_key("aa", 1)`, + }, + outkey: "abc", + expect: []ptinput.PlInputPt{ + ptinput.NewPlPoint(point.Metric, "n1", map[string]string{ + "a": "1", + }, map[string]any{ + "d": "x1", + "b": float64(2.0), + "aa": int64(1), + }, time.Time{}), + }, + }, + { + name: "default-use", + in: `{"a": "1", "b": 2, "c": {"d" : "x1"}}`, + allPl: map[string]string{ + "main.p": ` + d = load_json(_) + r = {} + for x in d["c"] { + r[x] = d["c"][x] + } + r["b"] = d["b"] + create_point("n1", {"a": d["a"]}, r, ts = 1, after_use="abc.p") + `, + "abc.p": `add_key("aa", 1)`, + }, + outkey: "abc", + expect: []ptinput.PlInputPt{ + ptinput.NewPlPoint(point.Metric, "n1", map[string]string{ + "a": "1", + }, map[string]any{ + "d": "x1", + "b": float64(2.0), + "aa": int64(1), + }, time.Unix(0, 1)), + }, + }, + { + name: "failed-use", + in: `{"a": "1", "b": 2, "c": {"d" : "x1"}}`, + allPl: map[string]string{ + "main.p": ` + d = load_json(_) + r = {} + for x in d["c"] { + r[x] = d["c"][x] + } + r["b"] = d["b"] + create_point("n1", {"a": d["a"]}, r, after_use="absc.p") + `, + "abc.p": `add_key("aa", 1)`, + }, + outkey: "abc", + fail: true, + expect: []ptinput.PlInputPt{ + ptinput.NewPlPoint(point.Metric, "n1", map[string]string{ + "a": "1", + }, map[string]any{ + "d": "x1", + "b": float64(2.0), + "aa": int64(1), + }, time.Time{}), + }, + }, + { + name: "L", + in: `{"a": "1", "b": 2, "c": {"d" : "x1"}}`, + allPl: map[string]string{ + "main.p": ` + d = load_json(_) + r = {} + for x in d["c"] { + r[x] = d["c"][x] + } + r["b"] = d["b"] + create_point("n1", {"a": d["a"]}, r) + create_point("n1", {"a": d["a"]}, r, category="L") + create_point("n1", {"a": d["a"]}, r, category="M") + create_point("n1", {"a": d["a"]}, r, category="T") + create_point("n1", {"a": d["a"]}, r, category="R") + create_point("n1", {"a": d["a"]}, r, category="N") + create_point("n1", {"a": d["a"]}, r, category="O") + create_point("n1", {"a": d["a"]}, r, category="CO") + create_point("n1", {"a": d["a"]}, r, category="S") + `, + }, + outkey: "abc", + expect: []ptinput.PlInputPt{ + ptinput.NewPlPoint(point.Metric, "n1", map[string]string{ + "a": "1", + }, map[string]any{ + "d": "x1", + "b": float64(2.0), + }, time.Time{}), + ptinput.NewPlPoint(point.Logging, "n1", map[string]string{ + "a": "1", + }, map[string]any{ + "d": "x1", + "b": float64(2.0), + }, time.Time{}), + ptinput.NewPlPoint(point.Metric, "n1", map[string]string{ + "a": "1", + }, map[string]any{ + "d": "x1", + "b": float64(2.0), + }, time.Time{}), + ptinput.NewPlPoint(point.Tracing, "n1", map[string]string{ + "a": "1", + }, map[string]any{ + "d": "x1", + "b": float64(2.0), + }, time.Time{}), + ptinput.NewPlPoint(point.RUM, "n1", map[string]string{ + "a": "1", + }, map[string]any{ + "d": "x1", + "b": float64(2.0), + }, time.Time{}), + ptinput.NewPlPoint(point.Network, "n1", map[string]string{ + "a": "1", + }, map[string]any{ + "d": "x1", + "b": float64(2.0), + }, time.Time{}), + ptinput.NewPlPoint(point.Object, "n1", map[string]string{ + "a": "1", + }, map[string]any{ + "d": "x1", + "b": float64(2.0), + }, time.Time{}), + ptinput.NewPlPoint(point.CustomObject, "n1", map[string]string{ + "a": "1", + }, map[string]any{ + "d": "x1", + "b": float64(2.0), + }, time.Time{}), + ptinput.NewPlPoint(point.Security, "n1", map[string]string{ + "a": "1", + }, map[string]any{ + "d": "x1", + "b": float64(2.0), + }, time.Time{}), + }, + }, + { + name: "L", + in: `{"a": "1", "b": 2, "c": {"d" : "x1"}}`, + allPl: map[string]string{ + "main.p": ` + d = load_json(_) + r = {} + for x in d["c"] { + r[x] = d["c"][x] + } + r["b"] = d["b"] + create_point("n1", {"a": d["a"]}, r) + create_point("n1", {"a": d["a"]}, r, category="logging") + create_point("n1", {"a": d["a"]}, r, category="metric") + create_point("n1", {"a": d["a"]}, r, category="tracing") + create_point("n1", {"a": d["a"]}, r, category="rum") + create_point("n1", {"a": d["a"]}, r, category="network") + create_point("n1", {"a": d["a"]}, r, category="object") + create_point("n1", {"a": d["a"]}, r, category="custom_object") + create_point("n1", {"a": d["a"]}, r, category="security") + `, + }, + outkey: "abc", + expect: []ptinput.PlInputPt{ + ptinput.NewPlPoint(point.Metric, "n1", map[string]string{ + "a": "1", + }, map[string]any{ + "d": "x1", + "b": float64(2.0), + }, time.Time{}), + ptinput.NewPlPoint(point.Logging, "n1", map[string]string{ + "a": "1", + }, map[string]any{ + "d": "x1", + "b": float64(2.0), + }, time.Time{}), + ptinput.NewPlPoint(point.Metric, "n1", map[string]string{ + "a": "1", + }, map[string]any{ + "d": "x1", + "b": float64(2.0), + }, time.Time{}), + ptinput.NewPlPoint(point.Tracing, "n1", map[string]string{ + "a": "1", + }, map[string]any{ + "d": "x1", + "b": float64(2.0), + }, time.Time{}), + ptinput.NewPlPoint(point.RUM, "n1", map[string]string{ + "a": "1", + }, map[string]any{ + "d": "x1", + "b": float64(2.0), + }, time.Time{}), + ptinput.NewPlPoint(point.Network, "n1", map[string]string{ + "a": "1", + }, map[string]any{ + "d": "x1", + "b": float64(2.0), + }, time.Time{}), + ptinput.NewPlPoint(point.Object, "n1", map[string]string{ + "a": "1", + }, map[string]any{ + "d": "x1", + "b": float64(2.0), + }, time.Time{}), + ptinput.NewPlPoint(point.CustomObject, "n1", map[string]string{ + "a": "1", + }, map[string]any{ + "d": "x1", + "b": float64(2.0), + }, time.Time{}), + ptinput.NewPlPoint(point.Security, "n1", map[string]string{ + "a": "1", + }, map[string]any{ + "d": "x1", + "b": float64(2.0), + }, time.Time{}), + }, + }, + } + + for idx, tc := range cases { + t.Run(tc.name, func(t *testing.T) { + pls, errs := NewTestingRunner2(tc.allPl) + if len(errs) != 0 { + if tc.fail { + t.Logf("[%d]expect error: %v", idx, errs) + } else { + t.Errorf("[%d] failed: %v", idx, errs) + } + return + } + runner, ok := pls["main.p"] + if !ok { + t.Fatal(ok) + } + + pt := ptinput.NewPlPoint( + point.Logging, "test", nil, map[string]any{"message": tc.in}, time.Now()) + errR := runScript(runner, pt) + + t.Log(pt.Fields()) + if errR != nil { + if tc.fail { + return + } + t.Fatal(errR) + } + + pts := pt.GetSubPoint() + + if len(pts) != len(tc.expect) { + t.Fatal("len(pt)!= len(tc.expect)") + } + for i, pt := range pts { + expect := tc.expect[i] + assert.Equal(t, expect.Tags(), pt.Tags()) + assert.Equal(t, expect.Fields(), pt.Fields()) + assert.Equal(t, expect.GetPtName(), pt.GetPtName()) + assert.Equal(t, expect.Category(), pt.Category()) + if !expect.PtTime().IsZero() { + assert.Equal(t, expect.PtTime(), pt.PtTime()) + } + } + }) + } +} diff --git a/pkg/inimpl/guancecloud/funcs/fn_datetime.go b/pkg/inimpl/guancecloud/ptinput/funcs/fn_datetime.go similarity index 55% rename from pkg/inimpl/guancecloud/funcs/fn_datetime.go rename to pkg/inimpl/guancecloud/ptinput/funcs/fn_datetime.go index 30830c1..65e0e58 100644 --- a/pkg/inimpl/guancecloud/funcs/fn_datetime.go +++ b/pkg/inimpl/guancecloud/ptinput/funcs/fn_datetime.go @@ -7,21 +7,25 @@ package funcs import ( "fmt" + "time" "github.com/GuanceCloud/platypus/pkg/ast" "github.com/GuanceCloud/platypus/pkg/engine/runtime" "github.com/GuanceCloud/platypus/pkg/errchain" - "github.com/GuanceCloud/platypus/pkg/inimpl/guancecloud/input" + "github.com/GuanceCloud/platypus/pkg/inimpl/guancecloud/ptinput" + timefmt "github.com/itchyny/timefmt-go" + conv "github.com/spf13/cast" ) func DateTimeChecking(ctx *runtime.Context, funcExpr *ast.CallExpr) *errchain.PlError { - if len(funcExpr.Param) != 3 { - return runtime.NewRunError(ctx, fmt.Sprintf( - "func %s expected 3 args", funcExpr.Name), funcExpr.NamePos) + if err := reindexFuncArgs(funcExpr, []string{ + "key", "precision", "fmt", "tz", + }, 3); err != nil { + return runtime.NewRunError(ctx, err.Error(), funcExpr.NamePos) } if _, err := getKeyName(funcExpr.Param[0]); err != nil { - return runtime.NewRunError(ctx, err.Error(), funcExpr.NamePos) + return runtime.NewRunError(ctx, err.Error(), funcExpr.Param[0].StartPos()) } switch funcExpr.Param[1].NodeType { //nolint:exhaustive @@ -39,13 +43,25 @@ func DateTimeChecking(ctx *runtime.Context, funcExpr *ast.CallExpr) *errchain.Pl "param `fmt` expect StringLiteral, got %s", funcExpr.Param[2].NodeType), funcExpr.Param[2].StartPos()) } + + if funcExpr.Param[3] != nil { + switch funcExpr.Param[3].NodeType { //nolint:exhaustive + case ast.TypeStringLiteral: + default: + return runtime.NewRunError(ctx, fmt.Sprintf( + "param `tz` expect StringLiteral, got %s", + funcExpr.Param[2].NodeType), funcExpr.Param[3].StartPos()) + } + } + + // switch return nil } func DateTime(ctx *runtime.Context, funcExpr *ast.CallExpr) *errchain.PlError { - if len(funcExpr.Param) != 3 { + if len(funcExpr.Param) < 3 { return runtime.NewRunError(ctx, fmt.Sprintf( - "func %s expected 3 args", funcExpr.Name), funcExpr.NamePos) + "func %s expected 3 or 4 args", funcExpr.Name), funcExpr.NamePos) } key, err := getKeyName(funcExpr.Param[0]) @@ -79,10 +95,49 @@ func DateTime(ctx *runtime.Context, funcExpr *ast.CallExpr) *errchain.PlError { return nil //nolint:nilerr } - if v, err := DateFormatHandle(cont.Value, precision, fmts); err != nil { - return runtime.NewRunError(ctx, err.Error(), funcExpr.NamePos) - } else if err := addKey2PtWithVal(ctx.InData(), key, v, ast.String, - input.KindPtDefault); err != nil { + ts := conv.ToInt64(cont.Value) + + switch precision { + case "us": + ts *= 1e3 + case "ms": + ts *= 1e6 + case "s": + ts *= 1e9 + default: + } + + t := time.Unix(0, ts) + + var tz string + if funcExpr.Param[3] != nil { + if funcExpr.Param[3].NodeType == ast.TypeStringLiteral { + tz = funcExpr.Param[3].StringLiteral.Val + } else { + return runtime.NewRunError(ctx, fmt.Sprintf( + "param `tz` expect StringLiteral, got %s", + funcExpr.Param[2].NodeType), funcExpr.Param[3].StartPos()) + } + } + + if tz != "" { + loc, err := time.LoadLocation(tz) + if err != nil { + return runtime.NewRunError(ctx, err.Error(), funcExpr.Param[2].StartPos()) + } + t = t.In(loc) + } + + if datetimeInnerFormat(fmts) { + if v, err := DateFormatHandle(&t, fmts); err != nil { + return runtime.NewRunError(ctx, err.Error(), funcExpr.NamePos) + } else if err := addKey2PtWithVal(ctx.InData(), key, v, ast.String, + ptinput.KindPtDefault); err != nil { + l.Debug(err) + return nil + } + } else if err := addKey2PtWithVal(ctx.InData(), key, timefmt.Format(t, fmts), ast.String, + ptinput.KindPtDefault); err != nil { l.Debug(err) return nil } diff --git a/pkg/inimpl/guancecloud/funcs/fn_datetime_test.go b/pkg/inimpl/guancecloud/ptinput/funcs/fn_datetime_test.go similarity index 53% rename from pkg/inimpl/guancecloud/funcs/fn_datetime_test.go rename to pkg/inimpl/guancecloud/ptinput/funcs/fn_datetime_test.go index a99099f..cbadd25 100644 --- a/pkg/inimpl/guancecloud/funcs/fn_datetime_test.go +++ b/pkg/inimpl/guancecloud/ptinput/funcs/fn_datetime_test.go @@ -9,7 +9,9 @@ import ( "testing" "time" - "github.com/stretchr/testify/assert" + "github.com/GuanceCloud/cliutils/point" + tu "github.com/GuanceCloud/cliutils/testutil" + "github.com/GuanceCloud/platypus/pkg/inimpl/guancecloud/ptinput" ) func TestDateTime(t *testing.T) { @@ -27,249 +29,280 @@ func TestDateTime(t *testing.T) { name: "ANSIC s", in: `{"a":{"timestamp": "1638253518", "second":2},"age":47}`, pl: ` - input = load_json(_); printf("%v", input["a"]["timestamp"]);ts = input["a"]["timestamp"]; - datetime(ts, 's', 'ANSIC') + json(_, a.timestamp) + datetime(a.timestamp, 's', 'ANSIC') `, - outkey: "ts", + outkey: "a.timestamp", expect: "Tue Nov 30 14:25:18 2021", - fail: false, }, { name: "ANSIC ms", in: `{"a":{"timestamp": "1638253518000", "second":2},"age":47}`, pl: ` - input = load_json(_); ts = input["a"]["timestamp"]; - datetime(ts, 'ms', 'ANSIC') + json(_, a.timestamp) + datetime(a.timestamp, 'ms', 'ANSIC') `, - outkey: "ts", + outkey: "a.timestamp", expect: "Tue Nov 30 14:25:18 2021", - fail: false, }, { name: "UnixDate s", in: `{"a":{"timestamp": "1638253518", "second":2},"age":47}`, pl: ` - input = load_json(_); ts = input["a"]["timestamp"]; - datetime(ts, 's', 'UnixDate') + json(_, a.timestamp) + datetime(a.timestamp, 's', 'UnixDate') `, - outkey: "ts", + outkey: "a.timestamp", expect: "Tue Nov 30 14:25:18 CST 2021", - fail: false, }, { name: "UnixDate ms", in: `{"a":{"timestamp": "1638253518999", "second":2},"age":47}`, pl: ` - input = load_json(_); ts = input["a"]["timestamp"]; - datetime(ts, 'ms', 'UnixDate') + json(_, a.timestamp) + datetime(a.timestamp, 'ms', 'UnixDate') `, - outkey: "ts", + outkey: "a.timestamp", expect: "Tue Nov 30 14:25:18 CST 2021", - fail: false, }, { name: "RubyDate ms", in: `{"a":{"timestamp": "1638253518999", "second":2},"age":47}`, pl: ` - input = load_json(_); ts = input["a"]["timestamp"]; - datetime(ts, 'ms', 'RubyDate') + json(_, a.timestamp) + datetime(a.timestamp, 'ms', 'RubyDate') `, - outkey: "ts", + outkey: "a.timestamp", expect: "Tue Nov 30 14:25:18 +0800 2021", - fail: false, }, { name: "RubyDate s", in: `{"a":{"timestamp": "1638253518", "second":2},"age":47}`, pl: ` - input = load_json(_); ts = input["a"]["timestamp"]; - datetime(ts, 's', 'RubyDate') + json(_, a.timestamp) + datetime(a.timestamp, 's', 'RubyDate') `, - outkey: "ts", + outkey: "a.timestamp", expect: "Tue Nov 30 14:25:18 +0800 2021", - fail: false, }, { name: "RFC822 ms", in: `{"a":{"timestamp": "1638253518999", "second":2},"age":47}`, pl: ` - input = load_json(_); ts = input["a"]["timestamp"]; - datetime(ts, 'ms', 'RFC822') + json(_, a.timestamp) + datetime(a.timestamp, 'ms', 'RFC822') `, - outkey: "ts", + outkey: "a.timestamp", expect: "30 Nov 21 14:25 CST", - fail: false, }, { name: "RFC822 s", in: `{"a":{"timestamp": "1638253518", "second":2},"age":47}`, pl: ` - input = load_json(_); ts = input["a"]["timestamp"]; - datetime(ts, 's', 'RFC822') + json(_, a.timestamp) + datetime(a.timestamp, 's', 'RFC822') `, - outkey: "ts", + outkey: "a.timestamp", expect: "30 Nov 21 14:25 CST", - fail: false, }, { name: "RFC822Z ms", in: `{"a":{"timestamp": "1638253518999", "second":2},"age":47}`, pl: ` - input = load_json(_); ts = input["a"]["timestamp"]; - datetime(ts, 'ms', 'RFC822Z') + json(_, a.timestamp) + datetime(a.timestamp, 'ms', 'RFC822Z') `, - outkey: "ts", + outkey: "a.timestamp", expect: "30 Nov 21 14:25 +0800", - fail: false, }, { name: "RFC822Z s", in: `{"a":{"timestamp": "1638253518", "second":2},"age":47}`, pl: ` - input = load_json(_); ts = input["a"]["timestamp"]; - datetime(ts, 's', 'RFC822Z') + json(_, a.timestamp) + datetime(a.timestamp, 's', 'RFC822Z') `, - outkey: "ts", + outkey: "a.timestamp", expect: "30 Nov 21 14:25 +0800", - fail: false, }, { name: "RFC850 ms", in: `{"a":{"timestamp": "1638253518999", "second":2},"age":47}`, pl: ` - input = load_json(_); ts = input["a"]["timestamp"]; - datetime(ts, 'ms', 'RFC850') + json(_, a.timestamp) + datetime(a.timestamp, 'ms', 'RFC850') `, - outkey: "ts", + outkey: "a.timestamp", expect: "Tuesday, 30-Nov-21 14:25:18 CST", - fail: false, }, { name: "RFC850 s", in: `{"a":{"timestamp": "1638253518", "second":2},"age":47}`, pl: ` - input = load_json(_); ts = input["a"]["timestamp"]; - datetime(ts, 's', 'RFC850') + json(_, a.timestamp) + datetime(a.timestamp, 's', 'RFC850') `, - outkey: "ts", + outkey: "a.timestamp", expect: "Tuesday, 30-Nov-21 14:25:18 CST", - fail: false, }, { name: "RFC1123 ms", in: `{"a":{"timestamp": "1638253518999", "second":2},"age":47}`, pl: ` - input = load_json(_); ts = input["a"]["timestamp"]; - datetime(ts, 'ms', 'RFC1123') + json(_, a.timestamp) + datetime(a.timestamp, 'ms', 'RFC1123') `, - outkey: "ts", + outkey: "a.timestamp", expect: "Tue, 30 Nov 2021 14:25:18 CST", - fail: false, }, { name: "RFC1123 s", in: `{"a":{"timestamp": "1638253518", "second":2},"age":47}`, pl: ` - input = load_json(_); ts = input["a"]["timestamp"]; - datetime(ts, 's', 'RFC1123') + json(_, a.timestamp) + datetime(a.timestamp, 's', 'RFC1123') `, - outkey: "ts", + outkey: "a.timestamp", expect: "Tue, 30 Nov 2021 14:25:18 CST", - fail: false, }, { name: "RFC1123Z ms", in: `{"a":{"timestamp": "1638253518999", "second":2},"age":47}`, pl: ` - input = load_json(_); ts = input["a"]["timestamp"]; - datetime(ts, 'ms', 'RFC1123Z') + json(_, a.timestamp) + datetime(a.timestamp, 'ms', 'RFC1123Z') `, - outkey: "ts", + outkey: "a.timestamp", expect: "Tue, 30 Nov 2021 14:25:18 +0800", - fail: false, }, { name: "RFC1123Z s", in: `{"a":{"timestamp": "1638253518", "second":2},"age":47}`, pl: ` - input = load_json(_); ts = input["a"]["timestamp"]; - datetime(ts, 's', 'RFC1123Z') + json(_, a.timestamp) + datetime(a.timestamp, 's', 'RFC1123Z') `, - outkey: "ts", + outkey: "a.timestamp", expect: "Tue, 30 Nov 2021 14:25:18 +0800", - fail: false, }, { name: "RFC3339 s", in: `{"a":{"timestamp": "1610960605", "second":2},"age":47}`, pl: ` - input = load_json(_); ts = input["a"]["timestamp"]; - datetime(ts, 's', 'RFC3339') + json(_, a.timestamp) + datetime(a.timestamp, 's', 'RFC3339') `, - outkey: "ts", + outkey: "a.timestamp", expect: "2021-01-18T17:03:25+08:00", - fail: false, }, { name: "RFC3339 ms", in: `{"a":{"timestamp": "1610960605000", "second":2},"age":47}`, pl: ` - input = load_json(_); ts = input["a"]["timestamp"]; - datetime(ts, 'ms', 'RFC3339') + json(_, a.timestamp) + datetime(a.timestamp, 'ms', 'RFC3339') `, - outkey: "ts", + outkey: "a.timestamp", expect: "2021-01-18T17:03:25+08:00", - fail: false, }, { name: "RFC3339Nano s", in: `{"a":{"timestamp": "1610960605", "second":2},"age":47}`, pl: ` - input = load_json(_); ts = input["a"]["timestamp"]; - datetime(ts, 's', 'RFC3339Nano') + json(_, a.timestamp) + datetime(a.timestamp, 's', 'RFC3339Nano') `, - outkey: "ts", + outkey: "a.timestamp", expect: "2021-01-18T17:03:25+08:00", - fail: false, }, { name: "RFC3339Nano ms", in: `{"a":{"timestamp": "1610960605001", "second":2},"age":47}`, pl: ` - input = load_json(_); ts = input["a"]["timestamp"]; - datetime(ts, 'ms', 'RFC3339Nano') + json(_, a.timestamp) + datetime(a.timestamp, 'ms', 'RFC3339Nano') `, - outkey: "ts", + outkey: "a.timestamp", expect: "2021-01-18T17:03:25.001+08:00", - fail: false, }, { name: "Kitchen ms", in: `{"a":{"timestamp": "1610960605001", "second":2},"age":47}`, pl: ` - input = load_json(_); ts = input["a"]["timestamp"]; - datetime(ts, 'ms', 'Kitchen') + json(_, a.timestamp) + datetime(a.timestamp, 'ms', 'Kitchen') `, - outkey: "ts", + outkey: "a.timestamp", expect: "5:03PM", - fail: true, }, { name: "Kitchen s", in: `{"a":{"timestamp": "1610960605", "second":2},"age":47}`, pl: ` - input = load_json(_); ts = input["a"]["timestamp"]; - datetime(ts, 's', 'Kitchen') + json(_, a.timestamp) + datetime(a.timestamp, 's', 'Kitchen') `, - outkey: "ts", + outkey: "a.timestamp", expect: "5:03PM", - fail: true, + }, + { + name: "udef_ms", + in: `{"a":{"timestamp": "1610960605000", "second":2},"age":47}`, + pl: ` + json(_, a.timestamp) + datetime(a.timestamp, 'ms', '%Y-%m-%d') + `, + outkey: "a.timestamp", + expect: "2021-01-18", + }, + { + name: "udef_us", + in: `{"a":{"timestamp": "1610960605000000", "second":2},"age":47}`, + pl: ` + json(_, a.timestamp) + datetime(a.timestamp, 'us', '%Y-%m-%d %H:%M:%S') + `, + outkey: "a.timestamp", + expect: "2021-01-18 17:03:25", + }, + { + name: "udef_ns", + in: `{"a":{"timestamp": "1610960605000000000", "second":2},"age":47}`, + pl: ` + json(_, a.timestamp) + datetime(a.timestamp, 'ns', '%Y-%m-%d %H:%M:%S') + `, + outkey: "a.timestamp", + expect: "2021-01-18 17:03:25", + }, + { + name: "udef_ns_tz_1", + in: `{"a":{"timestamp": "1610960605000000000", "second":2},"age":47}`, + pl: ` + json(_, a.timestamp) + datetime(a.timestamp, 'ns', '%Y-%m-%d %H:%M:%S', tz="Asia/Tokyo") + `, + outkey: "a.timestamp", + expect: "2021-01-18 18:03:25", + }, + { + name: "udef_ns_tz_2", + in: `{"a":{"timestamp": 1610960605000000000, "second":2},"age":47}`, + pl: ` + json(_, a.timestamp) + datetime(a.timestamp, 'ns', fmt='%Y-%m-%d %H:%M:%S', tz="UTC") + `, + outkey: "a.timestamp", + expect: "2021-01-18 09:03:25", }, } for idx, tc := range cases { t.Run(tc.name, func(t *testing.T) { runner, err := NewTestingRunner(tc.pl) + if tc.fail && err == nil { + t.Error("unknown error") + } if err != nil { if tc.fail { t.Logf("[%d]expect error: %s", idx, err) @@ -278,17 +311,19 @@ func TestDateTime(t *testing.T) { } return } + pt := ptinput.NewPlPoint( + point.Logging, "test", nil, map[string]any{"message": tc.in}, time.Now()) + errR := runScript(runner, pt) - _, _, f, _, _, err := runScript(runner, "test", nil, map[string]interface{}{ - "message": tc.in, - }, time.Now()) - if err != nil { - t.Error(err) - return + if errR != nil { + t.Fatal(errR) + } + + if tc.fail { + t.Logf("[%d]expect error: %s", idx, err) } - t.Log(f) - v := f[tc.outkey] - assert.Equal(t, tc.expect, v) + v, _, _ := pt.Get(tc.outkey) + tu.Equals(t, tc.expect, v) t.Logf("[%d] PASS", idx) }) } diff --git a/pkg/inimpl/guancecloud/ptinput/funcs/fn_decoder.go b/pkg/inimpl/guancecloud/ptinput/funcs/fn_decoder.go new file mode 100644 index 0000000..e3ed325 --- /dev/null +++ b/pkg/inimpl/guancecloud/ptinput/funcs/fn_decoder.go @@ -0,0 +1,120 @@ +// Unless explicitly stated otherwise all files in this repository are licensed +// under the MIT License. +// This product includes software developed at Guance Cloud (https://www.guance.com/). +// Copyright 2021-present Guance, Inc. + +package funcs + +import ( + "errors" + "fmt" + + "github.com/GuanceCloud/platypus/pkg/ast" + "github.com/GuanceCloud/platypus/pkg/engine/runtime" + "github.com/GuanceCloud/platypus/pkg/errchain" + "github.com/GuanceCloud/platypus/pkg/inimpl/guancecloud/ptinput" + "golang.org/x/text/encoding" + "golang.org/x/text/encoding/simplifiedchinese" + "golang.org/x/text/encoding/unicode" +) + +var errUnknownCharacterEncoding = errors.New("unknown character encoding") + +type Decoder struct { + decoder *encoding.Decoder +} + +func NewDecoder(enc string) (*Decoder, error) { + var decoder *encoding.Decoder + + switch enc { + case "'utf-8'": + decoder = unicode.UTF8.NewDecoder() + case "utf-16le", "'utf-16le'": + decoder = unicode.UTF16(unicode.LittleEndian, unicode.IgnoreBOM).NewDecoder() + case "utf-16be", "'utf-16be'": + decoder = unicode.UTF16(unicode.BigEndian, unicode.IgnoreBOM).NewDecoder() + case "gbk", "'gbk'": + decoder = simplifiedchinese.GBK.NewDecoder() + case "gb18030", "'gb18030'": + decoder = simplifiedchinese.GB18030.NewDecoder() + default: + return nil, errUnknownCharacterEncoding + } + + return &Decoder{decoder: decoder}, nil +} + +func Decode(ctx *runtime.Context, funcExpr *ast.CallExpr) *errchain.PlError { + var codeType string + + key, err := getKeyName(funcExpr.Param[0]) + if err != nil { + return runtime.NewRunError(ctx, err.Error(), funcExpr.Param[0].StartPos()) + } + + keyVal, err := ctx.GetKeyConv2Str(key) + if err != nil { + l.Debug(err) + return nil + } + switch funcExpr.Param[1].NodeType { //nolint:exhaustive + case ast.TypeAttrExpr, ast.TypeIdentifier: + key, err := getKeyName(funcExpr.Param[1]) + if err != nil { + return runtime.NewRunError(ctx, err.Error(), funcExpr.Param[1].StartPos()) + } + keyVal, err := ctx.GetKeyConv2Str(key) + if err != nil { + l.Debug(err) + return nil + } + codeType = keyVal + case ast.TypeStringLiteral: + codeType = funcExpr.Param[1].StringLiteral.Val + default: + return runtime.NewRunError(ctx, fmt.Sprintf( + "expect AttrExpr, Identifier or StringLiteral, got %s", + funcExpr.Param[1].NodeType), funcExpr.Param[1].StartPos()) + } + + encode, err := NewDecoder(codeType) + if err != nil { + l.Debug(err) + return nil + } + + newcont, err := encode.decoder.String(keyVal) + if err != nil { + l.Debug(err) + return nil + } + + if err := addKey2PtWithVal(ctx.InData(), key, newcont, ast.String, + ptinput.KindPtDefault); err != nil { + l.Debug(err) + return nil + } + + return nil +} + +func DecodeChecking(ctx *runtime.Context, funcExpr *ast.CallExpr) *errchain.PlError { + if len(funcExpr.Param) < 2 || len(funcExpr.Param) > 2 { + return runtime.NewRunError(ctx, fmt.Sprintf( + "func %s expected 2", funcExpr.Name), funcExpr.NamePos) + } + + if _, err := getKeyName(funcExpr.Param[0]); err != nil { + return runtime.NewRunError(ctx, err.Error(), funcExpr.Param[0].StartPos()) + } + + switch funcExpr.Param[1].NodeType { //nolint:exhaustive + case ast.TypeAttrExpr, ast.TypeIdentifier, ast.TypeStringLiteral: + default: + return runtime.NewRunError(ctx, fmt.Sprintf( + "expect AttrExpr, Identifier or StringLiteral, got %s", + funcExpr.Param[1].NodeType), funcExpr.Param[1].StartPos()) + } + return nil +} diff --git a/pkg/inimpl/guancecloud/ptinput/funcs/fn_decoder_test.go b/pkg/inimpl/guancecloud/ptinput/funcs/fn_decoder_test.go new file mode 100644 index 0000000..88c001b --- /dev/null +++ b/pkg/inimpl/guancecloud/ptinput/funcs/fn_decoder_test.go @@ -0,0 +1,82 @@ +// Unless explicitly stated otherwise all files in this repository are licensed +// under the MIT License. +// This product includes software developed at Guance Cloud (https://www.guance.com/). +// Copyright 2021-present Guance, Inc. + +package funcs + +import ( + "testing" + "time" + + "github.com/GuanceCloud/cliutils/point" + tu "github.com/GuanceCloud/cliutils/testutil" + "github.com/GuanceCloud/platypus/pkg/inimpl/guancecloud/ptinput" + "golang.org/x/text/encoding/simplifiedchinese" +) + +type funcCase struct { + name string + in string + script string + expected interface{} + key string + fail bool +} + +func TestDecode(t *testing.T) { + data := []string{"测试一下", "不知道", "测试一下123456", "哈哈哈哈哈", "-汪98阿萨德离开家"} + decodeDataSlice := make([]string, 10) + + for idx, cont := range data { + decodeData, _ := simplifiedchinese.GBK.NewEncoder().Bytes([]byte(cont)) + decodeDataSlice[idx] = string(decodeData) + } + + testCase := []*funcCase{ + { + in: decodeDataSlice[0], + script: `decode(_,"gbk")`, + key: "message", + }, + { + in: decodeDataSlice[1], + script: `decode(_,"gbk")`, + key: "message", + }, + { + in: decodeDataSlice[2], + script: `decode(_,"gbk")`, + key: "message", + }, + { + in: decodeDataSlice[3], + script: `decode(_,"gbk")`, + key: "message", + }, + { + in: decodeDataSlice[4], + script: `decode(_,"gbk")`, + key: "message", + }, + } + for idx, tc := range testCase { + t.Run(tc.name, func(t *testing.T) { + runner, err := NewTestingRunner(tc.script) + tu.Equals(t, nil, err) + + pt := ptinput.NewPlPoint( + point.Logging, "test", nil, map[string]any{"message": tc.in}, time.Now()) + errR := runScript(runner, pt) + + if errR != nil { + t.Fatal(errR) + } + + v, _, _ := pt.Get(tc.key) + tu.Equals(t, data[idx], v) + + t.Logf("[%d] PASS", idx) + }) + } +} diff --git a/pkg/inimpl/guancecloud/funcs/fn_default_time.go b/pkg/inimpl/guancecloud/ptinput/funcs/fn_default_time.go similarity index 84% rename from pkg/inimpl/guancecloud/funcs/fn_default_time.go rename to pkg/inimpl/guancecloud/ptinput/funcs/fn_default_time.go index 1dfb726..94e21c0 100644 --- a/pkg/inimpl/guancecloud/funcs/fn_default_time.go +++ b/pkg/inimpl/guancecloud/ptinput/funcs/fn_default_time.go @@ -7,12 +7,11 @@ package funcs import ( "fmt" - "time" "github.com/GuanceCloud/platypus/pkg/ast" "github.com/GuanceCloud/platypus/pkg/engine/runtime" "github.com/GuanceCloud/platypus/pkg/errchain" - "github.com/GuanceCloud/platypus/pkg/inimpl/guancecloud/input" + "github.com/GuanceCloud/platypus/pkg/inimpl/guancecloud/ptinput" ) func DefaultTimeChecking(ctx *runtime.Context, funcExpr *ast.CallExpr) *errchain.PlError { @@ -70,15 +69,10 @@ func DefaultTime(ctx *runtime.Context, funcExpr *ast.CallExpr) *errchain.PlError if nanots, err := TimestampHandle(cont, tz); err != nil { usePointTime(ctx, key, err) return nil - } else { - tn := time.Unix(nanots/int64(time.Second), - nanots%int64(time.Second)) - deletePtKey(ctx.InData(), key) - if err := setPointTime(ctx.InData(), tn); err != nil { - l.Debug(err) - return nil - } - deletePtKey(ctx.InData(), key) + } else if err := addKey2PtWithVal(ctx.InData(), key, nanots, + ast.Int, ptinput.KindPtDefault); err != nil { + l.Debug(err) + return nil } return nil @@ -86,7 +80,7 @@ func DefaultTime(ctx *runtime.Context, funcExpr *ast.CallExpr) *errchain.PlError func usePointTime(ctx *runtime.Context, key string, err error) { _ = addKey2PtWithVal(ctx.InData(), runtime.PlRunInfoField, fmt.Sprintf("time convert failed: %v", err), - ast.String, input.KindPtDefault) + ast.String, ptinput.KindPtDefault) - _ = setPointTime(ctx.InData(), pointTime(ctx.InData())) + _ = addKey2PtWithVal(ctx.InData(), key, pointTime(ctx.InData()), ast.Int, ptinput.KindPtDefault) } diff --git a/pkg/inimpl/guancecloud/funcs/fn_default_time_test.go b/pkg/inimpl/guancecloud/ptinput/funcs/fn_default_time_test.go similarity index 81% rename from pkg/inimpl/guancecloud/funcs/fn_default_time_test.go rename to pkg/inimpl/guancecloud/ptinput/funcs/fn_default_time_test.go index 6d4c079..0996293 100644 --- a/pkg/inimpl/guancecloud/funcs/fn_default_time_test.go +++ b/pkg/inimpl/guancecloud/ptinput/funcs/fn_default_time_test.go @@ -9,7 +9,9 @@ import ( "testing" "time" - "github.com/stretchr/testify/assert" + "github.com/GuanceCloud/cliutils/point" + tu "github.com/GuanceCloud/cliutils/testutil" + "github.com/GuanceCloud/platypus/pkg/inimpl/guancecloud/ptinput" ) func TestDefaultTime(t *testing.T) { @@ -27,7 +29,7 @@ func TestDefaultTime(t *testing.T) { name: "nginx log datetime, 02/Jan/2006:15:04:05 -0700", in: `{"time":"02/Dec/2021:11:55:34 +0800"}`, pl: ` - input = load_json(_); time=input["time"]; printf(time) + json(_, time) default_time(time) `, outkey: "time", @@ -38,7 +40,7 @@ func TestDefaultTime(t *testing.T) { name: "nginx log datetime, 02/Jan/2006:15:04:05 -0700", in: `{"time":"02/Dec/2021:11:55:34 +0800"}`, pl: ` - input = load_json(_); time=input["time"]; + json(_, time) default_time(time) `, outkey: "time", @@ -49,7 +51,7 @@ func TestDefaultTime(t *testing.T) { name: "redis log datetime-tokyo", in: `{"time":"02 Dec 2021 12:55:34.000"}`, pl: ` - input = load_json(_); time=input["time"]; + json(_, time) default_time(time, "Asia/Tokyo") `, outkey: "time", @@ -61,7 +63,7 @@ func TestDefaultTime(t *testing.T) { name: "redis log datetime-default tz", in: `{"time":"02 Dec 2021 11:55:34.000"}`, pl: ` - input = load_json(_); time=input["time"]; + json(_, time) default_time(time) `, outkey: "time", @@ -72,7 +74,7 @@ func TestDefaultTime(t *testing.T) { name: "redis log datetime no year", in: `{"time":"02 Dec 11:55:34.000"}`, pl: ` - input = load_json(_); time=input["time"]; + json(_, time) default_time(time) `, outkey: "time", @@ -83,7 +85,7 @@ func TestDefaultTime(t *testing.T) { name: "mysql, 171113 14:14:20", in: `{"time":"211202 11:55:34"}`, pl: ` - input = load_json(_); time=input["time"]; + json(_, time) default_time(time) `, outkey: "time", @@ -94,7 +96,7 @@ func TestDefaultTime(t *testing.T) { name: "gin, 2021/02/27 - 14:14:20", in: `{"time":"2021/12/02 - 11:55:34"}`, pl: ` - input = load_json(_); time=input["time"]; + json(_, time) default_time(time) `, outkey: "time", @@ -105,7 +107,7 @@ func TestDefaultTime(t *testing.T) { name: "apache, Tue May 18 06:25:05.176170 2021", in: `{"time":"Tue Dec 2 11:55:34.000000 2021"}`, pl: ` - input = load_json(_); time=input["time"]; + json(_, time) default_time(time) `, outkey: "time", @@ -116,7 +118,7 @@ func TestDefaultTime(t *testing.T) { name: "postgresql, 2021-05-27 06:54:14.760 UTC", in: `{"time":"2021-12-02 11:55:34.000 UTC"}`, pl: ` - input = load_json(_); time=input["time"]; + json(_, time) default_time(time) `, outkey: "time", @@ -127,7 +129,7 @@ func TestDefaultTime(t *testing.T) { name: "Dec 2, 2021 11:55:34 AM", in: `{"time":"Dec 2, 2021 11:55:34 AM"}`, pl: ` - input = load_json(_); time=input["time"]; + json(_, time) default_time(time) `, outkey: "time", @@ -138,7 +140,7 @@ func TestDefaultTime(t *testing.T) { name: "Dec 2, 2021", in: `{"time":"Dec 2, 2021"}`, pl: ` - input = load_json(_); time=input["time"]; + json(_, time) default_time(time) `, outkey: "time", @@ -149,7 +151,7 @@ func TestDefaultTime(t *testing.T) { name: "Dec 2, 2021 (tz: UTC)", in: `{"time":"Dec 2, 2021"}`, pl: ` - input = load_json(_); time=input["time"]; + json(_, time) default_time(time, "UTC") `, outkey: "time", @@ -160,7 +162,7 @@ func TestDefaultTime(t *testing.T) { name: "dec 2, '21 (tz: UTC)", in: `{"time":"Dec 2, '21"}`, pl: ` - input = load_json(_); time=input["time"]; + json(_, time) default_time(time, "UTC") `, outkey: "time", @@ -171,7 +173,7 @@ func TestDefaultTime(t *testing.T) { name: "Dec. 2, 2021 (tz: UTC)", in: `{"time":"Dec 2, 2021"}`, pl: ` - input = load_json(_); time=input["time"]; + json(_, time) default_time(time, "UTC") `, outkey: "time", @@ -182,7 +184,7 @@ func TestDefaultTime(t *testing.T) { name: "Dec. 2, 21 (tz: UTC)", in: `{"time":"Dec 2, 21"}`, pl: ` - input = load_json(_); time=input["time"]; + json(_, time) default_time(time, "UTC") `, outkey: "time", @@ -193,7 +195,7 @@ func TestDefaultTime(t *testing.T) { name: "Tue Dec 2 11:55:34 2021", in: `{"time":"Tue Dec 2 11:55:34 2021"}`, pl: ` - input = load_json(_); time=input["time"]; + json(_, time) default_time(time) `, outkey: "time", @@ -204,7 +206,7 @@ func TestDefaultTime(t *testing.T) { name: "Tue Dec 2 11:55:34 CST 2021", in: `{"time":"Tue Dec 2 11:55:34 CST 2021"}`, pl: ` - input = load_json(_); time=input["time"]; + json(_, time) default_time(time) `, outkey: "time", @@ -215,7 +217,7 @@ func TestDefaultTime(t *testing.T) { name: "Tue Dec 02 11:55:34 +0800 2021", in: `{"time":"Tue Dec 02 11:55:34 +0800 2021"}`, pl: ` - input = load_json(_); time=input["time"]; + json(_, time) default_time(time) `, outkey: "time", @@ -226,7 +228,7 @@ func TestDefaultTime(t *testing.T) { name: "Tuesday, 02-Dec-21 11:55:34 CST", in: `{"time":"Tuesday, 02-Dec-21 11:55:34 CST"}`, pl: ` - input = load_json(_); time=input["time"]; + json(_, time) default_time(time) `, outkey: "time", @@ -237,7 +239,7 @@ func TestDefaultTime(t *testing.T) { name: "Tue, 02 Dec 2021 11:55:34 CST", in: `{"time":"Tue, 02 Dec 2021 11:55:34 CST"}`, pl: ` - input = load_json(_); time=input["time"]; + json(_, time) default_time(time) `, outkey: "time", @@ -248,7 +250,7 @@ func TestDefaultTime(t *testing.T) { name: "Tue, 02 Dec 2021 11:55:34 +0800 (CST)", in: `{"time":"Tue, 02 Dec 2021 11:55:34 +0800 (CST)"}`, pl: ` - input = load_json(_); time=input["time"]; + json(_, time) default_time(time) `, outkey: "time", @@ -259,7 +261,7 @@ func TestDefaultTime(t *testing.T) { name: "Tue, 02 Dec 2021 11:55:34 +0800", in: `{"time":"Tue, 02 Dec 2021 11:55:34 +0800"}`, pl: ` - input = load_json(_); time=input["time"]; + json(_, time) default_time(time) `, outkey: "time", @@ -270,7 +272,7 @@ func TestDefaultTime(t *testing.T) { name: "Tue 02 Dec 2021 11:55:34 AM CST", in: `{"time":"Tue 02 Dec 2021 11:55:34 AM CST"}`, pl: ` - input = load_json(_); time=input["time"]; + json(_, time) default_time(time) `, outkey: "time", @@ -281,7 +283,7 @@ func TestDefaultTime(t *testing.T) { name: "Tue Dec 2 11:55:34 UTC+0800 2021", in: `{"time":"Tue 02 Dec 2021 11:55:34 AM CST"}`, pl: ` - input = load_json(_); time=input["time"]; + json(_, time) default_time(time) `, outkey: "time", @@ -292,7 +294,7 @@ func TestDefaultTime(t *testing.T) { name: "Tue, 2 Dec 2021 11:55:34 +0800", in: `{"time":"Tue, 2 Dec 2021 11:55:34 +0800"}`, pl: ` - input = load_json(_); time=input["time"]; + json(_, time) default_time(time) `, outkey: "time", @@ -303,7 +305,7 @@ func TestDefaultTime(t *testing.T) { name: "Tue Dec 02 2021 11:55:34 GMT+0800 (GMT Daylight Time)", in: `{"time":"Tue Dec 02 2021 11:55:34 GMT+0800 (GMT Daylight Time)"}`, pl: ` - input = load_json(_); time=input["time"]; + json(_, time) default_time(time) `, outkey: "time", @@ -314,7 +316,7 @@ func TestDefaultTime(t *testing.T) { name: "December 02, 2021 11:55:34am", in: `{"time":"December 02, 2021 11:55:34am"}`, pl: ` - input = load_json(_); time=input["time"]; + json(_, time) default_time(time) `, outkey: "time", @@ -325,7 +327,7 @@ func TestDefaultTime(t *testing.T) { name: "December 02, 2021 at 11:55:34am CST+08", in: `{"time":"December 02, 2021 at 11:55:34am CST+08"}`, pl: ` - input = load_json(_); time=input["time"]; + json(_, time) default_time(time) `, outkey: "time", @@ -336,7 +338,7 @@ func TestDefaultTime(t *testing.T) { name: "December 02, 2021 at 11:55:34am CST+08", in: `{"time":"December 02, 2021 at 11:55:34am CST+08"}`, pl: ` - input = load_json(_); time=input["time"]; + json(_, time) default_time(time) `, outkey: "time", @@ -347,7 +349,7 @@ func TestDefaultTime(t *testing.T) { name: "December 02, 2021 at 11:55am CST+08", in: `{"time":"December 02, 2021 at 11:55am CST+08"}`, pl: ` - input = load_json(_); time=input["time"]; + json(_, time) default_time(time) `, outkey: "time", @@ -358,7 +360,7 @@ func TestDefaultTime(t *testing.T) { name: "December 02, 2021, 11:55:34", in: `{"time":"December 02, 2021, 11:55:34"}`, pl: ` - input = load_json(_); time=input["time"]; + json(_, time) default_time(time) `, outkey: "time", @@ -369,7 +371,7 @@ func TestDefaultTime(t *testing.T) { name: "December 2, 2021", in: `{"time":"December 2, 2021"}`, pl: ` - input = load_json(_); time=input["time"]; + json(_, time) default_time(time) `, outkey: "time", @@ -380,7 +382,7 @@ func TestDefaultTime(t *testing.T) { name: "December 2th, 2021", in: `{"time":"December 2th, 2021"}`, pl: ` - input = load_json(_); time=input["time"]; + json(_, time) default_time(time) `, outkey: "time", @@ -391,7 +393,7 @@ func TestDefaultTime(t *testing.T) { name: "02 Dec 2021, 11:55", in: `{"time":"02 Dec 2021, 11:55"}`, pl: ` - input = load_json(_); time=input["time"]; + json(_, time) default_time(time) `, outkey: "time", @@ -402,7 +404,7 @@ func TestDefaultTime(t *testing.T) { name: "2 Dec 2021, 11:55", in: `{"time":"2 Dec 2021, 11:55"}`, pl: ` - input = load_json(_); time=input["time"]; + json(_, time) default_time(time) `, outkey: "time", @@ -413,7 +415,7 @@ func TestDefaultTime(t *testing.T) { name: "2 Dec 21", in: `{"time":"2 Dec 21"}`, pl: ` - input = load_json(_); time=input["time"]; + json(_, time) default_time(time) `, outkey: "time", @@ -424,7 +426,7 @@ func TestDefaultTime(t *testing.T) { name: "2 Dec 2021", in: `{"time":"2 Dec 2021"}`, pl: ` - input = load_json(_); time=input["time"]; + json(_, time) default_time(time) `, outkey: "time", @@ -435,7 +437,7 @@ func TestDefaultTime(t *testing.T) { name: "02 December 2021", in: `{"time":"02 December 2021"}`, pl: ` - input = load_json(_); time=input["time"]; + json(_, time) default_time(time) `, outkey: "time", @@ -446,7 +448,7 @@ func TestDefaultTime(t *testing.T) { name: "2 December 2021", in: `{"time":"2 December 2021"}`, pl: ` - input = load_json(_); time=input["time"]; + json(_, time) default_time(time) `, outkey: "time", @@ -457,7 +459,7 @@ func TestDefaultTime(t *testing.T) { name: "2021-Dec-02", in: `{"time":"2021-Dec-02"}`, pl: ` - input = load_json(_); time=input["time"]; + json(_, time) default_time(time) `, outkey: "time", @@ -469,7 +471,7 @@ func TestDefaultTime(t *testing.T) { name: "12/2/2021", in: `{"time":"12/2/2021"}`, pl: ` - input = load_json(_); time=input["time"]; + json(_, time) default_time(time) `, outkey: "time", @@ -480,7 +482,7 @@ func TestDefaultTime(t *testing.T) { name: "12/02/2021", in: `{"time":"12/02/2021"}`, pl: ` - input = load_json(_); time=input["time"]; + json(_, time) default_time(time) `, outkey: "time", @@ -491,7 +493,7 @@ func TestDefaultTime(t *testing.T) { name: "12/2/21", in: `{"time":"12/2/21"}`, pl: ` - input = load_json(_); time=input["time"]; + json(_, time) default_time(time) `, outkey: "time", @@ -502,7 +504,7 @@ func TestDefaultTime(t *testing.T) { name: "12/2/2021 11:55", in: `{"time":"12/2/2021 11:55"}`, pl: ` - input = load_json(_); time=input["time"]; + json(_, time) default_time(time) `, outkey: "time", @@ -513,7 +515,7 @@ func TestDefaultTime(t *testing.T) { name: "12/02/2021 11:55", in: `{"time":"12/02/2021 11:55"}`, pl: ` - input = load_json(_); time=input["time"]; + json(_, time) default_time(time) `, outkey: "time", @@ -524,7 +526,7 @@ func TestDefaultTime(t *testing.T) { name: "12/02/2021 11:55:43", in: `{"time":"12/02/2021 11:55:43"}`, pl: ` - input = load_json(_); time=input["time"]; + json(_, time) default_time(time) `, outkey: "time", @@ -535,7 +537,7 @@ func TestDefaultTime(t *testing.T) { name: "12/02/2021 11:55:43 AM", in: `{"time":"12/02/2021 11:55:43 AM"}`, pl: ` - input = load_json(_); time=input["time"]; + json(_, time) default_time(time) `, outkey: "time", @@ -546,7 +548,7 @@ func TestDefaultTime(t *testing.T) { name: "12/02/2021 11:55 AM", in: `{"time":"12/02/2021 11:55 AM"}`, pl: ` - input = load_json(_); time=input["time"]; + json(_, time) default_time(time) `, outkey: "time", @@ -557,7 +559,7 @@ func TestDefaultTime(t *testing.T) { name: "12/02/2021 11:55 11:55:43", in: `{"time":"12/02/2021 11:55:43"}`, pl: ` - input = load_json(_); time=input["time"]; + json(_, time) default_time(time) `, outkey: "time", @@ -568,7 +570,7 @@ func TestDefaultTime(t *testing.T) { name: "12/02/2021 11:55 11:55:43.9999999", in: `{"time":"12/02/2021 11:55:43.9999999"}`, pl: ` - input = load_json(_); time=input["time"]; + json(_, time) default_time(time) `, outkey: "time", @@ -580,7 +582,7 @@ func TestDefaultTime(t *testing.T) { name: "2021:12:2", in: `{"time":"2021:12:2"}`, pl: ` - input = load_json(_); time=input["time"]; + json(_, time) default_time(time) `, outkey: "time", @@ -591,7 +593,7 @@ func TestDefaultTime(t *testing.T) { name: "2021:12:02", in: `{"time":"2021:12:02"}`, pl: ` - input = load_json(_); time=input["time"]; + json(_, time) default_time(time) `, outkey: "time", @@ -602,7 +604,7 @@ func TestDefaultTime(t *testing.T) { name: "2021:12:2 11:55", in: `{"time":"2021:12:2 11:55"}`, pl: ` - input = load_json(_); time=input["time"]; + json(_, time) default_time(time) `, outkey: "time", @@ -612,7 +614,7 @@ func TestDefaultTime(t *testing.T) { name: "2021:12:02 11:55", in: `{"time":"2021:12:02 11:55"}`, pl: ` - input = load_json(_); time=input["time"]; + json(_, time) default_time(time) `, outkey: "time", @@ -622,7 +624,7 @@ func TestDefaultTime(t *testing.T) { name: "2021:12:02 11:55:43", in: `{"time":"2021:12:02 11:55:43"}`, pl: ` - input = load_json(_); time=input["time"]; + json(_, time) default_time(time) `, outkey: "time", @@ -632,7 +634,7 @@ func TestDefaultTime(t *testing.T) { name: "2021:12:2 12:55:43", in: `{"time":"2021:12:2 12:55:43"}`, pl: ` - input = load_json(_); time=input["time"]; + json(_, time) default_time(time, "Asia/Tokyo") `, outkey: "time", @@ -642,7 +644,7 @@ func TestDefaultTime(t *testing.T) { name: "2021:12:2 11:55:43", in: `{"time":"2021:12:2 11:55:43.89555"}`, pl: ` - input = load_json(_); time=input["time"]; + json(_, time) default_time(time) `, outkey: "time", @@ -652,7 +654,7 @@ func TestDefaultTime(t *testing.T) { name: "2021年11月02日", in: `{"time":"2021年12月02日"}`, pl: ` - input = load_json(_); time=input["time"]; + json(_, time) default_time(time) `, outkey: "time", @@ -663,7 +665,7 @@ func TestDefaultTime(t *testing.T) { name: "2021-12-2T11:55:43+0800", in: `{"time":"2021:12:2 11:55:43"}`, pl: ` - input = load_json(_); time=input["time"]; + json(_, time) default_time(time) `, outkey: "time", @@ -673,7 +675,7 @@ func TestDefaultTime(t *testing.T) { name: "2021-12-2T11:55:43+08:00", in: `{"time":"2021-12-2T11:55:43+08:00"}`, pl: ` - input = load_json(_); time=input["time"]; + json(_, time) default_time(time) `, outkey: "time", @@ -683,7 +685,7 @@ func TestDefaultTime(t *testing.T) { name: "2021-12-2T11:55:43", in: `{"time":"2021-12-2T11:55:43"}`, pl: ` - input = load_json(_); time=input["time"]; + json(_, time) default_time(time) `, outkey: "time", @@ -693,7 +695,7 @@ func TestDefaultTime(t *testing.T) { name: "2021-12-2T3:55:43Z", in: `{"time":"2021-12-2T3:55:43Z"}`, pl: ` - input = load_json(_); time=input["time"]; + json(_, time) default_time(time) `, outkey: "time", @@ -704,7 +706,7 @@ func TestDefaultTime(t *testing.T) { name: "2021-12-2 11:55:43.12223", in: `{"time":"2021-12-2 11:55:43.12223"}`, pl: ` - input = load_json(_); time=input["time"]; + json(_, time) default_time(time) `, outkey: "time", @@ -714,7 +716,7 @@ func TestDefaultTime(t *testing.T) { name: "2021-12-2 11:55:43.122230000", in: `{"time":"2021-12-2 11:55:43.122230000"}`, pl: ` - input = load_json(_); time=input["time"]; + json(_, time) default_time(time) `, outkey: "time", @@ -724,7 +726,7 @@ func TestDefaultTime(t *testing.T) { name: "2021-12-2 11:55", in: `{"time":"2021-12-2 11:55"}`, pl: ` - input = load_json(_); time=input["time"]; + json(_, time) default_time(time) `, outkey: "time", @@ -734,7 +736,7 @@ func TestDefaultTime(t *testing.T) { name: "2021-12-2 11:55:43", in: `{"time":"2021-12-2 11:55:43"}`, pl: ` - input = load_json(_); time=input["time"]; + json(_, time) default_time(time) `, outkey: "time", @@ -744,7 +746,7 @@ func TestDefaultTime(t *testing.T) { name: "2021-12-2 11:55:43 UTC", in: `{"time":"2021-12-2 11:55:43 UTC"}`, pl: ` - input = load_json(_); time=input["time"]; + json(_, time) default_time(time) `, outkey: "time", @@ -754,7 +756,7 @@ func TestDefaultTime(t *testing.T) { name: "2021-12-2 11:55:43 GMT", in: `{"time":"2021-12-2 11:55:43 GMT"}`, pl: ` - input = load_json(_); time=input["time"]; + json(_, time) default_time(time) `, outkey: "time", @@ -764,7 +766,7 @@ func TestDefaultTime(t *testing.T) { name: "2021-12-2 11:55:43 CST", in: `{"time":"2021-12-2 11:55:43 CST"}`, pl: ` - input = load_json(_); time=input["time"]; + json(_, time) default_time(time) `, outkey: "time", @@ -774,7 +776,7 @@ func TestDefaultTime(t *testing.T) { name: "2021-12-2 11:55:43 AM", in: `{"time":"2021-12-2 11:55:43 AM"}`, pl: ` - input = load_json(_); time=input["time"]; + json(_, time) default_time(time) `, outkey: "time", @@ -784,7 +786,7 @@ func TestDefaultTime(t *testing.T) { name: "2021-12-2 11:55:43 +0800", in: `{"time":"2021-12-2 11:55:43 +0800"}`, pl: ` - input = load_json(_); time=input["time"]; + json(_, time) default_time(time) `, outkey: "time", @@ -794,7 +796,7 @@ func TestDefaultTime(t *testing.T) { name: "2021-12-2 11:55:43 +0800 +08", in: `{"time":"2021-12-2 11:55:43 +0800 +08"}`, pl: ` - input = load_json(_); time=input["time"]; + json(_, time) default_time(time) `, outkey: "time", @@ -804,7 +806,7 @@ func TestDefaultTime(t *testing.T) { name: "2021-12-2 11:55:43 +08:00", in: `{"time":"2021-12-2 11:55:43 +08:00"}`, pl: ` - input = load_json(_); time=input["time"]; + json(_, time) default_time(time) `, outkey: "time", @@ -814,7 +816,7 @@ func TestDefaultTime(t *testing.T) { name: "2021-12-2 3:55:43.1 +0000 UTC", in: `{"time":"2021-12-2 3:55:43.1 +0000 UTC"}`, pl: ` - input = load_json(_); time=input["time"]; + json(_, time) default_time(time) `, outkey: "time", @@ -824,7 +826,7 @@ func TestDefaultTime(t *testing.T) { name: "2021-12-2 11:55:43.1 +0800 UTC", in: `{"time":"2021-12-2 11:55:43.1 +0800 UTC"}`, pl: ` - input = load_json(_); time=input["time"]; + json(_, time) default_time(time) `, outkey: "time", @@ -834,7 +836,7 @@ func TestDefaultTime(t *testing.T) { name: "2021-12-2 11:55:43 +0800 UTC", in: `{"time":"2021-12-2 11:55:43 +0800 UTC"}`, pl: ` - input = load_json(_); time=input["time"]; + json(_, time) default_time(time) `, outkey: "time", @@ -844,7 +846,7 @@ func TestDefaultTime(t *testing.T) { name: "2021-12-2 11:55:43 +0800 GMT", in: `{"time":"2021-12-2 11:55:43 +0800 GMT"}`, pl: ` - input = load_json(_); time=input["time"]; + json(_, time) default_time(time) `, outkey: "time", @@ -854,7 +856,7 @@ func TestDefaultTime(t *testing.T) { name: "2021-12-2 11:55:43 +0800 GMT m=+0.000000001", in: `{"time":"2021-12-2 11:55:43 +0800 GMT m=+0.000000001"}`, pl: ` - input = load_json(_); time=input["time"]; + json(_, time) default_time(time) `, outkey: "time", @@ -864,7 +866,7 @@ func TestDefaultTime(t *testing.T) { name: "2021-12-2 11:55:43.0001 +0800 GMT m=+0.000000001", in: `{"time":"2021-12-2 11:55:43.0001 +0800 GMT m=+0.000000001"}`, pl: ` - input = load_json(_); time=input["time"]; + json(_, time) default_time(time) `, outkey: "time", @@ -874,7 +876,7 @@ func TestDefaultTime(t *testing.T) { name: "2021-12-2 11:55:43+08:00", in: `{"time":"2021-12-2 11:55:43+08:00"}`, pl: ` - input = load_json(_); time=input["time"]; + json(_, time) default_time(time) `, outkey: "time", @@ -884,7 +886,7 @@ func TestDefaultTime(t *testing.T) { name: "2021-12-02", in: `{"time":"2021-12-02"}`, pl: ` - input = load_json(_); time=input["time"]; + json(_, time) default_time(time) `, outkey: "time", @@ -895,7 +897,7 @@ func TestDefaultTime(t *testing.T) { name: "2021-12", in: `{"time":"2021-12"}`, pl: ` - input = load_json(_); time=input["time"]; + json(_, time) default_time(time) `, outkey: "time", @@ -906,7 +908,7 @@ func TestDefaultTime(t *testing.T) { name: "2021", in: `{"time":"2021"}`, pl: ` - input = load_json(_); time=input["time"]; + json(_, time) default_time(time) `, outkey: "time", @@ -917,7 +919,7 @@ func TestDefaultTime(t *testing.T) { name: "2021-12-2 11:55:43,212", in: `{"time":"2021-12-2 11:55:43,212"}`, pl: ` - input = load_json(_); time=input["time"]; + json(_, time) default_time(time) `, outkey: "time", @@ -928,7 +930,7 @@ func TestDefaultTime(t *testing.T) { name: "12.2.2021", in: `{"time":"12.2.2021"}`, pl: ` - input = load_json(_); time=input["time"]; + json(_, time) default_time(time) `, outkey: "time", @@ -938,7 +940,7 @@ func TestDefaultTime(t *testing.T) { name: "12.02.2021", in: `{"time":"12.02.2021"}`, pl: ` - input = load_json(_); time=input["time"]; + json(_, time) default_time(time) `, outkey: "time", @@ -948,7 +950,7 @@ func TestDefaultTime(t *testing.T) { name: "12.02.21", in: `{"time":"12.02.21"}`, pl: ` - input = load_json(_); time=input["time"]; + json(_, time) default_time(time) `, outkey: "time", @@ -958,7 +960,7 @@ func TestDefaultTime(t *testing.T) { name: "2021.12", in: `{"time":"2021.12"}`, pl: ` - input = load_json(_); time=input["time"]; + json(_, time) default_time(time) `, outkey: "time", @@ -969,7 +971,7 @@ func TestDefaultTime(t *testing.T) { name: "2021.12.02", in: `{"time":"2021.12.02"}`, pl: ` - input = load_json(_); time=input["time"]; + json(_, time) default_time(time) `, outkey: "time", @@ -981,7 +983,7 @@ func TestDefaultTime(t *testing.T) { name: "20211202", in: `{"time":"20211202"}`, pl: ` - input = load_json(_); time=input["time"]; + json(_, time) default_time(time) `, outkey: "time", @@ -992,7 +994,7 @@ func TestDefaultTime(t *testing.T) { name: "20211202115543", in: `{"time":"20211202115543"}`, pl: ` - input = load_json(_); time=input["time"]; + json(_, time) default_time(time) `, outkey: "time", @@ -1004,7 +1006,7 @@ func TestDefaultTime(t *testing.T) { name: "1638417343", in: `{"time":"1638417343"}`, pl: ` - input = load_json(_); time=input["time"]; + json(_, time) default_time(time) `, outkey: "time", @@ -1015,7 +1017,7 @@ func TestDefaultTime(t *testing.T) { name: "1638417343001", in: `{"time":"1638417343001"}`, pl: ` - input = load_json(_); time=input["time"]; + json(_, time) default_time(time) `, outkey: "time", @@ -1026,7 +1028,7 @@ func TestDefaultTime(t *testing.T) { name: "1638417343001002", in: `{"time":"1638417343001002"}`, pl: ` - input = load_json(_); time=input["time"]; + json(_, time) default_time(time) `, outkey: "time", @@ -1037,7 +1039,7 @@ func TestDefaultTime(t *testing.T) { name: "1638417343001002003", in: `{"time":"1638417343001002003"}`, pl: ` - input = load_json(_); time=input["time"]; + json(_, time) default_time(time) `, outkey: "time", @@ -1048,7 +1050,7 @@ func TestDefaultTime(t *testing.T) { name: "Dec 2, 2021 (tz: UTC)", in: `{"time":"Dec 2, 2021"}`, pl: ` - input = load_json(_); time=input["time"]; + json(_, time) default_time(time, "+0") `, outkey: "time", @@ -1059,7 +1061,7 @@ func TestDefaultTime(t *testing.T) { name: "Tue Dec 2 11:55:34 CST 2021", in: `{"time":"Tue Dec 2 11:55:34 CST 2021"}`, pl: ` - input = load_json(_); time=input["time"]; + json(_, time) default_time(time, "+8") `, outkey: "time", @@ -1080,21 +1082,22 @@ func TestDefaultTime(t *testing.T) { return } - _, _, f, tn, _, err := runScript(runner, "test", nil, map[string]interface{}{ - "message": tc.in, - }, time.Now()) - if err != nil { - t.Error(err) - return + pt := ptinput.NewPlPoint( + point.Logging, "test", nil, map[string]any{"message": tc.in}, time.Now()) + errR := runScript(runner, pt) + if errR != nil { + t.Fatal(errR) } - t.Log(f) + + pt.KeyTime2Time() + var v interface{} if tc.outkey != "time" && tc.outkey != "" { - v = f[tc.outkey] + v, _, _ = pt.Get(tc.outkey) } else { - v = tn.UnixNano() + v = pt.PtTime().UnixNano() } - assert.Equal(t, tc.expect, v) + tu.Equals(t, tc.expect, v) t.Logf("[%d] PASS", idx) }) } diff --git a/pkg/inimpl/guancecloud/ptinput/funcs/fn_default_time_with_fmt.go b/pkg/inimpl/guancecloud/ptinput/funcs/fn_default_time_with_fmt.go new file mode 100644 index 0000000..a52b76c --- /dev/null +++ b/pkg/inimpl/guancecloud/ptinput/funcs/fn_default_time_with_fmt.go @@ -0,0 +1,115 @@ +// Unless explicitly stated otherwise all files in this repository are licensed +// under the MIT License. +// This product includes software developed at Guance Cloud (https://www.guance.com/). +// Copyright 2021-present Guance, Inc. + +package funcs + +import ( + "fmt" + "time" + + "github.com/GuanceCloud/platypus/pkg/ast" + "github.com/GuanceCloud/platypus/pkg/engine/runtime" + "github.com/GuanceCloud/platypus/pkg/errchain" + "github.com/GuanceCloud/platypus/pkg/inimpl/guancecloud/ptinput" +) + +func DefaultTimeWithFmtChecking(ctx *runtime.Context, funcExpr *ast.CallExpr) *errchain.PlError { + if len(funcExpr.Param) < 2 { + return runtime.NewRunError(ctx, fmt.Sprintf( + "func %s expected more than 2 args", funcExpr.Name), funcExpr.NamePos) + } + + switch funcExpr.Param[0].NodeType { //nolint:exhaustive + case ast.TypeAttrExpr, ast.TypeIdentifier: + default: + return runtime.NewRunError(ctx, fmt.Sprintf( + "param key expect AttrExpr or Identifier, got %s", + funcExpr.Param[0].NodeType), funcExpr.Param[0].StartPos()) + } + + switch funcExpr.Param[1].NodeType { //nolint:exhaustive + case ast.TypeStringLiteral: + default: + return runtime.NewRunError(ctx, fmt.Sprintf( + "param key expect StringLiteral, got %s", + funcExpr.Param[1].NodeType), funcExpr.Param[1].StartPos()) + } + + if len(funcExpr.Param) > 2 { + switch funcExpr.Param[2].NodeType { //nolint:exhaustive + case ast.TypeStringLiteral: + default: + return runtime.NewRunError(ctx, fmt.Sprintf( + "param key expect StringLiteral, got %s", + funcExpr.Param[2].NodeType), funcExpr.Param[2].StartPos()) + } + } + + return nil +} + +func DefaultTimeWithFmt(ctx *runtime.Context, funcExpr *ast.CallExpr) *errchain.PlError { + var err error + var goTimeFmt string + var tz string + var t time.Time + timezone := time.Local + + if len(funcExpr.Param) < 2 { + return runtime.NewRunError(ctx, fmt.Sprintf( + "func %s expected more than 2 args", funcExpr.Name), funcExpr.NamePos) + } + + key, err := getKeyName(funcExpr.Param[0]) + if err != nil { + return runtime.NewRunError(ctx, err.Error(), funcExpr.Param[0].StartPos()) + } + + switch funcExpr.Param[1].NodeType { //nolint:exhaustive + case ast.TypeStringLiteral: + goTimeFmt = funcExpr.Param[1].StringLiteral.Val + default: + return runtime.NewRunError(ctx, fmt.Sprintf( + "param key expect StringLiteral, got %s", + funcExpr.Param[1]), funcExpr.Param[1].StartPos()) + } + + if len(funcExpr.Param) > 2 { + switch funcExpr.Param[2].NodeType { //nolint:exhaustive + case ast.TypeStringLiteral: + tz = funcExpr.Param[2].StringLiteral.Val + default: + return runtime.NewRunError(ctx, fmt.Sprintf( + "param key expect StringLiteral, got %s", + funcExpr.Param[2].NodeType), funcExpr.Param[2].StartPos()) + } + } + + timeStr, err := ctx.GetKeyConv2Str(key) + if err != nil { + l.Debugf("key `%v' not exist, ignored", key) + return nil //nolint:nilerr + } + + if tz != "" { + timezone, err = time.LoadLocation(tz) + } + + if err == nil { + t, err = time.ParseInLocation(goTimeFmt, timeStr, timezone) + } + + if err != nil { + l.Debugf("time string: %s, time format: %s, timezone: %s, error msg: %s", + timeStr, goTimeFmt, tz, err) + return nil + } else { + if err := addKey2PtWithVal(ctx.InData(), key, t.UnixNano(), ast.Int, ptinput.KindPtDefault); err != nil { + l.Debug(err) + return nil + } + return nil + } +} diff --git a/pkg/inimpl/guancecloud/ptinput/funcs/fn_default_time_with_fmt_test.go b/pkg/inimpl/guancecloud/ptinput/funcs/fn_default_time_with_fmt_test.go new file mode 100644 index 0000000..f8a3766 --- /dev/null +++ b/pkg/inimpl/guancecloud/ptinput/funcs/fn_default_time_with_fmt_test.go @@ -0,0 +1,124 @@ +// Unless explicitly stated otherwise all files in this repository are licensed +// under the MIT License. +// This product includes software developed at Guance Cloud (https://www.guance.com/). +// Copyright 2021-present Guance, Inc. + +package funcs + +import ( + "testing" + "time" + + "github.com/GuanceCloud/cliutils/point" + tu "github.com/GuanceCloud/cliutils/testutil" + "github.com/GuanceCloud/platypus/pkg/inimpl/guancecloud/ptinput" +) + +func TestDefaultTimeWithFmt(t *testing.T) { + // local timezone: utc+0800 + cst := time.FixedZone("CST", 8*3600) + time.Local = cst + + cases := []struct { + name, pl string + in []string + outkey string + expect []interface{} + fail bool + }{ + { + name: "02/Jan/2006:15:04:05 -0700", + in: []string{ + `{"time":"02/Dec/2021:12:55:34 +0900"}`, + `{"time":"02/Dec/2021:11:55:34 +0800"}`, + }, + pl: ` + json(_, time) + default_time_with_fmt(time, "02/Jan/2006:15:04:05 -0700","Asia/Tokyo") + `, + outkey: "time", + expect: []interface{}{ + int64(1638417334000000000), + int64(1638417334000000000), + }, + fail: false, + }, + { + name: "02/Jan/2006:15:04:05 (Shanghai)", + in: []string{ + `{"time":"02/Dec/2021:11:55:34"}`, + }, + pl: ` + json(_, time) + default_time_with_fmt(time, "02/Jan/2006:15:04:05","Asia/Shanghai") + `, + outkey: "time", + expect: []interface{}{ + int64(1638417334000000000), + }, + fail: false, + }, + { + name: "02/Jan/2006:15:04:05 (Local Shanghai)", + in: []string{ + `{"time":"02/Dec/2021:11:55:34"}`, + }, + pl: ` + json(_, time) + default_time_with_fmt(time, "02/Jan/2006:15:04:05") + `, + outkey: "time", + expect: []interface{}{ + int64(1638417334000000000), + }, + fail: false, + }, + { + name: "02/Jan/2006:15:04:05 (Tokyo)", + in: []string{ + `{"time":"02/Dec/2021:12:55:34"}`, + }, + pl: ` + json(_, time) + default_time_with_fmt(time, "02/Jan/2006:15:04:05","Asia/Tokyo") + `, + outkey: "time", + expect: []interface{}{ + int64(1638417334000000000), + }, + fail: false, + }, + } + for idx, tc := range cases { + t.Run(tc.name, func(t *testing.T) { + runner, err := NewTestingRunner(tc.pl) + if err != nil { + if tc.fail { + t.Logf("[%d]expect error: %s", idx, err) + } else { + t.Errorf("[%d] failed: %s", idx, err) + } + return + } + for idxIn := 0; idxIn < len(tc.in); idxIn++ { + pt := ptinput.NewPlPoint( + point.Logging, "test", nil, map[string]any{"message": tc.in[idxIn]}, time.Now()) + errR := runScript(runner, pt) + if errR != nil { + t.Fatal(errR) + } + + pt.KeyTime2Time() + + var v interface{} + if tc.outkey != "time" && tc.outkey != "" { + v, _, _ = pt.Get(tc.outkey) + } else { + v = pt.PtTime().UnixNano() + } + tu.Equals(t, tc.expect[idxIn], v) + t.Logf("[%d] PASS", idx) + } + }) + } +} diff --git a/pkg/inimpl/guancecloud/ptinput/funcs/fn_delete.go b/pkg/inimpl/guancecloud/ptinput/funcs/fn_delete.go new file mode 100644 index 0000000..479f656 --- /dev/null +++ b/pkg/inimpl/guancecloud/ptinput/funcs/fn_delete.go @@ -0,0 +1,150 @@ +// Unless explicitly stated otherwise all files in this repository are licensed +// under the MIT License. +// This product includes software developed at Guance Cloud (https://www.guance.com/). +// Copyright 2021-present Guance, Inc. + +package funcs + +import ( + "fmt" + + "github.com/GuanceCloud/platypus/pkg/ast" + "github.com/GuanceCloud/platypus/pkg/engine/runtime" + "github.com/GuanceCloud/platypus/pkg/errchain" + "github.com/spf13/cast" +) + +func DeleteMapItemChecking(ctx *runtime.Context, funcExpr *ast.CallExpr) *errchain.PlError { + if err := reindexFuncArgs(funcExpr, []string{"src", "key"}, 2); err != nil { + return runtime.NewRunError(ctx, err.Error(), funcExpr.NamePos) + } + + switch funcExpr.Param[0].NodeType { //nolint:exhaustive + case ast.TypeIndexExpr, ast.TypeIdentifier: + default: + return runtime.NewRunError(ctx, fmt.Sprintf("param key expect IndexExpr or Identifier, got %s", + funcExpr.Param[0].NodeType), funcExpr.Param[0].StartPos()) + } + + switch funcExpr.Param[1].NodeType { //nolint:exhaustive + case ast.TypeStringLiteral: + default: + return runtime.NewRunError(ctx, fmt.Sprintf("param key expect StringLiteral, got %s", + funcExpr.Param[1].NodeType), funcExpr.Param[1].StartPos()) + } + + return nil +} + +func DeleteMapItem(ctx *runtime.Context, funcExpr *ast.CallExpr) *errchain.PlError { + var keyName string + switch funcExpr.Param[1].NodeType { //nolint:exhaustive + case ast.TypeStringLiteral: + keyName = funcExpr.Param[1].StringLiteral.Val + default: + return runtime.NewRunError(ctx, fmt.Sprintf("param key expect StringLiteral, got %s", + funcExpr.Param[1].NodeType), funcExpr.Param[1].StartPos()) + } + + switch funcExpr.Param[0].NodeType { //nolint:exhaustive + case ast.TypeIndexExpr: + indexExprDel(ctx, funcExpr.Param[0].IndexExpr, keyName) + return nil + case ast.TypeIdentifier: + key, err := getKeyName(funcExpr.Param[0]) + if err != nil { + return runtime.NewRunError(ctx, err.Error(), funcExpr.Param[0].StartPos()) + } + varb, err := ctx.GetKey(key) + if err != nil { + l.Debugf("key `%s' not exist, ignored", key) + return nil + } + + if varb.DType == ast.Map { + switch v := varb.Value.(type) { + case map[string]any: // for json map + delete(v, keyName) + return nil + default: + return nil + } + } + default: + return runtime.NewRunError(ctx, fmt.Sprintf("param key expect IndexExpr or Identifier, got %s", + funcExpr.Param[0].NodeType), funcExpr.Param[0].StartPos()) + } + + return nil +} + +func indexExprDel(ctx *runtime.Context, expr *ast.IndexExpr, keyDel string) { + key := expr.Obj.Name + + varb, err := ctx.GetKey(key) + if err != nil { + return + } + + switch varb.DType { //nolint:exhaustive + case ast.List: + switch varb.Value.(type) { + case []any: + default: + return + } + case ast.Map: + switch varb.Value.(type) { + case map[string]any: // for json map + default: + return + } + default: + return + } + + searchListAndMap(ctx, varb.Value, expr.Index, keyDel) +} + +func searchListAndMap(ctx *runtime.Context, obj any, index []*ast.Node, keyDel string) { + cur := obj + + for _, i := range index { + key, keyType, err := runtime.RunStmt(ctx, i) + if err != nil { + return + } + switch curVal := cur.(type) { + case map[string]any: + if keyType != ast.String { + return + } + var ok bool + cur, ok = curVal[key.(string)] + if !ok { + return + } + case []any: + if keyType != ast.Int { + return + } + keyInt := cast.ToInt(key) + + // 反转负数 + if keyInt < 0 { + keyInt = len(curVal) + keyInt + } + + if keyInt < 0 || keyInt >= len(curVal) { + return + } + cur = curVal[keyInt] + default: + return + } + } + + if v, ok := cur.(map[string]any); ok { + delete(v, keyDel) + } +} diff --git a/pkg/inimpl/guancecloud/ptinput/funcs/fn_delete_test.go b/pkg/inimpl/guancecloud/ptinput/funcs/fn_delete_test.go new file mode 100644 index 0000000..6828414 --- /dev/null +++ b/pkg/inimpl/guancecloud/ptinput/funcs/fn_delete_test.go @@ -0,0 +1,125 @@ +// Unless explicitly stated otherwise all files in this repository are licensed +// under the MIT License. +// This product includes software developed at Guance Cloud (https://www.guance.com/). +// Copyright 2021-present Guance, Inc. + +package funcs + +import ( + "testing" + "time" + + "github.com/GuanceCloud/cliutils/point" + "github.com/GuanceCloud/platypus/pkg/inimpl/guancecloud/ptinput" + "github.com/stretchr/testify/assert" +) + +func TestDelete(t *testing.T) { + cases := []struct { + name, pl, in string + expected interface{} + fail bool + outkey string + }{ + { + name: "delete_map", + pl: ` + a=load_json(_) + delete(a["a"][-1], "b") + add_key(a)`, + in: `{"a":[1, {"b":2}]}`, + expected: "{\"a\":[1,{}]}", + outkey: "a", + }, + { + pl: `j_map = load_json(_) + + delete(j_map["b"][-1], "c") + + delete(j_map, "a") + + add_key("j_map", j_map)`, + in: `{"a": "b", "b":[0, {"c": "d"}], "e": 1}`, + expected: "{\"b\":[0,{}],\"e\":1}", + outkey: "j_map", + }, + { + name: "delete_map", + pl: ` + a=load_json(_) + delete(a, "a") + add_key(a)`, + in: `{"a":[1, {"b":2}]}`, + expected: "{}", + outkey: "a", + }, + { + name: "delete_map", + pl: ` + a=load_json(_) + delete(a["a"], "b") + add_key(a)`, + in: `{"a":[1, {"b":2}]}`, + expected: "{\"a\":[1,{\"b\":2}]}", + outkey: "a", + }, + { + name: "delete_map", + pl: ` + a=load_json(_) + delete(b, "b") + add_key(a)`, + in: `{"a":[1, {"b":2}]}`, + expected: "{\"a\":[1,{\"b\":2}]}", + outkey: "a", + }, + { + name: "delete_map", + pl: ` + a=load_json(_) + delete(a["a"][-7], "b") + add_key(a)`, + in: `{"a":[1, {"b":2}]}`, + expected: "{\"a\":[1,{\"b\":2}]}", + outkey: "a", + }, + { + name: "delete_map", + pl: ` + a=load_json(_) + delete(b, x) + add_key(a)`, + in: `{"a":[1, {"b":2}]}`, + expected: "{\"a\":[1,{\"b\":2}]}", + outkey: "a", + fail: true, + }, + } + + for idx, tc := range cases { + t.Run(tc.name, func(t *testing.T) { + runner, err := NewTestingRunner(tc.pl) + if err != nil { + if tc.fail { + t.Logf("[%d]expect error: %s", idx, err) + } else { + t.Errorf("[%d] failed: %s", idx, err) + } + return + } + pt := ptinput.NewPlPoint( + point.Logging, "test", nil, map[string]any{"message": tc.in}, time.Now()) + errR := runScript(runner, pt) + if errR != nil { + t.Fatal(*errR) + } + + v, isTag, ok := pt.GetWithIsTag(tc.outkey) + + assert.Equal(t, true, ok) + assert.Equal(t, false, isTag) + assert.Equal(t, tc.expected, v) + t.Logf("[%d] PASS", idx) + }) + } +} diff --git a/pkg/inimpl/guancecloud/ptinput/funcs/fn_drop.go b/pkg/inimpl/guancecloud/ptinput/funcs/fn_drop.go new file mode 100644 index 0000000..f26d8aa --- /dev/null +++ b/pkg/inimpl/guancecloud/ptinput/funcs/fn_drop.go @@ -0,0 +1,23 @@ +// Unless explicitly stated otherwise all files in this repository are licensed +// under the MIT License. +// This product includes software developed at Guance Cloud (https://www.guance.com/). +// Copyright 2021-present Guance, Inc. + +package funcs + +import ( + "github.com/GuanceCloud/platypus/pkg/ast" + "github.com/GuanceCloud/platypus/pkg/engine/runtime" + "github.com/GuanceCloud/platypus/pkg/errchain" +) + +func DropChecking(ctx *runtime.Context, funcExpr *ast.CallExpr) *errchain.PlError { + return nil +} + +func Drop(ctx *runtime.Context, funcExpr *ast.CallExpr) *errchain.PlError { + if err := markPtDrop(ctx.InData()); err != nil { + l.Debug(err) + } + return nil +} diff --git a/pkg/inimpl/guancecloud/ptinput/funcs/fn_drop_origin_data.go b/pkg/inimpl/guancecloud/ptinput/funcs/fn_drop_origin_data.go new file mode 100644 index 0000000..d525130 --- /dev/null +++ b/pkg/inimpl/guancecloud/ptinput/funcs/fn_drop_origin_data.go @@ -0,0 +1,21 @@ +// Unless explicitly stated otherwise all files in this repository are licensed +// under the MIT License. +// This product includes software developed at Guance Cloud (https://www.guance.com/). +// Copyright 2021-present Guance, Inc. + +package funcs + +import ( + "github.com/GuanceCloud/platypus/pkg/ast" + "github.com/GuanceCloud/platypus/pkg/engine/runtime" + "github.com/GuanceCloud/platypus/pkg/errchain" +) + +func DropOriginDataChecking(ctx *runtime.Context, funcExpr *ast.CallExpr) *errchain.PlError { + return nil +} + +func DropOriginData(ctx *runtime.Context, funcExpr *ast.CallExpr) *errchain.PlError { + deletePtKey(ctx.InData(), "message") + return nil +} diff --git a/pkg/inimpl/guancecloud/ptinput/funcs/fn_drop_origin_data_test.go b/pkg/inimpl/guancecloud/ptinput/funcs/fn_drop_origin_data_test.go new file mode 100644 index 0000000..1450b15 --- /dev/null +++ b/pkg/inimpl/guancecloud/ptinput/funcs/fn_drop_origin_data_test.go @@ -0,0 +1,60 @@ +// Unless explicitly stated otherwise all files in this repository are licensed +// under the MIT License. +// This product includes software developed at Guance Cloud (https://www.guance.com/). +// Copyright 2021-present Guance, Inc. + +package funcs + +import ( + "testing" + "time" + + "github.com/GuanceCloud/cliutils/point" + "github.com/GuanceCloud/platypus/pkg/inimpl/guancecloud/ptinput" +) + +func TestDropOriginData(t *testing.T) { + cases := []struct { + name, pl, in string + key string + fail bool + }{ + { + name: "value type: string", + in: `162.62.81.1 - - [29/Nov/2021:07:30:50 +0000] "POST /?signature=b8d8ea×tamp=1638171049 HTTP/1.1" 200 413 "-" "Mozilla/4.0"`, + pl: ` + grok(_, "%{IPORHOST:client_ip} %{NOTSPACE} %{NOTSPACE} \\[%{HTTPDATE:time}\\] \"%{DATA} %{GREEDYDATA} HTTP/%{NUMBER}\" %{INT:status_code} %{INT:bytes}") + drop_origin_data() + `, + key: "message", + fail: false, + }, + } + + for idx, tc := range cases { + t.Run(tc.name, func(t *testing.T) { + runner, err := NewTestingRunner(tc.pl) + if err != nil { + if tc.fail { + t.Logf("[%d]expect error: %s", idx, err) + } else { + t.Errorf("[%d] failed: %s", idx, err) + } + return + } + pt := ptinput.NewPlPoint( + point.Logging, "test", nil, map[string]any{"message": tc.in}, time.Now()) + errR := runScript(runner, pt) + + if errR != nil { + t.Fatal(errR) + } + + if v, _, ok := pt.GetWithIsTag(tc.key); ok { + t.Errorf("[%d] failed: key `%s` value `%v`", idx, tc.key, v) + } else { + t.Logf("[%d] PASS", idx) + } + }) + } +} diff --git a/pkg/inimpl/guancecloud/ptinput/funcs/fn_drop_test.go b/pkg/inimpl/guancecloud/ptinput/funcs/fn_drop_test.go new file mode 100644 index 0000000..6498a6c --- /dev/null +++ b/pkg/inimpl/guancecloud/ptinput/funcs/fn_drop_test.go @@ -0,0 +1,68 @@ +// Unless explicitly stated otherwise all files in this repository are licensed +// under the MIT License. +// This product includes software developed at Guance Cloud (https://www.guance.com/). +// Copyright 2021-present Guance, Inc. + +package funcs + +import ( + "testing" + "time" + + "github.com/GuanceCloud/cliutils/point" + tu "github.com/GuanceCloud/cliutils/testutil" + "github.com/GuanceCloud/platypus/pkg/inimpl/guancecloud/ptinput" +) + +func TestDrop(t *testing.T) { + cases := []struct { + name, pl, in string + outkey string + expect interface{} + fail bool + }{ + { + name: "cast int", + in: `162.62.81.1 - - [29/Nov/2021:07:30:50 +0000] "123 /?signature=b8d8ea×tamp=1638171049 HTTP/1.1" 200 413 "-" "Mozilla/4.0"`, + pl: ` + grok(_, "%{IPORHOST:client_ip} %{NOTSPACE} %{NOTSPACE} \\[%{HTTPDATE:time}\\] \"%{DATA:data} %{GREEDYDATA} HTTP/%{NUMBER}\" %{INT:status_code} %{INT:bytes}") + drop() + cast(data, "int") + `, + outkey: "data", + expect: nil, + fail: false, + }, + } + + for idx, tc := range cases { + t.Run(tc.name, func(t *testing.T) { + runner, err := NewTestingRunner(tc.pl) + if err != nil { + if tc.fail { + t.Logf("[%d]expect error: %s", idx, err) + } else { + t.Errorf("[%d] failed: %s", idx, err) + } + return + } + pt := ptinput.NewPlPoint( + point.Logging, "test", nil, map[string]any{"message": tc.in}, time.Now()) + errR := runScript(runner, pt) + if errR != nil { + t.Fatal(*errR) + } + + if errR != nil { + t.Fatal(errR) + } + + if pt.Dropped() { + return + } + v, _, _ := pt.Get(tc.outkey) + tu.Equals(t, tc.expect, v) + t.Logf("[%d] PASS", idx) + }) + } +} diff --git a/pkg/inimpl/guancecloud/funcs/fn_dropkey.go b/pkg/inimpl/guancecloud/ptinput/funcs/fn_dropkey.go similarity index 100% rename from pkg/inimpl/guancecloud/funcs/fn_dropkey.go rename to pkg/inimpl/guancecloud/ptinput/funcs/fn_dropkey.go diff --git a/pkg/inimpl/guancecloud/funcs/fn_dropkey_test.go b/pkg/inimpl/guancecloud/ptinput/funcs/fn_dropkey_test.go similarity index 78% rename from pkg/inimpl/guancecloud/funcs/fn_dropkey_test.go rename to pkg/inimpl/guancecloud/ptinput/funcs/fn_dropkey_test.go index e1436f0..50c0c15 100644 --- a/pkg/inimpl/guancecloud/funcs/fn_dropkey_test.go +++ b/pkg/inimpl/guancecloud/ptinput/funcs/fn_dropkey_test.go @@ -8,6 +8,9 @@ package funcs import ( "testing" "time" + + "github.com/GuanceCloud/cliutils/point" + "github.com/GuanceCloud/platypus/pkg/inimpl/guancecloud/ptinput" ) func TestDropKey(t *testing.T) { @@ -40,15 +43,15 @@ drop_key(client_ip) return } - _, _, f, _, _, err := runScript(runner, "test", nil, map[string]interface{}{ - "message": tc.in, - }, time.Now()) - if err != nil { - t.Error(err) - return + pt := ptinput.NewPlPoint( + point.Logging, "test", nil, map[string]any{"message": tc.in}, time.Now()) + errR := runScript(runner, pt) + + if errR != nil { + t.Fatal(errR) } - t.Log(f) - if v, ok := f[tc.key]; ok { + + if v, _, ok := pt.GetWithIsTag(tc.key); ok { t.Errorf("[%d] failed: key `%s` value `%v`", idx, tc.key, v) } else { t.Logf("[%d] PASS", idx) diff --git a/pkg/inimpl/guancecloud/ptinput/funcs/fn_duration_precision.go b/pkg/inimpl/guancecloud/ptinput/funcs/fn_duration_precision.go new file mode 100644 index 0000000..bead367 --- /dev/null +++ b/pkg/inimpl/guancecloud/ptinput/funcs/fn_duration_precision.go @@ -0,0 +1,94 @@ +// Unless explicitly stated otherwise all files in this repository are licensed +// under the MIT License. +// This product includes software developed at Guance Cloud (https://www.guance.com/). +// Copyright 2021-present Guance, Inc. + +package funcs + +import ( + "fmt" + "reflect" + "strings" + + "github.com/GuanceCloud/platypus/pkg/ast" + "github.com/GuanceCloud/platypus/pkg/engine/runtime" + "github.com/GuanceCloud/platypus/pkg/errchain" + "github.com/GuanceCloud/platypus/pkg/inimpl/guancecloud/ptinput" + "github.com/spf13/cast" +) + +func DurationPrecisionChecking(ctx *runtime.Context, funcExpr *ast.CallExpr) *errchain.PlError { + return nil +} + +func DurationPrecision(ctx *runtime.Context, funcExpr *ast.CallExpr) *errchain.PlError { + if len(funcExpr.Param) != 3 { + return runtime.NewRunError(ctx, fmt.Sprintf( + "func `%s' expected 3 args", funcExpr.Name), funcExpr.NamePos) + } + + key, err := getKeyName(funcExpr.Param[0]) + if err != nil { + return runtime.NewRunError(ctx, err.Error(), funcExpr.Param[0].StartPos()) + } + varb, err := ctx.GetKey(key) + if err != nil { + return runtime.NewRunError(ctx, err.Error(), funcExpr.Param[0].StartPos()) + } + var tValue int64 + if varb.DType != ast.Int { + return runtime.NewRunError(ctx, "param value type expect int", + funcExpr.Param[0].StartPos()) + } else { + tValue = cast.ToInt64(varb.Value) + } + + oldNew := [2]int{} + for i := 1; i < 3; i += 1 { + var err error + switch funcExpr.Param[i].NodeType { //nolint:exhaustive + case ast.TypeStringLiteral: + if oldNew[i-1], err = precision(funcExpr.Param[i].StringLiteral.Val); err != nil { + l.Debug(err) + return nil + } + default: + return runtime.NewRunError(ctx, fmt.Sprintf( + "param key expect Identifier or AttrExpr, got `%s'", + reflect.TypeOf(funcExpr.Param[i]).String()), funcExpr.Param[i].StartPos()) + } + } + + delta := oldNew[1] - oldNew[0] + deltaAbs := delta + if delta < 0 { + deltaAbs = -delta + } + for i := 0; i < deltaAbs; i++ { + if delta < 0 { + tValue /= 10 + } else if delta > 0 { + tValue *= 10 + } + } + if err := addKey2PtWithVal(ctx.InData(), key, tValue, ast.Int, + ptinput.KindPtDefault); err != nil { + return runtime.NewRunError(ctx, err.Error(), funcExpr.NamePos) + } + return nil +} + +func precision(p string) (int, error) { + switch strings.ToLower(p) { + case "s": + return 0, nil + case "ms": + return 3, nil + case "us": + return 6, nil + case "ns": + return 9, nil + default: + return 0, fmt.Errorf("unknow precision: %s", p) + } +} diff --git a/pkg/inimpl/guancecloud/ptinput/funcs/fn_duration_precision_test.go b/pkg/inimpl/guancecloud/ptinput/funcs/fn_duration_precision_test.go new file mode 100644 index 0000000..655105b --- /dev/null +++ b/pkg/inimpl/guancecloud/ptinput/funcs/fn_duration_precision_test.go @@ -0,0 +1,102 @@ +// Unless explicitly stated otherwise all files in this repository are licensed +// under the MIT License. +// This product includes software developed at Guance Cloud (https://www.guance.com/). +// Copyright 2021-present Guance, Inc. + +package funcs + +import ( + "testing" + "time" + + "github.com/GuanceCloud/cliutils/point" + tu "github.com/GuanceCloud/cliutils/testutil" + "github.com/GuanceCloud/platypus/pkg/inimpl/guancecloud/ptinput" +) + +func TestDurationPrecision(t *testing.T) { + cases := []struct { + name, pl, in string + outkey string + expect interface{} + fail bool + }{ + { + name: "cast int", + in: `{"ts":12345}`, + pl: ` + json(_, ts) + cast(ts, "int") + duration_precision(ts, "ms", "ns") + `, + outkey: "ts", + expect: int64(12345000000), + fail: false, + }, + { + name: "cast int", + in: `{"ts":12345000}`, + pl: ` + json(_, ts) + cast(ts, "int") + duration_precision(ts, "ms", "s") + `, + outkey: "ts", + expect: int64(12345), + fail: false, + }, + { + name: "cast int", + in: `{"ts":12345000}`, + pl: ` + json(_, ts) + cast(ts, "int") + duration_precision(ts, "s", "s") + `, + outkey: "ts", + expect: int64(12345000), + fail: false, + }, + { + name: "cast int", + in: `{"ts":12345000}`, + pl: ` + json(_, ts) + cast(ts, "int") + duration_precision(ts, "ns", "us") + `, + outkey: "ts", + expect: int64(12345), + fail: false, + }, + } + + for idx, tc := range cases { + t.Run(tc.name, func(t *testing.T) { + runner, err := NewTestingRunner(tc.pl) + if err != nil { + if tc.fail { + t.Logf("[%d]expect error: %s", idx, err) + } else { + t.Errorf("[%d] failed: %s", idx, err) + } + return + } + + pt := ptinput.NewPlPoint( + point.Logging, "test", nil, map[string]any{"message": tc.in}, time.Now()) + errR := runScript(runner, pt) + + if errR != nil { + t.Fatal(errR) + } + + if tc.fail { + t.Logf("[%d]expect error: %s", idx, err) + } + v, _, _ := pt.Get(tc.outkey) + tu.Equals(t, tc.expect, v) + t.Logf("[%d] PASS", idx) + }) + } +} diff --git a/pkg/inimpl/guancecloud/funcs/fn_exit.go b/pkg/inimpl/guancecloud/ptinput/funcs/fn_exit.go similarity index 100% rename from pkg/inimpl/guancecloud/funcs/fn_exit.go rename to pkg/inimpl/guancecloud/ptinput/funcs/fn_exit.go diff --git a/pkg/inimpl/guancecloud/funcs/fn_exit_test.go b/pkg/inimpl/guancecloud/ptinput/funcs/fn_exit_test.go similarity index 71% rename from pkg/inimpl/guancecloud/funcs/fn_exit_test.go rename to pkg/inimpl/guancecloud/ptinput/funcs/fn_exit_test.go index bea1091..a18fe6d 100644 --- a/pkg/inimpl/guancecloud/funcs/fn_exit_test.go +++ b/pkg/inimpl/guancecloud/ptinput/funcs/fn_exit_test.go @@ -9,7 +9,9 @@ import ( "testing" "time" - "github.com/stretchr/testify/assert" + "github.com/GuanceCloud/cliutils/point" + tu "github.com/GuanceCloud/cliutils/testutil" + "github.com/GuanceCloud/platypus/pkg/inimpl/guancecloud/ptinput" ) func TestExit(t *testing.T) { @@ -44,20 +46,20 @@ func TestExit(t *testing.T) { } return } - _, _, f, _, drop, err := runScript(runner, "test", nil, map[string]interface{}{ - "message": tc.in, - }, time.Now()) - if err != nil { - t.Error(err) - return + + pt := ptinput.NewPlPoint( + point.Logging, "test", nil, map[string]any{"message": tc.in}, time.Now()) + errR := runScript(runner, pt) + + if errR != nil { + t.Fatal(errR.Error()) } - t.Log(f) - if drop { - return + + if tc.fail { + t.Logf("[%d]expect error: %s", idx, errR.Error()) } - t.Log(f) - v := f[tc.outkey] - assert.Equal(t, tc.expect, v) + v, _, _ := pt.Get(tc.outkey) + tu.Equals(t, tc.expect, v) t.Logf("[%d] PASS", idx) }) } diff --git a/pkg/inimpl/guancecloud/ptinput/funcs/fn_expr.go b/pkg/inimpl/guancecloud/ptinput/funcs/fn_expr.go new file mode 100644 index 0000000..0f583be --- /dev/null +++ b/pkg/inimpl/guancecloud/ptinput/funcs/fn_expr.go @@ -0,0 +1,22 @@ +// Unless explicitly stated otherwise all files in this repository are licensed +// under the MIT License. +// This product includes software developed at Guance Cloud (https://www.guance.com/). +// Copyright 2021-present Guance, Inc. + +package funcs + +import ( + "github.com/GuanceCloud/platypus/pkg/ast" + "github.com/GuanceCloud/platypus/pkg/engine/runtime" + "github.com/GuanceCloud/platypus/pkg/errchain" +) + +func ExprChecking(ctx *runtime.Context, funcExpr *ast.CallExpr) *errchain.PlError { + l.Debugf("warning: expr() is disabled") + return nil +} + +func Expr(ctx *runtime.Context, funcExpr *ast.CallExpr) *errchain.PlError { + l.Debugf("warning: expr() is disabled") + return nil +} diff --git a/pkg/inimpl/guancecloud/ptinput/funcs/fn_geo_ip.go b/pkg/inimpl/guancecloud/ptinput/funcs/fn_geo_ip.go new file mode 100644 index 0000000..ad4ed9a --- /dev/null +++ b/pkg/inimpl/guancecloud/ptinput/funcs/fn_geo_ip.go @@ -0,0 +1,76 @@ +// Unless explicitly stated otherwise all files in this repository are licensed +// under the MIT License. +// This product includes software developed at Guance Cloud (https://www.guance.com/). +// Copyright 2021-present Guance, Inc. + +package funcs + +import ( + "fmt" + + "github.com/GuanceCloud/platypus/pkg/ast" + "github.com/GuanceCloud/platypus/pkg/engine/runtime" + "github.com/GuanceCloud/platypus/pkg/errchain" + "github.com/GuanceCloud/platypus/pkg/inimpl/guancecloud/ptinput" + "github.com/GuanceCloud/platypus/pkg/inimpl/guancecloud/ptinput/ipdb" +) + +var ipdbInstance ipdb.IPdb + +var geoDefaultVal = "unknown" + +func Geo(ip string) (*ipdb.IPdbRecord, error) { + if ipdbInstance != nil { + return ipdbInstance.Geo(ip) + } else { + return &ipdb.IPdbRecord{}, nil + } +} + +func InitIPdb(instance ipdb.IPdb) { + ipdbInstance = instance +} + +func GeoIPChecking(ctx *runtime.Context, funcExpr *ast.CallExpr) *errchain.PlError { + if len(funcExpr.Param) != 1 { + return runtime.NewRunError(ctx, fmt.Sprintf( + "func `%s' expected 1 args", funcExpr.Name), funcExpr.NamePos) + } + + if _, err := getKeyName(funcExpr.Param[0]); err != nil { + return runtime.NewRunError(ctx, err.Error(), funcExpr.Param[0].StartPos()) + } + + return nil +} + +func GeoIP(ctx *runtime.Context, funcExpr *ast.CallExpr) *errchain.PlError { + if len(funcExpr.Param) != 1 { + return runtime.NewRunError(ctx, fmt.Sprintf( + "func `%s' expected 1 args", funcExpr.Name), funcExpr.NamePos) + } + key, err := getKeyName(funcExpr.Param[0]) + if err != nil { + return runtime.NewRunError(ctx, err.Error(), funcExpr.Param[0].StartPos()) + } + + ipStr, err := ctx.GetKeyConv2Str(key) + if err != nil { + l.Debugf("key `%v' not exist, ignored", key) + return nil //nolint:nilerr + } + + if dic, err := GeoIPHandle(ipStr); err != nil { + l.Debugf("GeoIPHandle: %s, ignored", err) + return nil + } else { + for k, v := range dic { + if err := addKey2PtWithVal(ctx.InData(), k, v, ast.String, ptinput.KindPtDefault); err != nil { + l.Debug(err) + return nil + } + } + } + + return nil +} diff --git a/pkg/inimpl/guancecloud/ptinput/funcs/fn_gep_ip_test.go b/pkg/inimpl/guancecloud/ptinput/funcs/fn_gep_ip_test.go new file mode 100644 index 0000000..183af00 --- /dev/null +++ b/pkg/inimpl/guancecloud/ptinput/funcs/fn_gep_ip_test.go @@ -0,0 +1,149 @@ +// Unless explicitly stated otherwise all files in this repository are licensed +// under the MIT License. +// This product includes software developed at Guance Cloud (https://www.guance.com/). +// Copyright 2021-present Guance, Inc. + +package funcs + +import ( + "testing" + "time" + + "github.com/GuanceCloud/cliutils/point" + "github.com/GuanceCloud/platypus/pkg/inimpl/guancecloud/ptinput" + "github.com/GuanceCloud/platypus/pkg/inimpl/guancecloud/ptinput/ipdb" + "github.com/stretchr/testify/assert" +) + +type mockGEO struct{} + +func (m *mockGEO) Init(dataDir string, config map[string]string) {} +func (m *mockGEO) SearchIsp(ip string) string { return "" } + +func (m *mockGEO) Geo(ip string) (*ipdb.IPdbRecord, error) { + return &ipdb.IPdbRecord{ + City: func() string { + if ip == "unknown-city" { + return geoDefaultVal + } else { + return "Shanghai" + } + }(), + Region: func() string { + if ip == "unknown-region" { + return geoDefaultVal + } else { + return "Shanghai" + } + }(), + Country: func() string { + if ip == "unknown-country-short" { + return geoDefaultVal + } else { + return "CN" + } + }(), + }, nil +} + +func TestGeoIpFunc(t *testing.T) { + ipdbInstance = &mockGEO{} + cases := []struct { + in string + script string + + expected map[string]string + + fail bool + }{ + { + in: `{"ip":"1.2.3.4-something", "second":2,"third":"abc","forth":true}`, + script: ` + json(_, ip) + geoip(ip)`, + expected: map[string]string{ + "city": "Shanghai", + "country": "CN", + "province": "Shanghai", + "isp": geoDefaultVal, + }, + }, + + { + in: `{"ip":"unknown-city", "second":2,"third":"abc","forth":true}`, + script: ` + json(_, ip) + geoip(ip)`, + expected: map[string]string{ + "city": geoDefaultVal, + "country": "CN", + "province": "Shanghai", + "isp": geoDefaultVal, + }, + }, + + { + in: `{"aa": {"ip":"116.228.89.xxx"}, "second":2,"third":"abc","forth":true}`, + script: ` + json(_, aa.ip) + geoip(aa.ip)`, + expected: map[string]string{ + "city": "Shanghai", + "country": "CN", + "province": "Shanghai", + "isp": geoDefaultVal, + }, + }, + + { + in: `{"aa": {"ip":"unknown-region"}, "second":2,"third":"abc","forth":true}`, + script: ` + json(_, aa.ip) + geoip(aa.ip)`, + expected: map[string]string{ + "city": "Shanghai", + "country": "CN", + "province": geoDefaultVal, + "isp": geoDefaultVal, + }, + }, + + { + in: `{"aa": {"ip":"unknown-country-short"}, "second":2,"third":"abc","forth":true}`, + script: ` + json(_, aa.ip) + geoip(aa.ip)`, + expected: map[string]string{ + "city": "Shanghai", + "country": geoDefaultVal, + "province": "Shanghai", + "isp": geoDefaultVal, + }, + }, + } + + for idx, tc := range cases { + t.Logf("case %d...", idx) + + runner, err := NewTestingRunner(tc.script) + if err != nil { + t.Errorf("[%d] failed: %s", idx, err) + return + } + + pt := ptinput.NewPlPoint( + point.Logging, "test", nil, map[string]any{"message": tc.in}, time.Now()) + errR := runScript(runner, pt) + + if errR != nil { + t.Fatal(errR.Error()) + } + + for k, v := range tc.expected { + r, isTag, ok := pt.GetWithIsTag(k) + assert.Equal(t, true, ok, "!ok") + assert.Equal(t, false, isTag) + assert.Equal(t, v, r, "%s != %s, output: %+#v", r, v) + } + } +} diff --git a/pkg/inimpl/guancecloud/funcs/fn_getkey.go b/pkg/inimpl/guancecloud/ptinput/funcs/fn_getkey.go similarity index 100% rename from pkg/inimpl/guancecloud/funcs/fn_getkey.go rename to pkg/inimpl/guancecloud/ptinput/funcs/fn_getkey.go diff --git a/pkg/inimpl/guancecloud/funcs/fn_getkey_test.go b/pkg/inimpl/guancecloud/ptinput/funcs/fn_getkey_test.go similarity index 81% rename from pkg/inimpl/guancecloud/funcs/fn_getkey_test.go rename to pkg/inimpl/guancecloud/ptinput/funcs/fn_getkey_test.go index abface8..701097a 100644 --- a/pkg/inimpl/guancecloud/funcs/fn_getkey_test.go +++ b/pkg/inimpl/guancecloud/ptinput/funcs/fn_getkey_test.go @@ -9,6 +9,8 @@ import ( "testing" "time" + "github.com/GuanceCloud/cliutils/point" + "github.com/GuanceCloud/platypus/pkg/inimpl/guancecloud/ptinput" "github.com/stretchr/testify/assert" ) @@ -55,14 +57,16 @@ func TestGetkey(t *testing.T) { } return } - _, _, f, _, _, err := runScript(runner, "test", nil, map[string]any{"message": tc.in}, time.Now()) + pt := ptinput.NewPlPoint( + point.Logging, "test", nil, map[string]any{"message": tc.in}, time.Now()) + errR := runScript(runner, pt) - if err == nil { - v := f["add_new_key"] + if errR == nil { + v, _, _ := pt.Get("add_new_key") assert.Equal(t, tc.expect, v) t.Logf("[%d] PASS", idx) } else { - t.Error(err) + t.Error(errR) } }) } diff --git a/pkg/inimpl/guancecloud/funcs/fn_grok.go b/pkg/inimpl/guancecloud/ptinput/funcs/fn_grok.go similarity index 95% rename from pkg/inimpl/guancecloud/funcs/fn_grok.go rename to pkg/inimpl/guancecloud/ptinput/funcs/fn_grok.go index 7aa40f9..2cbd1b6 100644 --- a/pkg/inimpl/guancecloud/funcs/fn_grok.go +++ b/pkg/inimpl/guancecloud/ptinput/funcs/fn_grok.go @@ -12,7 +12,7 @@ import ( "github.com/GuanceCloud/platypus/pkg/ast" "github.com/GuanceCloud/platypus/pkg/engine/runtime" "github.com/GuanceCloud/platypus/pkg/errchain" - "github.com/GuanceCloud/platypus/pkg/inimpl/guancecloud/input" + "github.com/GuanceCloud/platypus/pkg/inimpl/guancecloud/ptinput" ) func GrokChecking(ctx *runtime.Context, funcExpr *ast.CallExpr) *errchain.PlError { @@ -113,7 +113,7 @@ func Grok(ctx *runtime.Context, funcExpr *ast.CallExpr) *errchain.PlError { continue } } - if err := addKey2PtWithVal(ctx.InData(), k, v, dtype, input.KindPtDefault); err != nil { + if err := addKey2PtWithVal(ctx.InData(), k, v, dtype, ptinput.KindPtDefault); err != nil { l.Debug(err) ctx.Regs.ReturnAppend(false, ast.Bool) return nil diff --git a/pkg/inimpl/guancecloud/ptinput/funcs/fn_grok_test.go b/pkg/inimpl/guancecloud/ptinput/funcs/fn_grok_test.go new file mode 100644 index 0000000..3769da0 --- /dev/null +++ b/pkg/inimpl/guancecloud/ptinput/funcs/fn_grok_test.go @@ -0,0 +1,223 @@ +// Unless explicitly stated otherwise all files in this repository are licensed +// under the MIT License. +// This product includes software developed at Guance Cloud (https://www.guance.com/). +// Copyright 2021-present Guance, Inc. + +package funcs + +import ( + "testing" + "time" + + "github.com/GuanceCloud/cliutils/point" + tu "github.com/GuanceCloud/cliutils/testutil" + "github.com/GuanceCloud/platypus/pkg/inimpl/guancecloud/ptinput" +) + +func TestGrok(t *testing.T) { + cases := []struct { + name, pl, in string + expected interface{} + fail bool + outkey string + }{ + { + name: "normal_return_t", + pl: ` +add_pattern("_second", "(?:(?:[0-5]?[0-9]|60)(?:[:.,][0-9]+)?)") +add_pattern("_minute", "(?:[0-5][0-9])") +add_pattern("_hour", "(?:2[0123]|[01]?[0-9])") +add_pattern("time", "([^0-9]?)%{_hour:hour}:%{_minute:minute}(?::%{_second:second})([^0-9]?)") +add_key(grok_match_ok, grok(_, "%{time}"))`, + in: "12:13:14.123", + expected: true, + outkey: "grok_match_ok", + }, + { + name: "normal_return_f", + pl: ` +add_pattern("_second", "(?:(?:[0-5]?[0-9]|60)(?:[:.,][0-9]+)?)") +add_pattern("_minute", "(?:[0-5][0-9])") +add_pattern("_hour", "(?:2[0123]|[01]?[0-9])") +add_pattern("time", "([^0-9]?)%{_hour:hour}:%{_minute:minute}(?::%{_second:second})([^0-9]?)") +add_key(grok_match_ok, grok(_, "%{time}"))`, + in: "12 :13:14.123", + expected: false, + outkey: "grok_match_ok", + }, + { + name: "normal_return_sample_t", + pl: ` +add_key(grok_match_ok, grok(_, "12 :13:14.123"))`, + in: "12 :13:14.123", + expected: true, + outkey: "grok_match_ok", + }, + { + name: "normal_return_sample_f", + pl: ` +add_key(grok_match_ok, grok(_, "12 :13:14.123"))`, + in: "12:13:14.123", + expected: false, + outkey: "grok_match_ok", + }, + { + name: "normal", + pl: ` +add_pattern("_second", "(?:(?:[0-5]?[0-9]|60)(?:[:.,][0-9]+)?)") +add_pattern("_minute", "(?:[0-5][0-9])") +add_pattern("_hour", "(?:2[0123]|[01]?[0-9])") +add_pattern("time", "([^0-9]?)%{_hour:hour}:%{_minute:minute}(?::%{_second:second})([^0-9]?)") +grok(_, "%{time}")`, + in: "12:13:14.123", + expected: "14.123", + outkey: "second", + }, + { + name: "normal", + pl: ` +add_pattern("_second", "(?:(?:[0-5]?[0-9]|60)(?:[:.,][0-9]+)?)") +add_pattern("_minute", "(?:[0-5][0-9])") +add_pattern("_hour", "(?:2[0123]|[01]?[0-9])") +add_pattern("time", "([^0-9]?)%{_hour:hour}:%{_minute:minute}(?::%{_second:second})([^0-9]?)") +grok(_, "%{time}")`, + in: "12:13:14", + expected: "13", + outkey: "minute", + }, + { + name: "normal", + pl: ` +add_pattern("_second", "(?:(?:[0-5]?[0-9]|60)(?:[:.,][0-9]+)?)") +add_pattern("_minute", "(?:[0-5][0-9])") +add_pattern("_hour", "(?:2[0123]|[01]?[0-9])") +add_pattern("time", "([^0-9]?)%{_hour:hour}:%{_minute:minute}(?::%{_second:second})([^0-9]?)") +grok(_, "%{time}")`, + in: "12:13:14", + expected: "12", + outkey: "hour", + }, + { + name: "normal", + pl: ` +add_pattern("_second", "(?:(?:[0-5]?[0-9]|60)(?:[:.,][0-9]+)?)") +add_pattern("_minute", "(?:[0-5][0-9])") +add_pattern("_hour", "(?:2[0123]|[01]?[0-9])") +add_pattern("time", "([^0-9]?)%{_hour:hour}:%{_minute:minute}(?::%{_second:second})([^0-9]?)") +grok(_, "%{time}")`, + in: "12:13:14", + expected: "14", + outkey: "second", + }, + { + name: "normal", + pl: ` +add_pattern("time", "%{NUMBER:time:float}") +grok(_, '''%{time} +%{WORD:word:string} + %{WORD:code:int} +%{WORD:w1}''')`, + in: `1.1 +s + 123cvf +aa222`, + expected: int64(0), + outkey: "code", + }, + { + name: "normal", + pl: ` +add_pattern("time", "%{NUMBER:time:float}") +grok(_, '''%{time} +%{WORD:word:string} + %{WORD:code:int} +%{WORD:w1}''')`, + in: `1.1 +s + 123 +aa222`, + expected: int64(123), + outkey: "code", + }, + { + name: "normal", + pl: ` +add_pattern("time", "%{NUMBER:time:float}") +grok(_, '''%{time} +%{WORD:word:str} + %{WORD:code:int} +%{WORD:w1}''')`, + in: `1.1 +s + 123 +aa222`, + expected: int64(123), + outkey: "code", + }, + { + name: "normal", + pl: ` +add_pattern("_second", "(?:(?:[0-5]?[0-9]|60)(?:[:.,][0-9]+)?)") +add_pattern("_minute", "(?:[0-5][0-9])") +add_pattern("_hour", "(?:2[0123]|[01]?[0-9])") +add_pattern("time", "([^0-9]?)%{_hour:hour:string}:%{_minute:minute:int}(?::%{_second:second:float})([^0-9]?)") +grok(_, "%{WORD:date} %{time}")`, + in: "2021/1/11 2:13:14.123", + expected: float64(14.123), + outkey: "second", + }, + { + name: "trim_space", + in: " not_space ", + pl: `add_pattern("d", "[\\s\\S]*") + grok(_, "%{d:item}")`, + expected: "not_space", + outkey: "item", + }, + { + name: "trim_space, enable", + in: " not_space ", + pl: `add_pattern("d", "[\\s\\S]*") + grok(_, "%{d:item}", true)`, + expected: "not_space", + outkey: "item", + }, + { + name: "trim_space, disable", + in: " not_space ", + pl: `add_pattern("d", "[\\s\\S]*") + grok(_, "%{d:item}", false)`, + expected: " not_space ", + outkey: "item", + }, + } + + for idx, tc := range cases { + t.Run(tc.name, func(t *testing.T) { + runner, err := NewTestingRunner(tc.pl) + if err != nil { + if tc.fail { + t.Logf("[%d]expect error: %s", idx, err) + } else { + t.Errorf("[%d] failed: %s", idx, err) + } + return + } + + pt := ptinput.NewPlPoint( + point.Logging, "test", nil, map[string]any{"message": tc.in}, time.Now()) + errR := runScript(runner, pt) + + if errR != nil { + t.Fatal(errR.Error()) + } + + if tc.fail { + t.Logf("[%d]expect error: %s", idx, errR.Error()) + } + v, _, _ := pt.Get(tc.outkey) + tu.Equals(t, tc.expected, v) + t.Logf("[%d] PASS", idx) + }) + } +} diff --git a/pkg/inimpl/guancecloud/ptinput/funcs/fn_group.go b/pkg/inimpl/guancecloud/ptinput/funcs/fn_group.go new file mode 100644 index 0000000..208dc51 --- /dev/null +++ b/pkg/inimpl/guancecloud/ptinput/funcs/fn_group.go @@ -0,0 +1,165 @@ +// Unless explicitly stated otherwise all files in this repository are licensed +// under the MIT License. +// This product includes software developed at Guance Cloud (https://www.guance.com/). +// Copyright 2021-present Guance, Inc. + +package funcs + +import ( + "fmt" + + "github.com/GuanceCloud/platypus/pkg/ast" + "github.com/GuanceCloud/platypus/pkg/engine/runtime" + "github.com/GuanceCloud/platypus/pkg/errchain" + "github.com/GuanceCloud/platypus/pkg/inimpl/guancecloud/ptinput" +) + +func GroupChecking(ctx *runtime.Context, funcExpr *ast.CallExpr) *errchain.PlError { + if len(funcExpr.Param) < 3 || len(funcExpr.Param) > 4 { + return runtime.NewRunError(ctx, fmt.Sprintf( + "func `%s' expected 3 or 4 args", funcExpr.Name), funcExpr.NamePos) + } + + if _, err := getKeyName(funcExpr.Param[0]); err != nil { + return runtime.NewRunError(ctx, err.Error(), funcExpr.Param[0].StartPos()) + } + + var start, end float64 + + if len(funcExpr.Param) == 4 { + if _, err := getKeyName(funcExpr.Param[3]); err != nil { + return runtime.NewRunError(ctx, err.Error(), funcExpr.Param[3].StartPos()) + } + } + + var set []*ast.Node + if funcExpr.Param[1].NodeType == ast.TypeListInitExpr { + set = funcExpr.Param[1].ListInitExpr.List + } + + if len(set) != 2 { + return runtime.NewRunError(ctx, fmt.Sprintf( + "param between range value `%v' is not expected", set), + funcExpr.Param[1].StartPos()) + } + + switch set[0].NodeType { //nolint:exhaustive + case ast.TypeFloatLiteral: + start = set[0].FloatLiteral.Val + case ast.TypeIntegerLiteral: + start = float64(set[0].IntegerLiteral.Val) + + default: + return runtime.NewRunError(ctx, fmt.Sprintf( + "range value `%v' is not expected", set), set[0].StartPos()) + } + + switch set[1].NodeType { //nolint:exhaustive + case ast.TypeFloatLiteral: + end = set[1].FloatLiteral.Val + case ast.TypeIntegerLiteral: + end = float64(set[1].IntegerLiteral.Val) + default: + return runtime.NewRunError(ctx, fmt.Sprintf( + "range value `%v' is not expected", set), set[1].StartPos()) + } + + if start > end { + return runtime.NewRunError(ctx, fmt.Sprintf( + "range value start %v must le end %v", start, end), funcExpr.NamePos) + } + + return nil +} + +func Group(ctx *runtime.Context, funcExpr *ast.CallExpr) *errchain.PlError { + if len(funcExpr.Param) < 3 || len(funcExpr.Param) > 4 { + return runtime.NewRunError(ctx, fmt.Sprintf( + "func `%s' expected 3 or 4 args", funcExpr.Name), funcExpr.NamePos) + } + + key, err := getKeyName(funcExpr.Param[0]) + if err != nil { + return runtime.NewRunError(ctx, err.Error(), funcExpr.Param[0].StartPos()) + } + + cont, err := ctx.GetKey(key) + if err != nil { + l.Debugf("key `%v' not exist, ignored", key) + return nil //nolint:nilerr + } + + var start, end float64 + + var set []*ast.Node + if funcExpr.Param[1].NodeType == ast.TypeListInitExpr { + set = funcExpr.Param[1].ListInitExpr.List + } + + if len(set) != 2 { + return runtime.NewRunError(ctx, fmt.Sprintf( + "param between range value `%v' is not expected", set), funcExpr.Param[1].StartPos()) + } + + switch set[0].NodeType { //nolint:exhaustive + case ast.TypeIntegerLiteral: + start = float64(set[0].IntegerLiteral.Val) + case ast.TypeFloatLiteral: + start = set[0].FloatLiteral.Val + default: + return runtime.NewRunError(ctx, fmt.Sprintf("range value `%v' is not expected", set), + funcExpr.Param[1].StartPos()) + } + + switch set[1].NodeType { //nolint:exhaustive + case ast.TypeIntegerLiteral: + end = float64(set[1].IntegerLiteral.Val) + case ast.TypeFloatLiteral: + end = set[1].FloatLiteral.Val + default: + return runtime.NewRunError(ctx, fmt.Sprintf("range value `%v' is not expected", set), + funcExpr.Param[1].StartPos()) + } + + if start > end { + return runtime.NewRunError(ctx, fmt.Sprintf("range value start %v must le end %v", start, end), + funcExpr.Param[1].StartPos()) + } + + if GroupHandle(cont.Value, start, end) { + value := funcExpr.Param[2] + + var val any + var dtype ast.DType + + switch value.NodeType { //nolint:exhaustive + case ast.TypeFloatLiteral: + val = value.FloatLiteral.Val + dtype = ast.Float + case ast.TypeIntegerLiteral: + val = value.IntegerLiteral.Val + dtype = ast.Int + case ast.TypeStringLiteral: + val = value.StringLiteral.Val + dtype = ast.String + case ast.TypeBoolLiteral: + val = value.BoolLiteral.Val + dtype = ast.String + default: + l.Debugf("unknown group elements: %s", value.NodeType) + return runtime.NewRunError(ctx, "unsupported group type", + funcExpr.NamePos) + } + + if len(funcExpr.Param) == 4 { + if k, err := getKeyName(funcExpr.Param[3]); err == nil { + key = k + } else { + return runtime.NewRunError(ctx, err.Error(), funcExpr.Param[4].StartPos()) + } + } + _ = addKey2PtWithVal(ctx.InData(), key, val, dtype, ptinput.KindPtDefault) + } + + return nil +} diff --git a/pkg/inimpl/guancecloud/ptinput/funcs/fn_group_in.go b/pkg/inimpl/guancecloud/ptinput/funcs/fn_group_in.go new file mode 100644 index 0000000..2c0c37d --- /dev/null +++ b/pkg/inimpl/guancecloud/ptinput/funcs/fn_group_in.go @@ -0,0 +1,108 @@ +// Unless explicitly stated otherwise all files in this repository are licensed +// under the MIT License. +// This product includes software developed at Guance Cloud (https://www.guance.com/). +// Copyright 2021-present Guance, Inc. + +package funcs + +import ( + "fmt" + + "github.com/GuanceCloud/platypus/pkg/ast" + "github.com/GuanceCloud/platypus/pkg/engine/runtime" + "github.com/GuanceCloud/platypus/pkg/errchain" + "github.com/GuanceCloud/platypus/pkg/inimpl/guancecloud/ptinput" +) + +func GroupInChecking(ctx *runtime.Context, funcExpr *ast.CallExpr) *errchain.PlError { + if len(funcExpr.Param) < 3 || len(funcExpr.Param) > 4 { + return runtime.NewRunError(ctx, fmt.Sprintf( + "func %s expected 3 or 4 args", funcExpr.Name), funcExpr.NamePos) + } + + if _, err := getKeyName(funcExpr.Param[0]); err != nil { + return runtime.NewRunError(ctx, err.Error(), funcExpr.Param[0].StartPos()) + } + + if len(funcExpr.Param) == 4 { + switch funcExpr.Param[3].NodeType { //nolint:exhaustive + case ast.TypeAttrExpr, ast.TypeStringLiteral, ast.TypeIdentifier: + default: + return runtime.NewRunError(ctx, fmt.Sprintf( + "param new-key expect AttrExpr, StringLiteral or Identifier, got %s", + funcExpr.Param[3].NodeType), funcExpr.Param[3].StartPos()) + } + } + return nil +} + +func GroupIn(ctx *runtime.Context, funcExpr *ast.CallExpr) *errchain.PlError { + if len(funcExpr.Param) < 3 || len(funcExpr.Param) > 4 { + return runtime.NewRunError(ctx, fmt.Sprintf("func %s expected 3 or 4 args", funcExpr.Name), + funcExpr.NamePos) + } + + key, err := getKeyName(funcExpr.Param[0]) + if err != nil { + return runtime.NewRunError(ctx, err.Error(), funcExpr.Param[0].StartPos()) + } + + cont, err := ctx.GetKey(key) + if err != nil { + // l.Debugf("key '%v' not exist, ignored", key) + return nil //nolint:nilerr + } + + var set []any + switch funcExpr.Param[1].NodeType { //nolint:exhaustive + case ast.TypeIdentifier: + if v, err := ctx.GetKey(funcExpr.Param[1].Identifier.Name); err == nil && + v.DType == ast.List { + if v, ok := v.Value.([]any); ok { + set = v + } + } + case ast.TypeListInitExpr: + if v, dtype, err := runtime.RunListInitExpr(ctx, + funcExpr.Param[1].ListInitExpr); err == nil && dtype == ast.List { + if v, ok := v.([]any); ok { + set = v + } + } + } + + if GroupInHandle(cont.Value, set) { + value := funcExpr.Param[2] + + var val any + var dtype ast.DType + + switch value.NodeType { //nolint:exhaustive + case ast.TypeIntegerLiteral: + val = value.IntegerLiteral.Val + dtype = ast.Int + case ast.TypeFloatLiteral: + val = value.FloatLiteral.Val + dtype = ast.Float + case ast.TypeStringLiteral: + val = value.StringLiteral.Val + dtype = ast.String + case ast.TypeBoolLiteral: + val = value.BoolLiteral.Val + dtype = ast.String + default: + return nil + } + + if len(funcExpr.Param) == 4 { + if k, err := getKeyName(funcExpr.Param[3]); err == nil { + key = k + } else { + return runtime.NewRunError(ctx, err.Error(), funcExpr.Param[3].StartPos()) + } + } + _ = addKey2PtWithVal(ctx.InData(), key, val, dtype, ptinput.KindPtDefault) + } + + return nil +} diff --git a/pkg/inimpl/guancecloud/ptinput/funcs/fn_group_in_test.go b/pkg/inimpl/guancecloud/ptinput/funcs/fn_group_in_test.go new file mode 100644 index 0000000..432fd41 --- /dev/null +++ b/pkg/inimpl/guancecloud/ptinput/funcs/fn_group_in_test.go @@ -0,0 +1,110 @@ +// Unless explicitly stated otherwise all files in this repository are licensed +// under the MIT License. +// This product includes software developed at Guance Cloud (https://www.guance.com/). +// Copyright 2021-present Guance, Inc. + +package funcs + +import ( + "testing" + "time" + + "github.com/GuanceCloud/cliutils/point" + tu "github.com/GuanceCloud/cliutils/testutil" + "github.com/GuanceCloud/platypus/pkg/inimpl/guancecloud/ptinput" +) + +func TestGroupIn(t *testing.T) { + cases := []struct { + name, pl, in string + expected interface{} + fail bool + outkey string + }{ + { + name: "normal", + pl: `json(_, status) + group_in(status, [true], "ok", "newkey")`, + in: `{"status": true,"age":"47"}`, + expected: "ok", + outkey: "newkey", + }, + { + name: "normal", + pl: `json(_, status) + group_in(status, [true], "ok", "newkey")`, + in: `{"status": true,"age":"47"}`, + expected: "ok", + outkey: "newkey", + }, + { + name: "normal", + pl: `json(_, status) + group_in(status, [true], "ok", "newkey")`, + in: `{"status": "aa","age":"47"}`, + expected: "aa", + outkey: "status", + }, + { + name: "normal", + pl: `json(_, status) + group_in(status, ["aa"], "ok", "newkey")`, + in: `{"status": "aa","age":"47"}`, + expected: "ok", + outkey: "newkey", + }, + { + name: "normal", + in: `{"log_level": "test","age":"47"}`, + pl: `json(_, log_level) + group_in(log_level, [200, "test"], 119)`, + expected: int64(119), + outkey: "log_level", + }, + { + name: "normal", + in: `{"log_level": "test","age":"47"}`, + pl: `json(_, log_level) + group_in(log_level, [200, "test1"], 119)`, + expected: "test", + outkey: "log_level", + }, + { + name: "normal", + in: `{"log_level": "test","age":"47"}`, + pl: `json(_, log_level) + group_in(log_level, [200, "test"], 119, "hh")`, + expected: int64(119), + outkey: "hh", + }, + } + + for idx, tc := range cases { + t.Run(tc.name, func(t *testing.T) { + runner, err := NewTestingRunner(tc.pl) + if err != nil { + if tc.fail { + t.Logf("[%d]expect error: %s", idx, err) + } else { + t.Errorf("[%d] failed: %s", idx, err) + } + return + } + + pt := ptinput.NewPlPoint( + point.Logging, "test", nil, map[string]any{"message": tc.in}, time.Now()) + errR := runScript(runner, pt) + + if errR != nil { + t.Fatal(errR.Error()) + } + + if tc.fail { + t.Logf("[%d]expect error: %s", idx, errR.Error()) + } + v, _, _ := pt.Get(tc.outkey) + tu.Equals(t, tc.expected, v) + t.Logf("[%d] PASS", idx) + }) + } +} diff --git a/pkg/inimpl/guancecloud/ptinput/funcs/fn_group_test.go b/pkg/inimpl/guancecloud/ptinput/funcs/fn_group_test.go new file mode 100644 index 0000000..dc26d4b --- /dev/null +++ b/pkg/inimpl/guancecloud/ptinput/funcs/fn_group_test.go @@ -0,0 +1,102 @@ +// Unless explicitly stated otherwise all files in this repository are licensed +// under the MIT License. +// This product includes software developed at Guance Cloud (https://www.guance.com/). +// Copyright 2021-present Guance, Inc. + +package funcs + +import ( + "testing" + "time" + + "github.com/GuanceCloud/cliutils/point" + tu "github.com/GuanceCloud/cliutils/testutil" + "github.com/GuanceCloud/platypus/pkg/inimpl/guancecloud/ptinput" +) + +func TestGroup(t *testing.T) { + cases := []struct { + name, pl, in string + expected interface{} + fail bool + outkey string + }{ + { + name: "normal", + pl: `json(_, status) + group_between(status, [200, 400], false, newkey)`, + in: `{"status": 200,"age":47}`, + expected: false, + outkey: "newkey", + }, + { + name: "normal", + pl: `json(_, status) + group_between(status, [200, 400], 10, newkey)`, + in: `{"status": 200,"age":47}`, + expected: int64(10), + outkey: "newkey", + }, + { + name: "normal", + pl: `json(_, status) + group_between(status, [200, 400], "ok", newkey)`, + in: `{"status": 200,"age":47}`, + expected: "ok", + outkey: "newkey", + }, + { + name: "normal", + pl: `json(_, status) + group_between(status, [200, 299], "ok")`, + in: `{"status": 200,"age":47}`, + expected: "ok", + outkey: "status", + }, + { + name: "normal", + pl: `json(_, status) + group_between(status, [200, 299], "ok", newkey)`, + in: `{"status": 200,"age":47}`, + expected: "ok", + outkey: "newkey", + }, + { + name: "normal", + pl: `json(_, status) + group_between(status, [300, 400], "ok", newkey)`, + in: `{"status": 200,"age":47}`, + expected: nil, + outkey: "newkey", + }, + } + + for idx, tc := range cases { + t.Run(tc.name, func(t *testing.T) { + runner, err := NewTestingRunner(tc.pl) + if err != nil { + if tc.fail { + t.Logf("[%d]expect error: %s", idx, err) + } else { + t.Errorf("[%d] failed: %s", idx, err) + } + return + } + + pt := ptinput.NewPlPoint( + point.Logging, "test", nil, map[string]any{"message": tc.in}, time.Now()) + errR := runScript(runner, pt) + + if errR != nil { + t.Fatal(errR.Error()) + } + + if tc.fail { + t.Logf("[%d]expect error: %s", idx, errR.Error()) + } + v, _, _ := pt.Get(tc.outkey) + tu.Equals(t, tc.expected, v) + t.Logf("[%d] PASS", idx) + }) + } +} diff --git a/pkg/inimpl/guancecloud/ptinput/funcs/fn_json.go b/pkg/inimpl/guancecloud/ptinput/funcs/fn_json.go new file mode 100644 index 0000000..773148b --- /dev/null +++ b/pkg/inimpl/guancecloud/ptinput/funcs/fn_json.go @@ -0,0 +1,310 @@ +// Unless explicitly stated otherwise all files in this repository are licensed +// under the MIT License. +// This product includes software developed at Guance Cloud (https://www.guance.com/). +// Copyright 2021-present Guance, Inc. + +package funcs + +import ( + "encoding/json" + "fmt" + "reflect" + "strings" + + "github.com/GuanceCloud/platypus/pkg/ast" + "github.com/GuanceCloud/platypus/pkg/engine/runtime" + "github.com/GuanceCloud/platypus/pkg/errchain" + "github.com/GuanceCloud/platypus/pkg/inimpl/guancecloud/ptinput" +) + +func JSONChecking(ctx *runtime.Context, funcExpr *ast.CallExpr) *errchain.PlError { + if err := reindexFuncArgs(funcExpr, []string{ + "input", "json_path", "newkey", + "trim_space", "delete_after_extract", + }, 2); err != nil { + return runtime.NewRunError(ctx, err.Error(), funcExpr.NamePos) + } + + if _, err := getKeyName(funcExpr.Param[0]); err != nil { + return runtime.NewRunError(ctx, err.Error(), funcExpr.Param[0].StartPos()) + } + + lastIdxExpr := false + switch funcExpr.Param[1].NodeType { //nolint:exhaustive + case ast.TypeAttrExpr, ast.TypeIdentifier, ast.TypeIndexExpr: + var err error + lastIdxExpr, err = lastIsIndex(funcExpr.Param[1]) + if err != nil { + return runtime.NewRunError(ctx, err.Error(), funcExpr.Param[1].StartPos()) + } + default: + return runtime.NewRunError(ctx, fmt.Sprintf("expect AttrExpr, IndexExpr or Identifier, got %s", + funcExpr.Param[1].NodeType), funcExpr.Param[1].StartPos()) + } + + if funcExpr.Param[2] != nil { + switch funcExpr.Param[2].NodeType { //nolint:exhaustive + case ast.TypeAttrExpr, ast.TypeIdentifier, ast.TypeStringLiteral: + default: + return runtime.NewRunError(ctx, fmt.Sprintf("expect AttrExpr or Identifier, got %s", + funcExpr.Param[2].NodeType), funcExpr.Param[2].StartPos()) + } + } + + if funcExpr.Param[3] != nil { + switch funcExpr.Param[3].NodeType { //nolint:exhaustive + case ast.TypeBoolLiteral: + default: + return runtime.NewRunError(ctx, fmt.Sprintf("expect BoolLiteral, got %s", + funcExpr.Param[3].NodeType), funcExpr.Param[3].StartPos()) + } + } + + if funcExpr.Param[4] != nil { + switch funcExpr.Param[4].NodeType { //nolint:exhaustive + case ast.TypeBoolLiteral: + if funcExpr.Param[4].BoolLiteral.Val == lastIdxExpr { + return runtime.NewRunError(ctx, "does not support deleting elements in the list", + funcExpr.Param[4].StartPos()) + } + default: + return runtime.NewRunError(ctx, fmt.Sprintf("expect BoolLiteral, got %s", + funcExpr.Param[3].NodeType), funcExpr.Param[4].StartPos()) + } + } + + return nil +} + +func JSON(ctx *runtime.Context, funcExpr *ast.CallExpr) *errchain.PlError { + var jpath *ast.Node + + srcKey, err := getKeyName(funcExpr.Param[0]) + if err != nil { + return runtime.NewRunError(ctx, err.Error(), funcExpr.Param[0].StartPos()) + } + + switch funcExpr.Param[1].NodeType { //nolint:exhaustive + case ast.TypeAttrExpr, ast.TypeIdentifier, ast.TypeIndexExpr: + jpath = funcExpr.Param[1] + // TODO StringLiteral + default: + return runtime.NewRunError(ctx, fmt.Sprintf("expect AttrExpr or Identifier, got %s", + funcExpr.Param[1].NodeType), funcExpr.Param[1].StartPos()) + } + + targetKey, _ := getKeyName(jpath) + + if funcExpr.Param[2] != nil { + switch funcExpr.Param[2].NodeType { //nolint:exhaustive + case ast.TypeAttrExpr, ast.TypeIdentifier, ast.TypeStringLiteral: + targetKey, _ = getKeyName(funcExpr.Param[2]) + default: + return runtime.NewRunError(ctx, fmt.Sprintf("expect AttrExpr or Identifier, got %s", + funcExpr.Param[2].NodeType), funcExpr.Param[2].StartPos()) + } + } + + cont, err := ctx.GetKeyConv2Str(srcKey) + if err != nil { + l.Debug(err) + return nil + } + + deleteAfterExtract := false + if funcExpr.Param[4] != nil { + switch funcExpr.Param[4].NodeType { //nolint:exhaustive + case ast.TypeBoolLiteral: + deleteAfterExtract = funcExpr.Param[4].BoolLiteral.Val + default: + return runtime.NewRunError(ctx, fmt.Sprintf("expect BoolLiteral, got %s", + funcExpr.Param[3].NodeType), funcExpr.Param[4].StartPos()) + } + } + + v, dstS, err := GsonGet(cont, jpath, deleteAfterExtract) + if err != nil { + l.Debug(err) + return nil + } + + trimSpace := true + if funcExpr.Param[3] != nil { + switch funcExpr.Param[3].NodeType { //nolint:exhaustive + case ast.TypeBoolLiteral: + trimSpace = funcExpr.Param[3].BoolLiteral.Val + default: + return runtime.NewRunError(ctx, fmt.Sprintf("expect BoolLiteral, got %s", + funcExpr.Param[3].NodeType), funcExpr.Param[3].StartPos()) + } + } + + if vStr, ok := v.(string); ok && trimSpace { + v = strings.TrimSpace(vStr) + } + + var dtype ast.DType + switch v.(type) { + case bool: + dtype = ast.Bool + case float64: + dtype = ast.Float + case string: + dtype = ast.String + case []any: + dtype = ast.List + case map[string]any: + dtype = ast.Map + default: + return nil + } + if err := addKey2PtWithVal(ctx.InData(), targetKey, v, dtype, ptinput.KindPtDefault); err != nil { + l.Debug(err) + return nil + } + + if deleteAfterExtract { + if err := addKey2PtWithVal(ctx.InData(), srcKey, dstS, ast.String, + ptinput.KindPtDefault); err != nil { + l.Debug(err) + return nil + } + } + + return nil +} + +func GsonGet(s string, node *ast.Node, deleteAfter bool) (any, string, error) { + var m any + + err := json.Unmarshal([]byte(s), &m) + if err != nil { + return "", "", err + } + + val, err := jsonGet(m, node, deleteAfter) + if err != nil { + return "", "", err + } + + dst := s + if deleteAfter { + dstB, err := json.Marshal(m) + if err != nil { + return "", "", err + } + dst = string(dstB) + } + return val, dst, nil +} + +func jsonGet(val any, node *ast.Node, deleteAfter bool) (any, error) { + switch node.NodeType { //nolint:exhaustive + case ast.TypeStringLiteral: + return getByIdentifier(val, &ast.Identifier{ + Name: node.StringLiteral.Val, + }, deleteAfter) + case ast.TypeAttrExpr: + return getByAttr(val, node.AttrExpr, deleteAfter) + + case ast.TypeIdentifier: + return getByIdentifier(val, node.Identifier, deleteAfter) + + case ast.TypeIndexExpr: + child, err := getByIdentifier(val, node.IndexExpr.Obj, false) + if err != nil { + return nil, err + } + return getByIndex(child, node.IndexExpr, 0, deleteAfter) + default: + return nil, fmt.Errorf("json unsupport get from %s", node.NodeType) + } +} + +func getByAttr(val any, i *ast.AttrExpr, deleteAfter bool) (any, error) { + if i.Attr != nil { + child, err := jsonGet(val, i.Obj, false) + if err != nil { + return nil, err + } + return jsonGet(child, i.Attr, deleteAfter) + } else { + child, err := jsonGet(val, i.Obj, deleteAfter) + if err != nil { + return nil, err + } + return child, nil + } +} + +func getByIdentifier(val any, i *ast.Identifier, deleteAfter bool) (any, error) { + if i == nil { + return val, nil + } + + switch v := val.(type) { + case map[string]any: + if child, ok := v[i.Name]; !ok { + return nil, fmt.Errorf("%v not found", i.Name) + } else { + if deleteAfter { + delete(v, i.Name) + } + return child, nil + } + default: + return nil, fmt.Errorf("%v unsupport identifier get", reflect.TypeOf(v)) + } +} + +func getByIndex(val any, i *ast.IndexExpr, dimension int, deleteAfter bool) (any, error) { + switch v := val.(type) { + case []any: + if dimension >= len(i.Index) { + return nil, fmt.Errorf("dimension exceed") + } + + var index int + + switch i.Index[dimension].NodeType { //nolint:exhaustive + case ast.TypeIntegerLiteral: + index = int(i.Index[dimension].IntegerLiteral.Val) + case ast.TypeFloatLiteral: + index = int(i.Index[dimension].FloatLiteral.Val) + + default: + return nil, fmt.Errorf("index value is not int") + } + + if index < 0 { + index = len(v) + index + } + + if index < 0 || index >= len(v) { + return nil, fmt.Errorf("index out of range") + } + + child := v[index] + if dimension == len(i.Index)-1 { + return child, nil + } else { + return getByIndex(child, i, dimension+1, deleteAfter) + } + default: + return nil, fmt.Errorf("%v unsupport index get", reflect.TypeOf(v)) + } +} + +func lastIsIndex(expr *ast.Node) (bool, error) { + switch expr.NodeType { //nolint:exhaustive + case ast.TypeAttrExpr: + return lastIsIndex(expr.AttrExpr.Attr) + case ast.TypeIdentifier: + return false, nil + case ast.TypeIndexExpr: + return true, nil + default: + return false, fmt.Errorf("expect AttrExpr, IndexExpr or Identifier, got %s", + expr.NodeType) + } +} diff --git a/pkg/inimpl/guancecloud/ptinput/funcs/fn_json_test.go b/pkg/inimpl/guancecloud/ptinput/funcs/fn_json_test.go new file mode 100644 index 0000000..05c7460 --- /dev/null +++ b/pkg/inimpl/guancecloud/ptinput/funcs/fn_json_test.go @@ -0,0 +1,164 @@ +// Unless explicitly stated otherwise all files in this repository are licensed +// under the MIT License. +// This product includes software developed at Guance Cloud (https://www.guance.com/). +// Copyright 2021-present Guance, Inc. + +package funcs + +import ( + "testing" + "time" + + "github.com/GuanceCloud/cliutils/point" + tu "github.com/GuanceCloud/cliutils/testutil" + "github.com/GuanceCloud/platypus/pkg/inimpl/guancecloud/ptinput" + "github.com/stretchr/testify/assert" +) + +func TestJSON(t *testing.T) { + testCase := []*funcCase{ + { + in: `{ + "name": {"first": "Tom", "last": "Anderson"}, + "age":37, + "children": ["Sara","Alex","Jack"], + "fav.movie": "Deer Hunter", + "friends": [ + {"first": "Dale", "last": "Murphy", "age": 44, "nets": ["ig", "fb", "tw"]}, + {"first": "Roger", "last": "Craig", "age": 68, "nets": ["fb", "tw"]}, + {"first": "Jane", "last": "Murphy", "age": 47, "nets": ["ig", "tw"]} + ] + }`, + script: `json(_, name) + json(name, first)`, + expected: "Tom", + key: "first", + }, + { + in: `{ + "name": {"first": "Tom", "last": "Anderson"}, + "age":37, + "children": ["Sara","Alex","Jack"], + "fav.movie": "Deer Hunter", + "friends": [ + {"first": "Dale", "last": "Murphy", "age": 44, "nets": ["ig", "fb", "tw"]}, + {"first": "Roger", "last": "Craig", "age": 68, "nets": ["fb", "tw"]}, + {"first": "Jane", "last": "Murphy", "age": 47, "nets": ["ig", "tw"]} + ] + }`, + script: `json(_, friends) + json(friends, .[1].first, f_first)`, + expected: "Roger", + key: "f_first", + }, + { + in: `[ + {"first": "Dale", "last": "Murphy", "age": 44, "nets": ["ig", "fb", "tw"]}, + {"first": "Roger", "last": "Craig", "age": 68, "nets": ["fb", "tw"]}, + {"first": "Jane", "last": "Murphy", "age": 47, "nets": ["ig", "tw"]} + ]`, + script: `json(_, .[0].nets[-1])`, + expected: "tw", + key: "[0].nets[-1]", + }, + { + in: `[ + {"first": "Dale", "last": "Murphy", "age": 44, "nets": ["ig", "fb", "tw"]}, + {"first": "Roger", "last": "Craig", "age": 68, "nets": ["fb", "tw"]}, + {"first": "Jane", "last": "Murphy", "age": 47, "nets": ["ig", "tw"]} + ]`, + script: `json(_, .[1].age)`, + expected: float64(68), + key: "[1].age", + }, + { + name: "trim_space auto", + in: `{"item": " not_space "}`, + script: `json(_, item, item)`, + key: "item", + expected: "not_space", + }, + { + name: "trim_space disable", + in: `{"item": " not_space "}`, + script: `json(_, item, item, false)`, + key: "item", + expected: " not_space ", + }, + { + name: "trim_space enable", + in: `{"item": " not_space "}`, + script: `json(_, item, item, true)`, + key: "item", + expected: "not_space", + }, + { + name: "map_delete_after", + in: `{"item": " not_space "}`, + script: `json(_, item, item, true, true)`, + key: "message", + expected: "{}", + fail: false, + }, + { + name: "map_delete_after1", + in: `{"item": " not_space ", "item2":{"item3": [123]}}`, + script: `json(_, item2.item3, item, delete_after_extract = true)`, + key: "message", + expected: `{"item":" not_space ","item2":{}}`, + }, + { + name: "list_delete_after1", + in: `{"item": " not_space ", "item2": [[1,2,3,4,5],[6]]}`, + script: `json(_, .[0].item2[0][2].a[0], item, true, true)`, + key: "item", + expected: "1", + fail: true, + }, + { + name: "list_delete_after2", + in: `{"item": " not_space ", "item2": [[1,2,3,4,5],[6]]}`, + script: `json(_, .[0], item, true, true)`, + key: "item", + expected: "1", + fail: true, + }, + { + name: "list_delete_after3", + in: `{"item": " not_space ", "item2": [[1,2,3,4,5],[6]]}`, + script: `json(_, a[0][1], item, true, true)`, + key: "item", + expected: "1", + fail: true, + }, + } + + for idx, tc := range testCase { + t.Run(tc.name, func(t *testing.T) { + runner, err := NewTestingRunner(tc.script) + + if err != nil && tc.fail { + return + } else if err != nil || tc.fail { + tu.Equals(t, nil, err) + tu.Equals(t, tc.fail, err != nil) + } + + pt := ptinput.NewPlPoint( + point.Logging, "test", nil, map[string]any{"message": tc.in}, time.Now()) + errR := runScript(runner, pt) + if errR != nil { + t.Fatal(errR.Error()) + } + + r, _, ok := pt.GetWithIsTag(tc.key) + tu.Equals(t, true, ok) + if tc.key == "[2].age" { + t.Log(1) + } + assert.Equal(t, tc.expected, r) + + t.Logf("[%d] PASS", idx) + }) + } +} diff --git a/pkg/inimpl/guancecloud/ptinput/funcs/fn_jsonall.go b/pkg/inimpl/guancecloud/ptinput/funcs/fn_jsonall.go new file mode 100644 index 0000000..e8e6589 --- /dev/null +++ b/pkg/inimpl/guancecloud/ptinput/funcs/fn_jsonall.go @@ -0,0 +1,22 @@ +// Unless explicitly stated otherwise all files in this repository are licensed +// under the MIT License. +// This product includes software developed at Guance Cloud (https://www.guance.com/). +// Copyright 2021-present Guance, Inc. + +package funcs + +import ( + "github.com/GuanceCloud/platypus/pkg/ast" + "github.com/GuanceCloud/platypus/pkg/engine/runtime" + "github.com/GuanceCloud/platypus/pkg/errchain" +) + +func JSONAllChecking(ctx *runtime.Context, funcExpr *ast.CallExpr) *errchain.PlError { + l.Debugf("warning: json_all() is disabled") + return nil +} + +func JSONAll(ctx *runtime.Context, funcExpr *ast.CallExpr) *errchain.PlError { + l.Debugf("warning: json_all() is disabled") + return nil +} diff --git a/pkg/inimpl/guancecloud/ptinput/funcs/fn_kv.go b/pkg/inimpl/guancecloud/ptinput/funcs/fn_kv.go new file mode 100644 index 0000000..9e26d0b --- /dev/null +++ b/pkg/inimpl/guancecloud/ptinput/funcs/fn_kv.go @@ -0,0 +1,328 @@ +// Unless explicitly stated otherwise all files in this repository are licensed +// under the MIT License. +// This product includes software developed at Guance Cloud (https://www.guance.com/). +// Copyright 2021-present Guance, Inc. + +package funcs + +import ( + "fmt" + "regexp" + "strings" + "sync" + + "github.com/GuanceCloud/platypus/pkg/ast" + "github.com/GuanceCloud/platypus/pkg/engine/runtime" + "github.com/GuanceCloud/platypus/pkg/errchain" + "github.com/GuanceCloud/platypus/pkg/inimpl/guancecloud/ptinput" +) + +var ( + _defaultFieldSplitPattern = regexp.MustCompile(" ") + _defaultValueSplitPattern = regexp.MustCompile("=") +) + +var _regexpCache = reCache{ + m: map[string]*regexp.Regexp{}, +} + +type reCache struct { + m map[string]*regexp.Regexp + sync.RWMutex +} + +func (c *reCache) set(p string) error { + c.Lock() + defer c.Unlock() + + if c.m == nil { + return fmt.Errorf("nil ptr") + } + + if r, err := regexp.Compile(p); err != nil { + return err + } else { + c.m[p] = r + } + + return nil +} + +func (c *reCache) get(p string) (*regexp.Regexp, bool) { + c.RLock() + defer c.RUnlock() + if c.m != nil { + v, ok := c.m[p] + return v, ok + } + + return nil, false +} + +func KVSplitChecking(ctx *runtime.Context, funcExpr *ast.CallExpr) *errchain.PlError { + if err := reindexFuncArgs(funcExpr, []string{ + "key", "field_split_pattern", "value_split_pattern", + "trim_key", "trim_value", "include_keys", "prefix", + }, 1); err != nil { + return runtime.NewRunError(ctx, err.Error(), funcExpr.NamePos) + } + + if _, err := getKeyName(funcExpr.Param[0]); err != nil { + return runtime.NewRunError(ctx, err.Error(), funcExpr.Param[0].StartPos()) + } + + // field_split_pattern + if funcExpr.Param[1] != nil { + switch funcExpr.Param[1].NodeType { //nolint:exhaustive + case ast.TypeStringLiteral: + p := funcExpr.Param[1].StringLiteral.Val + if err := _regexpCache.set(p); err != nil { + return runtime.NewRunError(ctx, err.Error(), funcExpr.Param[1].StartPos()) + } + default: + return runtime.NewRunError(ctx, fmt.Sprintf("param field_split_pattern expect StringLiteral, got %s", + funcExpr.Param[0].NodeType), funcExpr.NamePos) + } + } + + // value_split_pattern + if funcExpr.Param[2] != nil { + switch funcExpr.Param[2].NodeType { //nolint:exhaustive + case ast.TypeStringLiteral: + p := funcExpr.Param[2].StringLiteral.Val + if err := _regexpCache.set(p); err != nil { + return runtime.NewRunError(ctx, err.Error(), funcExpr.Param[2].StartPos()) + } + default: + return runtime.NewRunError(ctx, fmt.Sprintf("param value_split_pattern expect StringLiteral, got %s", + funcExpr.Param[2].NodeType), funcExpr.NamePos) + } + } + + if funcExpr.Param[3] != nil { + switch funcExpr.Param[3].NodeType { //nolint:exhaustive + case ast.TypeStringLiteral: + default: + return runtime.NewRunError(ctx, fmt.Sprintf("param trim_key expect StringLiteral, got %s", + funcExpr.Param[3].NodeType), funcExpr.NamePos) + } + } + + if funcExpr.Param[4] != nil { + switch funcExpr.Param[4].NodeType { //nolint:exhaustive + case ast.TypeStringLiteral: + default: + return runtime.NewRunError(ctx, fmt.Sprintf("param trim_value expect StringLiteral, got %s", + funcExpr.Param[4].NodeType), funcExpr.NamePos) + } + } + + if funcExpr.Param[5] != nil { + switch funcExpr.Param[5].NodeType { //nolint:exhaustive + case ast.TypeListInitExpr, ast.TypeIdentifier: + default: + return runtime.NewRunError(ctx, fmt.Sprintf("param include_keys expect ListInitExpr or Identifier, got %s", + funcExpr.Param[5].NodeType), funcExpr.NamePos) + } + } + + if funcExpr.Param[6] != nil { + switch funcExpr.Param[6].NodeType { //nolint:exhaustive + case ast.TypeStringLiteral: + default: + return runtime.NewRunError(ctx, fmt.Sprintf("param prefix expect StringLiteral, got %s", + funcExpr.Param[6].NodeType), funcExpr.NamePos) + } + } + return nil +} + +func KVSplit(ctx *runtime.Context, funcExpr *ast.CallExpr) *errchain.PlError { + key, err := getKeyName(funcExpr.Param[0]) + if err != nil { + return runtime.NewRunError(ctx, err.Error(), funcExpr.Param[0].StartPos()) + } + + val, err := ctx.GetKeyConv2Str(key) + if err != nil { + ctx.Regs.ReturnAppend(false, ast.Bool) + return nil + } + + var fieldSplit, valueSplit *regexp.Regexp + + // field_split_pattern + if funcExpr.Param[1] != nil { + switch funcExpr.Param[1].NodeType { //nolint:exhaustive + case ast.TypeStringLiteral: + p := funcExpr.Param[1].StringLiteral.Val + var ok bool + + fieldSplit, ok = _regexpCache.get(p) + if !ok { + l.Debugf("field split pattern %s not found", p) + ctx.Regs.ReturnAppend(false, ast.Bool) + return nil + } + default: + return runtime.NewRunError(ctx, fmt.Sprintf("param field_split_pattern expect StringLiteral, got %s", + funcExpr.Param[0].NodeType), funcExpr.NamePos) + } + } + + if funcExpr.Param[2] != nil { + switch funcExpr.Param[2].NodeType { //nolint:exhaustive + case ast.TypeStringLiteral: + p := funcExpr.Param[2].StringLiteral.Val + + var ok bool + + valueSplit, ok = _regexpCache.get(p) + if !ok { + l.Debugf("value split pattern %s not found", p) + ctx.Regs.ReturnAppend(false, ast.Bool) + return nil + } + default: + return runtime.NewRunError(ctx, fmt.Sprintf("param value_split_pattern expect StringLiteral, got %s", + funcExpr.Param[2].NodeType), funcExpr.NamePos) + } + } + + var trimKey, trimValue string + if funcExpr.Param[3] != nil { + switch funcExpr.Param[3].NodeType { //nolint:exhaustive + case ast.TypeStringLiteral: + trimKey = funcExpr.Param[3].StringLiteral.Val + default: + return runtime.NewRunError(ctx, fmt.Sprintf("param trim_key expect StringLiteral, got %s", + funcExpr.Param[3].NodeType), funcExpr.NamePos) + } + } + + if funcExpr.Param[4] != nil { + switch funcExpr.Param[4].NodeType { //nolint:exhaustive + case ast.TypeStringLiteral: + trimValue = funcExpr.Param[4].StringLiteral.Val + default: + return runtime.NewRunError(ctx, fmt.Sprintf("param trim_value expect StringLiteral, got %s", + funcExpr.Param[4].NodeType), funcExpr.NamePos) + } + } + + var includeKeys []string + if funcExpr.Param[5] != nil { + switch funcExpr.Param[5].NodeType { //nolint:exhaustive + case ast.TypeListInitExpr, ast.TypeIdentifier: + v, dt, err := runtime.RunStmt(ctx, funcExpr.Param[5]) + if err != nil { + return err + } + if dt != ast.List { + break + } + switch v := v.(type) { + case []any: + for _, k := range v { + if k, ok := k.(string); ok { + includeKeys = append(includeKeys, k) + } + } + default: + } + + default: + return runtime.NewRunError(ctx, fmt.Sprintf("param include_keys expect ListInitExpr or Identifier, got %s", + funcExpr.Param[5].NodeType), funcExpr.NamePos) + } + } + + if len(includeKeys) == 0 { + ctx.Regs.ReturnAppend(false, ast.Bool) + return nil + } + + var prefix string + if funcExpr.Param[6] != nil { + switch funcExpr.Param[6].NodeType { //nolint:exhaustive + case ast.TypeStringLiteral: + prefix = funcExpr.Param[6].StringLiteral.Val + default: + return runtime.NewRunError(ctx, fmt.Sprintf("param prefix expect StringLiteral, got %s", + funcExpr.Param[6].NodeType), funcExpr.NamePos) + } + } + + result := kvSplit(val, includeKeys, fieldSplit, valueSplit, trimKey, trimValue, prefix) + if len(result) == 0 { + ctx.Regs.ReturnAppend(false, ast.Bool) + return nil + } + + for k, v := range result { + if err := addKey2PtWithVal(ctx.InData(), k, v, ast.String, ptinput.KindPtDefault); err != nil { + l.Debug(err) + } + } + + ctx.Regs.ReturnAppend(true, ast.Bool) + return nil +} + +func kvSplit(str string, includeKeys []string, fieldSplit, valueSplit *regexp.Regexp, + trimKey, trimValue, prefix string, +) map[string]string { + if str == "" { + return nil + } + + if fieldSplit == nil { + fieldSplit = _defaultFieldSplitPattern + } + + if valueSplit == nil { + valueSplit = _defaultValueSplitPattern + } + + ks := map[string]struct{}{} + + for _, v := range includeKeys { + ks[v] = struct{}{} + } + + result := map[string]string{} + fields := fieldSplit.Split(str, -1) + for _, field := range fields { + keyValue := valueSplit.Split(field, 2) + + if len(keyValue) == 2 { + // trim key + if tk := strings.Trim(keyValue[0], trimKey); tk != "" { + keyValue[0] = tk + } else { + continue + } + + // !include ? continue : ; + if len(ks) > 0 { + if _, ok := ks[keyValue[0]]; !ok { + continue + } + } + + // trim value + if trimValue != "" { + keyValue[1] = strings.Trim(keyValue[1], trimValue) + } + + // prefix + key + if prefix != "" { + keyValue[0] = prefix + keyValue[0] + } + + // append to result + result[keyValue[0]] = keyValue[1] + } + } + return result +} diff --git a/pkg/inimpl/guancecloud/ptinput/funcs/fn_kv_test.go b/pkg/inimpl/guancecloud/ptinput/funcs/fn_kv_test.go new file mode 100644 index 0000000..94a9cec --- /dev/null +++ b/pkg/inimpl/guancecloud/ptinput/funcs/fn_kv_test.go @@ -0,0 +1,108 @@ +// Unless explicitly stated otherwise all files in this repository are licensed +// under the MIT License. +// This product includes software developed at Guance Cloud (https://www.guance.com/). +// Copyright 2021-present Guance, Inc. + +package funcs + +import ( + "testing" + "time" + + "github.com/GuanceCloud/cliutils/point" + tu "github.com/GuanceCloud/cliutils/testutil" + "github.com/GuanceCloud/platypus/pkg/inimpl/guancecloud/ptinput" +) + +func TestKV(t *testing.T) { + cases := []struct { + name, pl, in string + expected []interface{} + fail bool + outkey []string + }{ + { + name: "normal", + pl: `kv_split(_)`, + in: `a=1 b=2 c=3`, + expected: []any{nil, nil, nil}, + outkey: []string{"a", "b", "c"}, + }, + { + name: "normal1", + pl: `kv_split(_, + include_keys=["a", "b","c"])`, + in: `a=1, b=2 c=3`, + expected: []any{"1,", "2", "3"}, + outkey: []string{"a", "b", "c"}, + }, + { + name: "trim_value", + pl: `kv_split(_, trim_value=",", + include_keys=["a", "b","c"])`, + in: `a=1, b=2 c=3`, + expected: []any{"1", "2", "3"}, + outkey: []string{"a", "b", "c"}, + }, + { + name: "trim_key_skip_empty_str_key", + pl: `kv_split(_, trim_value=",", trim_key="a", + include_keys=["a", "b","c"])`, + in: `a=1, b=2 c=3`, + expected: []any{nil, nil, "2", "3"}, + outkey: []string{"", "a", "b", "c"}, + }, + { + name: "prefix", + pl: `kv_split(_, prefix="prefix_",trim_value=",", trim_key="a", + include_keys=["a", "b","c"])`, + in: `a=1, b=2 c=3`, + expected: []any{nil, nil, "2", "3"}, + outkey: []string{"", "prefix_", "prefix_b", "prefix_c"}, + }, + { + name: "prefix", + pl: `kv_split(_, include_keys= ["b", ],prefix="prefix_",trim_value=",", trim_key="a")`, + in: `a=1, b=2 c=3`, + expected: []any{nil, nil, "2", nil}, + outkey: []string{"", "prefix_", "prefix_b", "prefix_c"}, + }, + { + name: "value_split_pattern", + pl: `kv_split(_, include_keys= ["b", ], field_split_pattern="\\+", value_split_pattern="::", + prefix="prefix_",trim_value=",", trim_key="a")`, + in: `a::1,+b::2+c::3`, + expected: []any{nil, nil, "2", nil}, + outkey: []string{"", "prefix_", "prefix_b", "prefix_c"}, + }, + } + + for idx, tc := range cases { + t.Run(tc.name, func(t *testing.T) { + runner, err := NewTestingRunner(tc.pl) + if err != nil { + if tc.fail { + t.Logf("[%d]expect error: %s", idx, err) + } else { + t.Errorf("[%d] failed: %s", idx, err) + } + return + } + pt := ptinput.NewPlPoint( + point.Logging, "test", nil, map[string]any{"message": tc.in}, time.Now()) + errR := runScript(runner, pt) + + if errR != nil { + t.Fatal(errR.Error()) + } + + for i := range tc.outkey { + // tu.Equals(t, nil, err) + t.Log(tc.outkey[i]) + v, _, _ := pt.Get(tc.outkey[i]) + tu.Equals(t, tc.expected[i], v) + t.Logf("[%d] PASS", idx) + } + }) + } +} diff --git a/pkg/inimpl/guancecloud/funcs/fn_len.go b/pkg/inimpl/guancecloud/ptinput/funcs/fn_len.go similarity index 100% rename from pkg/inimpl/guancecloud/funcs/fn_len.go rename to pkg/inimpl/guancecloud/ptinput/funcs/fn_len.go diff --git a/pkg/inimpl/guancecloud/funcs/fn_len_test.go b/pkg/inimpl/guancecloud/ptinput/funcs/fn_len_test.go similarity index 73% rename from pkg/inimpl/guancecloud/funcs/fn_len_test.go rename to pkg/inimpl/guancecloud/ptinput/funcs/fn_len_test.go index d08c93a..e4170db 100644 --- a/pkg/inimpl/guancecloud/funcs/fn_len_test.go +++ b/pkg/inimpl/guancecloud/ptinput/funcs/fn_len_test.go @@ -9,7 +9,9 @@ import ( "testing" "time" - "github.com/stretchr/testify/assert" + "github.com/GuanceCloud/cliutils/point" + tu "github.com/GuanceCloud/cliutils/testutil" + "github.com/GuanceCloud/platypus/pkg/inimpl/guancecloud/ptinput" ) func TestLen(t *testing.T) { @@ -56,17 +58,17 @@ func TestLen(t *testing.T) { } return } - _, _, f, _, _, err := runScript(runner, "test", nil, map[string]interface{}{ - "message": tc.in, - }, time.Now()) + pt := ptinput.NewPlPoint( + point.Logging, "test", nil, map[string]any{"message": tc.in}, time.Now()) + errR := runScript(runner, pt) - assert.Equal(t, nil, err) - - t.Log(f) - v := f[tc.outkey] - // assert.Equal(t, nil, err) - assert.Equal(t, tc.expected, v) + if errR != nil { + t.Fatal(errR.Error()) + } + v, _, _ := pt.Get(tc.outkey) + // tu.Equals(t, nil, err) + tu.Equals(t, tc.expected, v) t.Logf("[%d] PASS", idx) }) } diff --git a/pkg/inimpl/guancecloud/funcs/fn_load_json.go b/pkg/inimpl/guancecloud/ptinput/funcs/fn_load_json.go similarity index 88% rename from pkg/inimpl/guancecloud/funcs/fn_load_json.go rename to pkg/inimpl/guancecloud/ptinput/funcs/fn_load_json.go index 1840e27..cbd558e 100644 --- a/pkg/inimpl/guancecloud/funcs/fn_load_json.go +++ b/pkg/inimpl/guancecloud/ptinput/funcs/fn_load_json.go @@ -17,7 +17,7 @@ import ( func LoadJSONChecking(ctx *runtime.Context, funcExpr *ast.CallExpr) *errchain.PlError { if len(funcExpr.Param) != 1 { return runtime.NewRunError(ctx, fmt.Sprintf( - "func %s expected 1", funcExpr.Name), funcExpr.NamePos) + "func %s expects 1 arg", funcExpr.Name), funcExpr.NamePos) } return nil } @@ -35,7 +35,8 @@ func LoadJSON(ctx *runtime.Context, funcExpr *ast.CallExpr) *errchain.PlError { } errJ := json.Unmarshal([]byte(val.(string)), &m) if errJ != nil { - return runtime.NewRunError(ctx, errJ.Error(), funcExpr.Param[0].StartPos()) + ctx.Regs.ReturnAppend(nil, ast.Nil) + return nil } m, dtype = ast.DectDataType(m) diff --git a/pkg/inimpl/guancecloud/funcs/fn_load_json_test.go b/pkg/inimpl/guancecloud/ptinput/funcs/fn_load_json_test.go similarity index 83% rename from pkg/inimpl/guancecloud/funcs/fn_load_json_test.go rename to pkg/inimpl/guancecloud/ptinput/funcs/fn_load_json_test.go index 18a4184..7937588 100644 --- a/pkg/inimpl/guancecloud/funcs/fn_load_json_test.go +++ b/pkg/inimpl/guancecloud/ptinput/funcs/fn_load_json_test.go @@ -9,7 +9,9 @@ import ( "testing" "time" - "github.com/stretchr/testify/assert" + "github.com/GuanceCloud/cliutils/point" + tu "github.com/GuanceCloud/cliutils/testutil" + "github.com/GuanceCloud/platypus/pkg/inimpl/guancecloud/ptinput" ) func TestLoadJson(t *testing.T) { @@ -90,16 +92,17 @@ func TestLoadJson(t *testing.T) { } return } - _, _, f, _, _, err := runScript(runner, "test", nil, map[string]interface{}{ - "message": tc.in, - }, time.Now()) + pt := ptinput.NewPlPoint( + point.Logging, "test", nil, map[string]any{"message": tc.in}, time.Now()) + errR := runScript(runner, pt) - assert.Equal(t, nil, err) + if errR != nil { + t.Fatal(errR.Error()) + } - t.Log(f) - v := f[tc.outkey] - // assert.Equal(t, nil, err) - assert.Equal(t, tc.expected, v) + v, _, _ := pt.Get(tc.outkey) + // tu.Equals(t, nil, err) + tu.Equals(t, tc.expected, v) t.Logf("[%d] PASS", idx) }) diff --git a/pkg/inimpl/guancecloud/ptinput/funcs/fn_lowercase.go b/pkg/inimpl/guancecloud/ptinput/funcs/fn_lowercase.go new file mode 100644 index 0000000..8dc7f5b --- /dev/null +++ b/pkg/inimpl/guancecloud/ptinput/funcs/fn_lowercase.go @@ -0,0 +1,56 @@ +// Unless explicitly stated otherwise all files in this repository are licensed +// under the MIT License. +// This product includes software developed at Guance Cloud (https://www.guance.com/). +// Copyright 2021-present Guance, Inc. + +package funcs + +import ( + "fmt" + "strings" + + "github.com/GuanceCloud/platypus/pkg/ast" + "github.com/GuanceCloud/platypus/pkg/engine/runtime" + "github.com/GuanceCloud/platypus/pkg/errchain" + "github.com/GuanceCloud/platypus/pkg/inimpl/guancecloud/ptinput" +) + +func LowercaseChecking(ctx *runtime.Context, funcExpr *ast.CallExpr) *errchain.PlError { + if len(funcExpr.Param) != 1 { + return runtime.NewRunError(ctx, fmt.Sprintf( + "func %s expected 1 args", funcExpr.Name), funcExpr.NamePos) + } + + if _, err := getKeyName(funcExpr.Param[0]); err != nil { + return runtime.NewRunError(ctx, err.Error(), funcExpr.Param[0].StartPos()) + } + + return nil +} + +func Lowercase(ctx *runtime.Context, funcExpr *ast.CallExpr) *errchain.PlError { + if len(funcExpr.Param) != 1 { + return runtime.NewRunError(ctx, fmt.Sprintf( + "func %s expected 1 args", funcExpr.Name), funcExpr.NamePos) + } + + key, err := getKeyName(funcExpr.Param[0]) + if err != nil { + return runtime.NewRunError(ctx, err.Error(), funcExpr.Param[0].StartPos()) + } + + cont, err := ctx.GetKeyConv2Str(key) + if err != nil { + l.Debug(err) + return nil + } + + v := strings.ToLower(cont) + if err = addKey2PtWithVal(ctx.InData(), key, v, ast.String, + ptinput.KindPtDefault); err != nil { + l.Debug(err) + return nil + } + + return nil +} diff --git a/pkg/inimpl/guancecloud/ptinput/funcs/fn_lowercase_test.go b/pkg/inimpl/guancecloud/ptinput/funcs/fn_lowercase_test.go new file mode 100644 index 0000000..c609498 --- /dev/null +++ b/pkg/inimpl/guancecloud/ptinput/funcs/fn_lowercase_test.go @@ -0,0 +1,106 @@ +// Unless explicitly stated otherwise all files in this repository are licensed +// under the MIT License. +// This product includes software developed at Guance Cloud (https://www.guance.com/). +// Copyright 2021-present Guance, Inc. + +package funcs + +import ( + "testing" + "time" + + "github.com/GuanceCloud/cliutils/point" + tu "github.com/GuanceCloud/cliutils/testutil" + "github.com/GuanceCloud/platypus/pkg/inimpl/guancecloud/ptinput" +) + +func TestLowercase(t *testing.T) { + cases := []struct { + name, pl, in string + expected interface{} + fail bool + outkey string + }{ + { + name: "normal", + pl: `json(_, a.third) +lowercase(a.third) +`, + in: `{"a":{"first":2.3,"second":2,"third":"aBC","forth":true},"age":47}`, + expected: "abc", + outkey: "a.third", + }, + { + name: "normal", + pl: `json(_, a.third) +lowercase(a.third) +`, + in: `{"a":{"first":2.3,"second":2,"third":"aBC","forth":true,"age":"WWW"},"age":"wWW"}`, + expected: nil, + outkey: "a.age", + }, + { + name: "normal", + pl: `json(_, a.third) +lowercase(a.third) +`, + in: `{"a":{"first":2.3,"second":2,"third":"aBC","forth":true,"age":"WWW"},"age":"wWW"}`, + expected: nil, + outkey: "a.forth", + }, + { + name: "normal", + pl: `json(_, a.third) +lowercase(a.third) +`, + in: `{"a":{"first":"222SSd","second":2,"third":"aBC","forth":true,"age":"WWW"},"age":"wWW"}`, + expected: nil, + outkey: "a.first", + }, + { + name: "normal", + pl: `json(_, a.first) +lowercase(a.first) +`, + in: `{"a":{"first":"SSd","second":2,"third":"aBC","forth":true,"age":"WWW"},"age":"wWW"}`, + expected: "ssd", + outkey: "a.first", + }, + { + name: "normal", + pl: `json(_, age) +lowercase(age) +`, + in: `{"a":{"first":"SSd","second":2,"third":"aBC","forth":true,"age":"WWW"},"age":"wWW"}`, + expected: "www", + outkey: "age", + }, + } + for idx, tc := range cases { + t.Run(tc.name, func(t *testing.T) { + runner, err := NewTestingRunner(tc.pl) + if err != nil { + if tc.fail { + t.Logf("[%d]expect error: %s", idx, err) + } else { + t.Errorf("[%d] failed: %s", idx, err) + } + return + } + + pt := ptinput.NewPlPoint( + point.Logging, "test", nil, map[string]any{"message": tc.in}, time.Now()) + errR := runScript(runner, pt) + + if errR != nil { + t.Fatal(errR.Error()) + } + + v, _, _ := pt.Get(tc.outkey) + // tu.Equals(t, nil, err) + tu.Equals(t, tc.expected, v) + + t.Logf("[%d] PASS", idx) + }) + } +} diff --git a/pkg/inimpl/guancecloud/ptinput/funcs/fn_match.go b/pkg/inimpl/guancecloud/ptinput/funcs/fn_match.go new file mode 100644 index 0000000..050b998 --- /dev/null +++ b/pkg/inimpl/guancecloud/ptinput/funcs/fn_match.go @@ -0,0 +1,72 @@ +// Unless explicitly stated otherwise all files in this repository are licensed +// under the MIT License. +// This product includes software developed at Guance Cloud (https://www.guance.com/). +// Copyright 2021-present Guance, Inc. + +package funcs + +import ( + "fmt" + "regexp" + + "github.com/GuanceCloud/platypus/pkg/ast" + "github.com/GuanceCloud/platypus/pkg/engine/runtime" + "github.com/GuanceCloud/platypus/pkg/errchain" +) + +func MatchChecking(ctx *runtime.Context, funcExpr *ast.CallExpr) *errchain.PlError { + if len(funcExpr.Param) != 2 { + return runtime.NewRunError(ctx, fmt.Sprintf( + "func %s expects 2 args", funcExpr.Name), funcExpr.NamePos) + } + + param0 := funcExpr.Param[0] + if param0.NodeType != ast.TypeStringLiteral { + return runtime.NewRunError(ctx, fmt.Sprintf( + "expect StringLiteral, got %s", + param0.NodeType), param0.StartPos()) + } + + // 预先编译正则表达式 + re, err := regexp.Compile(param0.StringLiteral.Val) + if err != nil { + return runtime.NewRunError(ctx, fmt.Sprintf( + "func match: regexp compile failed: %s", err.Error()), param0.StartPos()) + } + funcExpr.Re = re + + param1 := funcExpr.Param[1] + if !isPlVarbOrFunc(param1) && param1.NodeType != ast.TypeStringLiteral { + return runtime.NewRunError(ctx, fmt.Sprintf("expect StringLiteral, Identifier or AttrExpr, got %s", + param1.NodeType), param1.StartPos()) + } + + return nil +} + +func Match(ctx *runtime.Context, funcExpr *ast.CallExpr) *errchain.PlError { + if len(funcExpr.Param) != 2 { + err := fmt.Errorf("func %s expects 2 args", funcExpr.Name) + l.Debug(err) + ctx.Regs.ReturnAppend(false, ast.Bool) + return nil + } + + if funcExpr.Re == nil { + ctx.Regs.ReturnAppend(false, ast.Bool) + return nil + } + + cnt, err := getStr(ctx, funcExpr.Param[1]) + if err != nil { + l.Debug(err) + ctx.Regs.ReturnAppend(false, ast.Bool) + return nil + } + + isMatch := funcExpr.Re.MatchString(cnt) + + ctx.Regs.ReturnAppend(isMatch, ast.Bool) + + return nil +} diff --git a/pkg/inimpl/guancecloud/ptinput/funcs/fn_match_test.go b/pkg/inimpl/guancecloud/ptinput/funcs/fn_match_test.go new file mode 100644 index 0000000..24e9b2a --- /dev/null +++ b/pkg/inimpl/guancecloud/ptinput/funcs/fn_match_test.go @@ -0,0 +1,77 @@ +// Unless explicitly stated otherwise all files in this repository are licensed +// under the MIT License. +// This product includes software developed at Guance Cloud (https://www.guance.com/). +// Copyright 2021-present Guance, Inc. + +package funcs + +import ( + "testing" + "time" + + "github.com/GuanceCloud/cliutils/point" + "github.com/GuanceCloud/platypus/pkg/inimpl/guancecloud/ptinput" + "github.com/stretchr/testify/assert" +) + +func TestMatch(t *testing.T) { + cases := []struct { + name, pl, in string + expected interface{} + fail bool + outkey string + }{ + { + name: "normal", + pl: `abc = "sss" + add_key(abc, match("\\w+", abc))`, + in: `test`, + expected: true, + outkey: "abc", + }, + { + name: "normal", + pl: `abc = "sss" + add_key(abc, match("sss", abc))`, + in: `test`, + expected: true, + outkey: "abc", + }, + { + name: "normal", + pl: `abc = "sss" + add_key(abc, match(abc, abc))`, + in: `test`, + expected: true, + outkey: "abc", + fail: true, + }, + } + + for idx, tc := range cases { + t.Run(tc.name, func(t *testing.T) { + runner, err := NewTestingRunner(tc.pl) + if err != nil { + if tc.fail { + t.Logf("[%d]expect error: %s", idx, err) + } else { + t.Errorf("[%d] failed: %s", idx, err) + } + return + } + pt := ptinput.NewPlPoint( + point.Logging, "test", nil, map[string]any{"message": tc.in}, time.Now()) + errR := runScript(runner, pt) + + if errR != nil { + t.Fatal(errR.Error()) + } + + v, _, _ := pt.Get(tc.outkey) + // tu.Equals(t, nil, err) + assert.Equal(t, tc.expected, v) + + t.Logf("[%d] PASS", idx) + }) + } +} diff --git a/pkg/inimpl/guancecloud/ptinput/funcs/fn_nullif.go b/pkg/inimpl/guancecloud/ptinput/funcs/fn_nullif.go new file mode 100644 index 0000000..2e206c2 --- /dev/null +++ b/pkg/inimpl/guancecloud/ptinput/funcs/fn_nullif.go @@ -0,0 +1,63 @@ +// Unless explicitly stated otherwise all files in this repository are licensed +// under the MIT License. +// This product includes software developed at Guance Cloud (https://www.guance.com/). +// Copyright 2021-present Guance, Inc. + +package funcs + +import ( + "fmt" + "reflect" + + "github.com/GuanceCloud/platypus/pkg/ast" + "github.com/GuanceCloud/platypus/pkg/engine/runtime" + "github.com/GuanceCloud/platypus/pkg/errchain" +) + +func NullIfChecking(ctx *runtime.Context, funcExpr *ast.CallExpr) *errchain.PlError { + if len(funcExpr.Param) != 2 { + return runtime.NewRunError(ctx, fmt.Sprintf("func %s expected 2 args", funcExpr.Name), + funcExpr.NamePos) + } + + if _, err := getKeyName(funcExpr.Param[0]); err != nil { + return runtime.NewRunError(ctx, err.Error(), funcExpr.Param[0].StartPos()) + } + + return nil +} + +func NullIf(ctx *runtime.Context, funcExpr *ast.CallExpr) *errchain.PlError { + if len(funcExpr.Param) != 2 { + return runtime.NewRunError(ctx, fmt.Sprintf( + "func %s expected 2 args", funcExpr.Name), funcExpr.NamePos) + } + + key, err := getKeyName(funcExpr.Param[0]) + if err != nil { + return runtime.NewRunError(ctx, err.Error(), funcExpr.Param[0].StartPos()) + } + + cont, err := ctx.GetKey(key) + if err != nil { + // l.Debugf("key `%v' not exist, ignored", key) + return nil //nolint:nilerr + } + + var val interface{} + + switch funcExpr.Param[1].NodeType { //nolint:exhaustive + case ast.TypeStringLiteral, ast.TypeFloatLiteral, ast.TypeIntegerLiteral, + ast.TypeBoolLiteral, ast.TypeNilLiteral: + if v, _, err := runtime.RunStmt(ctx, funcExpr.Param[1]); err == nil { + val = v + } + } + + // todo key string + if reflect.DeepEqual(cont.Value, val) { + deletePtKey(ctx.InData(), key) + } + + return nil +} diff --git a/pkg/inimpl/guancecloud/ptinput/funcs/fn_nullif_test.go b/pkg/inimpl/guancecloud/ptinput/funcs/fn_nullif_test.go new file mode 100644 index 0000000..fe254ef --- /dev/null +++ b/pkg/inimpl/guancecloud/ptinput/funcs/fn_nullif_test.go @@ -0,0 +1,99 @@ +// Unless explicitly stated otherwise all files in this repository are licensed +// under the MIT License. +// This product includes software developed at Guance Cloud (https://www.guance.com/). +// Copyright 2021-present Guance, Inc. + +package funcs + +import ( + "testing" + "time" + + "github.com/GuanceCloud/cliutils/point" + tu "github.com/GuanceCloud/cliutils/testutil" + "github.com/GuanceCloud/platypus/pkg/inimpl/guancecloud/ptinput" +) + +func TestNullIf(t *testing.T) { + cases := []struct { + name, pl, in string + expected interface{} + fail bool + outkey string + }{ + { + name: "normal", + pl: `json(_, a.first, a_first) + nullif(a_first, "1")`, + in: `{"a":{"first": 1,"second":2,"third":"aBC","forth":true},"age":47}`, + expected: float64(1), + outkey: "a_first", + }, + { + name: "normal", + pl: `json(_, a.first, a_first) + nullif(a_first, 1)`, + in: `{"a":{"first": "1","second":2,"third":"aBC","forth":true},"age":47}`, + expected: "1", + outkey: "a_first", + }, + { + name: "normal", + pl: `json(_, a.first, a_first) + nullif(a_first, "")`, + in: `{"a":{"first": "","second":2,"third":"aBC","forth":true},"age":47}`, + expected: nil, + outkey: "a_first", + }, + { + name: "normal", + pl: `json(_, a.first, a_first) + nullif(a_first, nil)`, + in: `{"a":{"first": null,"second":2,"third":"aBC","forth":true},"age":47}`, + expected: nil, + outkey: "a_first", + }, + { + name: "normal", + pl: `json(_, a.first, a_first) + nullif(a_first, true)`, + in: `{"a":{"first": true,"second":2,"third":"aBC","forth":true},"age":47}`, + expected: nil, + outkey: "a_first", + }, + { + name: "normal", + pl: `json(_, a.first, a_first) + nullif(a_first, 2.3)`, + in: `{"a":{"first": 2.3, "second":2,"third":"aBC","forth":true},"age":47}`, + expected: nil, + outkey: "a_first", + }, + } + + for idx, tc := range cases { + t.Run(tc.name, func(t *testing.T) { + runner, err := NewTestingRunner(tc.pl) + if err != nil { + if tc.fail { + t.Logf("[%d]expect error: %s", idx, err) + } else { + t.Errorf("[%d] failed: %s", idx, err) + } + return + } + pt := ptinput.NewPlPoint( + point.Logging, "test", nil, map[string]any{"message": tc.in}, time.Now()) + errR := runScript(runner, pt) + + if errR != nil { + t.Fatal(errR.Error()) + } + + v, _, _ := pt.Get(tc.outkey) + tu.Equals(t, tc.expected, v) + + t.Logf("[%d] PASS", idx) + }) + } +} diff --git a/pkg/inimpl/guancecloud/ptinput/funcs/fn_parse_date.go b/pkg/inimpl/guancecloud/ptinput/funcs/fn_parse_date.go new file mode 100644 index 0000000..7195b3f --- /dev/null +++ b/pkg/inimpl/guancecloud/ptinput/funcs/fn_parse_date.go @@ -0,0 +1,158 @@ +// Unless explicitly stated otherwise all files in this repository are licensed +// under the MIT License. +// This product includes software developed at Guance Cloud (https://www.guance.com/). +// Copyright 2021-present Guance, Inc. + +package funcs + +import ( + "strconv" + "time" + + "github.com/GuanceCloud/platypus/pkg/ast" + "github.com/GuanceCloud/platypus/pkg/engine/runtime" + "github.com/GuanceCloud/platypus/pkg/errchain" + "github.com/GuanceCloud/platypus/pkg/inimpl/guancecloud/ptinput" + "github.com/spf13/cast" +) + +func ParseDateChecking(ctx *runtime.Context, funcExpr *ast.CallExpr) *errchain.PlError { + if err := reindexFuncArgs(funcExpr, []string{ + "key", "yy", "MM", "dd", "hh", "mm", "ss", "ms", "us", "ns", "zone", + }, 0); err != nil { + return runtime.NewRunError(ctx, err.Error(), funcExpr.NamePos) + } + return nil +} + +func ParseDate(ctx *runtime.Context, funcExpr *ast.CallExpr) *errchain.PlError { + now := time.Now() + var key string + + var zone *time.Location + + var yy int + var MM time.Month + var dd, hh, mm, ss, ms, us, ns int + + key, err := getKeyName(funcExpr.Param[0]) + if err != nil { + return runtime.NewRunError(ctx, err.Error(), funcExpr.Param[0].StartPos()) + } + + // year + if x := getArgStrConv2Int(ctx, funcExpr.Param[1]); true { + yy, err = fixYear(now, x) + if err != nil { + return runtime.NewRunError(ctx, err.Error(), funcExpr.Param[1].StartPos()) + } + } + + // month + if x := getArgStr(ctx, funcExpr.Param[2]); true { + MM, err = fixMonth(now, x) + if err != nil { + return runtime.NewRunError(ctx, err.Error(), funcExpr.Param[2].StartPos()) + } + } + + // day + if x := getArgStrConv2Int(ctx, funcExpr.Param[3]); true { + dd, err = fixDay(now, x) + if err != nil { + return runtime.NewRunError(ctx, err.Error(), funcExpr.Param[3].StartPos()) + } + } + + // hour + if x := getArgStrConv2Int(ctx, funcExpr.Param[4]); true { + hh, err = fixHour(now, x) + if err != nil { + return runtime.NewRunError(ctx, err.Error(), funcExpr.Param[4].StartPos()) + } + } + // mm + if x := getArgStrConv2Int(ctx, funcExpr.Param[5]); true { + mm, err = fixMinute(now, x) + if err != nil { + return runtime.NewRunError(ctx, err.Error(), funcExpr.Param[5].StartPos()) + } + } + + // ss + if x := getArgStrConv2Int(ctx, funcExpr.Param[6]); true { + ss, err = fixSecond(x) + if err != nil { + return runtime.NewRunError(ctx, err.Error(), funcExpr.Param[6].StartPos()) + } + } + // ms + if x := getArgStrConv2Int(ctx, funcExpr.Param[7]); true { + if x == DefaultInt { + ms = 0 + } else { + ms = int(x) + } + } + + // us + if x := getArgStrConv2Int(ctx, funcExpr.Param[8]); true { + if x == DefaultInt { + us = 0 + } else { + us = int(x) + } + } + + // ns + if x := getArgStrConv2Int(ctx, funcExpr.Param[9]); true { + if x == DefaultInt { + ns = 0 + } else { + ns = int(x) + } + } + + if x := getArgStr(ctx, funcExpr.Param[10]); true { + if x == "" { + zone = time.UTC + } else { + zone, err = tz(x) + if err != nil { + return runtime.NewRunError(ctx, err.Error(), funcExpr.Param[10].StartPos()) + } + } + } + + t := time.Date(yy, MM, dd, hh, mm, ss, ms*1000*1000+us*1000+ns, zone) + + _ = addKey2PtWithVal(ctx.InData(), key, t.UnixNano(), ast.Int, ptinput.KindPtDefault) + + return nil +} + +func getArgStr(ctx *runtime.Context, node *ast.Node) string { + if node == nil { + return "" + } + + if v, dtype, err := runtime.RunStmt(ctx, node); err == nil { + if dtype == ast.String { + return cast.ToString(v) + } + } + return "" +} + +func getArgStrConv2Int(ctx *runtime.Context, node *ast.Node) int64 { + str := getArgStr(ctx, node) + if str == "" { + return DefaultInt + } + + v, err := strconv.ParseInt(str, 10, 64) //nolint: gomnd + if err != nil { + return DefaultInt + } + return v +} diff --git a/pkg/inimpl/guancecloud/ptinput/funcs/fn_parse_date_test.go b/pkg/inimpl/guancecloud/ptinput/funcs/fn_parse_date_test.go new file mode 100644 index 0000000..782fec0 --- /dev/null +++ b/pkg/inimpl/guancecloud/ptinput/funcs/fn_parse_date_test.go @@ -0,0 +1,278 @@ +// Unless explicitly stated otherwise all files in this repository are licensed +// under the MIT License. +// This product includes software developed at Guance Cloud (https://www.guance.com/). +// Copyright 2021-present Guance, Inc. + +package funcs + +import ( + "testing" + "time" + + "github.com/GuanceCloud/cliutils/point" + tu "github.com/GuanceCloud/cliutils/testutil" + "github.com/GuanceCloud/platypus/pkg/inimpl/guancecloud/ptinput" +) + +func TestParseDate(t *testing.T) { + now := time.Now() + collectTime := now.Add(time.Hour * 12345) + cases := []struct { + name string + pl, in string + outKey string + expected int64 + fail bool + }{ + { + name: "normal", + pl: ` +grok(_, "%{INT:hour}:%{INT:min}:%{INT:sec}\\.%{INT:msec}") +parse_date("time", "", "", "", hour, min, sec, msec, "", "", "+8") +`, + in: "16:40:03.290", + outKey: "time", + expected: time.Date(now.Year(), now.Month(), now.Day(), + 16, 40, 0o3, 290*1000*1000, time.FixedZone("UTC+8", 8*60*60)).UnixNano(), + fail: false, + }, + + { + name: "normal", + pl: ` +grok(_, "%{INT:hour}:%{INT:min}:%{INT:sec}\\.%{INT:msec}") +parse_date(key="time", hh=hour, mm=min, ss=sec, ms=msec, zone="+8") +`, + in: "16:40:03.290", + outKey: "time", + expected: time.Date(now.Year(), now.Month(), now.Day(), + 16, 40, 0o3, 290*1000*1000, time.FixedZone("UTC+8", 8*60*60)).UnixNano(), + fail: false, + }, + + { + name: "normal", + pl: ` +grok(_, "%{INT:year}-%{INT:month}-%{INT:day} %{INT:hour}:%{INT:min}:%{INT:sec}\\.%{INT:msec}") +parse_date(key="time", yy=year, MM=month, dd=day, hh=hour, mm=min, ss=sec, ms=msec, zone="+8") +`, + in: "2020-12-12 16:40:03.290", + outKey: "time", + expected: time.Date(2020, 12, 12, + 16, 40, 0o3, 290*1000*1000, time.FixedZone("UTC+8", 8*60*60)).UnixNano(), + fail: false, + }, + + { + name: "normal", + pl: ` +grok(_, "%{NOTSPACE:wd}\\s+%{NOTSPACE:month}\\s+%{INT:day} %{INT:hour}:%{INT:min}:%{INT:sec}\\s+%{NOTSPACE:tz}\\s+%{INT:year}") +parse_date(key="time", yy=year, MM=month, dd=day, hh=hour, mm=min, ss=sec, zone=tz) +`, + in: "Mon Sep 6 16:40:03 CST 2021", + outKey: "time", + expected: time.Date(2021, 9, 6, + 16, 40, 0o3, 0, time.FixedZone("UTC+8", 8*60*60)).UnixNano(), + fail: false, + }, + + { + name: "normal", + pl: ` +grok(_, "%{NOTSPACE:wd}\\s+%{NOTSPACE:month}\\s+%{INT:day}\\s+%{INT:hour}:%{INT:min}:%{INT:sec}\\s+%{NOTSPACE:tz}\\s+%{INT:year}") +parse_date(key="time", yy=year, MM=month, dd=day, hh=hour, mm=min, ss=sec, zone=tz) +`, + in: "Mon Sep 6 16:40:03 CST 2021", + outKey: "time", + expected: time.Date(2021, 9, 6, + 16, 40, 0o3, 0, time.FixedZone("UTC+8", 8*60*60)).UnixNano(), + fail: false, + }, + + { + name: "partial year", + pl: ` +grok(_, "%{NOTSPACE:wd}\\s+%{NOTSPACE:month}\\s+%{INT:day}\\s+%{INT:hour}:%{INT:min}:%{INT:sec}\\s+%{NOTSPACE:tz}\\s+%{INT:year}") +parse_date(key="time", yy=year, MM=month, dd=day, hh=hour, mm=min, ss=sec, zone=tz) +`, + in: "Mon Sep 6 16:40:03 CST 00", + outKey: "time", + expected: time.Date(2000, 9, 6, + 16, 40, 0o3, 0, time.FixedZone("UTC+8", 8*60*60)).UnixNano(), + }, + + { + name: "partial year", + pl: ` +grok(_, "%{NOTSPACE:wd}\\s+%{NOTSPACE:month}\\s+%{INT:day}\\s+%{INT:hour}:%{INT:min}:%{INT:sec}\\s+%{NOTSPACE:tz}\\s+%{INT:year}") +parse_date(key="time", yy=year, MM=month, dd=day, hh=hour, mm=min, ss=sec, zone=tz) +`, + in: "Mon Sep 6 16:40:03 CST 09", + outKey: "time", + expected: time.Date(2009, 9, 6, + 16, 40, 0o3, 0, time.FixedZone("UTC+8", 8*60*60)).UnixNano(), + }, + + { + name: "us", + pl: ` +grok(_, "%{INT:hour}:%{INT:min}:%{INT:sec}\\.%{INT:us}") +parse_date("time", "", "", "", hour, min, sec, "", us, "", "+8") +`, + in: "16:40:03.290290", + outKey: "time", + expected: time.Date(now.Year(), now.Month(), now.Day(), + 16, 40, 0o3, 290290*1000, time.FixedZone("UTC+8", 8*60*60)).UnixNano(), + }, + + { + name: "ns", + pl: ` +grok(_, "%{INT:hour}:%{INT:min}:%{INT:sec}\\.%{INT:ns}") +parse_date("time", "", "", "", hour, min, sec, "", "", ns, "+8") +`, + in: "16:40:03.290290330", + outKey: "time", + expected: time.Date(now.Year(), now.Month(), now.Day(), + 16, 40, 0o3, 290290330, time.FixedZone("UTC+8", 8*60*60)).UnixNano(), + }, + + { + name: "ns for kwargs", + pl: ` +grok(_, "%{INT:hour}:%{INT:min}:%{INT:sec}\\.%{INT:ns}") +parse_date(key="time", hh=hour, mm=min, ss=sec, ns=ns, zone="CST") +`, + in: "16:40:03.290290330", + outKey: "time", + expected: time.Date(now.Year(), now.Month(), now.Day(), + 16, 40, 0o3, 290290330, time.FixedZone("UTC+8", 8*60*60)).UnixNano(), + }, + + { + name: "missed second: use 0", + pl: ` +grok(_, "%{INT:hour}:%{INT:min}") +parse_date(key="time", hh=hour, mm=min, zone="CST")`, + in: "16:40", + outKey: "time", + expected: time.Date(now.Year(), now.Month(), now.Day(), + 16, 40, 0o0, 0, time.FixedZone("UTC+8", 8*60*60)).UnixNano(), + }, + + { + name: "normal", + pl: ` +grok(_, "%{INT:hour}:%{INT:min}:%{INT:sec}\\.%{INT:msec}") +parse_date("time", "", "", "", hour, min, sec, msec, "", "", "") +`, + in: "16:40:03.290", + outKey: "time", + expected: time.Date(now.Year(), now.Month(), now.Day(), + 16, 40, 0o3, 290*1000*1000, time.FixedZone("UTC", 0)).UnixNano(), + fail: false, + }, + + { + name: "invalid hour", + pl: ` +grok(_, "%{NOTSPACE:wd}\\s+%{NOTSPACE:month}\\s+%{INT:day}\\s+%{INT:hour}:%{INT:min}:%{INT:sec}\\s+%{NOTSPACE:tz}\\s+%{INT:year}") +parse_date(key="time", yy=year, MM=month, dd=day, hh=hour, mm=min, ss=sec, zone=tz) +`, + in: "Mon Sep 6 25:40:03 CST 2021", + expected: collectTime.UnixNano(), + fail: true, + }, + + { + name: "invalid hour", + pl: ` +grok(_, "%{NOTSPACE:wd}\\s+%{NOTSPACE:month}\\s+%{INT:day}\\s+%{INT:hour}:%{INT:min}:%{INT:sec}\\s+%{NOTSPACE:tz}\\s+%{INT:year}") +parse_date(key="time", yy=year, MM=month, dd=day, hh=hour, mm=min, ss=sec, zone=tz) +`, + in: "Mon Sep 6 -2:40:03 CST 2021", + expected: collectTime.UnixNano(), + fail: true, + }, + + { + name: "invalid minute", + pl: ` +grok(_, "%{NOTSPACE:wd}\\s+%{NOTSPACE:month}\\s+%{INT:day}\\s+%{INT:hour}:%{INT:min}:%{INT:sec}\\s+%{NOTSPACE:tz}\\s+%{INT:year}") +parse_date(key="time", yy=year, MM=month, dd=day, hh=hour, mm=min, ss=sec, zone=tz) +`, + in: "Mon Sep 6 12:61:03 CST 2021", + expected: collectTime.UnixNano(), + fail: true, + }, + + { + name: "invalid minute", + pl: ` +grok(_, "%{NOTSPACE:wd}\\s+%{NOTSPACE:month}\\s+%{INT:day}\\s+%{INT:hour}:%{INT:min}:%{INT:sec}\\s+%{NOTSPACE:tz}\\s+%{INT:year}") +parse_date(key="time", yy=year, MM=month, dd=day, hh=hour, mm=min, ss=sec, zone=tz) +`, + in: "Mon Sep 6 12:60:03 CST 2021", + expected: collectTime.UnixNano(), + fail: true, + }, + + { + name: "invalid second", + pl: ` +grok(_, "%{NOTSPACE:wd}\\s+%{NOTSPACE:month}\\s+%{INT:day}\\s+%{INT:hour}:%{INT:min}:%{INT:sec}\\s+%{NOTSPACE:tz}\\s+%{INT:year}") +parse_date(key="time", yy=year, MM=month, dd=day, hh=hour, mm=min, ss=sec, zone=tz) +`, + in: "Mon Sep 6 12:11:61 CST 2021", + expected: collectTime.UnixNano(), + fail: true, + }, + + { + name: "invalid minute", + pl: ` +grok(_, "%{NOTSPACE:wd}\\s+%{NOTSPACE:month}\\s+%{INT:day}\\s+%{INT:hour}:%{INT:min}:%{INT:sec}\\s+%{NOTSPACE:tz}\\s+%{INT:year}") +parse_date(key="time", yy=year, MM=month, dd=day, hh=hour, mm=min, ss=sec, zone=tz) +`, + in: "Mon Sep 6 12:12:-1 CST 2021", + expected: collectTime.UnixNano(), + fail: true, + }, + } + + for idx, tc := range cases { + t.Run(tc.name, func(t *testing.T) { + runner, err := NewTestingRunner(tc.pl) + if err != nil { + if tc.fail { + t.Logf("[%d]expect error: %s", idx, err) + } else { + t.Errorf("[%d] failed: %s", idx, err) + } + return + } + + pt := ptinput.NewPlPoint( + point.Logging, "test", nil, map[string]any{"message": tc.in}, time.Now()) + errR := runScript(runner, pt) + + if errR != nil { + if tc.fail { + t.Logf("[%d]expect error: %s", idx, errR.Error()) + } else { + t.Error(errR.Error()) + } + } else { + pt.KeyTime2Time() + var v interface{} + if tc.outKey != "time" && tc.outKey != "" { + v, _, _ = pt.Get(tc.outKey) + } else { + v = pt.PtTime().UnixNano() + } + tu.Equals(t, tc.expected, v) + t.Logf("[%d] PASS", idx) + } + }) + } +} diff --git a/pkg/inimpl/guancecloud/ptinput/funcs/fn_parse_duration.go b/pkg/inimpl/guancecloud/ptinput/funcs/fn_parse_duration.go new file mode 100644 index 0000000..4fff777 --- /dev/null +++ b/pkg/inimpl/guancecloud/ptinput/funcs/fn_parse_duration.go @@ -0,0 +1,69 @@ +// Unless explicitly stated otherwise all files in this repository are licensed +// under the MIT License. +// This product includes software developed at Guance Cloud (https://www.guance.com/). +// Copyright 2021-present Guance, Inc. + +package funcs + +import ( + "fmt" + "time" + + "github.com/GuanceCloud/platypus/pkg/ast" + "github.com/GuanceCloud/platypus/pkg/engine/runtime" + "github.com/GuanceCloud/platypus/pkg/errchain" + "github.com/GuanceCloud/platypus/pkg/inimpl/guancecloud/ptinput" +) + +func ParseDurationChecking(ctx *runtime.Context, funcExpr *ast.CallExpr) *errchain.PlError { + if len(funcExpr.Param) != 1 { + return runtime.NewRunError(ctx, fmt.Sprintf( + "func %s expects 1 arg", funcExpr.Name), funcExpr.NamePos) + } + if _, err := getKeyName(funcExpr.Param[0]); err != nil { + return runtime.NewRunError(ctx, err.Error(), funcExpr.Param[0].StartPos()) + } + return nil +} + +func ParseDuration(ctx *runtime.Context, funcExpr *ast.CallExpr) *errchain.PlError { + if len(funcExpr.Param) != 1 { + l.Debugf("parse_duration(): invalid param") + + return runtime.NewRunError(ctx, fmt.Sprintf( + "func %s expects 1 arg", funcExpr.Name), funcExpr.NamePos) + } + + key, err := getKeyName(funcExpr.Param[0]) + if err != nil { + return runtime.NewRunError(ctx, err.Error(), funcExpr.Param[0].StartPos()) + } + + cont, err := ctx.GetKey(key) + if err != nil { + l.Debug(err) + return nil + } + + duStr, ok := cont.Value.(string) + if !ok { + l.Debug(err) + return nil + // return runtime.NewRunError(ctx, "parse_duration() expect string arg", + // funcExpr.Param[0].StartPos()) + } + + l.Debugf("parse duration %s", duStr) + du, err := time.ParseDuration(duStr) + if err != nil { + l.Debug(err) + return nil + } + + if err := addKey2PtWithVal(ctx.InData(), key, int64(du), ast.Int, + ptinput.KindPtDefault); err != nil { + l.Debug(err) + return nil + } + return nil +} diff --git a/pkg/inimpl/guancecloud/ptinput/funcs/fn_parse_duration_test.go b/pkg/inimpl/guancecloud/ptinput/funcs/fn_parse_duration_test.go new file mode 100644 index 0000000..e29ae2e --- /dev/null +++ b/pkg/inimpl/guancecloud/ptinput/funcs/fn_parse_duration_test.go @@ -0,0 +1,171 @@ +// Unless explicitly stated otherwise all files in this repository are licensed +// under the MIT License. +// This product includes software developed at Guance Cloud (https://www.guance.com/). +// Copyright 2021-present Guance, Inc. + +package funcs + +import ( + "testing" + "time" + + "github.com/GuanceCloud/cliutils/point" + tu "github.com/GuanceCloud/cliutils/testutil" + "github.com/GuanceCloud/platypus/pkg/inimpl/guancecloud/ptinput" +) + +func TestParseDuration(t *testing.T) { + cases := []struct { + name string + pl, in string + outKey string + expected interface{} + fail bool + }{ + { + name: "normal", + pl: "json(_, `str`)\n" + + "parse_duration(`str`)", + in: `{"str": "1s"}`, + outKey: "str", + expected: int64(time.Second), + fail: false, + }, + + { + name: "normal", + pl: "json(_, `str`)\n" + + "parse_duration(`str`)", + in: `{"str": "1ms"}`, + outKey: "str", + expected: int64(time.Millisecond), + fail: false, + }, + + { + name: "normal", + pl: "json(_, `str`)\n" + + "parse_duration(`str`)", + in: `{"str": "1us"}`, + outKey: "str", + expected: int64(time.Microsecond), + fail: false, + }, + + { + name: "normal", + pl: "json(_, `str`)\n" + + "parse_duration(`str`)", + in: `{"str": "1µs"}`, + outKey: "str", + expected: int64(time.Microsecond), + fail: false, + }, + + { + name: "normal", + pl: "json(_, `str`)\n" + + "parse_duration(`str`)", + in: `{"str": "1m"}`, + outKey: "str", + expected: int64(time.Minute), + fail: false, + }, + + { + name: "normal", + pl: "json(_, `str`)\n" + + "parse_duration(`str`)", + in: `{"str": "1h"}`, + outKey: "str", + expected: int64(time.Hour), + fail: false, + }, + + { + name: "normal", + pl: "json(_, `str`)\n" + + "parse_duration(`str`)", + in: `{"str": "-23h"}`, + outKey: "str", + expected: -23 * int64(time.Hour), + fail: false, + }, + + { + name: "normal", + pl: "json(_, `str`)\n" + + "parse_duration(`str`)", + in: `{"str": "-23ns"}`, + outKey: "str", + expected: int64(-23), + fail: false, + }, + + { + name: "normal", + pl: "json(_, `str`)\n" + + "parse_duration(`str`)", + in: `{"str": "-2.3s"}`, + outKey: "str", + expected: int64(time.Second*-2 - 300*time.Millisecond), + fail: false, + }, + + { + name: "invalid input string", + pl: "json(_, `str`)\n" + + "parse_duration(`str`)", + in: `{"str": "1uuus"}`, + outKey: "str", + expected: "1uuus", + fail: false, + }, + + { + name: "too many args", + pl: "json(_, `str`)\n" + + "parse_duration(`str`)", + in: `{"str": "1uuus"}`, + fail: true, + }, + + { + name: "invalid input type", + pl: "json(_, `str`)\n" + + "parse_duration(`str`)", + in: `{"str": 1}`, + fail: true, + }, + } + + for idx, tc := range cases { + t.Run(tc.name, func(t *testing.T) { + runner, err := NewTestingRunner(tc.pl) + if err != nil { + if tc.fail { + t.Logf("[%d]expect error: %s", idx, err) + } else { + t.Errorf("[%d] failed: %s", idx, err) + } + return + } + + pt := ptinput.NewPlPoint( + point.Logging, "test", nil, map[string]any{"message": tc.in}, time.Now()) + errR := runScript(runner, pt) + + if errR != nil { + t.Fatal(errR) + } + + if v, _, err := pt.Get(tc.outKey); err != nil { + if !tc.fail { + t.Errorf("[%d]expect error: %s", idx, err) + } + } else { + tu.Equals(t, tc.expected, v) + } + }) + } +} diff --git a/pkg/inimpl/guancecloud/ptinput/funcs/fn_parse_int.go b/pkg/inimpl/guancecloud/ptinput/funcs/fn_parse_int.go new file mode 100644 index 0000000..57f573f --- /dev/null +++ b/pkg/inimpl/guancecloud/ptinput/funcs/fn_parse_int.go @@ -0,0 +1,110 @@ +// Unless explicitly stated otherwise all files in this repository are licensed +// under the MIT License. +// This product includes software developed at Guance Cloud (https://www.guance.com/). +// Copyright 2021-present Guance, Inc. + +package funcs + +import ( + "strconv" + + "github.com/GuanceCloud/platypus/pkg/ast" + "github.com/GuanceCloud/platypus/pkg/engine/runtime" + "github.com/GuanceCloud/platypus/pkg/errchain" +) + +func ParseIntChecking(ctx *runtime.Context, funcExpr *ast.CallExpr) *errchain.PlError { + if err := reindexFuncArgs(funcExpr, []string{ + "val", "base", + }, 2); err != nil { + return runtime.NewRunError(ctx, err.Error(), funcExpr.NamePos) + } + + return nil +} + +func ParseInt(ctx *runtime.Context, funcExpr *ast.CallExpr) *errchain.PlError { + val, _, err := runtime.RunStmt(ctx, funcExpr.Param[0]) + if err != nil { + return err + } + + if val == nil { + ctx.Regs.ReturnAppend(int64(0), ast.Int) + return nil + } + + valStr, ok := val.(string) + if !ok { + ctx.Regs.ReturnAppend(int64(0), ast.Int) + return nil + } + + base, _, err := runtime.RunStmt(ctx, funcExpr.Param[1]) + if err != nil { + return err + } + + if base == nil { + ctx.Regs.ReturnAppend(int64(0), ast.Int) + return nil + } + + baseInt, ok := base.(int64) + if !ok { + ctx.Regs.ReturnAppend(int64(0), ast.Int) + return nil + } + + v, _ := strconv.ParseInt(valStr, int(baseInt), 64) + ctx.Regs.ReturnAppend(v, ast.Int) + return nil +} + +func FormatIntChecking(ctx *runtime.Context, funcExpr *ast.CallExpr) *errchain.PlError { + if err := reindexFuncArgs(funcExpr, []string{ + "val", "base", + }, 2); err != nil { + return runtime.NewRunError(ctx, err.Error(), funcExpr.NamePos) + } + + return nil +} + +func FormatInt(ctx *runtime.Context, funcExpr *ast.CallExpr) *errchain.PlError { + val, _, err := runtime.RunStmt(ctx, funcExpr.Param[0]) + if err != nil { + return err + } + + if val == nil { + ctx.Regs.ReturnAppend("", ast.Int) + return nil + } + + valStr, ok := val.(int64) + if !ok { + ctx.Regs.ReturnAppend("", ast.Int) + return nil + } + + base, _, err := runtime.RunStmt(ctx, funcExpr.Param[1]) + if err != nil { + return err + } + + if base == nil { + ctx.Regs.ReturnAppend("", ast.Int) + return nil + } + + baseInt, ok := base.(int64) + if !ok { + ctx.Regs.ReturnAppend("", ast.Int) + return nil + } + + v := strconv.FormatInt(valStr, int(baseInt)) + ctx.Regs.ReturnAppend(v, ast.String) + return nil +} diff --git a/pkg/inimpl/guancecloud/ptinput/funcs/fn_parse_int_test.go b/pkg/inimpl/guancecloud/ptinput/funcs/fn_parse_int_test.go new file mode 100644 index 0000000..95a5c75 --- /dev/null +++ b/pkg/inimpl/guancecloud/ptinput/funcs/fn_parse_int_test.go @@ -0,0 +1,164 @@ +// Unless explicitly stated otherwise all files in this repository are licensed +// under the MIT License. +// This product includes software developed at Guance Cloud (https://www.guance.com/). +// Copyright 2021-present Guance, Inc. + +package funcs + +import ( + "testing" + "time" + + "github.com/GuanceCloud/cliutils/point" + tu "github.com/GuanceCloud/cliutils/testutil" + "github.com/GuanceCloud/platypus/pkg/inimpl/guancecloud/ptinput" +) + +func TestParseInt(t *testing.T) { + cases := []struct { + name, pl, in string + expected interface{} + fail bool + outkey string + }{ + { + name: "format", + pl: `a = 17 + b = format_int(a, 16) + if b != "11" { + add_key(abc, b) + exit() + } + c = parse_int(b, 16) + if c != a { + add_key(abc, c) + exit() + } else { + add_key(abc, "ok") + } + `, + in: `test`, + expected: "ok", + outkey: "abc", + }, + { + name: "parse", + pl: `a = "11" # 0x11 + b = parse_int(a, 16) + if b != 17 { + add_key(abc, b) + exit() + } + c = format_int(b, 16) + if c != a { + add_key(abc, c) + exit() + } else { + add_key(abc, "ok") + } + `, + in: `test`, + expected: "ok", + outkey: "abc", + }, + { + name: "spanid-format", + pl: `a = 7665324064912355185 + b = format_int(a, 16) + if b != "6a60b39fd95aaf71" { + add_key(abc, b) + exit() + } + c = parse_int(b, 16) + if c != a { + add_key(abc, c) + exit() + } else { + add_key(abc, "ok") + } + `, + in: `test`, + expected: "ok", + outkey: "abc", + }, + { + name: "spanid-format-all-in-one", + pl: `a = "7665324064912355185" + b = format_int(parse_int(a, 10), 16) + if b != "6a60b39fd95aaf71" { + add_key(abc, b) + exit() + } else { + add_key(abc, "ok") + } + `, + in: `test`, + expected: "ok", + outkey: "abc", + }, + { + name: "parse", + pl: `a = "6a60b39fd95aaf71" + b = parse_int(a, 16) + if b != 7665324064912355185 { + add_key(abc, b) + exit() + } + c = format_int(b, 16) + if c != a { + add_key(abc, c) + } else { + add_key(abc, "ok") + } + `, + in: `test`, + expected: "ok", + outkey: "abc", + }, + { + name: "parsex", + pl: `a = "0x6a60b39fd95aaf71" + b = parse_int(a, 0) + if b != 7665324064912355185 { + add_key(abc, b) + exit() + } + c = format_int(b, 16) + if "0x"+c != a { + add_key(abc, c) + } else { + add_key(abc, "ok") + } + `, + in: `test`, + expected: "ok", + outkey: "abc", + }, + } + + for idx, tc := range cases { + t.Run(tc.name, func(t *testing.T) { + runner, err := NewTestingRunner(tc.pl) + if err != nil { + if tc.fail { + t.Logf("[%d]expect error: %s", idx, err) + } else { + t.Errorf("[%d] failed: %s", idx, err) + } + return + } + pt := ptinput.NewPlPoint( + point.Logging, "test", nil, map[string]any{"message": tc.in}, time.Now()) + errR := runScript(runner, pt) + + if errR != nil { + t.Fatal(errR.Error()) + } + + v, _, _ := pt.Get(tc.outkey) + // tu.Equals(t, nil, err) + tu.Equals(t, tc.expected, v) + t.Logf("[%d] PASS", idx) + }) + } +} diff --git a/pkg/inimpl/guancecloud/funcs/fn_rename.go b/pkg/inimpl/guancecloud/ptinput/funcs/fn_rename.go similarity index 100% rename from pkg/inimpl/guancecloud/funcs/fn_rename.go rename to pkg/inimpl/guancecloud/ptinput/funcs/fn_rename.go diff --git a/pkg/inimpl/guancecloud/funcs/fn_rename_test.go b/pkg/inimpl/guancecloud/ptinput/funcs/fn_rename_test.go similarity index 86% rename from pkg/inimpl/guancecloud/funcs/fn_rename_test.go rename to pkg/inimpl/guancecloud/ptinput/funcs/fn_rename_test.go index 3dee858..dc357aa 100644 --- a/pkg/inimpl/guancecloud/funcs/fn_rename_test.go +++ b/pkg/inimpl/guancecloud/ptinput/funcs/fn_rename_test.go @@ -9,6 +9,8 @@ import ( "testing" "time" + "github.com/GuanceCloud/cliutils/point" + "github.com/GuanceCloud/platypus/pkg/inimpl/guancecloud/ptinput" "github.com/stretchr/testify/assert" ) @@ -87,14 +89,17 @@ func TestRename(t *testing.T) { } return } - _, _, f, _, _, err := runScript(runner, "test", nil, map[string]interface{}{ - "message": tc.in, - }, time.Now()) - assert.Equal(t, nil, err) + pt := ptinput.NewPlPoint( + point.Logging, "test", nil, map[string]any{"message": tc.in}, time.Now()) + errR := runScript(runner, pt) - t.Log(f) - v, ok := f[tc.outkey] + if errR != nil { + t.Fatal(errR.Error()) + } + + v, isTag, ok := pt.GetWithIsTag(tc.outkey) + assert.Equal(t, false, isTag) assert.Equal(t, true, ok) assert.Equal(t, tc.expected, v) t.Logf("[%d] PASS", idx) diff --git a/pkg/inimpl/guancecloud/funcs/fn_replace.go b/pkg/inimpl/guancecloud/ptinput/funcs/fn_replace.go similarity index 84% rename from pkg/inimpl/guancecloud/funcs/fn_replace.go rename to pkg/inimpl/guancecloud/ptinput/funcs/fn_replace.go index 40bd091..bff0a4b 100644 --- a/pkg/inimpl/guancecloud/funcs/fn_replace.go +++ b/pkg/inimpl/guancecloud/ptinput/funcs/fn_replace.go @@ -13,7 +13,7 @@ import ( "github.com/GuanceCloud/platypus/pkg/ast" "github.com/GuanceCloud/platypus/pkg/engine/runtime" "github.com/GuanceCloud/platypus/pkg/errchain" - "github.com/GuanceCloud/platypus/pkg/inimpl/guancecloud/input" + "github.com/GuanceCloud/platypus/pkg/inimpl/guancecloud/ptinput" ) func ReplaceChecking(ctx *runtime.Context, funcExpr *ast.CallExpr) *errchain.PlError { @@ -26,24 +26,40 @@ func ReplaceChecking(ctx *runtime.Context, funcExpr *ast.CallExpr) *errchain.PlE return runtime.NewRunError(ctx, err.Error(), funcExpr.Param[0].StartPos()) } + var pattern string switch funcExpr.Param[1].NodeType { //nolint:exhaustive case ast.TypeStringLiteral: + pattern = funcExpr.Param[1].StringLiteral.Val default: return runtime.NewRunError(ctx, fmt.Sprintf( "expect StringLiteral, got %s", funcExpr.Param[1].NodeType), funcExpr.Param[1].StartPos()) } + reg, err := regexp.Compile(pattern) + if err != nil { + return runtime.NewRunError(ctx, fmt.Sprintf("regular expression %s parse err: %s", + reflect.TypeOf(funcExpr.Param[1]).String(), err.Error()), funcExpr.Param[1].StartPos()) + } + + funcExpr.PrivateData = reg + switch funcExpr.Param[2].NodeType { //nolint:exhaustive case ast.TypeStringLiteral: default: - return runtime.NewRunError(ctx, fmt.Sprintf("expect StringLiteral, got %s", + return runtime.NewRunError(ctx, fmt.Sprintf( + "expect StringLiteral, got %s", funcExpr.Param[2].NodeType), funcExpr.Param[2].StartPos()) } return nil } func Replace(ctx *runtime.Context, funcExpr *ast.CallExpr) *errchain.PlError { + reg, ok := funcExpr.PrivateData.(*regexp.Regexp) + if !ok { + return runtime.NewRunError(ctx, "regexp obj not found", funcExpr.Param[1].StartPos()) + } + if len(funcExpr.Param) != 3 { return runtime.NewRunError(ctx, fmt.Sprintf( "func %s expects 3 args", funcExpr.Name), funcExpr.NamePos) @@ -54,15 +70,7 @@ func Replace(ctx *runtime.Context, funcExpr *ast.CallExpr) *errchain.PlError { return runtime.NewRunError(ctx, err.Error(), funcExpr.Param[0].StartPos()) } - var pattern, dz string - - switch funcExpr.Param[1].NodeType { //nolint:exhaustive - case ast.TypeStringLiteral: - pattern = funcExpr.Param[1].StringLiteral.Val - default: - return runtime.NewRunError(ctx, fmt.Sprintf("expect StringLiteral, got %s", - funcExpr.Param[1].NodeType), funcExpr.Param[1].StartPos()) - } + var dz string switch funcExpr.Param[2].NodeType { //nolint:exhaustive case ast.TypeStringLiteral: @@ -72,11 +80,6 @@ func Replace(ctx *runtime.Context, funcExpr *ast.CallExpr) *errchain.PlError { funcExpr.Param[2].NodeType), funcExpr.Param[2].StartPos()) } - reg, err := regexp.Compile(pattern) - if err != nil { - return runtime.NewRunError(ctx, fmt.Sprintf("regular expression %s parse err: %s", - reflect.TypeOf(funcExpr.Param[1]).String(), err.Error()), funcExpr.Param[1].StartPos()) - } cont, err := ctx.GetKeyConv2Str(key) if err != nil { l.Debugf("key `%v' not exist, ignored", key) @@ -85,7 +88,7 @@ func Replace(ctx *runtime.Context, funcExpr *ast.CallExpr) *errchain.PlError { newCont := reg.ReplaceAllString(cont, dz) if err := addKey2PtWithVal(ctx.InData(), key, newCont, ast.String, - input.KindPtDefault); err != nil { + ptinput.KindPtDefault); err != nil { l.Debug(err) return nil } diff --git a/pkg/inimpl/guancecloud/ptinput/funcs/fn_replace_test.go b/pkg/inimpl/guancecloud/ptinput/funcs/fn_replace_test.go new file mode 100644 index 0000000..5a7494b --- /dev/null +++ b/pkg/inimpl/guancecloud/ptinput/funcs/fn_replace_test.go @@ -0,0 +1,121 @@ +// Unless explicitly stated otherwise all files in this repository are licensed +// under the MIT License. +// This product includes software developed at Guance Cloud (https://www.guance.com/). +// Copyright 2021-present Guance, Inc. + +package funcs + +import ( + "testing" + "time" + + "github.com/GuanceCloud/cliutils/point" + tu "github.com/GuanceCloud/cliutils/testutil" + "github.com/GuanceCloud/platypus/pkg/inimpl/guancecloud/ptinput" +) + +func TestReplace(t *testing.T) { + cases := []struct { + name string + pl, in string + outKey string + expected interface{} + fail bool + }{ + { + name: `normal`, + pl: "json(_, `str`)\n" + + "replace(`str`, \"(1[0-9]{2})[0-9]{4}([0-9]{4})\", \"$1****$2\")", + in: `{"str": "13789123014"}`, + outKey: "str", + fail: false, + expected: "137****3014", + }, + + { + name: `normal`, + pl: "json(_, `str`)\n" + + "replace(`str`, \"([a-z]*) \\\\w*\", \"$1 ***\")", + in: `{"str": "zhang san"}`, + outKey: "str", + expected: "zhang ***", + fail: false, + }, + + { + name: `normal`, + pl: "json(_, `str`)\n" + + "replace(`str`, \"([1-9]{4})[0-9]{10}([0-9]{4})\", \"$1**********$2\")", + in: `{"str": "362201200005302565"}`, + outKey: "str", + expected: "3622**********2565", + fail: false, + }, + + { + name: `normal`, + pl: "json(_, `str`)\n" + + "replace(`str`, '([\u4e00-\u9fa5])[\u4e00-\u9fa5]([\u4e00-\u9fa5])', \"$1＊$2\")", + in: `{"str": "小阿卡"}`, + outKey: "str", + expected: "小＊卡", + fail: false, + }, + { + name: `normal`, + pl: "json(_, `str`)\n" + + "replace(str1, '([\u4e00-\u9fa5])[\u4e00-\u9fa5]([\u4e00-\u9fa5])', \"$1＊$2\")", + in: `{"str": "小阿卡"}`, + outKey: "str", + expected: "小阿卡", + fail: false, + }, + { + name: `not enough args`, + pl: "json(_, `str`)\n" + + "replace(`str`, '([\u4e00-\u9fa5])[\u4e00-\u9fa5]([\u4e00-\u9fa5])')", + in: `{"str": "小阿卡"}`, + fail: true, + }, + { + name: `invalid arg type`, + pl: "json(_, `str`)\n" + + "replace(`str`, 2, \"$1＊$2\")", + in: `{"str": "小阿卡"}`, + fail: true, + }, + } + + for idx, tc := range cases { + t.Run(tc.name, func(t *testing.T) { + runner, err := NewTestingRunner(tc.pl) + if err != nil { + if tc.fail { + t.Logf("[%d]expect error: %s", idx, err) + } else { + t.Errorf("[%d] failed: %s", idx, err) + } + return + } + + pt := ptinput.NewPlPoint( + point.Logging, "test", nil, map[string]any{"message": tc.in}, time.Now()) + errR := runScript(runner, pt) + + if errR != nil { + t.Fatal(errR.Error()) + } + + if v, _, ok := pt.GetWithIsTag(tc.outKey); !ok { + if !tc.fail { + t.Errorf("[%d]expect error: %s", idx, errR.Error()) + } + } else { + tu.Equals(t, tc.expected, v) + t.Logf("[%d] PASS", idx) + } + + t.Logf("[%d] PASS", idx) + }) + } +} diff --git a/pkg/inimpl/guancecloud/ptinput/funcs/fn_sample.go b/pkg/inimpl/guancecloud/ptinput/funcs/fn_sample.go new file mode 100644 index 0000000..9017825 --- /dev/null +++ b/pkg/inimpl/guancecloud/ptinput/funcs/fn_sample.go @@ -0,0 +1,80 @@ +// Unless explicitly stated otherwise all files in this repository are licensed +// under the MIT License. +// This product includes software developed at Guance Cloud (https://www.guance.com/). +// Copyright 2021-present Guance, Inc. + +package funcs + +import ( + "fmt" + "time" + + "github.com/GuanceCloud/platypus/pkg/ast" + "github.com/GuanceCloud/platypus/pkg/engine/runtime" + "github.com/GuanceCloud/platypus/pkg/errchain" +) + +func SampleChecking(ctx *runtime.Context, funcExpr *ast.CallExpr) *errchain.PlError { + if len(funcExpr.Param) != 1 { + return runtime.NewRunError(ctx, fmt.Sprintf( + "func %s expects 1 arg", funcExpr.Name), funcExpr.NamePos) + } + + switch funcExpr.Param[0].NodeType { //nolint:exhaustive + case ast.TypeIntegerLiteral, ast.TypeFloatLiteral, ast.TypeArithmeticExpr: + default: + return runtime.NewRunError(ctx, fmt.Sprintf( + "expect NumberLiteral or ArithmeticExpr, got %s", funcExpr.Param[0].NodeType), + funcExpr.Param[0].StartPos()) + } + + return nil +} + +func Sample(ctx *runtime.Context, funcExpr *ast.CallExpr) *errchain.PlError { + if len(funcExpr.Param) != 1 { + return runtime.NewRunError(ctx, fmt.Sprintf( + "func %s expects 1 arg", funcExpr.Name), funcExpr.NamePos) + } + + var probability float64 + + switch funcExpr.Param[0].NodeType { //nolint:exhaustive + case ast.TypeArithmeticExpr: + res, dtype, err := runtime.RunArithmeticExpr(ctx, funcExpr.Param[0].ArithmeticExpr) + if err != nil { + return err + } + if dtype != ast.Float && dtype != ast.Int { + return runtime.NewRunError(ctx, + "data type of the result of arithmetic expression is neither float nor int", + funcExpr.Param[0].StartPos()) + } + p, ok := res.(float64) + if !ok { + return runtime.NewRunError(ctx, "failed to convert expression evaluation result to float", + funcExpr.Param[0].StartPos()) + } + probability = p + + case ast.TypeFloatLiteral: + probability = funcExpr.Param[0].FloatLiteral.Val + case ast.TypeIntegerLiteral: + probability = float64(funcExpr.Param[0].IntegerLiteral.Val) + default: + return runtime.NewRunError(ctx, fmt.Sprintf( + "expect NumberLiteral or ArithmeticExpr, got %s", funcExpr.Param[0].NodeType), + funcExpr.Param[0].StartPos()) + } + + if probability < 0 || probability > 1 { + l.Error("sampling probability should be in the range [0, 1]") + return nil + // return runtime.NewRunError(ctx, + // "sampling probability should be in the range [0, 1]", funcExpr.NamePos) + } + + res := time.Now().UnixMicro()%100 <= int64(probability*100) + ctx.Regs.ReturnAppend(res, ast.Bool) + return nil +} diff --git a/pkg/inimpl/guancecloud/ptinput/funcs/fn_sample_test.go b/pkg/inimpl/guancecloud/ptinput/funcs/fn_sample_test.go new file mode 100644 index 0000000..ccef5dc --- /dev/null +++ b/pkg/inimpl/guancecloud/ptinput/funcs/fn_sample_test.go @@ -0,0 +1,99 @@ +// Unless explicitly stated otherwise all files in this repository are licensed +// under the MIT License. +// This product includes software developed at Guance Cloud (https://www.guance.com/). +// Copyright 2021-present Guance, Inc. + +package funcs + +import ( + "testing" + "time" + + "github.com/GuanceCloud/cliutils/point" + "github.com/GuanceCloud/platypus/pkg/inimpl/guancecloud/ptinput" +) + +func TestSample(t *testing.T) { + cases := []struct { + name, pl, in string + fail bool + outKey string + }{ + { + name: "fifty-fifty", + in: `dummy input`, + pl: ` +if sample(0.5) { + add_key(hello, "world") +} + `, + fail: false, + outKey: "hello", + }, + { + name: "definite", + in: `dummy input`, + pl: ` +if sample(1) { + add_key(hello, "world") +} + `, + fail: false, + outKey: "hello", + }, + { + name: "expression as arg", + in: `dummy input`, + pl: ` +if sample(2 * 0.1) { + add_key(hello, "world") +} + `, + fail: false, + outKey: "hello", + }, + { + name: "negative probability", + in: `dummy input`, + pl: ` +sample(-0.5) + `, + fail: true, + }, + { + name: "probability out of range", + in: `dummy input`, + pl: ` +sample(2) + `, + fail: true, + }, + } + + for idx, tc := range cases { + t.Run(tc.name, func(t *testing.T) { + runner, err := NewTestingRunner(tc.pl) + if err != nil { + if tc.fail { + t.Logf("[%d]expect error: %s", idx, err) + } else { + t.Errorf("[%d] failed: %s", idx, err) + } + return + } + + pt := ptinput.NewPlPoint( + point.Logging, "test", nil, map[string]any{"message": tc.in}, time.Now()) + errR := runScript(runner, pt) + + if errR != nil { + t.Fatal(errR.Error()) + } + + if v, _, err := pt.Get(tc.outKey); err == nil { + t.Logf("k/v pair `%s = %s` has been added to output", tc.outKey, v) + } + t.Logf("[%d] PASS", idx) + }) + } +} diff --git a/pkg/inimpl/guancecloud/funcs/fn_set_measurement.go b/pkg/inimpl/guancecloud/ptinput/funcs/fn_set_measurement.go similarity index 100% rename from pkg/inimpl/guancecloud/funcs/fn_set_measurement.go rename to pkg/inimpl/guancecloud/ptinput/funcs/fn_set_measurement.go diff --git a/pkg/inimpl/guancecloud/funcs/fn_set_mesaurement_test.go b/pkg/inimpl/guancecloud/ptinput/funcs/fn_set_mesaurement_test.go similarity index 83% rename from pkg/inimpl/guancecloud/funcs/fn_set_mesaurement_test.go rename to pkg/inimpl/guancecloud/ptinput/funcs/fn_set_mesaurement_test.go index 149c4dd..7a03190 100644 --- a/pkg/inimpl/guancecloud/funcs/fn_set_mesaurement_test.go +++ b/pkg/inimpl/guancecloud/ptinput/funcs/fn_set_mesaurement_test.go @@ -9,6 +9,8 @@ import ( "testing" "time" + "github.com/GuanceCloud/cliutils/point" + "github.com/GuanceCloud/platypus/pkg/inimpl/guancecloud/ptinput" "github.com/stretchr/testify/assert" ) @@ -28,6 +30,7 @@ func TestSetMeasurement(t *testing.T) { grok(_, "%{IPORHOST:client_ip} %{NOTSPACE} %{NOTSPACE} \\[%{HTTPDATE:time}\\] \"%{DATA:data} %{GREEDYDATA} HTTP/%{NUMBER}\" %{INT:status_code} %{INT:bytes}") cast(data, "int") set_measurement(client_ip) + `, del: false, out: "client_ip", @@ -62,18 +65,17 @@ func TestSetMeasurement(t *testing.T) { } return } + pt := ptinput.NewPlPoint( + point.Logging, "test", nil, map[string]any{"message": tc.in}, time.Now()) + errR := runScript(runner, pt) - m, tags, _, _, _, err := runScript(runner, "test", nil, map[string]interface{}{ - "message": tc.in, - }, time.Now()) - if err != nil { - t.Error(err) - return + if errR != nil { + t.Fatal(errR.Error()) } - t.Log(t) - _, ok := tags[tc.out] + + _, _, ok := pt.GetWithIsTag(tc.out) assert.Equal(t, tc.del, !ok) - assert.Equal(t, tc.expect, m) + assert.Equal(t, tc.expect, pt.GetPtName()) t.Logf("[%d] PASS", idx) }) } diff --git a/pkg/inimpl/guancecloud/funcs/fn_set_tag.go b/pkg/inimpl/guancecloud/ptinput/funcs/fn_set_tag.go similarity index 90% rename from pkg/inimpl/guancecloud/funcs/fn_set_tag.go rename to pkg/inimpl/guancecloud/ptinput/funcs/fn_set_tag.go index 1182b11..dc6d326 100644 --- a/pkg/inimpl/guancecloud/funcs/fn_set_tag.go +++ b/pkg/inimpl/guancecloud/ptinput/funcs/fn_set_tag.go @@ -11,7 +11,7 @@ import ( "github.com/GuanceCloud/platypus/pkg/ast" "github.com/GuanceCloud/platypus/pkg/engine/runtime" "github.com/GuanceCloud/platypus/pkg/errchain" - "github.com/GuanceCloud/platypus/pkg/inimpl/guancecloud/input" + "github.com/GuanceCloud/platypus/pkg/inimpl/guancecloud/ptinput" ) func SetTagChecking(ctx *runtime.Context, funcExpr *ast.CallExpr) *errchain.PlError { @@ -59,7 +59,7 @@ func SetTag(ctx *runtime.Context, funcExpr *ast.CallExpr) *errchain.PlError { v, err := ctx.GetKey(key) if err != nil { l.Debug(err) - _ = addKey2PtWithVal(ctx.InData(), key, "", ast.String, input.KindPtTag) + _ = addKey2PtWithVal(ctx.InData(), key, "", ast.String, ptinput.KindPtTag) return nil } @@ -67,7 +67,7 @@ func SetTag(ctx *runtime.Context, funcExpr *ast.CallExpr) *errchain.PlError { dtype = v.DType } - _ = addKey2PtWithVal(ctx.InData(), key, val, dtype, input.KindPtTag) + _ = addKey2PtWithVal(ctx.InData(), key, val, dtype, ptinput.KindPtTag) return nil } diff --git a/pkg/inimpl/guancecloud/funcs/fn_set_tag_test.go b/pkg/inimpl/guancecloud/ptinput/funcs/fn_set_tag_test.go similarity index 84% rename from pkg/inimpl/guancecloud/funcs/fn_set_tag_test.go rename to pkg/inimpl/guancecloud/ptinput/funcs/fn_set_tag_test.go index 8965d87..79ea712 100644 --- a/pkg/inimpl/guancecloud/funcs/fn_set_tag_test.go +++ b/pkg/inimpl/guancecloud/ptinput/funcs/fn_set_tag_test.go @@ -9,6 +9,8 @@ import ( "testing" "time" + "github.com/GuanceCloud/cliutils/point" + "github.com/GuanceCloud/platypus/pkg/inimpl/guancecloud/ptinput" "github.com/stretchr/testify/assert" ) @@ -26,6 +28,7 @@ func TestSetTag(t *testing.T) { set_tag(client_ip) grok(_, "%{IPORHOST:client_ip} %{NOTSPACE} %{NOTSPACE} \\[%{HTTPDATE:time}\\] \"%{DATA:data} %{GREEDYDATA} HTTP/%{NUMBER}\" %{INT:status_code} %{INT:bytes}") cast(data, "int") + `, outtag: "client_ip", expect: "162.62.81.1", @@ -61,7 +64,8 @@ func TestSetTag(t *testing.T) { name: "set_tag 3", in: `{"str_a": "2", "str_b": "3"}`, pl: ` - input = load_json(_); str_a = input["str_a"]; str_b=input["str_b"]; + json(_, str_a) + json(_, str_b) set_tag(str_a, str_b) `, outtag: "str_a", @@ -72,7 +76,8 @@ func TestSetTag(t *testing.T) { name: "set_tag 3 int", in: `{"str_a": "2", "str_b": 3}`, pl: ` - input = load_json(_); str_a = input["str_a"]; str_b=input["str_b"]; + json(_, str_a) + json(_, str_b) set_tag(str_a, str_b) `, outtag: "str_a", @@ -92,15 +97,17 @@ func TestSetTag(t *testing.T) { } return } - _, tags, _, _, _, err := runScript(runner, "test", nil, map[string]interface{}{ - "message": tc.in, - }, time.Now()) - if err != nil { - t.Error(err) - return + pt := ptinput.NewPlPoint( + point.Logging, "test", nil, map[string]any{"message": tc.in}, time.Now()) + errR := runScript(runner, pt) + + if errR != nil { + t.Fatal(errR.Error()) } - t.Log(tags) - v := tags[tc.outtag] + + v, isTag, ok := pt.GetWithIsTag(tc.outtag) + assert.Equal(t, true, ok) + assert.Equal(t, true, isTag) assert.Equal(t, tc.expect, v) t.Logf("[%d] PASS", idx) }) diff --git a/pkg/inimpl/guancecloud/funcs/fn_sql_cover.go b/pkg/inimpl/guancecloud/ptinput/funcs/fn_sql_cover.go similarity index 94% rename from pkg/inimpl/guancecloud/funcs/fn_sql_cover.go rename to pkg/inimpl/guancecloud/ptinput/funcs/fn_sql_cover.go index 0029c20..4adeb0c 100644 --- a/pkg/inimpl/guancecloud/funcs/fn_sql_cover.go +++ b/pkg/inimpl/guancecloud/ptinput/funcs/fn_sql_cover.go @@ -12,7 +12,7 @@ import ( "github.com/GuanceCloud/platypus/pkg/ast" "github.com/GuanceCloud/platypus/pkg/engine/runtime" "github.com/GuanceCloud/platypus/pkg/errchain" - "github.com/GuanceCloud/platypus/pkg/inimpl/guancecloud/input" + "github.com/GuanceCloud/platypus/pkg/inimpl/guancecloud/ptinput" ) func SQLCoverChecking(ctx *runtime.Context, funcExpr *ast.CallExpr) *errchain.PlError { @@ -52,7 +52,7 @@ func SQLCover(ctx *runtime.Context, funcExpr *ast.CallExpr) *errchain.PlError { return nil } if err := addKey2PtWithVal(ctx.InData(), key, v, ast.String, - input.KindPtDefault); err != nil { + ptinput.KindPtDefault); err != nil { l.Debug(err) return nil } diff --git a/pkg/inimpl/guancecloud/funcs/fn_sql_cover_test.go b/pkg/inimpl/guancecloud/ptinput/funcs/fn_sql_cover_test.go similarity index 66% rename from pkg/inimpl/guancecloud/funcs/fn_sql_cover_test.go rename to pkg/inimpl/guancecloud/ptinput/funcs/fn_sql_cover_test.go index 5d5d6f5..b3b2fce 100644 --- a/pkg/inimpl/guancecloud/funcs/fn_sql_cover_test.go +++ b/pkg/inimpl/guancecloud/ptinput/funcs/fn_sql_cover_test.go @@ -9,7 +9,9 @@ import ( "testing" "time" - "github.com/stretchr/testify/assert" + "github.com/GuanceCloud/cliutils/point" + tu "github.com/GuanceCloud/cliutils/testutil" + "github.com/GuanceCloud/platypus/pkg/inimpl/guancecloud/ptinput" ) func TestSqlCover(t *testing.T) { @@ -22,27 +24,27 @@ func TestSqlCover(t *testing.T) { }{ { name: "normal", - pl: `input = load_json(_) ; str1xxx =input["str1xxx"]; - sql_cover(str1xxx)`, - in: `{"str1xxx": "select abc from def where x > 3 and y < 5"}`, - outKey: "str1xxx", + pl: `json(_, str_msg) + sql_cover(str_msg)`, + in: `{"str_msg": "select abc from def where x > 3 and y < 5"}`, + outKey: "str_msg", expected: "select abc from def where x > ? and y < ?", fail: false, }, { name: "normal", - pl: `input = load_json(_) ; str1xxx =input["str1xxx"]; - sql_cover(str1xxx)`, - in: `{"str1xxx": "SELECT $func$INSERT INTO table VALUES ('a', 1, 2)$func$ FROM users"}`, + pl: `json(_, str_msg) + sql_cover(str_msg)`, + in: `{"str_msg": "SELECT $func$INSERT INTO table VALUES ('a', 1, 2)$func$ FROM users"}`, fail: true, }, { name: "normal", - pl: `input = load_json(_) ; str1xxx =input["str1xxx"]; - sql_cover(str1xxx)`, - in: `{"str1xxx": "SELECT Codi , Nom_CA AS Nom, Descripció_CAT AS Descripció FROM ProtValAptitud WHERE Vigent=1 ORDER BY Ordre, Codi"}`, - outKey: "str1xxx", + pl: `json(_, str_msg) + sql_cover(str_msg)`, + in: `{"str_msg": "SELECT Codi , Nom_CA AS Nom, Descripció_CAT AS Descripció FROM ProtValAptitud WHERE Vigent=1 ORDER BY Ordre, Codi"}`, + outKey: "str_msg", expected: "SELECT Codi, Nom_CA, Descripció_CAT FROM ProtValAptitud WHERE Vigent = ? ORDER BY Ordre, Codi", fail: false, }, @@ -53,10 +55,10 @@ func TestSqlCover(t *testing.T) { }, { name: "normal", - pl: `input = load_json(_) ; str1xxx =input["str1xxx"]; - sql_cover(str1xxx)`, - in: `{"str1xxx": "SELECT ('/uffd')"}`, - outKey: "str1xxx", + pl: `json(_, str_msg) + sql_cover(str_msg)`, + in: `{"str_msg": "SELECT ('/uffd')"}`, + outKey: "str_msg", expected: "SELECT ( ? )", fail: false, }, @@ -92,20 +94,20 @@ select abc from def where x > 3 and y < 5`, return } - _, _, f, _, _, err := runScript(runner, "test", nil, map[string]interface{}{ - "message": tc.in, - }, time.Now()) - if err != nil { - t.Fatal(err) + pt := ptinput.NewPlPoint( + point.Logging, "test", nil, map[string]any{"message": tc.in}, time.Now()) + errR := runScript(runner, pt) + + if errR != nil { + t.Fatal(errR.Error()) } - t.Log(f) - if v, ok := f[tc.outKey]; !ok { + if v, _, err := pt.Get(tc.outKey); err != nil { if !tc.fail { - t.Errorf("[%d]expect error: %s", idx, err) + t.Errorf("[%d]expect error: %s", idx, errR.Error()) } } else { - assert.Equal(t, tc.expected, v) + tu.Equals(t, tc.expected, v) t.Logf("[%d] PASS", idx) } }) diff --git a/pkg/inimpl/guancecloud/funcs/fn_strfmt.go b/pkg/inimpl/guancecloud/ptinput/funcs/fn_strfmt.go similarity index 95% rename from pkg/inimpl/guancecloud/funcs/fn_strfmt.go rename to pkg/inimpl/guancecloud/ptinput/funcs/fn_strfmt.go index da4a1fd..268e0b4 100644 --- a/pkg/inimpl/guancecloud/funcs/fn_strfmt.go +++ b/pkg/inimpl/guancecloud/ptinput/funcs/fn_strfmt.go @@ -11,7 +11,7 @@ import ( "github.com/GuanceCloud/platypus/pkg/ast" "github.com/GuanceCloud/platypus/pkg/engine/runtime" "github.com/GuanceCloud/platypus/pkg/errchain" - "github.com/GuanceCloud/platypus/pkg/inimpl/guancecloud/input" + "github.com/GuanceCloud/platypus/pkg/inimpl/guancecloud/ptinput" ) func StrfmtChecking(ctx *runtime.Context, funcExpr *ast.CallExpr) *errchain.PlError { @@ -62,7 +62,7 @@ func Strfmt(ctx *runtime.Context, funcExpr *ast.CallExpr) *errchain.PlError { strfmt := fmt.Sprintf(fmts, outdata...) if err := addKey2PtWithVal(ctx.InData(), key, strfmt, ast.String, - input.KindPtDefault); err != nil { + ptinput.KindPtDefault); err != nil { l.Debug(err) return nil } diff --git a/pkg/inimpl/guancecloud/funcs/fn_strfmt_test.go b/pkg/inimpl/guancecloud/ptinput/funcs/fn_strfmt_test.go similarity index 64% rename from pkg/inimpl/guancecloud/funcs/fn_strfmt_test.go rename to pkg/inimpl/guancecloud/ptinput/funcs/fn_strfmt_test.go index b3d6375..98ed4d9 100644 --- a/pkg/inimpl/guancecloud/funcs/fn_strfmt_test.go +++ b/pkg/inimpl/guancecloud/ptinput/funcs/fn_strfmt_test.go @@ -9,6 +9,9 @@ import ( "testing" "time" + "github.com/GuanceCloud/cliutils/point" + tu "github.com/GuanceCloud/cliutils/testutil" + "github.com/GuanceCloud/platypus/pkg/inimpl/guancecloud/ptinput" "github.com/stretchr/testify/assert" ) @@ -21,19 +24,12 @@ func TestStrfmt(t *testing.T) { fail bool }{ { - name: "normal1", + name: "normal", pl: ` -input = load_json(_) -input = input["a"] -printf("%v", input) -a_second = input["second"] -a_third = input["third"] - +json(_, a.second, a_second) +json(_, a.third, a_third) cast(a_second, "int") -a_second = get_key(a_second) - -a_forth = input["forth"] - +json(_, a.forth, a_forth) strfmt(bb, "%d %s %v", a_second, a_third, a_forth) `, in: `{"a":{"first":2.3,"second":2,"third":"abc","forth":true},"age":47}`, @@ -43,13 +39,10 @@ strfmt(bb, "%d %s %v", a_second, a_third, a_forth) }, { - name: "normal2", + name: "normal", pl: ` - input = load_json(_) - input = input["a"] -a_first = input["first"] - -cast(a_first, "float");a_first=get_key(a_first) +json(_, a.first, a_first) +cast(a_first, "float") strfmt(bb, "%.4f", a_first) `, in: `{"a":{"first":2.3,"second":2,"third":"abc","forth":true},"age":47}`, @@ -59,13 +52,10 @@ strfmt(bb, "%.4f", a_first) }, { - name: "normal3", + name: "normal", pl: ` - input = load_json(_) - input = input["a"] -a_first = input["first"] - -cast(a_first, "float");a_first=get_key(a_first) +json(_, a.first, a_first) +cast(a_first, "float") strfmt(bb, "%.4f%d", a_first, 3) `, in: `{"a":{"first":2.3,"second":2,"third":"abc","forth":true},"age":47}`, @@ -75,13 +65,10 @@ strfmt(bb, "%.4f%d", a_first, 3) }, { - name: "normal4", + name: "normal", pl: ` - input = load_json(_) - input = input["a"] -a_first = input["first"] - -cast(a_first, "float");a_first=get_key(a_first) +json(_, a.first, a_first) +cast(a_first, "float") strfmt(bb, "%.4f%.1f", a_first, 3.5) `, in: `{"a":{"first":2.3,"second":2,"third":"abc","forth":true},"age":47}`, @@ -91,12 +78,9 @@ strfmt(bb, "%.4f%.1f", a_first, 3.5) }, { - name: "normal5", + name: "normal", pl: ` - input = load_json(_) - input = input["a"] -a_forth = input["forth"] - +json(_, a.forth, a_forth) strfmt(bb, "%v%s", a_forth, "tone") `, in: `{"a":{"first":2.3,"second":2,"third":"abcd","forth":true},"age":47}`, @@ -108,10 +92,8 @@ strfmt(bb, "%v%s", a_forth, "tone") { name: "not enough arg", pl: ` - input = load_json(_) - input = input["a"] - a_first = input["first"] - cast(a_first, "float");a_first=get_key(a_first) + json(_, a.first, a_first) + cast(a_first, "float") strfmt(bb) `, in: `{"a":{"first":2.3,"second":2,"third":"abc","forth":true},"age":47}`, @@ -121,10 +103,8 @@ strfmt(bb, "%v%s", a_forth, "tone") { name: "incorrect arg type", pl: ` - input = load_json(_) - input = input["a"] - a_first = input["first"] - cast(a_first, "float");a_first=get_key(a_first) + json(_, a.first, a_first) + cast(a_first, "float") strfmt(bb, 1) `, in: `{"a":{"first":2.3,"second":2,"third":"abc","forth":true},"age":47}`, @@ -144,22 +124,22 @@ strfmt(bb, "%v%s", a_forth, "tone") return } - m, _, f, _, _, err := runScript(runner, "test", nil, map[string]interface{}{ - "message": tc.in, - }, time.Now()) - if err != nil { - t.Fatal(err) + pt := ptinput.NewPlPoint( + point.Logging, "test", nil, map[string]any{"message": tc.in}, time.Now()) + errR := runScript(runner, pt) + + if errR != nil { + t.Fatal(errR.Error()) } - assert.Equal(t, "test", m) + assert.Equal(t, "test", pt.GetPtName()) - t.Log(f) - if v, ok := f[tc.outKey]; !ok { + if v, _, err := pt.Get(tc.outKey); err != nil { if !tc.fail { t.Errorf("[%d]expect error: %s", idx, err) } } else { - assert.Equal(t, tc.expected, v) + tu.Equals(t, tc.expected, v) t.Logf("[%d] PASS", idx) } }) diff --git a/pkg/inimpl/guancecloud/ptinput/funcs/fn_timestamp.go b/pkg/inimpl/guancecloud/ptinput/funcs/fn_timestamp.go new file mode 100644 index 0000000..023cc76 --- /dev/null +++ b/pkg/inimpl/guancecloud/ptinput/funcs/fn_timestamp.go @@ -0,0 +1,48 @@ +// Unless explicitly stated otherwise all files in this repository are licensed +// under the MIT License. +// This product includes software developed at Guance Cloud (https://www.guance.com/). +// Copyright 2021-present Guance, Inc. + +package funcs + +import ( + "time" + + "github.com/GuanceCloud/platypus/pkg/ast" + "github.com/GuanceCloud/platypus/pkg/engine/runtime" + "github.com/GuanceCloud/platypus/pkg/errchain" +) + +func TimestampChecking(ctx *runtime.Context, funcExpr *ast.CallExpr) *errchain.PlError { + err := reindexFuncArgs(funcExpr, []string{"precision"}, 0) + if err != nil { + return runtime.NewRunError(ctx, err.Error(), funcExpr.NamePos) + } + return nil +} + +func Timestamp(ctx *runtime.Context, funcExpr *ast.CallExpr) *errchain.PlError { + var precision string + if funcExpr.Param[0] != nil { + v, _, err := runtime.RunStmt(ctx, funcExpr.Param[0]) + if err != nil { + return err + } + if v, ok := v.(string); ok { + precision = v + } + } + var ts int64 + switch precision { + case "us": + ts = time.Now().UnixMicro() + case "ms": + ts = time.Now().UnixMilli() + case "s": + ts = time.Now().Unix() + default: + ts = time.Now().UnixNano() + } + ctx.Regs.ReturnAppend(ts, ast.Int) + return nil +} diff --git a/pkg/inimpl/guancecloud/ptinput/funcs/fn_timstamp_test.go b/pkg/inimpl/guancecloud/ptinput/funcs/fn_timstamp_test.go new file mode 100644 index 0000000..e17ad02 --- /dev/null +++ b/pkg/inimpl/guancecloud/ptinput/funcs/fn_timstamp_test.go @@ -0,0 +1,100 @@ +// Unless explicitly stated otherwise all files in this repository are licensed +// under the MIT License. +// This product includes software developed at Guance Cloud (https://www.guance.com/). +// Copyright 2021-present Guance, Inc. + +package funcs + +import ( + "math" + "testing" + "time" + + "github.com/GuanceCloud/cliutils/point" + "github.com/GuanceCloud/platypus/pkg/inimpl/guancecloud/ptinput" + "github.com/stretchr/testify/assert" +) + +func TestTimestamp(t *testing.T) { + cases := []struct { + name, pl, in string + outkey string + expect interface{} + fail bool + }{ + { + name: "timestamp_default", + in: `time_now`, + pl: `add_key(time1, timestamp())`, + outkey: "time1", + expect: time.Now().UnixNano(), + fail: false, + }, + { + name: "time_now_ns", + in: `time_now`, + pl: `add_key(time1, timestamp("ns"))`, + outkey: "time1", + expect: time.Now().UnixNano(), + fail: false, + }, + { + name: "time_now_us", + in: `time_now`, + pl: `add_key(time1, timestamp("us")* 1000)`, + outkey: "time1", + expect: time.Now().UnixNano(), + fail: false, + }, + { + name: "time_now_ms", + in: `time_now`, + pl: `add_key(time1, timestamp("ms")*1000000)`, + outkey: "time1", + expect: time.Now().UnixNano(), + fail: false, + }, + { + name: "time_now_s", + in: `time_now`, + pl: `add_key(time1, timestamp("s")*1000000000)`, + outkey: "time1", + expect: time.Now().UnixNano(), + }, + } + + for idx, tc := range cases { + t.Run(tc.name, func(t *testing.T) { + runner, err := NewTestingRunner(tc.pl) + if err != nil { + if tc.fail { + t.Logf("[%d]expect error: %s", idx, err) + } else { + t.Errorf("[%d] failed: %s", idx, err) + } + return + } + + pt := ptinput.NewPlPoint( + point.Logging, "test", nil, map[string]any{"message": tc.in}, time.Now()) + errR := runScript(runner, pt) + + if errR != nil { + t.Fatal(errR.Error()) + } + + v, _, _ := pt.Get(tc.outkey) + tsAct, ok := v.(int64) + if !ok { + t.Fatalf("type of %s is not int64", tc.outkey) + } + switch expect := tc.expect.(type) { + case int64: + assert.GreaterOrEqual(t, 1e9, math.Abs(float64(expect-tsAct))) + default: + t.Fatal("undefined action") + } + t.Logf("[%d] PASS", idx) + }) + } +} diff --git a/pkg/inimpl/guancecloud/funcs/fn_trim.go b/pkg/inimpl/guancecloud/ptinput/funcs/fn_trim.go similarity index 95% rename from pkg/inimpl/guancecloud/funcs/fn_trim.go rename to pkg/inimpl/guancecloud/ptinput/funcs/fn_trim.go index ee8d282..7c698f2 100644 --- a/pkg/inimpl/guancecloud/funcs/fn_trim.go +++ b/pkg/inimpl/guancecloud/ptinput/funcs/fn_trim.go @@ -12,7 +12,7 @@ import ( "github.com/GuanceCloud/platypus/pkg/ast" "github.com/GuanceCloud/platypus/pkg/engine/runtime" "github.com/GuanceCloud/platypus/pkg/errchain" - "github.com/GuanceCloud/platypus/pkg/inimpl/guancecloud/input" + "github.com/GuanceCloud/platypus/pkg/inimpl/guancecloud/ptinput" ) func TrimChecking(ctx *runtime.Context, funcExpr *ast.CallExpr) *errchain.PlError { @@ -65,7 +65,7 @@ func Trim(ctx *runtime.Context, funcExpr *ast.CallExpr) *errchain.PlError { } if err = addKey2PtWithVal(ctx.InData(), key, val, ast.String, - input.KindPtDefault); err != nil { + ptinput.KindPtDefault); err != nil { l.Debug(err) return nil } diff --git a/pkg/inimpl/guancecloud/funcs/fn_trim_test.go b/pkg/inimpl/guancecloud/ptinput/funcs/fn_trim_test.go similarity index 76% rename from pkg/inimpl/guancecloud/funcs/fn_trim_test.go rename to pkg/inimpl/guancecloud/ptinput/funcs/fn_trim_test.go index bf7a3f4..0640704 100644 --- a/pkg/inimpl/guancecloud/funcs/fn_trim_test.go +++ b/pkg/inimpl/guancecloud/ptinput/funcs/fn_trim_test.go @@ -9,7 +9,9 @@ import ( "testing" "time" - "github.com/stretchr/testify/assert" + "github.com/GuanceCloud/cliutils/point" + tu "github.com/GuanceCloud/cliutils/testutil" + "github.com/GuanceCloud/platypus/pkg/inimpl/guancecloud/ptinput" ) func TestTrim(t *testing.T) { @@ -65,17 +67,16 @@ func TestTrim(t *testing.T) { } return } + pt := ptinput.NewPlPoint( + point.Logging, "test", nil, map[string]any{"message": tc.in}, time.Now()) + errR := runScript(runner, pt) - _, _, f, _, _, err := runScript(runner, "test", nil, map[string]interface{}{ - "message": tc.in, - }, time.Now()) - if err != nil { - t.Fatal(err) + if errR != nil { + t.Fatal(errR.Error()) } - t.Log(f) - v := f[tc.outkey] - assert.Equal(t, tc.expect, v) + v, _, _ := pt.Get(tc.outkey) + tu.Equals(t, tc.expect, v) t.Logf("[%d] PASS", idx) }) } diff --git a/pkg/inimpl/guancecloud/funcs/fn_uppercase.go b/pkg/inimpl/guancecloud/ptinput/funcs/fn_uppercase.go similarity index 93% rename from pkg/inimpl/guancecloud/funcs/fn_uppercase.go rename to pkg/inimpl/guancecloud/ptinput/funcs/fn_uppercase.go index e2b283b..9f37bc5 100644 --- a/pkg/inimpl/guancecloud/funcs/fn_uppercase.go +++ b/pkg/inimpl/guancecloud/ptinput/funcs/fn_uppercase.go @@ -12,7 +12,7 @@ import ( "github.com/GuanceCloud/platypus/pkg/ast" "github.com/GuanceCloud/platypus/pkg/engine/runtime" "github.com/GuanceCloud/platypus/pkg/errchain" - "github.com/GuanceCloud/platypus/pkg/inimpl/guancecloud/input" + "github.com/GuanceCloud/platypus/pkg/inimpl/guancecloud/ptinput" ) func UppercaseChecking(ctx *runtime.Context, funcExpr *ast.CallExpr) *errchain.PlError { @@ -46,7 +46,7 @@ func Uppercase(ctx *runtime.Context, funcExpr *ast.CallExpr) *errchain.PlError { v := strings.ToUpper(cont) if err := addKey2PtWithVal(ctx.InData(), key, v, ast.String, - input.KindPtDefault); err != nil { + ptinput.KindPtDefault); err != nil { l.Debug(err) return nil } diff --git a/pkg/inimpl/guancecloud/funcs/fn_uppercase_test.go b/pkg/inimpl/guancecloud/ptinput/funcs/fn_uppercase_test.go similarity index 69% rename from pkg/inimpl/guancecloud/funcs/fn_uppercase_test.go rename to pkg/inimpl/guancecloud/ptinput/funcs/fn_uppercase_test.go index d0d8304..c3087a0 100644 --- a/pkg/inimpl/guancecloud/funcs/fn_uppercase_test.go +++ b/pkg/inimpl/guancecloud/ptinput/funcs/fn_uppercase_test.go @@ -10,7 +10,9 @@ import ( "testing" "time" - "github.com/stretchr/testify/assert" + "github.com/GuanceCloud/cliutils/point" + tu "github.com/GuanceCloud/cliutils/testutil" + "github.com/GuanceCloud/platypus/pkg/inimpl/guancecloud/ptinput" ) func TestUppercase(t *testing.T) { @@ -22,24 +24,22 @@ func TestUppercase(t *testing.T) { fail bool }{ { - name: "normal1", + name: "normal", pl: ` -input = load_json(_) -third = input["a"]["third"] -uppercase(third) +json(_, a.third) +uppercase(a.third) `, in: `{"a":{"first":2.3,"second":2,"third":"abc","forth":true},"age":47}`, - outKey: "third", + outKey: "a.third", expected: "ABC", fail: false, }, { - name: "normal2", + name: "normal", pl: ` - input = load_json(_) - age = input["age"] - uppercase(age) +json(_, age) +uppercase(age) `, in: `{"a":{"first":2.3,"second":2,"third":"abc","forth":true},"age":47}`, outKey: "age", @@ -48,13 +48,13 @@ uppercase(third) }, { - name: "normal3", + name: "normal", pl: ` - input = load_json(_) - forth = input["a"]["forth"];uppercase(forth) +json(_, a.forth) +uppercase(a.forth) `, in: `{"a":{"first":2.3,"second":2,"third":"abc","forth":"1a2B3c/d"},"age":47}`, - outKey: "forth", + outKey: "a.forth", expected: strings.ToUpper("1a2B3C/d"), fail: false, }, @@ -62,9 +62,8 @@ uppercase(third) { name: "too many args", pl: ` - input = load_json(_) - forth = input["a"]["forth"] - uppercase(forth, "someArg") + json(_, a.forth) + uppercase(a.forth, "someArg") `, in: `{"a":{"first":2.3,"second":2,"third":"abc","forth":"1a2B3c/d"},"age":47}`, fail: true, @@ -73,8 +72,7 @@ uppercase(third) { name: "invalid arg type", pl: ` - input = load_json(_) - forth = input["a"]["forth"] + json(_, a.forth) uppercase("hello") `, in: `{"a":{"first":2.3,"second":2,"third":"abc","forth":"1a2B3c/d"},"age":47}`, @@ -94,19 +92,20 @@ uppercase(third) return } - _, _, f, _, _, err := runScript(runner, "test", nil, map[string]interface{}{ - "message": tc.in, - }, time.Now()) - if err != nil { - t.Fatal(err) + pt := ptinput.NewPlPoint( + point.Logging, "test", nil, map[string]any{"message": tc.in}, time.Now()) + errR := runScript(runner, pt) + + if errR != nil { + t.Fatal(errR.Error()) } - if v, ok := f[tc.outKey]; !ok { + if v, _, err := pt.Get(tc.outKey); err != nil { if !tc.fail { t.Errorf("[%d]expect error: %s", idx, err) } } else { - assert.Equal(t, tc.expected, v) + tu.Equals(t, tc.expected, v) t.Logf("[%d] PASS", idx) } }) diff --git a/pkg/inimpl/guancecloud/ptinput/funcs/fn_url_parse.go b/pkg/inimpl/guancecloud/ptinput/funcs/fn_url_parse.go new file mode 100644 index 0000000..db0b79a --- /dev/null +++ b/pkg/inimpl/guancecloud/ptinput/funcs/fn_url_parse.go @@ -0,0 +1,69 @@ +// Unless explicitly stated otherwise all files in this repository are licensed +// under the MIT License. +// This product includes software developed at Guance Cloud (https://www.guance.com/). +// Copyright 2021-present Guance, Inc. + +package funcs + +import ( + "fmt" + "net/url" + "strings" + + "github.com/GuanceCloud/platypus/pkg/ast" + "github.com/GuanceCloud/platypus/pkg/engine/runtime" + "github.com/GuanceCloud/platypus/pkg/errchain" +) + +func URLParseChecking(ctx *runtime.Context, funcExpr *ast.CallExpr) *errchain.PlError { + if len(funcExpr.Param) != 1 { + return runtime.NewRunError(ctx, fmt.Sprintf( + "func %s expects 1 arg", funcExpr.Name), funcExpr.NamePos) + } + if funcExpr.Param[0].NodeType != ast.TypeIdentifier && funcExpr.Param[0].NodeType != ast.TypeAttrExpr { + return runtime.NewRunError(ctx, fmt.Sprintf( + "expect Identifier or AttrExpr, got %s", funcExpr.Param[0].NodeType), funcExpr.Param[0].StartPos()) + } + return nil +} + +func URLParse(ctx *runtime.Context, funcExpr *ast.CallExpr) *errchain.PlError { + if len(funcExpr.Param) != 1 { + return runtime.NewRunError(ctx, fmt.Sprintf( + "func %s expects 1 arg", funcExpr.Name), funcExpr.NamePos) + } + if funcExpr.Param[0].NodeType != ast.TypeIdentifier && funcExpr.Param[0].NodeType != ast.TypeAttrExpr { + return runtime.NewRunError(ctx, fmt.Sprintf( + "expect Identifier or AttrExpr, got %s", funcExpr.Param[0].NodeType), + funcExpr.Param[0].StartPos()) + } + key, err := getKeyName(funcExpr.Param[0]) + if err != nil { + return runtime.NewRunError(ctx, err.Error(), funcExpr.Param[0].StartPos()) + } + u, err := ctx.GetKeyConv2Str(key) + if err != nil { + l.Debug(err) + return nil + } + + uu, err := url.Parse(u) + if err != nil { + return runtime.NewRunError(ctx, fmt.Sprintf( + "failed to parse url %s: %s", u, err.Error()), funcExpr.NamePos) + } + + params := make(map[string]any) + for k, vs := range uu.Query() { + params[k] = strings.Join(vs, ",") + } + res := map[string]any{ + "scheme": uu.Scheme, + "host": uu.Host, + "port": uu.Port(), + "path": uu.Path, + "params": params, + } + ctx.Regs.ReturnAppend(res, ast.Map) + return nil +} diff --git a/pkg/inimpl/guancecloud/ptinput/funcs/fn_url_parse_test.go b/pkg/inimpl/guancecloud/ptinput/funcs/fn_url_parse_test.go new file mode 100644 index 0000000..ea2d3bd --- /dev/null +++ b/pkg/inimpl/guancecloud/ptinput/funcs/fn_url_parse_test.go @@ -0,0 +1,150 @@ +// Unless explicitly stated otherwise all files in this repository are licensed +// under the MIT License. +// This product includes software developed at Guance Cloud (https://www.guance.com/). +// Copyright 2021-present Guance, Inc. + +package funcs + +import ( + "testing" + "time" + + "github.com/GuanceCloud/cliutils/point" + tu "github.com/GuanceCloud/cliutils/testutil" + "github.com/GuanceCloud/platypus/pkg/inimpl/guancecloud/ptinput" +) + +func TestURLParse(t *testing.T) { + cases := []struct { + name string + pl, in string + outKey string + expected interface{} + fail bool + }{ + { + name: "port", + pl: ` +json(_, url) +m = url_parse(url) +add_key(scheme, m["scheme"]) +`, + in: `{"url": "https://www.baidu.com"}`, + outKey: "scheme", + expected: "https", + fail: false, + }, + { + name: "host", + pl: ` +json(_, url) +m = url_parse(url) +add_key(host, m["host"]) +`, + in: `{"url": "http://127.0.0.1:9529"}`, + outKey: "host", + expected: "127.0.0.1:9529", + fail: false, + }, + { + name: "port", + pl: ` +json(_, url) +m = url_parse(url) +add_key(port, m["port"]) +`, + in: `{"url": "http://127.0.0.1:9529"}`, + outKey: "port", + expected: "9529", + fail: false, + }, + { + name: "path", + pl: ` +json(_, url) +m = url_parse(url) +add_key(path, m["path"]) +`, + in: `{"url": "http://127.0.0.1:9529/v1/metrics"}`, + outKey: "path", + expected: "/v1/metrics", + fail: false, + }, + { + name: "arg1", + pl: ` +json(_, url) +m = url_parse(url) +add_key(a, m["params"]["arg1"]) +`, + in: `{"url": "http://127.0.0.1:9529/v1/metrics?arg1=v1&arg2=v2"}`, + outKey: "a", + expected: "v1", + fail: false, + }, + { + name: "arg2", + pl: ` +json(_, url) +m = url_parse(url) +add_key(a, m["params"]["arg2"]) +`, + in: `{"url": "http://127.0.0.1:9529/v1/metrics?arg1=v1&arg2=v2&arg2=v3"}`, + outKey: "a", + expected: "v2,v3", + fail: false, + }, + { + name: "invalid url", + pl: ` +json(_, url) +m = url_parse(url) +add_key(p, m["path"]) +`, + in: `{"url": "/var/log/datakit/log"}`, + fail: true, + }, + { + name: "two many args", + pl: ` +json(_, url) +m = url_parse(url, 2) +`, + in: `{"url": "http://127.0.0.1:9529/v1/metrics?arg1=v1&arg2=v2"}`, + fail: true, + }, + } + + for idx, tc := range cases { + t.Run(tc.name, func(t *testing.T) { + runner, err := NewTestingRunner(tc.pl) + if err != nil { + if tc.fail { + t.Logf("[%d]expect error: %s", idx, err) + } else { + t.Errorf("[%d] failed: %s", idx, err) + } + return + } + + var pt = ptinput.NewPlPoint(point.Logging, "test", nil, map[string]any{"message": tc.in}, time.Now()) + errR := runScript(runner, pt) + if errR != nil { + t.Fatal(errR) + } + + if v, istag, ok := pt.GetWithIsTag(tc.outKey); !ok { + if !tc.fail { + t.Errorf("[%d]key %s, error: %s", idx, tc.outKey, err) + } + } else { + if istag { + t.Errorf("key %s should be a field", tc.outKey) + } else { + tu.Equals(t, tc.expected, v) + t.Logf("[%d] PASS", idx) + } + } + }) + } +} diff --git a/pkg/inimpl/guancecloud/funcs/fn_urldecode.go b/pkg/inimpl/guancecloud/ptinput/funcs/fn_urldecode.go similarity index 93% rename from pkg/inimpl/guancecloud/funcs/fn_urldecode.go rename to pkg/inimpl/guancecloud/ptinput/funcs/fn_urldecode.go index c370662..b090c35 100644 --- a/pkg/inimpl/guancecloud/funcs/fn_urldecode.go +++ b/pkg/inimpl/guancecloud/ptinput/funcs/fn_urldecode.go @@ -11,7 +11,7 @@ import ( "github.com/GuanceCloud/platypus/pkg/ast" "github.com/GuanceCloud/platypus/pkg/engine/runtime" "github.com/GuanceCloud/platypus/pkg/errchain" - "github.com/GuanceCloud/platypus/pkg/inimpl/guancecloud/input" + "github.com/GuanceCloud/platypus/pkg/inimpl/guancecloud/ptinput" ) func URLDecodeChecking(ctx *runtime.Context, funcExpr *ast.CallExpr) *errchain.PlError { @@ -44,7 +44,7 @@ func URLDecode(ctx *runtime.Context, funcExpr *ast.CallExpr) *errchain.PlError { if v, err := UrldecodeHandle(cont); err != nil { return runtime.NewRunError(ctx, err.Error(), funcExpr.NamePos) - } else if err := addKey2PtWithVal(ctx.InData(), key, v, ast.String, input.KindPtDefault); err != nil { + } else if err := addKey2PtWithVal(ctx.InData(), key, v, ast.String, ptinput.KindPtDefault); err != nil { l.Debug(err) return nil } diff --git a/pkg/inimpl/guancecloud/funcs/fn_urldecode_test.go b/pkg/inimpl/guancecloud/ptinput/funcs/fn_urldecode_test.go similarity index 79% rename from pkg/inimpl/guancecloud/funcs/fn_urldecode_test.go rename to pkg/inimpl/guancecloud/ptinput/funcs/fn_urldecode_test.go index c9e5e10..70184b5 100644 --- a/pkg/inimpl/guancecloud/funcs/fn_urldecode_test.go +++ b/pkg/inimpl/guancecloud/ptinput/funcs/fn_urldecode_test.go @@ -9,7 +9,9 @@ import ( "testing" "time" - "github.com/stretchr/testify/assert" + "github.com/GuanceCloud/cliutils/point" + tu "github.com/GuanceCloud/cliutils/testutil" + "github.com/GuanceCloud/platypus/pkg/inimpl/guancecloud/ptinput" ) func TestURLDecode(t *testing.T) { @@ -22,7 +24,7 @@ func TestURLDecode(t *testing.T) { }{ { name: "normal", - pl: `input = load_json(_); url = input["url"] + pl: `json(_, url) url_decode(url)`, in: `{"url":"http%3a%2f%2fwww.baidu.com%2fs%3fwd%3d%e6%b5%8b%e8%af%95"}`, outKey: "url", @@ -32,7 +34,7 @@ func TestURLDecode(t *testing.T) { { name: "normal", - pl: `input = load_json(_); url = input["url"] + pl: `json(_, url) url_decode(url)`, in: `{"url":"https:%2F%2Fkubernetes.io%2Fdocs%2Freference%2Faccess-authn-authz%2Fbootstrap-tokens%2F"}`, outKey: "url", @@ -42,7 +44,7 @@ func TestURLDecode(t *testing.T) { { name: "non-existed key", - pl: `input = load_json(_); url = input["url"] + pl: `json(_, url) url_decode(link)`, in: `{"url":"https:%2F%2Fkubernetes.io%2Fdocs%2Freference%2Faccess-authn-authz%2Fbootstrap-tokens%2F"}`, outKey: "link", @@ -70,19 +72,20 @@ func TestURLDecode(t *testing.T) { return } - _, _, f, _, _, err := runScript(runner, "test", nil, map[string]interface{}{ - "message": tc.in, - }, time.Now()) - if err != nil { - t.Fatal(err) + pt := ptinput.NewPlPoint( + point.Logging, "test", nil, map[string]any{"message": tc.in}, time.Now()) + errR := runScript(runner, pt) + + if errR != nil { + t.Fatal(errR.Error()) } - if v, ok := f[tc.outKey]; !ok { + if v, _, err := pt.Get(tc.outKey); err != nil { if !tc.fail { t.Errorf("[%d]expect error", idx) } } else { - assert.Equal(t, tc.expected, v) + tu.Equals(t, tc.expected, v) t.Logf("[%d] PASS", idx) } }) diff --git a/pkg/inimpl/guancecloud/funcs/fn_use.go b/pkg/inimpl/guancecloud/ptinput/funcs/fn_use.go similarity index 93% rename from pkg/inimpl/guancecloud/funcs/fn_use.go rename to pkg/inimpl/guancecloud/ptinput/funcs/fn_use.go index 1d7c2da..8434791 100644 --- a/pkg/inimpl/guancecloud/funcs/fn_use.go +++ b/pkg/inimpl/guancecloud/ptinput/funcs/fn_use.go @@ -52,12 +52,11 @@ func Use(ctx *runtime.Context, funcExpr *ast.CallExpr) *errchain.PlError { } default: return runtime.NewRunError(ctx, fmt.Sprintf( - "param key expects AttrExpr or Identifier, got %s", + "param key expects StringLiteral, got %s", funcExpr.Param[0].NodeType), funcExpr.Param[0].StartPos()) } - err := runtime.RefRunScript(ctx, refScript) - if err != nil { + if err := runtime.RefRunScript(ctx, refScript); err != nil { return err.ChainAppend(ctx.Name(), funcExpr.NamePos) } return nil diff --git a/pkg/inimpl/guancecloud/ptinput/funcs/fn_use_test.go b/pkg/inimpl/guancecloud/ptinput/funcs/fn_use_test.go new file mode 100644 index 0000000..b41b2c5 --- /dev/null +++ b/pkg/inimpl/guancecloud/ptinput/funcs/fn_use_test.go @@ -0,0 +1,62 @@ +// Unless explicitly stated otherwise all files in this repository are licensed +// under the MIT License. +// This product includes software developed at Guance Cloud (https://www.guance.com/). +// Copyright 2021-present Guance, Inc. + +package funcs + +import ( + "testing" + "time" + + "github.com/GuanceCloud/cliutils/point" + "github.com/GuanceCloud/platypus/pkg/inimpl/guancecloud/ptinput" + "github.com/stretchr/testify/assert" +) + +func TestUse(t *testing.T) { + case1 := map[string]string{ + "a.p": "if true {use(\"b.p\")}", + "b.p": "add_key(b,1)", + "d.p": "use(\"c.p\")", + "c.p": "use(\"a.p\") use(\"d.p\") use(\"fcName.p\")", + } + + ret := [2][]string{ + {"a.p", "b.p"}, + {"d.p", "c.p"}, + } + + timenow := time.Now() + + ret1, ret2 := NewTestingRunner2(case1) + assert.Equal(t, len(ret[0]), len(ret1)) + assert.Equal(t, len(ret[1]), len(ret2)) + + for _, v := range ret[0] { + if _, ok := ret1[v]; !ok { + t.Error(v) + } + } + + for _, v := range ret[1] { + if _, ok := ret2[v]; !ok { + t.Error(v) + } + } + + for _, name := range ret[0] { + pt := ptinput.NewPlPoint( + point.Network, "default", map[string]string{"ax": "1"}, nil, timenow) + + errR := runScript(ret1[name], pt) + + if errR != nil { + t.Fatal(errR) + } + assert.Equal(t, map[string]string{"ax": "1"}, pt.Tags()) + assert.Equal(t, map[string]interface{}{"b": int64(1)}, pt.Fields()) + assert.Equal(t, false, pt.Dropped()) + assert.Equal(t, "default", pt.GetPtName()) + } +} diff --git a/pkg/inimpl/guancecloud/ptinput/funcs/fn_useragent.go b/pkg/inimpl/guancecloud/ptinput/funcs/fn_useragent.go new file mode 100644 index 0000000..679097e --- /dev/null +++ b/pkg/inimpl/guancecloud/ptinput/funcs/fn_useragent.go @@ -0,0 +1,54 @@ +// Unless explicitly stated otherwise all files in this repository are licensed +// under the MIT License. +// This product includes software developed at Guance Cloud (https://www.guance.com/). +// Copyright 2021-present Guance, Inc. + +package funcs + +import ( + "fmt" + + "github.com/GuanceCloud/platypus/pkg/ast" + "github.com/GuanceCloud/platypus/pkg/engine/runtime" + "github.com/GuanceCloud/platypus/pkg/errchain" + "github.com/GuanceCloud/platypus/pkg/inimpl/guancecloud/ptinput" +) + +func UserAgentChecking(ctx *runtime.Context, funcExpr *ast.CallExpr) *errchain.PlError { + if len(funcExpr.Param) != 1 { + return runtime.NewRunError(ctx, fmt.Sprintf( + "func `%s' expects 1 args", funcExpr.Name), funcExpr.NamePos) + } + if _, err := getKeyName(funcExpr.Param[0]); err != nil { + return runtime.NewRunError(ctx, err.Error(), funcExpr.Param[0].StartPos()) + } + return nil +} + +func UserAgent(ctx *runtime.Context, funcExpr *ast.CallExpr) *errchain.PlError { + if len(funcExpr.Param) != 1 { + return runtime.NewRunError(ctx, fmt.Sprintf( + "func `%s' expects 1 args", funcExpr.Name), funcExpr.NamePos) + } + + key, err := getKeyName(funcExpr.Param[0]) + if err != nil { + return runtime.NewRunError(ctx, err.Error(), funcExpr.Param[0].StartPos()) + } + + cont, err := ctx.GetKeyConv2Str(key) + if err != nil { + l.Debugf("key `%v' not exist, ignored", key) + return nil //nolint:nilerr + } + + dic, dicT := UserAgentHandle(cont) + + for k, val := range dic { + if dtype, ok := dicT[k]; ok { + _ = addKey2PtWithVal(ctx.InData(), k, val, dtype, ptinput.KindPtDefault) + } + } + + return nil +} diff --git a/pkg/inimpl/guancecloud/ptinput/funcs/fn_useragent_test.go b/pkg/inimpl/guancecloud/ptinput/funcs/fn_useragent_test.go new file mode 100644 index 0000000..315bfe9 --- /dev/null +++ b/pkg/inimpl/guancecloud/ptinput/funcs/fn_useragent_test.go @@ -0,0 +1,140 @@ +// Unless explicitly stated otherwise all files in this repository are licensed +// under the MIT License. +// This product includes software developed at Guance Cloud (https://www.guance.com/). +// Copyright 2021-present Guance, Inc. + +package funcs + +import ( + "testing" + "time" + + "github.com/GuanceCloud/cliutils/point" + tu "github.com/GuanceCloud/cliutils/testutil" + "github.com/GuanceCloud/platypus/pkg/inimpl/guancecloud/ptinput" +) + +func TestUserAgent(t *testing.T) { + cases := []struct { + name string + pl, in string + expected map[string]interface{} + fail bool + }{ + { + name: "normal", + pl: `json(_, userAgent) + user_agent(userAgent)`, + in: ` +{ + "userAgent" : "Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/36.0.1985.125 Safari/537.36", + "second" : 2, + "third" : "abc", + "forth" : true +} +`, + expected: map[string]interface{}{ + "isMobile": false, + "isBot": false, + "os": "Windows 7", + "browser": "Chrome", + "browserVer": "36.0.1985.125", + "engine": "AppleWebKit", + "engineVer": "537.36", + "ua": "Windows", + }, + fail: false, + }, + { + name: "normal", + pl: `json(_, userAgent) + user_agent(userAgent)`, + in: ` +{ + "userAgent" : "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/605.1.15 (KHTML, like Gecko) Version/15.1 Safari/605.1.15" +} +`, + expected: map[string]interface{}{ + "isMobile": false, + "isBot": false, + "os": "Intel Mac OS X 10_15_7", + "browser": "Safari", + "browserVer": "15.1", + "engine": "AppleWebKit", + "engineVer": "605.1.15", + "ua": "Macintosh", + }, + fail: false, + }, + + { + name: "normal", + pl: `json(_, userAgent) + user_agent(agent)`, + in: ` +{ + "userAgent" : "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/605.1.15 (KHTML, like Gecko) Version/15.1 Safari/605.1.15" +} +`, + expected: map[string]interface{}{}, + fail: false, + }, + + { + name: "invalid arg type", + pl: `json(_, userAgent) + user_agent("userAgent")`, + in: ` + { + "userAgent" : "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/605.1.15 (KHTML, like Gecko) Version/15.1 Safari/605.1.15" + } + `, + expected: map[string]interface{}{}, + fail: false, + }, + + { + name: "too many args", + pl: `json(_, userAgent) + user_agent(userAgent, someArg)`, + in: ` + { + "userAgent" : "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/605.1.15 (KHTML, like Gecko) Version/15.1 Safari/605.1.15" + } + `, + fail: true, + }, + } + for idx, tc := range cases { + t.Run(tc.name, func(t *testing.T) { + runner, err := NewTestingRunner(tc.pl) + if err != nil { + if tc.fail { + t.Logf("[%d]expect error: %s", idx, err) + } else { + t.Errorf("[%d] failed: %s", idx, err) + } + return + } + + pt := ptinput.NewPlPoint( + point.Logging, "test", nil, map[string]any{"message": tc.in}, time.Now()) + errR := runScript(runner, pt) + + if errR != nil { + if tc.fail { + t.Logf("[%d]expect error: %s", idx, err) + } else { + t.Fatal(errR) + } + } else { + fieldsToCompare := make(map[string]interface{}) + for k := range tc.expected { + fieldsToCompare[k], _, _ = pt.Get(k) + } + tu.Equals(t, tc.expected, fieldsToCompare) + t.Logf("[%d] PASS", idx) + } + }) + } +} diff --git a/pkg/inimpl/guancecloud/ptinput/funcs/fn_valid_json.go b/pkg/inimpl/guancecloud/ptinput/funcs/fn_valid_json.go new file mode 100644 index 0000000..28efc78 --- /dev/null +++ b/pkg/inimpl/guancecloud/ptinput/funcs/fn_valid_json.go @@ -0,0 +1,41 @@ +// Unless explicitly stated otherwise all files in this repository are licensed +// under the MIT License. +// This product includes software developed at Guance Cloud (https://www.guance.com/). +// Copyright 2021-present Guance, Inc. + +package funcs + +import ( + "encoding/json" + "fmt" + + "github.com/GuanceCloud/platypus/pkg/ast" + "github.com/GuanceCloud/platypus/pkg/engine/runtime" + "github.com/GuanceCloud/platypus/pkg/errchain" +) + +func ValidJSONChecking(ctx *runtime.Context, funcExpr *ast.CallExpr) *errchain.PlError { + if len(funcExpr.Param) != 1 { + return runtime.NewRunError(ctx, fmt.Sprintf( + "func %s expects 1 arg", funcExpr.Name), funcExpr.NamePos) + } + return nil +} + +func ValidJSON(ctx *runtime.Context, funcExpr *ast.CallExpr) *errchain.PlError { + val, _, err := runtime.RunStmt(ctx, funcExpr.Param[0]) + if err != nil { + return err + } + + if val != nil { + if v, ok := val.(string); ok { + valid := json.Valid([]byte(v)) + ctx.Regs.ReturnAppend(valid, ast.Bool) + return nil + } + } + + ctx.Regs.ReturnAppend(false, ast.Bool) + return nil +} diff --git a/pkg/inimpl/guancecloud/ptinput/funcs/fn_valid_json_test.go b/pkg/inimpl/guancecloud/ptinput/funcs/fn_valid_json_test.go new file mode 100644 index 0000000..56f8cc6 --- /dev/null +++ b/pkg/inimpl/guancecloud/ptinput/funcs/fn_valid_json_test.go @@ -0,0 +1,100 @@ +// Unless explicitly stated otherwise all files in this repository are licensed +// under the MIT License. +// This product includes software developed at Guance Cloud (https://www.guance.com/). +// Copyright 2021-present Guance, Inc. + +package funcs + +import ( + "testing" + "time" + + "github.com/GuanceCloud/cliutils/point" + tu "github.com/GuanceCloud/cliutils/testutil" + "github.com/GuanceCloud/platypus/pkg/inimpl/guancecloud/ptinput" +) + +func TestValidJson(t *testing.T) { + cases := []struct { + name, pl, in string + outkey string + expect interface{} + fail bool + }{ + { + name: "map", + in: `{"a":{"first": [2.2, 1.1], "ff": "[2.2, 1.1]","second":2,"third":"aBC","forth":true},"age":47}`, + pl: ` + if valid_json(_) { + d = load_json(_) + add_key("abc", d["a"]["first"][0]) + } + `, + outkey: "abc", + expect: 2.2, + }, + { + name: "map", + in: `{"a"??:{"first": [2.2, 1.1], "ff": "[2.2, 1.1]","second":2,"third":"aBC","forth":true},"age":47}`, + pl: ` + if valid_json(_) { + } else { + d = load_json(_) + add_key("abc", d["a"]["first"][0]) + } + `, + outkey: "abc", + expect: 2.2, + fail: true, + }, + { + name: "map", + in: ``, + pl: "add_key(`in`, valid_json(_))", + outkey: "in", + expect: false, + }, + { + name: "for-compatibility-with-older-versions", + in: `{"a":{"first": [2.2, 1.1], "ff": "[2.2, 1.1]","second":2,"third":"aBC","forth":true},"age":47}`, + pl: ` + if vaild_json(_) { + d = load_json(_) + add_key("abc", d["a"]["first"][0]) + } + `, + outkey: "abc", + expect: 2.2, + }, + } + + for idx, tc := range cases { + t.Run(tc.name, func(t *testing.T) { + runner, err := NewTestingRunner(tc.pl) + if err != nil { + if tc.fail { + t.Logf("[%d]expect error: %s", idx, err) + } else { + t.Errorf("[%d] failed: %s", idx, err) + } + return + } + + pt := ptinput.NewPlPoint( + point.Logging, "test", nil, map[string]any{"message": tc.in}, time.Now()) + errR := runScript(runner, pt) + + t.Log(pt.Fields()) + if errR != nil { + if tc.fail { + return + } + t.Fatal(errR) + } else { + v, _, _ := pt.Get(tc.outkey) + tu.Equals(t, tc.expect, v) + t.Logf("[%d] PASS", idx) + } + }) + } +} diff --git a/pkg/inimpl/guancecloud/ptinput/funcs/fn_value_type.go b/pkg/inimpl/guancecloud/ptinput/funcs/fn_value_type.go new file mode 100644 index 0000000..0fd5621 --- /dev/null +++ b/pkg/inimpl/guancecloud/ptinput/funcs/fn_value_type.go @@ -0,0 +1,47 @@ +// Unless explicitly stated otherwise all files in this repository are licensed +// under the MIT License. +// This product includes software developed at Guance Cloud (https://www.guance.com/). +// Copyright 2021-present Guance, Inc. + +package funcs + +import ( + "github.com/GuanceCloud/platypus/pkg/ast" + "github.com/GuanceCloud/platypus/pkg/engine/runtime" + "github.com/GuanceCloud/platypus/pkg/errchain" +) + +func ValueTypeChecking(ctx *runtime.Context, funcExpr *ast.CallExpr) *errchain.PlError { + if err := reindexFuncArgs(funcExpr, []string{ + "val", + }, 1); err != nil { + return runtime.NewRunError(ctx, err.Error(), funcExpr.NamePos) + } + + return nil +} + +func ValueType(ctx *runtime.Context, funcExpr *ast.CallExpr) *errchain.PlError { + if _, dtype, err := runtime.RunStmt(ctx, funcExpr.Param[0]); err != nil { + ctx.Regs.ReturnAppend("", ast.String) + return nil + } else { + var v string + switch dtype { //nolint:exhaustive + case ast.Bool: + v = "bool" + case ast.Int: + v = "int" + case ast.Float: + v = "float" + case ast.String: + v = "str" + case ast.List: + v = "list" + case ast.Map: + v = "map" + } + ctx.Regs.ReturnAppend(v, ast.String) + return nil + } +} diff --git a/pkg/inimpl/guancecloud/ptinput/funcs/fn_value_type_test.go b/pkg/inimpl/guancecloud/ptinput/funcs/fn_value_type_test.go new file mode 100644 index 0000000..6902b3a --- /dev/null +++ b/pkg/inimpl/guancecloud/ptinput/funcs/fn_value_type_test.go @@ -0,0 +1,181 @@ +// Unless explicitly stated otherwise all files in this repository are licensed +// under the MIT License. +// This product includes software developed at Guance Cloud (https://www.guance.com/). +// Copyright 2021-present Guance, Inc. + +package funcs + +import ( + "testing" + "time" + + "github.com/GuanceCloud/cliutils/point" + tu "github.com/GuanceCloud/cliutils/testutil" + "github.com/GuanceCloud/platypus/pkg/inimpl/guancecloud/ptinput" +) + +func TestVauleType(t *testing.T) { + cases := []struct { + name, pl, in string + outkey string + expect interface{} + fail bool + }{ + { + name: "map", + in: `{"a":{"first": [2.2, 1.1], "ff": "[2.2, 1.1]","second":2,"third":"aBC","forth":true},"age":47}`, + pl: ` + d = load_json(_) + add_key("val_type", value_type(d)) + `, + outkey: "val_type", + expect: "map", + fail: false, + }, + { + name: "list", + in: `{"a":{"first": [2.2, 1.1], "ff": "[2.2, 1.1]","second":2,"third":"aBC","forth":true},"age":47}`, + pl: ` + d = load_json(_) + + if value_type(d) == "map" && "a" in d && + value_type(d["a"]) == "map" && "first" in d["a"] { + add_key("val_type", value_type(d["a"]["first"])) + } + `, + outkey: "val_type", + expect: "list", + fail: false, + }, + { + name: "map_2", + in: `{"a":{"first": [2.2, 1.1], "ff": "[2.2, 1.1]","second":2,"third":"aBC","forth":true},"age":47}`, + pl: ` + d = load_json(_) + + if value_type(d) == "map" && "a" in d { + add_key("val_type", value_type(d["a"])) + } + `, + outkey: "val_type", + expect: "map", + fail: false, + }, + { + name: "list-not-in", + in: `{"a":{"first": [2.2, 1.1], "ff": "[2.2, 1.1]","second":2,"third":"aBC","forth":true},"age":47}`, + pl: ` + d = load_json(_) + + if "a" in d && "first" in d["a"] { + add_key("val_type", value_type(d["a"]["fist"])) # "fist" not in d["a"] + } + `, + outkey: "val_type", + expect: "", + fail: false, + }, + { + name: "int->float", + in: `{"a":{"first": [2.2, 1], "ff": "[2.2, 1.1]","second":2,"third":"aBC","forth":true},"age":47}`, + pl: ` + d = load_json(_) + add_key("val_type", value_type(d["a"]["first"][1])) + `, + outkey: "val_type", + expect: "float", // not int + fail: false, + }, + { + name: "float", + in: `{"a":{"first": [2.2, 1], "ff": "[2.2, 1.1]","second":2,"third":"aBC","forth":true},"age":47}`, + pl: ` + d = load_json(_) + add_key("val_type", value_type(d["a"]["first"][0])) + `, + outkey: "val_type", + expect: "float", + fail: false, + }, + { + name: "int", + in: ``, + pl: ` + d = {"a" : 1} + add_key("val_type", value_type(d["a"])) + `, + outkey: "val_type", + expect: "int", + fail: false, + }, + { + name: "bool", + in: `{"a":{"first": [true, 1], "ff": "[2.2, 1.1]","second":2,"third":"aBC","forth":true},"age":47}`, + pl: ` + d = load_json(_) + add_key("val_type", value_type(d["a"]["first"][0])) + `, + outkey: "val_type", + expect: "bool", + fail: false, + }, + { + name: "str", + in: `{"a":{"first": ["true", 1], "ff": "[2.2, 1.1]","second":2,"third":"aBC","forth":true},"age":47}`, + pl: ` + d = load_json(_) + add_key("val_type", value_type(d["a"]["first"][0])) + `, + outkey: "val_type", + expect: "str", + fail: false, + }, + { + name: "empty_nil", + in: ``, + pl: ` + add_key("val_type", value_type(nil)) + `, + outkey: "val_type", + expect: "", + fail: false, + }, + { + name: "empty_nil", + in: ``, + pl: ` + add_key("val_type", value_type(x)) + `, + outkey: "val_type", + expect: "", + fail: false, + }, + } + + for idx, tc := range cases { + t.Run(tc.name, func(t *testing.T) { + runner, err := NewTestingRunner(tc.pl) + if err != nil { + if tc.fail { + t.Logf("[%d]expect error: %s", idx, err) + } else { + t.Errorf("[%d] failed: %s", idx, err) + } + return + } + + pt := ptinput.NewPlPoint( + point.Logging, "test", nil, map[string]any{"message": tc.in}, time.Now()) + errR := runScript(runner, pt) + + t.Log(pt.Fields()) + if errR != nil { + t.Fatal(errR) + } else { + v, _, _ := pt.Get(tc.outkey) + tu.Equals(t, tc.expect, v) + t.Logf("[%d] PASS", idx) + } + }) + } +} diff --git a/pkg/inimpl/guancecloud/funcs/fn_xml.go b/pkg/inimpl/guancecloud/ptinput/funcs/fn_xml.go similarity index 96% rename from pkg/inimpl/guancecloud/funcs/fn_xml.go rename to pkg/inimpl/guancecloud/ptinput/funcs/fn_xml.go index 45f1ae0..8541d07 100644 --- a/pkg/inimpl/guancecloud/funcs/fn_xml.go +++ b/pkg/inimpl/guancecloud/ptinput/funcs/fn_xml.go @@ -13,7 +13,7 @@ import ( "github.com/GuanceCloud/platypus/pkg/ast" "github.com/GuanceCloud/platypus/pkg/engine/runtime" "github.com/GuanceCloud/platypus/pkg/errchain" - "github.com/GuanceCloud/platypus/pkg/inimpl/guancecloud/input" + "github.com/GuanceCloud/platypus/pkg/inimpl/guancecloud/ptinput" "github.com/antchfx/xmlquery" ) @@ -105,7 +105,7 @@ func XML(ctx *runtime.Context, funcExpr *ast.CallExpr) *errchain.PlError { } if err := addKey2PtWithVal(ctx.InData(), fieldName, dest.InnerText(), ast.String, - input.KindPtDefault); err != nil { + ptinput.KindPtDefault); err != nil { l.Debug(err) return nil } diff --git a/pkg/inimpl/guancecloud/funcs/fn_xml_test.go b/pkg/inimpl/guancecloud/ptinput/funcs/fn_xml_test.go similarity index 85% rename from pkg/inimpl/guancecloud/funcs/fn_xml_test.go rename to pkg/inimpl/guancecloud/ptinput/funcs/fn_xml_test.go index 80ae63a..58db1ef 100644 --- a/pkg/inimpl/guancecloud/funcs/fn_xml_test.go +++ b/pkg/inimpl/guancecloud/ptinput/funcs/fn_xml_test.go @@ -9,7 +9,9 @@ import ( "testing" "time" - "github.com/stretchr/testify/assert" + "github.com/GuanceCloud/cliutils/point" + tu "github.com/GuanceCloud/cliutils/testutil" + "github.com/GuanceCloud/platypus/pkg/inimpl/guancecloud/ptinput" ) func TestXML(t *testing.T) { @@ -105,20 +107,23 @@ func TestXML(t *testing.T) { for idx, tc := range testCases { t.Run(tc.name, func(t *testing.T) { runner, err := NewTestingRunner(tc.script) - assert.Equal(t, nil, err) - _, _, f, _, _, err := runScript(runner, "test", nil, map[string]interface{}{ - "message": tc.in, - }, time.Now()) + tu.Equals(t, nil, err) - assert.Equal(t, nil, err) + pt := ptinput.NewPlPoint( + point.Logging, "test", nil, map[string]any{"message": tc.in}, time.Now()) + errR := runScript(runner, pt) - r, ok := f[tc.key] - if !ok && tc.fail { + if errR != nil { + t.Fatal(errR) + } + + r, isTag, ok := pt.GetWithIsTag(tc.key) + if !ok && !isTag && tc.fail { t.Logf("[%d] failed as expected", idx) return } - assert.Equal(t, true, ok) - assert.Equal(t, tc.expected, r) + tu.Equals(t, nil, err) + tu.Equals(t, tc.expected, r) t.Logf("[%d] PASS", idx) }) } diff --git a/pkg/inimpl/guancecloud/ptinput/funcs/func_query_refer_table_test.go b/pkg/inimpl/guancecloud/ptinput/funcs/func_query_refer_table_test.go new file mode 100644 index 0000000..be1b39d --- /dev/null +++ b/pkg/inimpl/guancecloud/ptinput/funcs/func_query_refer_table_test.go @@ -0,0 +1,268 @@ +// Unless explicitly stated otherwise all files in this repository are licensed +// under the MIT License. +// This product includes software developed at Guance Cloud (https://www.guance.com/). +// Copyright 2021-present Guance, Inc. + +package funcs + +import ( + "context" + "net/http" + "net/http/httptest" + "testing" + "time" + + "github.com/GuanceCloud/cliutils/point" + "github.com/GuanceCloud/platypus/pkg/inimpl/guancecloud/ptinput" + "github.com/GuanceCloud/platypus/pkg/inimpl/guancecloud/ptinput/refertable" + "github.com/stretchr/testify/assert" +) + +func TestQueryReferTable(t *testing.T) { + files := map[string]string{ + "a.json": testTableData, + } + server := newJSONDataServer(files) + defer server.Close() + url := server.URL + + ctx := context.Background() + ctx, cancel := context.WithCancel(ctx) + defer cancel() + refertable.InitReferTableRunner(ctx, url+"?name=a.json", "", time.Second*5, false, false) + + ok := refertable.InitFinished(time.Second * 5) + if !ok { + t.Fatal("init refer table timeout") + } + + testCase := []struct { + name string + in string + + script string + + key []string + expected []any + fail bool + }{ + { + name: "test query", + in: `test query"`, + script: `add_key(f1, 123) + query_refer_table("table1", key = "f1", value= f1)`, + key: []string{"key1", "key2", "f1", "f2"}, + expected: []any{"a", float64(123), int64(123), false}, + }, + { + name: "test query multi", + in: `test query multi"`, + script: `add_key(f1, 123) + mquery_refer_table("table1", ["f1"], values= [f1])`, + key: []string{"key1", "key2", "f1", "f2"}, + expected: []any{"a", float64(123), int64(123), false}, + }, + { + name: "test int", + in: `test int `, + script: `# add_key(f1, 123) + query_refer_table("table1", "f1", 123)`, + key: []string{"key1", "key2", "f1", "f2"}, + expected: []any{"a", float64(123), int64(123), false}, + }, + { + name: "test string", + in: `test string`, + script: ` + query_refer_table("table1", "key1", "a")`, + key: []string{"key1", "key2", "f1", "f2"}, + expected: []any{"a", float64(123), int64(123), false}, + }, + { + name: "test string 2", + in: `test string`, + script: ` + t = "table1" + k = "key1" + v = "a" + query_refer_table(t, k, v)`, + key: []string{"key1", "key2", "f1", "f2"}, + expected: []any{"a", float64(123), int64(123), false}, + }, + { + name: "test float", + in: `test float`, + script: ` + # add_key(f1, 123) + query_refer_table("table1", "key2", 123.)`, + key: []string{"key1", "key2", "f1", "f2"}, + expected: []any{"a", float64(123), int64(123), false}, + }, + { + name: "test bool", + in: `test bool`, + script: ` + # add_key(f1, 123) + query_refer_table("table1", "f2", false)`, + key: []string{"key1", "key2", "f1", "f2"}, + expected: []any{"a", float64(123), int64(123), false}, + }, + { + name: "test float but int", + in: `test float but int`, + script: ` + # add_key(f1, 123) + query_refer_table("table1", "key2", 123)`, + key: []string{"key1", "key2", "f1", "f2"}, + expected: []any{nil, nil, nil, nil}, + }, + { + name: "test query, key not find", + in: `test query, key not find"`, + script: `#add_key(f1, 123) + query_refer_table("table1", "f1", f1)`, + key: []string{"key1", "key2", "f1", "f2"}, + expected: []any{nil, nil, nil, nil}, + }, + { + name: "test query, positional keyword", + in: `test query, positional keyword`, + script: `#add_key(f1, 123) + query_refer_table("table1", "f1", value=f1)`, + key: []string{"key1", "key2", "f1", "f2"}, + expected: []any{nil, nil, nil, nil}, + fail: true, + }, + { + name: "test-multi", + in: `test-multi"`, + script: ` + key = "f2" + value = "ab" + mquery_refer_table("table1", ["key1", key], [value, false])`, + key: []string{"key1", "key2", "f1", "f2"}, + expected: []any{"ab", float64(1234), int64(123), false}, + }, + } + + for _, tc := range testCase { + t.Run(tc.name, func(t *testing.T) { + runner, err := NewTestingRunner(tc.script) + if err != nil { + t.Error(err) + return + } + + pt := ptinput.NewPlPoint( + point.Logging, "test", nil, map[string]any{"message": tc.in}, time.Now()) + errR := runScript(runner, pt) + + if errR != nil { + t.Fatal(errR.Error()) + } + + for idxK, key := range tc.key { + v, _, ok := pt.GetWithIsTag(key) + if !ok { + if len(tc.expected) != 0 { + t.Logf("key: %s, value exp: %v act: nil", + key, tc.expected[idxK]) + } + } + assert.Equal(t, tc.expected[idxK], v) + } + }) + } +} + +func newJSONDataServer(files map[string]string) *httptest.Server { + server := httptest.NewServer(http.HandlerFunc( + func(w http.ResponseWriter, r *http.Request) { + switch r.Method { + case http.MethodGet: + default: + w.WriteHeader(http.StatusBadRequest) + return + } + name := r.FormValue("name") + data := files[name] + w.Write([]byte(data)) + w.WriteHeader(http.StatusOK) + }, + )) + return server +} + +var testTableData = ` +[ + { + "table_name": "table1", + "column_name": [ + "key1", + "key2", + "f1", + "f2" + ], + "column_type": [ + "string", + "float", + "int", + "bool" + ], + "row_data": [ + [ + "a", + 123, + "123", + "false" + ], + [ + "ab", + "1234", + "123", + "true" + ], + [ + "ab", + "1234", + "123", + "false" + ] + ] + }, + { + "table_name": "table2", + "primary_key": [ + "key1", + "key2" + ], + "column_name": [ + "key1", + "key2", + "f1", + "f2" + ], + "column_type": [ + "string", + "float", + "int", + "bool" + ], + "row_data": [ + [ + "a", + 123, + "123", + "true" + ], + [ + "a", + "1234", + "123", + "true" + ] + ] + } +] + +` diff --git a/pkg/inimpl/guancecloud/funcs/funcs_b_test.go b/pkg/inimpl/guancecloud/ptinput/funcs/funcs_b_test.go similarity index 98% rename from pkg/inimpl/guancecloud/funcs/funcs_b_test.go rename to pkg/inimpl/guancecloud/ptinput/funcs/funcs_b_test.go index 0dd70f7..f73c878 100644 --- a/pkg/inimpl/guancecloud/funcs/funcs_b_test.go +++ b/pkg/inimpl/guancecloud/ptinput/funcs/funcs_b_test.go @@ -12,7 +12,7 @@ import ( "time" "github.com/stretchr/testify/assert" - tu "gitlab.jiagouyun.com/cloudcare-tools/cliutils/testutil" + tu "github.com/GuanceCloud/cliutils/testutil" ) type funcCase struct { @@ -155,14 +155,14 @@ func TestDefaultTimeFunc(t *testing.T) { t.Run(tt.name, func(t *testing.T) { p, err := NewPipeline(tt.script) - assert.Equal(t, err, nil) + tu.Equals(t, err, nil) p.Run(tt.data) r, err := p.getContent(tt.key) - assert.Equal(t, err, nil) + tu.Equals(t, err, nil) - assert.Equal(t, r, tt.expected) + tu.Equals(t, r, tt.expected) // t.Logf("[passed? %v]out: %s <> %v", ok, tt.data, p.Output) }) @@ -360,16 +360,16 @@ func TestGroupFunc(t *testing.T) { for _, tt := range testCase { t.Run(tt.name, func(t *testing.T) { p, err := NewPipeline(tt.script) - assert.Equal(t, p.lastErr, err) + tu.Equals(t, p.lastErr, err) p.Run(tt.data) r, err := p.getContent(tt.key) if tt.expected != nil { - assert.Equal(t, nil, err) + tu.Equals(t, nil, err) } - assert.Equal(t, tt.expected, r) + tu.Equals(t, tt.expected, r) }) } } diff --git a/pkg/inimpl/guancecloud/funcs/funcs_benchmark_test.go b/pkg/inimpl/guancecloud/ptinput/funcs/funcs_benchmark_test.go similarity index 88% rename from pkg/inimpl/guancecloud/funcs/funcs_benchmark_test.go rename to pkg/inimpl/guancecloud/ptinput/funcs/funcs_benchmark_test.go index 80f70db..1d0ac74 100644 --- a/pkg/inimpl/guancecloud/funcs/funcs_benchmark_test.go +++ b/pkg/inimpl/guancecloud/ptinput/funcs/funcs_benchmark_test.go @@ -8,6 +8,9 @@ package funcs import ( "testing" "time" + + "github.com/GuanceCloud/cliutils/point" + "github.com/GuanceCloud/platypus/pkg/inimpl/guancecloud/ptinput" ) /* @@ -159,10 +162,12 @@ func BenchmarkParseLog(b *testing.B) { data := `2017-12-29T12:33:33.095243Z 2 Query SELECT TABLE_SCHEMA, TABLE_NAME FROM INFORMATION_SCHEMA.TABLES WHERE CREATE_OPTIONS LIKE '%partitioned%'` for n := 0; n < b.N; n++ { - if _, _, _, _, _, err := runScript(runner, "test", nil, map[string]interface{}{ - "message": data, - }, time.Now()); err != nil { - b.Error(err) + pt := ptinput.NewPlPoint( + point.Logging, "test", nil, map[string]any{"message": data}, time.Now()) + errR := runScript(runner, pt) + + if errR != nil { + b.Fatal(errR) } } } @@ -190,10 +195,12 @@ func BenchmarkParseLog_tz(b *testing.B) { data := `2017-12-29T12:33:33.095243Z 1.1.1.1 2 ` for n := 0; n < b.N; n++ { - if _, _, _, _, _, err := runScript(runner, "test", nil, map[string]interface{}{ - "message": data, - }, time.Now()); err != nil { - b.Error(err) + pt := ptinput.NewPlPoint( + point.Logging, "test", nil, map[string]any{"message": data}, time.Now()) + errR := runScript(runner, pt) + + if errR != nil { + b.Fatal(errR) } } } @@ -212,10 +219,12 @@ grok(_, "%{IPORHOST:client_ip} %{NOTSPACE:http_ident} %{NOTSPACE:http_auth} \\[% // data := `fe80:d::127.0.0.1 - - [21/Jul/2021:14:14:38 +0800] "GET /?1 HTTP/1.1" 200 2178 "-" "Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/91.0.4472.164 Safari/537.36"` for n := 0; n < b.N; n++ { - if _, _, _, _, _, err := runScript(runner, "test", nil, map[string]interface{}{ - "message": data, - }, time.Now()); err != nil { - b.Error(err) + pt := ptinput.NewPlPoint( + point.Logging, "test", nil, map[string]any{"message": data}, time.Now()) + errR := runScript(runner, pt) + + if errR != nil { + b.Fatal(errR) } } } @@ -260,10 +269,12 @@ default_time(time) data := `127.0.0.1 - - [21/Jul/2021:14:14:38 +0800] "GET /?1 HTTP/1.1" 200 2178 "-" "Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/91.0.4472.164 Safari/537.36"` b.ResetTimer() for n := 0; n < b.N; n++ { - if _, _, _, _, _, err := runScript(runner, "test", nil, map[string]interface{}{ - "message": data, - }, time.Now()); err != nil { - b.Error(err) + pt := ptinput.NewPlPoint( + point.Logging, "test", nil, map[string]any{"message": data}, time.Now()) + errR := runScript(runner, pt) + + if errR != nil { + b.Fatal(errR) } } } @@ -286,10 +297,12 @@ func BenchmarkParseLogWithTfmt(b *testing.B) { data := `2021-07-20T12:33:33.095243Z 2 Query SELECT TABLE_SCHEMA, TABLE_NAME FROM INFORMATION_SCHEMA.TABLES WHERE CREATE_OPTIONS LIKE '%partitioned%'` for n := 0; n < b.N; n++ { - if _, _, _, _, _, err := runScript(runner, "test", nil, map[string]interface{}{ - "message": data, - }, time.Now()); err != nil { - b.Error(err) + pt := ptinput.NewPlPoint( + point.Logging, "test", nil, map[string]any{"message": data}, time.Now()) + errR := runScript(runner, pt) + + if errR != nil { + b.Fatal(errR) } } } @@ -313,10 +326,12 @@ func BenchmarkParseLogWithTfmt_tz(b *testing.B) { data := `2021-07-20T12:33:33.095243Z 2 Query SELECT TABLE_SCHEMA, TABLE_NAME FROM INFORMATION_SCHEMA.TABLES WHERE CREATE_OPTIONS LIKE '%partitioned%'` for n := 0; n < b.N; n++ { - if _, _, _, _, _, err := runScript(runner, "test", nil, map[string]interface{}{ - "message": data, - }, time.Now()); err != nil { - b.Error(err) + pt := ptinput.NewPlPoint( + point.Logging, "test", nil, map[string]any{"message": data}, time.Now()) + errR := runScript(runner, pt) + + if errR != nil { + b.Fatal(errR) } } } @@ -335,10 +350,12 @@ func BenchmarkParseLogWithTfmt_NoAddPattern(b *testing.B) { data := `2021-07-20T12:33:33.095243Z 2 Query SELECT TABLE_SCHEMA, TABLE_NAME FROM INFORMATION_SCHEMA.TABLES WHERE CREATE_OPTIONS LIKE '%partitioned%'` for n := 0; n < b.N; n++ { - if _, _, _, _, _, err := runScript(runner, "test", nil, map[string]interface{}{ - "message": data, - }, time.Now()); err != nil { - b.Error(err) + pt := ptinput.NewPlPoint( + point.Logging, "test", nil, map[string]any{"message": data}, time.Now()) + errR := runScript(runner, pt) + + if errR != nil { + b.Fatal(errR) } } } diff --git a/pkg/inimpl/guancecloud/ptinput/funcs/funcs_mquery_refer_table.go b/pkg/inimpl/guancecloud/ptinput/funcs/funcs_mquery_refer_table.go new file mode 100644 index 0000000..aad742c --- /dev/null +++ b/pkg/inimpl/guancecloud/ptinput/funcs/funcs_mquery_refer_table.go @@ -0,0 +1,191 @@ +// Unless explicitly stated otherwise all files in this repository are licensed +// under the MIT License. +// This product includes software developed at Guance Cloud (https://www.guance.com/). +// Copyright 2021-present Guance, Inc. + +package funcs + +import ( + "fmt" + "reflect" + + "github.com/GuanceCloud/platypus/pkg/ast" + "github.com/GuanceCloud/platypus/pkg/engine/runtime" + "github.com/GuanceCloud/platypus/pkg/errchain" + "github.com/GuanceCloud/platypus/pkg/inimpl/guancecloud/ptinput" + "github.com/GuanceCloud/platypus/pkg/inimpl/guancecloud/ptinput/refertable" + "github.com/spf13/cast" +) + +func MQueryReferTableChecking(ctx *runtime.Context, funcExpr *ast.CallExpr) *errchain.PlError { + err := reindexFuncArgs(funcExpr, []string{"table_name", "keys", "values"}, 3) + if err != nil { + return runtime.NewRunError(ctx, err.Error(), funcExpr.NamePos) + } + + if _, err := getKeyName(funcExpr.Param[0]); err != nil { + return runtime.NewRunError(ctx, err.Error(), funcExpr.Param[0].StartPos()) + } + + switch funcExpr.Param[1].NodeType { //nolint:exhaustive + case ast.TypeListInitExpr: + for _, v := range funcExpr.Param[1].ListInitExpr.List { + switch v.NodeType { //nolint:exhaustive + case ast.TypeStringLiteral, ast.TypeIdentifier, ast.TypeAttrExpr: + default: + return runtime.NewRunError(ctx, fmt.Sprintf( + "expect StringLiteral, Identifier or AttrExpr in FuncArgList, got %s", + reflect.TypeOf(v).String()), funcExpr.Param[1].StartPos()) + } + } + case ast.TypeIdentifier, ast.TypeAttrExpr: + default: + return runtime.NewRunError(ctx, fmt.Sprintf( + "param key expects FuncArgList, Identifier or AttrExpr, got %s", + funcExpr.Param[1].NodeType), funcExpr.Param[1].StartPos()) + } + + switch funcExpr.Param[2].NodeType { //nolint:exhaustive + case ast.TypeListInitExpr: + for _, v := range funcExpr.Param[2].ListInitExpr.List { + switch v.NodeType { //nolint:exhaustive + case ast.TypeIdentifier, ast.TypeAttrExpr, + ast.TypeStringLiteral, ast.TypeBoolLiteral, + ast.TypeFloatLiteral, ast.TypeIntegerLiteral: + default: + return runtime.NewRunError(ctx, fmt.Sprintf( + "expect Identifier, AttrExpr, "+ + "StringLiteral, NumberLiteral or BoolLiteral in FuncArgList, got %s", + v.NodeType), funcExpr.Param[2].StartPos()) + } + } + case ast.TypeIdentifier, ast.TypeAttrExpr: + default: + return runtime.NewRunError(ctx, fmt.Sprintf( + "param key expects StringLiteral, NumberLiteral, BoolLiteral, AttrExpr or Identifier, got %s", + funcExpr.Param[2].NodeType), funcExpr.Param[2].StartPos()) + } + + return nil +} + +func MQueryReferTableMulti(ctx *runtime.Context, funcExpr *ast.CallExpr) *errchain.PlError { + var tableName string + + tname, dtype, err := runtime.RunStmt(ctx, funcExpr.Param[0]) + if err != nil { + return err + } + if dtype != ast.String { + return runtime.NewRunError(ctx, "param expect string", + funcExpr.Param[0].StartPos()) + } + + tableName = tname.(string) + + var colName []string + + switch funcExpr.Param[1].NodeType { //nolint:exhaustive + case ast.TypeListInitExpr: + for _, v := range funcExpr.Param[1].ListInitExpr.List { + switch v.NodeType { //nolint:exhaustive + case ast.TypeStringLiteral: + colName = append(colName, v.StringLiteral.Val) + case ast.TypeIdentifier, ast.TypeAttrExpr: + key, _ := getKeyName(v) + val, err := ctx.GetKey(key) + if err != nil { + l.Error(err) + return runtime.NewRunError(ctx, err.Error(), funcExpr.Param[1].StartPos()) + } + switch val.DType { //nolint:exhaustive + case ast.String: + colName = append(colName, cast.ToString(val.Value)) + default: + err := fmt.Errorf("unsupported column name value type %s", val.DType) + l.Error(err) + return runtime.NewRunError(ctx, err.Error(), funcExpr.Param[1].StartPos()) + } + } + } + case ast.TypeIdentifier, ast.TypeAttrExpr: + key, _ := getKeyName(funcExpr.Param[1]) + + val, err := ctx.GetKey(key) + if err != nil { + l.Error(err) + return runtime.NewRunError(ctx, err.Error(), funcExpr.Param[1].StartPos()) + } + switch val.DType { //nolint:exhaustive + case ast.String: + colName = append(colName, cast.ToString(val.Value)) + default: + err := fmt.Errorf("unsupported column name value type %s", val.DType) + l.Error(err) + return runtime.NewRunError(ctx, err.Error(), funcExpr.Param[1].StartPos()) + } + } + + var colValue []any + switch funcExpr.Param[2].NodeType { //nolint:exhaustive + case ast.TypeListInitExpr: + for _, v := range funcExpr.Param[2].ListInitExpr.List { + switch v.NodeType { //nolint:exhaustive + case ast.TypeStringLiteral: + colValue = append(colValue, v.StringLiteral.Val) + case ast.TypeFloatLiteral: + colValue = append(colValue, v.FloatLiteral.Val) + case ast.TypeIntegerLiteral: + colValue = append(colValue, v.IntegerLiteral.Val) + case ast.TypeBoolLiteral: + colValue = append(colValue, v.BoolLiteral.Val) + case ast.TypeIdentifier, ast.TypeAttrExpr: + key, _ := getKeyName(v) + val, err := ctx.GetKey(key) + if err != nil { + l.Debug(err) + return runtime.NewRunError(ctx, err.Error(), funcExpr.Param[2].StartPos()) + } + colValue = append(colValue, val.Value) + } + } + case ast.TypeIdentifier, ast.TypeAttrExpr: + key, _ := getKeyName(funcExpr.Param[2]) + val, err := ctx.GetKey(key) + if err != nil { + l.Error(err) + return runtime.NewRunError(ctx, err.Error(), funcExpr.Param[2].StartPos()) + } + switch val.DType { //nolint:exhaustive + case ast.List: + if colval, ok := val.Value.([]any); ok { + colValue = colval + } + default: + err := fmt.Errorf("unsupported column value type %s", reflect.TypeOf(val).String()) + l.Error(err) + return runtime.NewRunError(ctx, err.Error(), funcExpr.Param[2].StartPos()) + } + } + + if vMap, ok := refertable.QueryReferTable(tableName, + colName, colValue, nil); ok { + for k, v := range vMap { + var dtype ast.DType + switch v.(type) { + case string: + dtype = ast.String + case bool: + dtype = ast.Bool + case int64: + dtype = ast.Int + case float64: + dtype = ast.Float + default: + return nil + } + _ = addKey2PtWithVal(ctx.InData(), k, v, dtype, ptinput.KindPtDefault) + } + } + return nil +} diff --git a/pkg/inimpl/guancecloud/ptinput/funcs/funcs_query_refer_table.go b/pkg/inimpl/guancecloud/ptinput/funcs/funcs_query_refer_table.go new file mode 100644 index 0000000..b9de582 --- /dev/null +++ b/pkg/inimpl/guancecloud/ptinput/funcs/funcs_query_refer_table.go @@ -0,0 +1,158 @@ +// Unless explicitly stated otherwise all files in this repository are licensed +// under the MIT License. +// This product includes software developed at Guance Cloud (https://www.guance.com/). +// Copyright 2021-present Guance, Inc. + +package funcs + +import ( + "fmt" + + "github.com/GuanceCloud/platypus/pkg/ast" + "github.com/GuanceCloud/platypus/pkg/engine/runtime" + "github.com/GuanceCloud/platypus/pkg/errchain" + "github.com/GuanceCloud/platypus/pkg/inimpl/guancecloud/ptinput" + "github.com/GuanceCloud/platypus/pkg/inimpl/guancecloud/ptinput/refertable" + "github.com/spf13/cast" +) + +func QueryReferTableChecking(ctx *runtime.Context, funcExpr *ast.CallExpr) *errchain.PlError { + err := reindexFuncArgs(funcExpr, []string{"table_name", "key", "value"}, 3) + if err != nil { + return runtime.NewRunError(ctx, err.Error(), funcExpr.NamePos) + } + + if _, err := getKeyName(funcExpr.Param[0]); err != nil { + return runtime.NewRunError(ctx, err.Error(), funcExpr.Param[0].StartPos()) + } + + switch funcExpr.Param[1].NodeType { //nolint:exhaustive + case ast.TypeStringLiteral, ast.TypeIdentifier, ast.TypeAttrExpr: + default: + return runtime.NewRunError(ctx, fmt.Sprintf("param key expects StringLiteral, Identifier or AttrExpr, got %s", + funcExpr.Param[1]), funcExpr.Param[1].StartPos()) + } + + switch funcExpr.Param[2].NodeType { //nolint:exhaustive + case ast.TypeIdentifier, ast.TypeAttrExpr, + ast.TypeStringLiteral, ast.TypeBoolLiteral, + ast.TypeFloatLiteral, ast.TypeIntegerLiteral: + default: + return runtime.NewRunError(ctx, fmt.Sprintf( + "param key expects StringLiteral, NumberLiteral, BoolLiteral, AttrExpr or Identifier, got %s", + funcExpr.Param[2]), funcExpr.Param[2].StartPos()) + } + + // TODO: pos param 4: selected([]string) + + // if len(funcExpr.Param) == 4 { + // switch v := funcExpr.Param[3].(type) { + // case parser.FuncArgList: + // for _, item := range v { + // switch item.(type) { + // case *parser.StringLiteral: + // default: + // return fmt.Errorf("param key expects StringLiteral, got %s", + // reflect.TypeOf(funcExpr.Param[2]).String()) + // } + // } + // case nil: + // default: + // return fmt.Errorf("param key expects FuncArgList, got %s", + // reflect.TypeOf(funcExpr.Param[3]).String()) + // } + // } + + return nil +} + +func QueryReferTable(ctx *runtime.Context, funcExpr *ast.CallExpr) *errchain.PlError { + var tableName string + + tname, dtype, err := runtime.RunStmt(ctx, funcExpr.Param[0]) + if err != nil { + return err + } + if dtype != ast.String { + return runtime.NewRunError(ctx, "param expect string", funcExpr.Param[0].StartPos()) + } + + tableName = tname.(string) + + var colName string + switch funcExpr.Param[1].NodeType { //nolint:exhaustive + case ast.TypeStringLiteral: + colName = funcExpr.Param[1].StringLiteral.Val + case ast.TypeIdentifier, ast.TypeAttrExpr: + key, _ := getKeyName(funcExpr.Param[1]) + val, err := ctx.GetKey(key) + if err != nil { + l.Debug(err) + return runtime.NewRunError(ctx, err.Error(), funcExpr.Param[1].StartPos()) + } + switch val.DType { //nolint:exhaustive + case ast.String: + colName = cast.ToString(val.Value) + default: + err := fmt.Errorf("unsupported column name value type %s", val.DType) + l.Debug(err) + return runtime.NewRunError(ctx, err.Error(), funcExpr.Param[1].StartPos()) + } + default: + return nil + } + + var colValue any + switch funcExpr.Param[2].NodeType { //nolint:exhaustive + case ast.TypeIdentifier, ast.TypeAttrExpr: + key, _ := getKeyName(funcExpr.Param[2]) + if val, err := ctx.GetKey(key); err != nil { + l.Debugf("key %s not found: %v", key, err) + return nil + } else { + colValue = val.Value + } + case ast.TypeStringLiteral: + colValue = funcExpr.Param[2].StringLiteral.Val + case ast.TypeIntegerLiteral: + colValue = funcExpr.Param[2].IntegerLiteral.Val + case ast.TypeFloatLiteral: + colValue = funcExpr.Param[2].FloatLiteral.Val + case ast.TypeBoolLiteral: + colValue = funcExpr.Param[2].BoolLiteral.Val + } + + // TODO: pos param 4: selected([]string) + + // var selectd []string + // switch v := funcExpr.Param[3].(type) { + // case parser.FuncArgList: + // for _, item := range v { + // if item, ok := item.(*parser.StringLiteral); ok { + // selectd = append(selectd, item.Val) + // } + // } + // case nil: + // } + + if vMap, ok := refertable.QueryReferTable(tableName, + []string{colName}, []any{colValue}, nil); ok { + for k, v := range vMap { + var dtype ast.DType + switch v.(type) { + case string: + dtype = ast.String + case bool: + dtype = ast.Bool + case int64: + dtype = ast.Int + case float64: + dtype = ast.Float + default: + return nil + } + _ = addKey2PtWithVal(ctx.InData(), k, v, dtype, ptinput.KindPtDefault) + } + } + return nil +} diff --git a/pkg/inimpl/guancecloud/funcs/funcs_test.go b/pkg/inimpl/guancecloud/ptinput/funcs/funcs_test.go similarity index 96% rename from pkg/inimpl/guancecloud/funcs/funcs_test.go rename to pkg/inimpl/guancecloud/ptinput/funcs/funcs_test.go index 1693ba2..600556c 100644 --- a/pkg/inimpl/guancecloud/funcs/funcs_test.go +++ b/pkg/inimpl/guancecloud/ptinput/funcs/funcs_test.go @@ -10,8 +10,8 @@ import ( "testing" "time" - tu "gitlab.jiagouyun.com/cloudcare-tools/cliutils/testutil" - "github.com/GuanceCloud/platypus/parser" + tu "github.com/GuanceCloud/cliutils/testutil" + "gitlab.jiagouyun.com/cloudcare-tools/datakit/internal/pipeline/parser" ) func assertEqual(t *testing.T, a, b interface{}) { @@ -126,13 +126,13 @@ func TestStringfFunc(t *testing.T) { strfmt(bb, "%d %s %v", a.second, a.third, a.forth) ` p, err := NewPipeline(script) - assert.Assert(t, err == nil, "") + tu.Assert(t, err == nil, "") p.Run(js) t.Logf("%+#v", p.Output) v, _ := p.getContent("bb") - assert.Equal(t, "2 abc true", v) + tu.Equals(t, "2 abc true", v) } func TestUppercaseFunc(t *testing.T) { diff --git a/pkg/inimpl/guancecloud/funcs/handle.go b/pkg/inimpl/guancecloud/ptinput/funcs/handle.go similarity index 86% rename from pkg/inimpl/guancecloud/funcs/handle.go rename to pkg/inimpl/guancecloud/ptinput/funcs/handle.go index eb71eae..01cba58 100644 --- a/pkg/inimpl/guancecloud/funcs/handle.go +++ b/pkg/inimpl/guancecloud/ptinput/funcs/handle.go @@ -13,6 +13,7 @@ import ( "time" "github.com/GuanceCloud/platypus/pkg/ast" + "github.com/GuanceCloud/platypus/pkg/inimpl/guancecloud/ptinput/ip2isp" "github.com/araddon/dateparse" "github.com/mssola/user_agent" conv "github.com/spf13/cast" @@ -138,24 +139,26 @@ func UserAgentHandle(str string) (map[string]interface{}, map[string]ast.DType) return res, retType } -func DateFormatHandle(data interface{}, precision string, fmts string) (string, error) { - v := conv.ToInt64(data) - - var t time.Time - switch precision { - case "s": - t = time.Unix(v, 0) - case "ms": - num := v * int64(time.Millisecond) - t = time.Unix(0, num) +func GeoIPHandle(ip string) (map[string]string, error) { + record, err := Geo(ip) + if err != nil { + return nil, err } - for key, value := range dateFormatStr { - if key == fmts { - return t.Format(value), nil - } - } + res := make(map[string]string) + + res["city"] = record.City + res["province"] = record.Region + res["country"] = record.Country + res["isp"] = ip2isp.SearchISP(ip) + + return res, nil +} +func DateFormatHandle(t *time.Time, fmts string) (string, error) { + if timefmt, ok := dateFormatStr[fmts]; ok { + return t.Format(timefmt), nil + } return "", fmt.Errorf("format pattern %v no support", fmts) } @@ -253,6 +256,13 @@ var dateFormatStr = map[string]string{ "StampNano": time.StampNano, } +func datetimeInnerFormat(fmt string) bool { + if _, ok := dateFormatStr[fmt]; ok { + return ok + } + return false +} + func JSONParse(jsonStr string) map[string]interface{} { res := make(map[string]interface{}) jsonObj := gjson.Parse(jsonStr) @@ -302,6 +312,33 @@ func isArray(obj gjson.Result) bool { return obj.IsArray() } +var monthMaps = map[string]time.Month{ + "january": time.January, + "february": time.February, + "march": time.March, + "april": time.April, + "june": time.June, + "july": time.July, + "august": time.August, + "september": time.September, + "october": time.October, + "november": time.November, + "december": time.December, + + "jan": time.January, + "feb": time.February, + "mar": time.March, + "apr": time.April, + "may": time.May, + "jun": time.June, + "jul": time.July, + "aug": time.August, + "sep": time.September, + "oct": time.October, + "nov": time.November, + "dec": time.December, +} + var timezoneList = map[string]string{ "-11": "Pacific/Midway", "-10": "Pacific/Honolulu", diff --git a/pkg/inimpl/guancecloud/ptinput/funcs/handle_test.go b/pkg/inimpl/guancecloud/ptinput/funcs/handle_test.go new file mode 100644 index 0000000..8fa5435 --- /dev/null +++ b/pkg/inimpl/guancecloud/ptinput/funcs/handle_test.go @@ -0,0 +1,27 @@ +// Unless explicitly stated otherwise all files in this repository are licensed +// under the MIT License. +// This product includes software developed at Guance Cloud (https://www.guance.com/). +// Copyright 2021-present Guance, Inc. + +package funcs + +import ( + "fmt" + T "testing" + "time" + + "github.com/stretchr/testify/assert" +) + +func TestTimestampHandle(t *T.T) { + t.Run("test-tz+0", func(t *T.T) { + for i := 1970; i < 1980; i++ { + x := fmt.Sprintf("Thu Jan 16 10:05:19 %d", i) + ts, err := TimestampHandle(x, "+0") + + assert.NoError(t, err) + + t.Logf("%s : %v", x, time.Unix(0, ts)) + } + }) +} diff --git a/pkg/inimpl/guancecloud/funcs/ifelse_test.go b/pkg/inimpl/guancecloud/ptinput/funcs/ifelse_test.go similarity index 86% rename from pkg/inimpl/guancecloud/funcs/ifelse_test.go rename to pkg/inimpl/guancecloud/ptinput/funcs/ifelse_test.go index bd18f43..0090e6d 100644 --- a/pkg/inimpl/guancecloud/funcs/ifelse_test.go +++ b/pkg/inimpl/guancecloud/ptinput/funcs/ifelse_test.go @@ -9,7 +9,9 @@ import ( "testing" "time" - "github.com/stretchr/testify/assert" + "github.com/GuanceCloud/cliutils/point" + tu "github.com/GuanceCloud/cliutils/testutil" + "github.com/GuanceCloud/platypus/pkg/inimpl/guancecloud/ptinput" ) func TestIfelse(t *testing.T) { @@ -158,15 +160,16 @@ if invalid_status_code != nil { return } - _, _, f, _, _, err := runScript(runner, "test", nil, map[string]interface{}{ - "message": tc.in, - }, time.Now()) - if err != nil { - t.Fatal(err) + pt := ptinput.NewPlPoint( + point.Logging, "test", nil, map[string]any{"message": tc.in}, time.Now()) + errR := runScript(runner, pt) + + if errR != nil { + t.Fatal(errR) } - v := f["add_new_key"] - assert.Equal(t, tc.expect, v) + v, _, _ := pt.GetWithIsTag("add_new_key") + tu.Equals(t, tc.expect, v) t.Logf("[%d] PASS", idx) }) } diff --git a/pkg/inimpl/guancecloud/ptinput/funcs/md/add_key.en.md b/pkg/inimpl/guancecloud/ptinput/funcs/md/add_key.en.md new file mode 100644 index 0000000..d7fd47b --- /dev/null +++ b/pkg/inimpl/guancecloud/ptinput/funcs/md/add_key.en.md @@ -0,0 +1,27 @@ +### `add_key()` {#fn-add-key} + +Function prototype: `fn add_key(key, value)` + +Function description: Add a key to point + +Function parameters: + +- `key`: key name +- `value`: key value + +Example: + +```python +# input: {"age": 17, "name": "zhangsan", "height": 180} + +# script +add_key(city, "shanghai") + +# result +{ + "age": 17, + "height": 180, + "name": "zhangsan", + "city": "shanghai" +} +``` diff --git a/pkg/inimpl/guancecloud/ptinput/funcs/md/add_key.md b/pkg/inimpl/guancecloud/ptinput/funcs/md/add_key.md new file mode 100644 index 0000000..6bcef87 --- /dev/null +++ b/pkg/inimpl/guancecloud/ptinput/funcs/md/add_key.md @@ -0,0 +1,27 @@ +### `add_key()` {#fn-add-key} + +函数原型：`fn add_key(key, value)` + +函数说明：往 point 中增加一个字段 + +函数参数 + +- `key`: 新增的 key 名称 +- `value`：作为 key 的值 + +示例： + +```python +# 待处理数据：{"age": 17, "name": "zhangsan", "height": 180} + +# 处理脚本 +add_key(city, "shanghai") + +# 处理结果 +{ + "age": 17, + "height": 180, + "name": "zhangsan", + "city": "shanghai" +} +``` diff --git a/pkg/inimpl/guancecloud/ptinput/funcs/md/add_pattern.en.md b/pkg/inimpl/guancecloud/ptinput/funcs/md/add_pattern.en.md new file mode 100644 index 0000000..7d2d782 --- /dev/null +++ b/pkg/inimpl/guancecloud/ptinput/funcs/md/add_pattern.en.md @@ -0,0 +1,54 @@ +### `add_pattern()` {#fn-add-pattern} + +Function prototype: `fn add_pattern(name: str, pattern: str)` + +Function description: Create custom grok patterns. The grok pattern has scope restrictions, such as a new scope will be generated in the if else statement, and the pattern is only valid within this scope. This function cannot overwrite existing grok patterns in the same scope or in the previous scope + +Function parameters: + +- `name`: pattern naming +- `pattern`: custom pattern content + +Example: + +```python +# input data: "11,abc,end1", "22,abc,end1", "33,abc,end3" + +# script +add_pattern("aa", "\\d{2}") +grok(_, "%{aa:aa}") +if false { + +} else { + add_pattern("bb", "[a-z]{3}") + if aa == "11" { + add_pattern("cc", "end1") + grok(_, "%{aa:aa},%{bb:bb},%{cc:cc}") + } elif aa == "22" { + # Using pattern cc here will cause compilation failure: no pattern found for %{cc} + grok(_, "%{aa:aa},%{bb:bb},%{INT:cc}") + } elif aa == "33" { + add_pattern("bb", "[\\d]{5}") # Overwriting bb here fails + add_pattern("cc", "end3") + grok(_, "%{aa:aa},%{bb:bb},%{cc:cc}") + } +} + +# result +{ + "aa": "11" + "bb": "abc" + "cc": "end1" + "message": "11,abc,end1" +} +{ + "aa": "22" + "message": "22,abc,end1" +} +{ + "aa": "33" + "bb": "abc" + "cc": "end3" + "message": "33,abc,end3" +} +``` diff --git a/pkg/inimpl/guancecloud/ptinput/funcs/md/add_pattern.md b/pkg/inimpl/guancecloud/ptinput/funcs/md/add_pattern.md new file mode 100644 index 0000000..75209ed --- /dev/null +++ b/pkg/inimpl/guancecloud/ptinput/funcs/md/add_pattern.md @@ -0,0 +1,54 @@ +### `add_pattern()` {#fn-add-pattern} + +函数原型：`fn add_pattern(name: str, pattern: str)` + +函数说明：创建自定义 grok 模式。grok 模式有作用域限制，如在 if else 语句内将产生新的作用域，该 pattern 仅在此作用域内有效。该函数不可覆盖同一作用域或者上一作用域已经存在的 grok 模式 + +参数： + +- `name`：模式命名 +- `pattern`: 自定义模式内容 + +示例： + +```python +# 待处理数据："11,abc,end1", "22,abc,end1", "33,abc,end3" + +# pipline 脚本 +add_pattern("aa", "\\d{2}") +grok(_, "%{aa:aa}") +if false { + +} else { + add_pattern("bb", "[a-z]{3}") + if aa == "11" { + add_pattern("cc", "end1") + grok(_, "%{aa:aa},%{bb:bb},%{cc:cc}") + } elif aa == "22" { + # 此处使用 pattern cc 将导致编译失败：no pattern found for %{cc} + grok(_, "%{aa:aa},%{bb:bb},%{INT:cc}") + } elif aa == "33" { + add_pattern("bb", "[\\d]{5}") # 此处覆盖 bb 失败 + add_pattern("cc", "end3") + grok(_, "%{aa:aa},%{bb:bb},%{cc:cc}") + } +} + +# 处理结果 +{ + "aa": "11" + "bb": "abc" + "cc": "end1" + "message": "11,abc,end1" +} +{ + "aa": "22" + "message": "22,abc,end1" +} +{ + "aa": "33" + "bb": "abc" + "cc": "end3" + "message": "33,abc,end3" +} +``` diff --git a/pkg/inimpl/guancecloud/ptinput/funcs/md/adjust_timezone.en.md b/pkg/inimpl/guancecloud/ptinput/funcs/md/adjust_timezone.en.md new file mode 100644 index 0000000..063c09b --- /dev/null +++ b/pkg/inimpl/guancecloud/ptinput/funcs/md/adjust_timezone.en.md @@ -0,0 +1,59 @@ +### `adjust_timezone()` {#fn-adjust-timezone} + +Function prototype: `fn adjust_timezone(key: int, minute: int)` + +Function parameters: + +- `key`: Nanosecond timestamp, such as the timestamp obtained by the `default_time(time)` function +- `minute`: The return value allows the number of minutes (integer) beyond the current time, the value range is [0, 15], the default value is 2 minutes + +Function description: Make the difference between the incoming timestamp minus the timestamp of the function execution time within (-60+minute, minute] minutes; it is not applicable to data whose time difference exceeds this range, otherwise it will result in wrong data being obtained. Calculation process: + +1. Add hours to the value of key to make it within the current hour +2. At this time, calculate the difference between the two minutes. The value range of the two minutes is [0, 60), and the difference range is between (-60,0] and [0, 60) +3. If the difference is less than or equal to -60 + minute, add 1 hour, and if the difference is greater than minute, subtract 1 hour +4. The default value of minute is 2, and the range of the difference is allowed to be (-58, 2], if it is 11:10 at this time, the log time is 3:12:00.001, and the final result is 10:12:00.001; if at this time is 11:59:1.000, the log time is 3:01:1.000, and the final result is 12:01:1.000 + +Example: + +```json +# input data 1 +{ + "time":"11 Jul 2022 12:49:20.937", + "second":2, + "third":"abc", + "forth":true +} +``` + +Script： + +```python +json(_, time) # Extract the time field (if the time zone in the container is UTC+0000) +default_time(time) # Convert the extracted time field into a timestamp + # (Use local time zone UTC+0800/UTC+0900... parsing for data without time zone) +adjust_timezone(time) + # Automatically (re)select time zone, calibrate time offset + +``` + +Execute `datakit pipeline -P .p -F --date`: + +```json +# output 1 +{ + "message": "{\n \"time\":\"11 Jul 2022 12:49:20.937\",\n \"second\":2,\n \"third\":\"abc\",\n \"forth\":true\n}", + "status": "unknown", + "time": "2022-07-11T20:49:20.937+08:00" +} +``` + +local time: `2022-07-11T20:55:10.521+08:00` + +The times obtained by using only `default_time` and parsing according to the default local time zone (UTC+8) are: + +- Output result of input 1： `2022-07-11T12:49:20.937+08:00` + +After using `adjust_timezone` will get: + +- Output result of input 1： `2022-07-11T20:49:20.937+08:00` diff --git a/pkg/inimpl/guancecloud/ptinput/funcs/md/adjust_timezone.md b/pkg/inimpl/guancecloud/ptinput/funcs/md/adjust_timezone.md new file mode 100644 index 0000000..840c033 --- /dev/null +++ b/pkg/inimpl/guancecloud/ptinput/funcs/md/adjust_timezone.md @@ -0,0 +1,58 @@ +### `adjust_timezone()` {#fn-adjust-timezone} + +函数原型：`fn adjust_timezone(key: int, minute: int)` + +函数参数 + +- `key`: 纳秒时间戳，如 `default_time(time)` 函数处理后得到的时间戳 +- `minute`: 返回值允许超出当前时间的分钟数（整数），取值范围 [0, 15], 默认值为 2 分钟 + +函数说明：使得传入的时间戳减去函数执行时刻的时间戳的差值在（-60+minute, minute] 分钟内；不适用于时间差超出此范围的数据，否则将导致获取到错误的数据。计算流程： + +1. 为 key 的值加上数小时使其处于当前小时内 +2. 此时计算两者分钟差，两者分钟数值范围为 [0, 60)，差值范围在 (-60,0] 和 [0, 60) +3. 差值小于等于 -60 + minute 的加 1 小时，大于 minute 的减 1 小时 +4. minute 默认值为 2，则差的范围允许在 (-58, 2]，若此时为 11:10，日志时间为 3:12:00.001，最终结果为 10:12:00.001；若此时为 11:59:1.000, 日志时间为 3:01:1.000，最终结果为 12:01:1.000 + +示例： + +```json +# 输入 1 +{ + "time":"11 Jul 2022 12:49:20.937", + "second":2, + "third":"abc", + "forth":true +} +``` + +脚本： + +```python +json(_, time) # 提取 time 字段 (若容器中时区 UTC+0000) +default_time(time) # 将提取到的 time 字段转换成时间戳 + # (对无时区数据使用本地时区 UTC+0800/UTC+0900..。解析) +adjust_timezone(time) + # 自动(重新)选择时区，校准时间偏差 +``` + +执行 `datakit pipeline -P .p -F --date`: + +```json +# 输出 1 +{ + "message": "{\n \"time\":\"11 Jul 2022 12:49:20.937\",\n \"second\":2,\n \"third\":\"abc\",\n \"forth\":true\n}", + "status": "unknown", + "time": "2022-07-11T20:49:20.937+08:00" +} +``` + +本机时间：`2022-07-11T20:55:10.521+08:00` + +仅使用 `default_time` 按照默认本机时区（UTC+8）解析得到的时间分别为： + +- 输入 1 结果： `2022-07-11T12:49:20.937+08:00` + +使用 `adjust_timezone` 后将得到： + +- 输入 1 结果： `2022-07-11T20:49:20.937+08:00` diff --git a/pkg/inimpl/guancecloud/ptinput/funcs/md/agg_create.en.md b/pkg/inimpl/guancecloud/ptinput/funcs/md/agg_create.en.md new file mode 100644 index 0000000..c87ec4c --- /dev/null +++ b/pkg/inimpl/guancecloud/ptinput/funcs/md/agg_create.en.md @@ -0,0 +1,20 @@ +### `agg_create()` {#fn-agg-create} + +Function prototype: `fn agg_create(bucket: str, on_interval: str = "60s", on_count: int = 0, keep_value: bool = false, const_tags: map[string]string = nil, category: str = "M")` + +Function description: Create an aggregation measurement, set the time or number of times through `on_interval` or `on_count` as the aggregation period, upload the aggregated data after the aggregation is completed, and choose whether to keep the last aggregated data + +Function parameters: + +- `bucket`: String type, as an aggregated field, if the bucket has already been created, the function will not perform any operations. +- `on_interval`：The default value is `60s`, which takes time as the aggregation period, and the unit is `s`, and the parameter takes effect when the value is greater than `0`; it cannot be combined with `on_count` less than or equal to 0. +- `on_count`: The default value is `0`, the number of processed points is used as the aggregation period, and the parameter takes effect when the value is greater than `0` +- `keep_value`: The default value is `false` +- `const_tags`: Custom tags, empty by default +- `category`: Data category for aggregated data, optional parameter, the default value is "M", indicating the indicator category data. + +示例： + +```python +agg_create("cpu_agg_info", interval = 60) +``` diff --git a/pkg/inimpl/guancecloud/ptinput/funcs/md/agg_create.md b/pkg/inimpl/guancecloud/ptinput/funcs/md/agg_create.md new file mode 100644 index 0000000..fa894fc --- /dev/null +++ b/pkg/inimpl/guancecloud/ptinput/funcs/md/agg_create.md @@ -0,0 +1,20 @@ +### `agg_create()` {#fn-agg-create} + +函数原型：`fn agg_create(bucket: str, on_interval: str = "60s", on_count: int = 0, keep_value: bool = false, const_tags: map[string]string = nil, category: str = "M")` + +函数说明：创建一个用于聚合的指标集，通过 `on_interval` 或 `on_count` 设置时间或次数作为聚合周期，聚合结束后将上传聚合数据，可以选择是否保留上一次聚合的数据 + +函数参数： + +- `bucket`: 字符串类型，作为聚合出的指标的指标集名，如果该 bucket 已经创建，则函数不执行任何操作 +- `on_interval`：默认值 `60s`, 以时间作为聚合周期，单位 `s`，值大于 `0` 时参数生效；不能同时与 `on_count` 小于等于 0； +- `on_count`: 默认值 `0`，以处理的点数作为聚合周期，值大于 `0` 时参数生效 +- `keep_value`: 默认值 `false` +- `const_tags`: 自定义的 tags，默认为空 +- `category`: 聚合数据的数据类别，可选参数，默认值为 "M"，表示指标类别数据。 + +示例： + +```python +agg_create("cpu_agg_info", on_interval = "30s") +``` diff --git a/pkg/inimpl/guancecloud/ptinput/funcs/md/agg_metric.en.md b/pkg/inimpl/guancecloud/ptinput/funcs/md/agg_metric.en.md new file mode 100644 index 0000000..a95b4a4 --- /dev/null +++ b/pkg/inimpl/guancecloud/ptinput/funcs/md/agg_metric.en.md @@ -0,0 +1,57 @@ +### `agg_metric()` {#fn-agg-metric} + +[:octicons-tag-24: Version-1.5.10](../datakit/changelog.md#cl-1.5.10) + +Function prototype: `fn agg_metric(bucket: str, new_field: str, agg_fn: str, agg_by: []string, agg_field: str, category: str = "M")` + +Function description: According to the field name in the input data, the value is automatically taken as the label of the aggregated data, and the aggregated data is stored in the corresponding bucket. + +Function parameters: + +- `bucket`: String type, the bucket created by the agg_create function, if the bucket has not been created, the function will not perform any operations. +- `new_field`： The name of the field in the aggregated data, the data type of its value is `float`. +- `agg_fn`: Aggregation function, can be one of `"avg"`, `"sum"`, `"min"`, `"max"`, `"set"`. +- `agg_by`: The name of the field in the input data will be used as the tag of the aggregated data, and the value of these fields can only be string type data. +- `agg_field`: The field name in the input data, automatically obtain the field value for aggregation. +- `category`: Data category for aggregated data, optional parameter, the default value is "M", indicating the indicator category data. + +Example: + +Take `logging` category data as an example: + +multiple logs： +``` +1 +``` + +``` +2 +``` + +``` +3 +``` + +script: + +```python +agg_create("cpu_agg_info", interval=10, const_tags={"tag1":"value_user_define_tag"}) + +set_tag("tag1", "value1") + +field1 = _ + +cast(field1, "int") + +agg_metric("cpu_agg_info", "agg_field_1", "sum", ["tag1", "host"], "field1") +``` + +metric output: + +``` +{ + "host": "your_hostname", + "tag1": "value1", + "agg_field_1": 6, +} +``` diff --git a/pkg/inimpl/guancecloud/ptinput/funcs/md/agg_metric.md b/pkg/inimpl/guancecloud/ptinput/funcs/md/agg_metric.md new file mode 100644 index 0000000..6bc07d8 --- /dev/null +++ b/pkg/inimpl/guancecloud/ptinput/funcs/md/agg_metric.md @@ -0,0 +1,56 @@ +### `agg_metric()` {#fn-agg-metric} + +函数原型：`fn agg_metric(bucket: str, new_field: str, agg_fn: str, agg_by: []string, agg_field: str, category: str = "M")` + +函数说明：根据输入的数据中的字段的名，自动取值后作为聚合数据的 tag，并将这些聚合数据存储在对应的 bucket 中 + +函数参数： + +- `bucket`: 字符串类型，函数 `agg_create` 创建出的对应指标集合的 bucket，如果该 bucket 未被创建，则函数不执行任何操作 +- `new_field`： 聚合出的数据中的指标名，其值的数据类型为 `float` +- `agg_fn`: 聚合函数，可以是 `"avg"`,`"sum"`,`"min"`,`"max"`,`"set"` 中的一种 +- `agg_by`: 输入的数据中的字段的名，将作为聚合出的数据的 tag，这些字段的值只能是字符串类型的数据 +- `agg_field`: 输入的数据中的字段名，自动获取字段值进行聚合 +- `category`: 聚合数据的数据类别，可选参数，默认值为 "M"，表示指标类别数据。 + +示例： + +以日志类别数据为例： + +多个输入日志： + +``` not-set +1 +``` + +``` not-set +2 +``` + +``` not-set +3 +``` + +脚本： + +```python +agg_create("cpu_agg_info", interval=10, const_tags={"tag1":"value_user_define_tag"}) + +set_tag("tag1", "value1") + +field1 = _ + +cast(field1, "int") + +agg_metric("cpu_agg_info", "agg_field_1", "sum", ["tag1", "host"], "field1") +``` + +指标输出： + +``` not-set +{ + "host": "your_hostname", + "tag1": "value1", + "agg_field_1": 6, +} +``` diff --git a/pkg/inimpl/guancecloud/ptinput/funcs/md/append.en.md b/pkg/inimpl/guancecloud/ptinput/funcs/md/append.en.md new file mode 100644 index 0000000..7d69ad3 --- /dev/null +++ b/pkg/inimpl/guancecloud/ptinput/funcs/md/append.en.md @@ -0,0 +1,25 @@ +### `append()` {#fn-append} + +Function prototype: `fn append(arr, elem) arr` + +Function description: Add the element elem to the end of the array arr. + +Function parameters: + +- `arr`: array +- `elem`: element being added. + +Example: + +```python +# Example 1 +abc = ["1", "2"] +abc = append(abc, 5.1) +# abc = ["1", "2", 5.1] + +# Example 2 +a = [1, 2] +b = [3, 4] +c = append(a, b) +# c = [1, 2, [3, 4]] +``` diff --git a/pkg/inimpl/guancecloud/ptinput/funcs/md/append.md b/pkg/inimpl/guancecloud/ptinput/funcs/md/append.md new file mode 100644 index 0000000..43dfee6 --- /dev/null +++ b/pkg/inimpl/guancecloud/ptinput/funcs/md/append.md @@ -0,0 +1,25 @@ +### `append()` {#fn-append} + +函数原型：`fn append(arr, elem) arr` + +函数说明：往数组 arr 末尾添加元素 elem。 + +参数： + +- `arr`: 要添加元素的数组。 +- `elem`: 添加的元素。 + +示例： + +```python +# 例 1 +abc = ["1", "2"] +abc = append(abc, 5.1) +# abc = ["1", "2", 5.1] + +# 例 2 +a = [1, 2] +b = [3, 4] +c = append(a, b) +# c = [1, 2, [3, 4]] +``` diff --git a/pkg/inimpl/guancecloud/ptinput/funcs/md/b64dec.en.md b/pkg/inimpl/guancecloud/ptinput/funcs/md/b64dec.en.md new file mode 100644 index 0000000..6e0ea1f --- /dev/null +++ b/pkg/inimpl/guancecloud/ptinput/funcs/md/b64dec.en.md @@ -0,0 +1,22 @@ +### `b64dec()` {#fn-b64dec} + +Function prototype: `fn b64dec(key: str)` + +Function description: Base64 decodes the string data obtained on the specified field + +Function parameters: + +- `key`: fields to extract + +Example: + +```python +# input data {"str": "aGVsbG8sIHdvcmxk"} +json(_, `str`) +b64enc(`str`) + +# result +# { +# "str": "hello, world" +# } +``` diff --git a/pkg/inimpl/guancecloud/ptinput/funcs/md/b64dec.md b/pkg/inimpl/guancecloud/ptinput/funcs/md/b64dec.md new file mode 100644 index 0000000..63dcd52 --- /dev/null +++ b/pkg/inimpl/guancecloud/ptinput/funcs/md/b64dec.md @@ -0,0 +1,22 @@ +### `b64dec()` {#fn-b64dec} + +函数原型：`fn b64dec(key: str)` + +函数说明：对指定字段上获取的字符串数据进行 base64 解码 + +函数参数 + +- `key`: 待提取字段 + +示例： + +```python +# 待处理数据 {"str": "aGVsbG8sIHdvcmxk"} +json(_, `str`) +b64enc(`str`) + +# 处理结果 +# { +# "str": "hello, world" +# } +``` diff --git a/pkg/inimpl/guancecloud/ptinput/funcs/md/b64enc.en.md b/pkg/inimpl/guancecloud/ptinput/funcs/md/b64enc.en.md new file mode 100644 index 0000000..d549b89 --- /dev/null +++ b/pkg/inimpl/guancecloud/ptinput/funcs/md/b64enc.en.md @@ -0,0 +1,22 @@ +### `b64enc()` {#fn-b64enc} + +Function prototype: `fn b64enc(key: str)` + +Function description: Base64 encode the string data obtained on the specified field + +Function parameters: + +- `key`: key name + +Example: + +```python +# input data {"str": "hello, world"} +json(_, `str`) +b64enc(`str`) + +# result +# { +# "str": "aGVsbG8sIHdvcmxk" +# } +``` diff --git a/pkg/inimpl/guancecloud/ptinput/funcs/md/b64enc.md b/pkg/inimpl/guancecloud/ptinput/funcs/md/b64enc.md new file mode 100644 index 0000000..4d8d3f1 --- /dev/null +++ b/pkg/inimpl/guancecloud/ptinput/funcs/md/b64enc.md @@ -0,0 +1,22 @@ +### `b64enc()` {#fn-b64enc} + +函数原型：`fn b64enc(key: str)` + +函数说明：对指定字段上获取的字符串数据进行 base64 编码 + +函数参数 + +- `key`: 待提取字段 + +示例： + +```python +# 待处理数据 {"str": "hello, world"} +json(_, `str`) +b64enc(`str`) + +# 处理结果 +# { +# "str": "aGVsbG8sIHdvcmxk" +# } +``` diff --git a/pkg/inimpl/guancecloud/ptinput/funcs/md/cast.en.md b/pkg/inimpl/guancecloud/ptinput/funcs/md/cast.en.md new file mode 100644 index 0000000..96bd89c --- /dev/null +++ b/pkg/inimpl/guancecloud/ptinput/funcs/md/cast.en.md @@ -0,0 +1,25 @@ +### `cast()` {#fn-cast} + +Function prototype: `fn cast(key, dst_type: str)` + +Function description: Convert the key value to the specified type + +Function parameters: + +- `key`: key name +- `type`：The target type of conversion, support `\"str\", \"float\", \"int\", \"bool\"` + +Example: + +```python +# input data: {"first": 1,"second":2,"third":"aBC","forth":true} + +# script +json(_, first) +cast(first, "str") + +# result +{ + "first": "1" +} +``` diff --git a/pkg/inimpl/guancecloud/ptinput/funcs/md/cast.md b/pkg/inimpl/guancecloud/ptinput/funcs/md/cast.md new file mode 100644 index 0000000..96eeae4 --- /dev/null +++ b/pkg/inimpl/guancecloud/ptinput/funcs/md/cast.md @@ -0,0 +1,25 @@ +### `cast()` {#fn-cast} + +函数原型：`fn cast(key, dst_type: str)` + +函数说明：将 key 值转换成指定类型 + +函数参数 + +- `key`: 已提取的某字段 +- `type`：转换的目标类型，支持 `\"str\", \"float\", \"int\", \"bool\"` 这几种，目标类型需要用英文状态双引号括起来 + +示例： + +```python +# 待处理数据：{"first": 1,"second":2,"third":"aBC","forth":true} + +# 处理脚本 +json(_, first) +cast(first, "str") + +# 处理结果 +{ + "first": "1" +} +``` diff --git a/pkg/inimpl/guancecloud/ptinput/funcs/md/cidr.en.md b/pkg/inimpl/guancecloud/ptinput/funcs/md/cidr.en.md new file mode 100644 index 0000000..c54d751 --- /dev/null +++ b/pkg/inimpl/guancecloud/ptinput/funcs/md/cidr.en.md @@ -0,0 +1,25 @@ +### `cidr()` {#fn-cidr} + +Function prototype: `fn cidr(ip: str, prefix: str) bool` + +Function description: Determine whether the IP is in a CIDR block + +Function parameters: + +- `ip`: IP address +- `prefix`： IP prefix, such as `192.0.2.1/24` + +Example: + +```python +# script +ip = "192.0.2.233" +if cidr(ip, "192.0.2.1/24") { + add_key(ip_prefix, "192.0.2.1/24") +} + +# result +{ + "ip_prefix": "192.0.2.1/24" +} +``` diff --git a/pkg/inimpl/guancecloud/ptinput/funcs/md/cidr.md b/pkg/inimpl/guancecloud/ptinput/funcs/md/cidr.md new file mode 100644 index 0000000..ba6f862 --- /dev/null +++ b/pkg/inimpl/guancecloud/ptinput/funcs/md/cidr.md @@ -0,0 +1,28 @@ +### `cidr()` {#fn-cidr} + +函数原型：`fn cidr(ip: str, prefix: str) bool` + +函数说明： 判断 IP 是否在某个 CIDR 块 + +函数参数 + +- `ip`: IP 地址 +- `prefix`： IP 前缀，如 `192.0.2.1/24` + +示例： + +```python +# 待处理数据： + +# 处理脚本 + +ip = "192.0.2.233" +if cidr(ip, "192.0.2.1/24") { + add_key(ip_prefix, "192.0.2.1/24") +} + +# 处理结果 +{ + "ip_prefix": "192.0.2.1/24" +} +``` diff --git a/pkg/inimpl/guancecloud/ptinput/funcs/md/conv_traceid_w3c_to_dd.en.md b/pkg/inimpl/guancecloud/ptinput/funcs/md/conv_traceid_w3c_to_dd.en.md new file mode 100644 index 0000000..3ebb8f5 --- /dev/null +++ b/pkg/inimpl/guancecloud/ptinput/funcs/md/conv_traceid_w3c_to_dd.en.md @@ -0,0 +1,31 @@ +### `conv_traceid_w3c_to_dd()` {#fn-conv-traceid-w3c-to-dd} + +Function prototype: `fn conv_traceid_w3c_to_dd(key)` + +Function description: Convert a hex-encoded 128-bit/64-bit W3C Trace ID string(length 32 characters or 16 characters) to a decimal-encoded 64-bit DataDog Trace ID string. + +Function parameters: + +- `key`: 128-bit/64-bit Trace ID to convert + +Example: + +```python + +# script input: + +"18962fdd9eea517f2ae0771ea69d6e16" + +# script: + +grok(_, "%{NOTSPACE:trace_id}") + +conv_traceid_w3c_to_dd(trace_id) + +# result: + +{ + "trace_id": "3089600317904219670", +} + +``` diff --git a/pkg/inimpl/guancecloud/ptinput/funcs/md/conv_traceid_w3c_to_dd.md b/pkg/inimpl/guancecloud/ptinput/funcs/md/conv_traceid_w3c_to_dd.md new file mode 100644 index 0000000..deab807 --- /dev/null +++ b/pkg/inimpl/guancecloud/ptinput/funcs/md/conv_traceid_w3c_to_dd.md @@ -0,0 +1,31 @@ +### `conv_traceid_w3c_to_dd()` {#fn-conv-traceid-w3c-to-dd} + +函数原型：`fn conv_traceid_w3c_to_dd(key)` + +函数说明：将 16 进制编码的 128-bit/64-bit W3C Trace ID 字符串（长度 32 个字符或 16 个字符）转换为 10 进制编码的 64-bit DataDog Trace ID 字符串。 + +函数参数 + +- `key`: 待转换的 128-bit/64-bit Trace ID + +示例： + +```python + +# script input: + +"18962fdd9eea517f2ae0771ea69d6e16" + +# script: + +grok(_, "%{NOTSPACE:trace_id}") + +conv_traceid_w3c_to_dd(trace_id) + +# result: + +{ + "trace_id": "3089600317904219670", +} + +``` diff --git a/pkg/inimpl/guancecloud/ptinput/funcs/md/cover.en.md b/pkg/inimpl/guancecloud/ptinput/funcs/md/cover.en.md new file mode 100644 index 0000000..0229490 --- /dev/null +++ b/pkg/inimpl/guancecloud/ptinput/funcs/md/cover.en.md @@ -0,0 +1,22 @@ +### `cover()` {#fn-cover} + +Function prototype: `fn cover(key: str, range: list)` + +Function description: Perform data desensitization by range on the string data obtained on the specified field + +Function parameters: + +- `key`: Key name +- `range`: The index range of the desensitized string (`[start,end]`) Both start and end support negative subscripts, which are used to express the semantics of tracing back from the end. The interval is reasonable. If end is greater than the maximum length of the string, it will default to the maximum length + +Example: + +```python +# input data {"str": "13789123014"} +json(_, `str`) +cover(`str`, [8, 9]) + +# input data {"abc": "13789123014"} +json(_, abc) +cover(abc, [2, 4]) +``` diff --git a/pkg/inimpl/guancecloud/ptinput/funcs/md/cover.md b/pkg/inimpl/guancecloud/ptinput/funcs/md/cover.md new file mode 100644 index 0000000..200c96c --- /dev/null +++ b/pkg/inimpl/guancecloud/ptinput/funcs/md/cover.md @@ -0,0 +1,22 @@ +### `cover()` {#fn-cover} + +函数原型：`fn cover(key: str, range: list)` + +函数说明：对指定字段上获取的字符串数据，按范围进行数据脱敏处理 + +函数参数 + +- `key`: 待提取字段 +- `range`: 脱敏字符串的索引范围（`[start,end]`） start 和 end 均支持负数下标，用来表达从尾部往前追溯的语义。区间合理即可，end 如果大于字符串最大长度会默认成最大长度 + +示例： + +```python +# 待处理数据 {"str": "13789123014"} +json(_, `str`) +cover(`str`, [8, 9]) + +# 待处理数据 {"abc": "13789123014"} +json(_, abc) +cover(abc, [2, 4]) +``` diff --git a/pkg/inimpl/guancecloud/ptinput/funcs/md/create_point.en.md b/pkg/inimpl/guancecloud/ptinput/funcs/md/create_point.en.md new file mode 100644 index 0000000..852b65d --- /dev/null +++ b/pkg/inimpl/guancecloud/ptinput/funcs/md/create_point.en.md @@ -0,0 +1,25 @@ +### `create_point()` {#fn-create-point} + +Function prototype: `fn create_point(name, tags, fields, ts = 0, category = "M", after_use = "")` + +Function description: Create new data and output + +Function parameters: + +- `name`: point name, which is regarded as the name of the metric set, log source, etc. +- `tags`: data tags +- `fields`: data fields +- `ts`: optional parameter, unix nanosecond timestamp, defaults to current time +- `category`: optional parameter, data category, supports category name and name abbreviation, such as metric category can be filled with `M` or `metric`, log is `L` or `logging` +- `after_use`: optional parameter, after the point is created, execute the specified pl script on the created point; if the original data type is L, the created data category is M, and the script under the L category is executed at this time + +Example: + +```py +# input +''' +{"a": "b"} +''' +fields = load_json(_) +create_point("name_pt", {"a": "b"}, fields) +``` diff --git a/pkg/inimpl/guancecloud/ptinput/funcs/md/create_point.md b/pkg/inimpl/guancecloud/ptinput/funcs/md/create_point.md new file mode 100644 index 0000000..6bf0db3 --- /dev/null +++ b/pkg/inimpl/guancecloud/ptinput/funcs/md/create_point.md @@ -0,0 +1,26 @@ +### `create_point()` {#fn-create-point} + +函数原型：`fn create_point(name, tags, fields, ts = 0, category = "M", after_use = "")` + +函数说明：创建新的数据并输出 + +函数参数： + +- `name`: point name，视为指标集的名、日志 source 等 +- `tags`: 数据标签 +- `fields`: 数据字段 +- `ts`: 可选参数，unix 纳秒时间戳，默认为当前时间 +- `category`: 可选参数，数据类别，支持类别名称和名称简写，如指标类别可以填写 `M` 或 `metric`，日志则是 `L` 或 `logging` +- `after_use`: 可选参数，创建 point 后，对创建的 point 执行指定的 pl 脚本；如果原始数据类型是 L，被创建的数据的类别为 M，此时执行的还是 L 类别下的脚本 + + +示例： + +```py +# input +''' +{"a": "b"} +''' +fields = load_json(_) +create_point("name_pt", {"a": "b"}, fields) +``` diff --git a/pkg/inimpl/guancecloud/ptinput/funcs/md/datetime.en.md b/pkg/inimpl/guancecloud/ptinput/funcs/md/datetime.en.md new file mode 100644 index 0000000..f0d9e41 --- /dev/null +++ b/pkg/inimpl/guancecloud/ptinput/funcs/md/datetime.en.md @@ -0,0 +1,103 @@ +### `datetime()` {#fn-datetime} + +Function prototype: `fn datetime(key, precision: str, fmt: str, tz: str = "")` + +Function description: Convert timestamp to specified date format + +Function parameters: + +- `key`: Extracted timestamp (required parameter) +- `precision`: Input timestamp precision (s, ms, us, ns) +- `fmt`: date format, provides built-in date format and supports custom date format +- `tz`: time zone (optional parameter), convert the timestamp to the time in the specified time zone, the default time zone of the host is used + +Built-in date formats: + +| Built-in format | date | description | +| - | - | - | +| "ANSI-C" | "Mon Jan _2 15:04:05 2006" | | +| "UnixDate" | "Mon Jan _2 15:04:05 MST 2006" | | +| "RubyDate" | "Mon Jan 02 15:04:05 -0700 2006" | | +| "RFC822" | "02 Jan 06 15:04 MST" | | +| "RFC822Z" | "02 Jan 06 15:04 -0700" | RFC822 with numeric zone | +| "RFC850" | "Monday, 02-Jan-06 15:04:05 MST" | | +| "RFC1123" | "Mon, 02 Jan 2006 15:04:05 MST" | | +| "RFC1123Z" | "Mon, 02 Jan 2006 15:04:05 -0700" | RFC1123 with numeric zone | +| "RFC3339" | "2006-01-02T15:04:05Z07:00" | | +| "RFC3339Nano" | "2006-01-02T15:04:05.999999999Z07:00" | | +| "Kitchen" | "3:04PM" | | + + +Custom date format: + +The output date format can be customized through the combination of placeholders + +| character | example | description | +| - | - | - | +| a | %a | week abbreviation, such as `Wed` | +| A | %A | The full letter of the week, such as `Wednesday`| +| b | %b | month abbreviation, such as `Mar` | +| B | %B | The full letter of the month, such as `March` | +| C | %c | century, current year divided by 100 | +| **d** | %d | day of the month; range `[01, 31]` | +| e | %e | day of the month; range `[1, 31]`, pad with spaces | +| **H** | %H | hour, using 24-hour clock; range `[00, 23]` | +| I | %I | hour, using 12-hour clock; range `[01, 12]` | +| j | %j | day of the year, range `[001, 365]` | +| k | %k | hour, using 24-hour clock; range `[0, 23]` | +| l | %l | hour, using 12-hour clock; range `[1, 12]`, padding with spaces | +| **m** | %m | month, range `[01, 12]` | +| **M** | %M | minutes, range `[00, 59]` | +| n | %n | represents a newline character `\n` | +| p | %p | `AM` or `PM` | +| P | %P | `am` or `pm` | +| s | %s | seconds since 1970-01-01 00:00:00 UTC | +| **S** | %S | seconds, range `[00, 60]` | +| t | %t | represents the tab character `\t` | +| u | %u | day of the week, Monday is 1, range `[1, 7]` | +| w | %w | day of the week, 0 for Sunday, range `[0, 6]` | +| y | %y | year in range `[00, 99]` | +| **Y** | %Y | decimal representation of the year| +| **z** | %z | RFC 822/ISO 8601:1988 style time zone (e.g. `-0600` or `+0800` etc.) | +| Z | %Z | time zone abbreviation, such as `CST` | +| % | %% | represents the character `%` | + +Example: + +```python +# input data: +# { +# "a":{ +# "timestamp": "1610960605000", +# "second":2 +# }, +# "age":47 +# } + +# script +json(_, a.timestamp) +datetime(a.timestamp, 'ms', 'RFC3339') +``` + + +```python +# script +ts = timestamp() +datetime(ts, 'ns', fmt='%Y-%m-%d %H:%M:%S', tz="UTC") + +# output +{ + "ts": "2023-03-08 06:43:39" +} +``` + +```python +# script +ts = timestamp() +datetime(ts, 'ns', '%m/%d/%y %H:%M:%S %z', "Asia/Tokyo") + +# output +{ + "ts": "03/08/23 15:44:59 +0900" +} +``` diff --git a/pkg/inimpl/guancecloud/ptinput/funcs/md/datetime.md b/pkg/inimpl/guancecloud/ptinput/funcs/md/datetime.md new file mode 100644 index 0000000..e41c3b7 --- /dev/null +++ b/pkg/inimpl/guancecloud/ptinput/funcs/md/datetime.md @@ -0,0 +1,101 @@ +### `datetime()` {#fn-datetime} + +函数原型：`fn datetime(key, precision: str, fmt: str, tz: str = "")` + +函数说明：将时间戳转成指定日期格式 + +函数参数： + +- `key`: 已经提取的时间戳 +- `precision`：输入的时间戳精度(s, ms, us, ns) +- `fmt`：日期格式，提供内置日期格式且支持自定义日期格式 +- `tz`: 时区 (可选参数)，将时间戳转换为指定时区的时间，默认使用主机的时区 + +内置日期格式： + +| 内置格式 | 日期 | 描述 | +| --- | --- | --- | +| "ANSI-C" | "Mon Jan _2 15:04:05 2006" | | +| "UnixDate" | "Mon Jan _2 15:04:05 MST 2006" | | +| "RubyDate" | "Mon Jan 02 15:04:05 -0700 2006" | | +| "RFC822" | "02 Jan 06 15:04 MST" | | +| "RFC822Z" | "02 Jan 06 15:04 -0700" | RFC822 with numeric zone | +| "RFC850" | "Monday, 02-Jan-06 15:04:05 MST" | | +| "RFC1123" | "Mon, 02 Jan 2006 15:04:05 MST" | | +| "RFC1123Z" | "Mon, 02 Jan 2006 15:04:05 -0700" | RFC1123 with numeric zone | +| "RFC3339" | "2006-01-02T15:04:05Z07:00" | | +| "RFC3339Nano" | "2006-01-02T15:04:05.999999999Z07:00" | | +| "Kitchen" | "3:04PM" | | + +自定义日期格式： + +可通过占位符的组合自定义输出日期格式 + +| 字符 | 示例 | 描述 | +| --- | --- | --- | +| a | %a | 星期的缩写，如 `Wed` | +| A | %A | 星期的全写，如 `Wednesday` | +| b | %b | 月份缩写，如 `Mar` | +| B | %B | 月份的全写，如 `March` | +| C | %c | 世纪数，当前年份除 100 | +| **d** | %d | 一个月内的第几天；范围 `[01, 31]` | +| e | %e | 一个月内的第几天；范围 `[1, 31]`，使用空格填充 | +| **H** | %H | 小时，使用 24 小时制； 范围 `[00, 23]` | +| I | %I | 小时，使用 12 小时制； 范围 `[01, 12]` | +| j | %j | 一年内的第几天，范围 `[001, 365]` | +| k | %k | 小时，使用 24 小时制； 范围 `[0, 23]` | +| l | %l | 小时，使用 12 小时制； 范围 `[1, 12]`，使用空格填充 | +| **m** | %m | 月份，范围 `[01, 12]` | +| **M** | %M | 分钟，范围 `[00, 59]` | +| n | %n | 表示换行符 `\n` | +| p | %p | `AM` 或 `PM` | +| P | %P | `am` 或 `pm` | +| s | %s | 自 1970-01-01 00:00:00 UTC 来的的秒数 | +| **S** | %S | 秒数，范围 `[00, 60]` | +| t | %t | 表示制表符 `\t` | +| u | %u | 星期几，星期一为 1，范围 `[1, 7]` | +| w | %w | 星期几，星期天为 0, 范围 `[0, 6]` | +| y | %y | 年份，范围 `[00, 99]` | +| **Y** | %Y | 年份的十进制表示 | +| **z** | %z | RFC 822/ISO 8601:1988 风格的时区 (如： `-0600` 或 `+0100` 等) | +| Z | %Z | 时区缩写，如 `CST` | +| % | %% | 表示字符 `%` | + +示例： + +```python +# 待处理数据： +# { +# "a":{ +# "timestamp": "1610960605000", +# "second":2 +# }, +# "age":47 +# } + +# 处理脚本 +json(_, a.timestamp) +datetime(a.timestamp, 'ms', 'RFC3339') +``` + +```python +# 处理脚本 +ts = timestamp() +datetime(ts, 'ns', fmt='%Y-%m-%d %H:%M:%S', tz="UTC") + +# 输出 +{ + "ts": "2023-03-08 06:43:39" +} +``` + +```python +# 处理脚本 +ts = timestamp() +datetime(ts, 'ns', '%m/%d/%y %H:%M:%S %z', "Asia/Tokyo") + +# 输出 +{ + "ts": "03/08/23 15:44:59 +0900" +} +``` diff --git a/pkg/inimpl/guancecloud/ptinput/funcs/md/decode.en.md b/pkg/inimpl/guancecloud/ptinput/funcs/md/decode.en.md new file mode 100644 index 0000000..92645c8 --- /dev/null +++ b/pkg/inimpl/guancecloud/ptinput/funcs/md/decode.en.md @@ -0,0 +1,16 @@ +### `decode()` {#fn-decode} + +Function prototype: `fn decode(text: str, text_encode: str)` + +Function description: Convert text to UTF8 encoding to deal with the problem that the original log is not UTF8 encoded. Currently supported encodings are utf-16le/utf-16be/gbk/gb18030 (these encoding names can only be lowercase) + +Example: + +```python +decode("wwwwww", "gbk") + +# Extracted data(drop: false, cost: 33.279µs): +# { +# "message": "wwwwww", +# } +``` diff --git a/pkg/inimpl/guancecloud/ptinput/funcs/md/decode.md b/pkg/inimpl/guancecloud/ptinput/funcs/md/decode.md new file mode 100644 index 0000000..8ecdb47 --- /dev/null +++ b/pkg/inimpl/guancecloud/ptinput/funcs/md/decode.md @@ -0,0 +1,14 @@ +### `decode()` {#fn-decode} + +函数原型：`fn decode(text: str, text_encode: str)` + +函数说明：把 text 变成 UTF8 编码，以处理原始日志为非 UTF8 编码的问题。目前支持的编码为 utf-16le/utf-16be/gbk/gb18030（这些编码名只能小写） + +```python +decode("wwwwww", "gbk") + +# Extracted data(drop: false, cost: 33.279µs): +# { +# "message": "wwwwww", +# } +``` diff --git a/pkg/inimpl/guancecloud/ptinput/funcs/md/default_time.en.md b/pkg/inimpl/guancecloud/ptinput/funcs/md/default_time.en.md new file mode 100644 index 0000000..08d1797 --- /dev/null +++ b/pkg/inimpl/guancecloud/ptinput/funcs/md/default_time.en.md @@ -0,0 +1,79 @@ +### `default_time()` {#fn-defalt-time} + +Function prototype: `fn default_time(key: str, timezone: str = "")` + +Function description: Use an extracted field as the timestamp of the final data + +Function parameters: + +- `key`: key name +- `timezone`: Specifies the time zone used by the time text to be formatted, optional parameter, the default is the current system time zone, time zone example `+8/-8/+8:30` + +The pending data supports the following formatting times + + +| date format | date format | date format | date format | +| ----- | ---- | ---- | ---- | +| `2014-04-26 17:24:37.3186369` | `May 8, 2009 5:57:51 PM` | `2012-08-03 18:31:59.257000000` | `oct 7, 1970` | +| `2014-04-26 17:24:37.123` | `oct 7, '70` | `2013-04-01 22:43` | `oct. 7, 1970` | +| `2013-04-01 22:43:22` | `oct. 7, 70` | `2014-12-16 06:20:00 UTC` | `Mon Jan 2 15:04:05 2006` | +| `2014-12-16 06:20:00 GMT` | `Mon Jan 2 15:04:05 MST 2006` | `2014-04-26 05:24:37 PM` | `Mon Jan 02 15:04:05 -0700 2006` | +| `2014-04-26 13:13:43 +0800` | `Monday, 02-Jan-06 15:04:05 MST` | `2014-04-26 13:13:43 +0800 +08` | `Mon, 02 Jan 2006 15:04:05 MST` | +| `2014-04-26 13:13:44 +09:00` | `Tue, 11 Jul 2017 16:28:13 +0200 (CEST)` | `2012-08-03 18:31:59.257000000 +0000 UTC` | `Mon, 02 Jan 2006 15:04:05 -0700` | +| `2015-09-30 18:48:56.35272715 +0000 UTC` | `Thu, 4 Jan 2018 17:53:36 +0000` | `2015-02-18 00:12:00 +0000 GMT` | `Mon 30 Sep 2018 09:09:09 PM UTC` | +| `2015-02-18 00:12:00 +0000 UTC` | `Mon Aug 10 15:44:11 UTC+0100 2015` | `2015-02-08 03:02:00 +0300 MSK m=+0.000000001` | `Thu, 4 Jan 2018 17:53:36 +0000` | +| `2015-02-08 03:02:00.001 +0300 MSK m=+0.000000001` | `Fri Jul 03 2015 18:04:07 GMT+0100 (GMT Daylight Time)` | `2017-07-19 03:21:51+00:00` | `September 17, 2012 10:09am` | +| `2014-04-26` | `September 17, 2012 at 10:09am PST-08` | `2014-04` | `September 17, 2012, 10:10:09` | +| `2014` | `2014:3:31` | `2014-05-11 08:20:13,787` | `2014:03:31` | +| `3.31.2014` | `2014:4:8 22:05` | `03.31.2014` | `2014:04:08 22:05` | +| `08.21.71` | `2014:04:2 03:00:51` | `2014.03` | `2014:4:02 03:00:51` | +| `2014.03.30` | `2012:03:19 10:11:59` | `20140601` | `2012:03:19 10:11:59.3186369` | +| `20140722105203` | `2014 年 04 月 08 日 ` | `1332151919` | `2006-01-02T15:04:05+0000` | +| `1384216367189` | `2009-08-12T22:15:09-07:00` | `1384216367111222` | `2009-08-12T22:15:09` | +| `1384216367111222333` | `2009-08-12T22:15:09Z` | + + +Example JSON extraction: + +```python +# raw json +{ + "time":"06/Jan/2017:16:16:37 +0000", + "second":2, + "third":"abc", + "forth":true +} + +# script +json(_, time) # extract time field +default_time(time) # convert the extracted time field into a timestamp + +# result +{ + "time": 1483719397000000000, +} +``` + +Text extraction example: + +```python +# raw log text +# 2021-01-11T17:43:51.887+0800 DEBUG io io/io.go:458 post cost 6.87021ms + +# script +grok(_, '%{TIMESTAMP_ISO8601:log_time}') # Extract the log time and name the field log_time +default_time(log_time) # Convert the extracted log_time field into a timestamp + +# result +{ + "log_time": 1610358231887000000, +} + +# For the data collected by logging, it is better to name the time field as time, otherwise the logging collector will fill it with the current time +rename("time", log_time) + +# result +{ + "time": 1610358231887000000, +} +``` diff --git a/pkg/inimpl/guancecloud/ptinput/funcs/md/default_time.md b/pkg/inimpl/guancecloud/ptinput/funcs/md/default_time.md new file mode 100644 index 0000000..92077fa --- /dev/null +++ b/pkg/inimpl/guancecloud/ptinput/funcs/md/default_time.md @@ -0,0 +1,80 @@ +### `default_time()` {#fn-defalt-time} + +函数原型：`fn default_time(key: str, timezone: str = "")` + +函数说明：以提取的某个字段作为最终数据的时间戳 + +函数参数 + +- `key`: 指定的 key， key 的数据类型需要为字符串类型 +- `timezone`: 指定待格式化的时间文本所使用的时区，可选参数，默认为当前系统时区，时区示例 `+8/-8/+8:30` + +待处理数据支持以下格式化时间 + + +| 日期格式 | 日期格式 | 日期格式 | 日期格式 | +| ----- | ---- | ---- | ---- | +| `2014-04-26 17:24:37.3186369` | `May 8, 2009 5:57:51 PM` | `2012-08-03 18:31:59.257000000` | `oct 7, 1970` | +| `2014-04-26 17:24:37.123` | `oct 7, '70` | `2013-04-01 22:43` | `oct. 7, 1970` | +| `2013-04-01 22:43:22` | `oct. 7, 70` | `2014-12-16 06:20:00 UTC` | `Mon Jan 2 15:04:05 2006` | +| `2014-12-16 06:20:00 GMT` | `Mon Jan 2 15:04:05 MST 2006` | `2014-04-26 05:24:37 PM` | `Mon Jan 02 15:04:05 -0700 2006` | +| `2014-04-26 13:13:43 +0800` | `Monday, 02-Jan-06 15:04:05 MST` | `2014-04-26 13:13:43 +0800 +08` | `Mon, 02 Jan 2006 15:04:05 MST` | +| `2014-04-26 13:13:44 +09:00` | `Tue, 11 Jul 2017 16:28:13 +0200 (CEST)` | `2012-08-03 18:31:59.257000000 +0000 UTC` | `Mon, 02 Jan 2006 15:04:05 -0700` | +| `2015-09-30 18:48:56.35272715 +0000 UTC` | `Thu, 4 Jan 2018 17:53:36 +0000` | `2015-02-18 00:12:00 +0000 GMT` | `Mon 30 Sep 2018 09:09:09 PM UTC` | +| `2015-02-18 00:12:00 +0000 UTC` | `Mon Aug 10 15:44:11 UTC+0100 2015` | `2015-02-08 03:02:00 +0300 MSK m=+0.000000001` | `Thu, 4 Jan 2018 17:53:36 +0000` | +| `2015-02-08 03:02:00.001 +0300 MSK m=+0.000000001` | `Fri Jul 03 2015 18:04:07 GMT+0100 (GMT Daylight Time)` | `2017-07-19 03:21:51+00:00` | `September 17, 2012 10:09am` | +| `2014-04-26` | `September 17, 2012 at 10:09am PST-08` | `2014-04` | `September 17, 2012, 10:10:09` | +| `2014` | `2014:3:31` | `2014-05-11 08:20:13,787` | `2014:03:31` | +| `3.31.2014` | `2014:4:8 22:05` | `03.31.2014` | `2014:04:08 22:05` | +| `08.21.71` | `2014:04:2 03:00:51` | `2014.03` | `2014:4:02 03:00:51` | +| `2014.03.30` | `2012:03:19 10:11:59` | `20140601` | `2012:03:19 10:11:59.3186369` | +| `20140722105203` | `2014 年 04 月 08 日 ` | `1332151919` | `2006-01-02T15:04:05+0000` | +| `1384216367189` | `2009-08-12T22:15:09-07:00` | `1384216367111222` | `2009-08-12T22:15:09` | +| `1384216367111222333` | `2009-08-12T22:15:09Z` | + + +JSON 提取示例： + +```python +# 原始 json +{ + "time":"06/Jan/2017:16:16:37 +0000", + "second":2, + "third":"abc", + "forth":true +} + +# pipeline 脚本 +json(_, time) # 提取 time 字段 +default_time(time) # 将提取到的 time 字段转换成时间戳 + +# 处理结果 +{ + "time": 1483719397000000000, +} +``` + +文本提取示例： + +```python +# 原始日志文本 +2021-01-11T17:43:51.887+0800 DEBUG io io/io.go:458 post cost 6.87021ms + +# pipeline 脚本 +grok(_, '%{TIMESTAMP_ISO8601:log_time}') # 提取日志时间，并将字段命名为 log_time +default_time(log_time) # 将提取到的 log_time 字段转换成时间戳 + +# 处理结果 +{ + "log_time": 1610358231887000000, +} + +# 对于 logging 采集的数据，最好将时间字段命名为 time，否则 logging 采集器会以当前时间填充 +rename("time", log_time) + +# 处理结果 +{ + "time": 1610358231887000000, +} +``` + diff --git a/pkg/inimpl/guancecloud/ptinput/funcs/md/delete.en.md b/pkg/inimpl/guancecloud/ptinput/funcs/md/delete.en.md new file mode 100644 index 0000000..9deac0d --- /dev/null +++ b/pkg/inimpl/guancecloud/ptinput/funcs/md/delete.en.md @@ -0,0 +1,25 @@ +### `delete()` {#fn-delete} + +Function prototype: `fn delete(src: map[string]any, key: str)` + +Function description: Delete the key in the JSON map + +```python + +# input +# {"a": "b", "b":[0, {"c": "d"}], "e": 1} + +# script +j_map = load_json(_) + +delete(j_map["b"][-1], "c") + +delete(j_map, "a") + +add_key("j_map", j_map) + +# result: +# { +# "j_map": "{\"b\":[0,{}],\"e\":1}", +# } +``` diff --git a/pkg/inimpl/guancecloud/ptinput/funcs/md/delete.md b/pkg/inimpl/guancecloud/ptinput/funcs/md/delete.md new file mode 100644 index 0000000..b7aa545 --- /dev/null +++ b/pkg/inimpl/guancecloud/ptinput/funcs/md/delete.md @@ -0,0 +1,25 @@ +### `delete()` {#fn-delete} + +函数原型：`fn delete(src: map[string]any, key: str)` + +函数说明： 删除 JSON map 中的 key + +```python + +# input +# {"a": "b", "b":[0, {"c": "d"}], "e": 1} + +# script +j_map = load_json(_) + +delete(j_map["b"][-1], "c") + +delete(j_map, "a") + +add_key("j_map", j_map) + +# result: +# { +# "j_map": "{\"b\":[0,{}],\"e\":1}", +# } +``` diff --git a/pkg/inimpl/guancecloud/ptinput/funcs/md/drop.en.md b/pkg/inimpl/guancecloud/ptinput/funcs/md/drop.en.md new file mode 100644 index 0000000..d44f7f7 --- /dev/null +++ b/pkg/inimpl/guancecloud/ptinput/funcs/md/drop.en.md @@ -0,0 +1,23 @@ +### `drop()` {#fn-drop} + +Function prototype: `fn drop()` + +Function description: Discard the entire log without uploading + +Example: + +```python +# in << {"str_a": "2", "str_b": "3"} +json(_, str_a) +if str_a == "2"{ + drop() + exit() +} +json(_, str_b) + +# Extracted data(drop: true, cost: 30.02µs): +# { +# "message": "{\"str_a\": \"2\", \"str_b\": \"3\"}", +# "str_a": "2" +# } +``` diff --git a/pkg/inimpl/guancecloud/ptinput/funcs/md/drop.md b/pkg/inimpl/guancecloud/ptinput/funcs/md/drop.md new file mode 100644 index 0000000..d394c3d --- /dev/null +++ b/pkg/inimpl/guancecloud/ptinput/funcs/md/drop.md @@ -0,0 +1,22 @@ +### `drop()` {#fn-drop} + +函数原型：`fn drop()` + +函数说明：丢弃整条日志，不进行上传 + +```python +# in << {"str_a": "2", "str_b": "3"} +json(_, str_a) +if str_a == "2"{ + drop() + exit() +} +json(_, str_b) + +# Extracted data(drop: true, cost: 30.02µs): +# { +# "message": "{\"str_a\": \"2\", \"str_b\": \"3\"}", +# "str_a": "2" +# } +``` + diff --git a/pkg/inimpl/guancecloud/ptinput/funcs/md/drop_key.en.md b/pkg/inimpl/guancecloud/ptinput/funcs/md/drop_key.en.md new file mode 100644 index 0000000..6699b25 --- /dev/null +++ b/pkg/inimpl/guancecloud/ptinput/funcs/md/drop_key.en.md @@ -0,0 +1,26 @@ +### `drop_key()` {#fn-drop-key} + +Function prototype: `fn drop_key(key)` + +Function description: Delete key + +Function parameters: + +- `key`: key to be deleted + +Example： + +```python +# data = "{\"age\": 17, \"name\": \"zhangsan\", \"height\": 180}" + +json(_, age,) +json(_, name) +json(_, height) +drop_key(height) + +# result +# { +# "age": 17, +# "name": "zhangsan" +# } +``` diff --git a/pkg/inimpl/guancecloud/ptinput/funcs/md/drop_key.md b/pkg/inimpl/guancecloud/ptinput/funcs/md/drop_key.md new file mode 100644 index 0000000..55fd218 --- /dev/null +++ b/pkg/inimpl/guancecloud/ptinput/funcs/md/drop_key.md @@ -0,0 +1,28 @@ +### `drop_key()` {#fn-drop-key} + +函数原型：`fn drop_key(key)` + +函数说明：删除已提取字段 + +函数参数： + +- `key`: 待删除字段名 + +示例： + +```python +# data = "{\"age\": 17, \"name\": \"zhangsan\", \"height\": 180}" + +# 处理脚本 +json(_, age,) +json(_, name) +json(_, height) +drop_key(height) + +# 处理结果 +{ + "age": 17, + "name": "zhangsan" +} +``` + diff --git a/pkg/inimpl/guancecloud/ptinput/funcs/md/drop_origin_data.en.md b/pkg/inimpl/guancecloud/ptinput/funcs/md/drop_origin_data.en.md new file mode 100644 index 0000000..353cd00 --- /dev/null +++ b/pkg/inimpl/guancecloud/ptinput/funcs/md/drop_origin_data.en.md @@ -0,0 +1,14 @@ +### `drop_origin_data()` {#fn-drop-origin-data} + +Function prototype: `fn drop_origin_data()` + +Function description: Discard the initial text, otherwise the initial text is placed in the message field + +Example: + +```python +# input data: {"age": 17, "name": "zhangsan", "height": 180} + +# delete message field +drop_origin_data() +``` diff --git a/pkg/inimpl/guancecloud/ptinput/funcs/md/drop_origin_data.md b/pkg/inimpl/guancecloud/ptinput/funcs/md/drop_origin_data.md new file mode 100644 index 0000000..2469ebe --- /dev/null +++ b/pkg/inimpl/guancecloud/ptinput/funcs/md/drop_origin_data.md @@ -0,0 +1,15 @@ +### `drop_origin_data()` {#fn-drop-origin-data} + +函数原型：`fn drop_origin_data()` + +函数说明：丢弃初始化文本，否则初始文本放在 message 字段中 + +示例： + +```python +# 待处理数据：{"age": 17, "name": "zhangsan", "height": 180} + +# 结果集中删除 message 内容 +drop_origin_data() +``` + diff --git a/pkg/inimpl/guancecloud/ptinput/funcs/md/duration_precision.en.md b/pkg/inimpl/guancecloud/ptinput/funcs/md/duration_precision.en.md new file mode 100644 index 0000000..7792186 --- /dev/null +++ b/pkg/inimpl/guancecloud/ptinput/funcs/md/duration_precision.en.md @@ -0,0 +1,20 @@ +### `duration_precision()` {#fn-duration-precision} + +Function prototype: `fn duration_precision(key, old_precision: str, new_precision: str)` + +Function description: Perform duration precision conversion, and specify the current precision and target precision through Function parameters:. Support conversion between s, ms, us, ns. + +Example: + +```python +# in << {"ts":12345} +json(_, ts) +cast(ts, "int") +duration_precision(ts, "ms", "ns") + +# Extracted data(drop: false, cost: 33.279µs): +# { +# "message": "{\"ts\":12345}", +# "ts": 12345000000 +# } +``` diff --git a/pkg/inimpl/guancecloud/ptinput/funcs/md/duration_precision.md b/pkg/inimpl/guancecloud/ptinput/funcs/md/duration_precision.md new file mode 100644 index 0000000..742817f --- /dev/null +++ b/pkg/inimpl/guancecloud/ptinput/funcs/md/duration_precision.md @@ -0,0 +1,18 @@ +### `duration_precision()` {#fn-duration-precision} + +函数原型：`fn duration_precision(key, old_precision: str, new_precision: str)` + +函数说明：进行 duration 精度的转换，通过参数指定当前精度和目标精度。支持在 s, ms, us, ns 间转换。 + +```python +# in << {"ts":12345} +json(_, ts) +cast(ts, "int") +duration_precision(ts, "ms", "ns") + +# Extracted data(drop: false, cost: 33.279µs): +# { +# "message": "{\"ts\":12345}", +# "ts": 12345000000 +# } +``` diff --git a/pkg/inimpl/guancecloud/ptinput/funcs/md/exit.en.md b/pkg/inimpl/guancecloud/ptinput/funcs/md/exit.en.md new file mode 100644 index 0000000..ce5d5aa --- /dev/null +++ b/pkg/inimpl/guancecloud/ptinput/funcs/md/exit.en.md @@ -0,0 +1,20 @@ +### `exit()` {#fn-exit} + +Function prototype: `fn exit()` + +Function description: End the parsing of the current log, if the function drop() is not called, the parsed part will still be output + +```python +# in << {"str_a": "2", "str_b": "3"} +json(_, str_a) +if str_a == "2"{ + exit() +} +json(_, str_b) + +# Extracted data(drop: false, cost: 48.233µs): +# { +# "message": "{\"str_a\": \"2\", \"str_b\": \"3\"}", +# "str_a": "2" +# } +``` diff --git a/pkg/inimpl/guancecloud/ptinput/funcs/md/exit.md b/pkg/inimpl/guancecloud/ptinput/funcs/md/exit.md new file mode 100644 index 0000000..ae11c8c --- /dev/null +++ b/pkg/inimpl/guancecloud/ptinput/funcs/md/exit.md @@ -0,0 +1,21 @@ +### `exit()` {#fn-exit} + +函数原型：`fn exit()` + +函数说明：结束当前一条日志的解析，若未调用函数 drop() 仍会输出已经解析的部分 + +```python +# in << {"str_a": "2", "str_b": "3"} +json(_, str_a) +if str_a == "2"{ + exit() +} +json(_, str_b) + +# Extracted data(drop: false, cost: 48.233µs): +# { +# "message": "{\"str_a\": \"2\", \"str_b\": \"3\"}", +# "str_a": "2" +# } +``` + diff --git a/pkg/inimpl/guancecloud/ptinput/funcs/md/format_int.en.md b/pkg/inimpl/guancecloud/ptinput/funcs/md/format_int.en.md new file mode 100644 index 0000000..ad697e4 --- /dev/null +++ b/pkg/inimpl/guancecloud/ptinput/funcs/md/format_int.en.md @@ -0,0 +1,46 @@ +### `format_int()` {#fn-format-int} + +Function prototype: `fn format_int(val: int, base: int) str` + +Function description: Converts a numeric value to a numeric string in the specified base. + +Function parameters: + +- `val`: The number to be converted. +- `base`: Base, ranging from 2 to 36; when the base is greater than 10, lowercase letters a to z are used to represent values 10 and later. + +Example: + +```python +# script0 +a = 7665324064912355185 +b = format_int(a, 16) +if b != "6a60b39fd95aaf71" { + add_key(abc, b) +} else { + add_key(abc, "ok") +} + +# result +''' +{ + "abc": "ok" +} +''' + +# script1 +a = "7665324064912355185" +b = format_int(parse_int(a, 10), 16) +if b != "6a60b39fd95aaf71" { + add_key(abc, b) +} else { + add_key(abc, "ok") +} + +# result +''' +{ + "abc": "ok" +} +''' +``` diff --git a/pkg/inimpl/guancecloud/ptinput/funcs/md/format_int.md b/pkg/inimpl/guancecloud/ptinput/funcs/md/format_int.md new file mode 100644 index 0000000..71ef6e9 --- /dev/null +++ b/pkg/inimpl/guancecloud/ptinput/funcs/md/format_int.md @@ -0,0 +1,46 @@ +### `format_int()` {#fn-format-int} + +函数原型：`fn format_int(val: int, base: int) str` + +函数说明：将数值转换为指定进制的数值字符串。 + +参数： + +- `val`: 待转换的整数 +- `base`: 进制，范围 2 到 36；进制大于 10 时使用小写字母 a 到 z 表示 10 及以后的数值。 + +示例： + +```python +# script0 +a = 7665324064912355185 +b = format_int(a, 16) +if b != "6a60b39fd95aaf71" { + add_key(abc, b) +} else { + add_key(abc, "ok") +} + +# result +''' +{ + "abc": "ok" +} +''' + +# script1 +a = "7665324064912355185" +b = format_int(parse_int(a, 10), 16) +if b != "6a60b39fd95aaf71" { + add_key(abc, b) +} else { + add_key(abc, "ok") +} + +# result +''' +{ + "abc": "ok" +} +''' +``` diff --git a/pkg/inimpl/guancecloud/ptinput/funcs/md/geoip.en.md b/pkg/inimpl/guancecloud/ptinput/funcs/md/geoip.en.md new file mode 100644 index 0000000..ba9e238 --- /dev/null +++ b/pkg/inimpl/guancecloud/ptinput/funcs/md/geoip.en.md @@ -0,0 +1,34 @@ +### `geoip()` {#fn-geoip} + +Function prototype: `fn geoip(ip: str)` + +Function description: Append more IP information to IP. `geoip()` will generate additional fields, such as: + +- `isp`: operator +- `city`: city +- `province`: province +- `country`: country + +Function parameters: + +- `ip`: The extracted IP field supports both IPv4 and IPv6 + +Example: + +```python +# input data: {"ip":"1.2.3.4"} + +# script +json(_, ip) +geoip(ip) + +# result +{ + "city" : "Brisbane", + "country" : "AU", + "ip" : "1.2.3.4", + "province" : "Queensland", + "isp" : "unknown" + "message" : "{\"ip\": \"1.2.3.4\"}", +} +``` diff --git a/pkg/inimpl/guancecloud/ptinput/funcs/md/geoip.md b/pkg/inimpl/guancecloud/ptinput/funcs/md/geoip.md new file mode 100644 index 0000000..bce78a9 --- /dev/null +++ b/pkg/inimpl/guancecloud/ptinput/funcs/md/geoip.md @@ -0,0 +1,34 @@ +### `geoip()` {#fn-geoip} + +函数原型：`fn geoip(ip: str)` + +函数说明：在 IP 上追加更多 IP 信息。 `geoip()` 会额外产生多个字段，如： + +- `isp`: 运营商 +- `city`: 城市 +- `province`: 省份 +- `country`: 国家 + +参数： + +- `ip`: 已经提取出来的 IP 字段，支持 IPv4 和 IPv6 + +示例： + +```python +# 待处理数据：{"ip":"1.2.3.4"} + +# 处理脚本 +json(_, ip) +geoip(ip) + +# 处理结果 +{ + "city" : "Brisbane", + "country" : "AU", + "ip" : "1.2.3.4", + "province" : "Queensland", + "isp" : "unknown" + "message" : "{\"ip\": \"1.2.3.4\"}", +} +``` diff --git a/pkg/inimpl/guancecloud/ptinput/funcs/md/get_key.en.md b/pkg/inimpl/guancecloud/ptinput/funcs/md/get_key.en.md new file mode 100644 index 0000000..1f73427 --- /dev/null +++ b/pkg/inimpl/guancecloud/ptinput/funcs/md/get_key.en.md @@ -0,0 +1,40 @@ +### `get_key()` {#fn-get-key} + +Function prototype: `fn get_key(key)` + +Function description: Read the value of key from the input point + +Function parameters: + +- `key_name`: key name + +Example: + +```python +add_key("city", "shanghai") + +# Here you can directly access the value of the key with the same name in point through city +if city == "shanghai" { + add_key("city_1", city) +} + +# Due to the right associativity of assignment, get the value whose key is "city" first, +# Then create a variable named city +city = city + " --- ningbo" + " --- " + + "hangzhou" + " --- suzhou ---" + "" + +# get_key gets the value of "city" from point +# If there is a variable named city, it cannot be obtained directly from point +if city != get_key("city") { + add_key("city_2", city) +} + +# result +""" +{ + "city": "shanghai", + "city_1": "shanghai", + "city_2": "shanghai --- ningbo --- hangzhou --- suzhou ---" +} +""" +``` diff --git a/pkg/inimpl/guancecloud/ptinput/funcs/md/get_key.md b/pkg/inimpl/guancecloud/ptinput/funcs/md/get_key.md new file mode 100644 index 0000000..fe60e49 --- /dev/null +++ b/pkg/inimpl/guancecloud/ptinput/funcs/md/get_key.md @@ -0,0 +1,40 @@ +### `get_key()` {#fn-get-key} + +函数原型：`fn get_key(key)` + +函数说明：从输入 point 中读取 key 的值，而不是堆栈上的变量的值 + +函数参数 + +- `key_name`: key 的名称 + +示例： + +```python +add_key("city", "shanghai") + +# 此处可以直接通过 city 访问获取 point 中的同名 key 的值 +if city == "shanghai" { + add_key("city_1", city) +} + +# 由于赋值的右结合性，先获取 key 为 "city" 的值， +# 而后创建名为 city 的变量 +city = city + " --- ningbo" + " --- " + + "hangzhou" + " --- suzhou ---" + "" + +# get_key 从 point 中获取 "city" 的值 +# 存在名为 city 的变量，则无法直接从 point 中获取 +if city != get_key("city") { + add_key("city_2", city) +} + +# 处理结果 +""" +{ + "city": "shanghai", + "city_1": "shanghai", + "city_2": "shanghai --- ningbo --- hangzhou --- suzhou ---" +} +""" +``` diff --git a/pkg/inimpl/guancecloud/ptinput/funcs/md/grok.en.md b/pkg/inimpl/guancecloud/ptinput/funcs/md/grok.en.md new file mode 100644 index 0000000..5cf0b6f --- /dev/null +++ b/pkg/inimpl/guancecloud/ptinput/funcs/md/grok.en.md @@ -0,0 +1,51 @@ +### `grok()` {#fn-grok} + +Function prototype: `fn grok(input: str, pattern: str, trim_space: bool = true) bool` + +Function description: Extract the contents of the text string `input` by `pattern`, and return true when pattern matches input successfully, otherwise return false. + +Function parameters: + +- `input`：The text to be extracted can be the original text (`_`) or a `key` after the initial extraction +- `pattern`: grok expression, the data type of the specified key is supported in the expression: bool, float, int, string (corresponding to Pipeline's str, can also be written as str), the default is string +- `trim_space`: Delete the leading and trailing blank characters in the extracted characters, the default value is true +```python +grok(_, pattern) #Use the entered text directly as raw data +grok(key, pattern) # For a key that has been extracted before, do grok again +``` + +示例： + +```python +# input data: "12/01/2021 21:13:14.123" + +# script +add_pattern("_second", "(?:(?:[0-5]?[0-9]|60)(?:[:.,][0-9]+)?)") +add_pattern("_minute", "(?:[0-5][0-9])") +add_pattern("_hour", "(?:2[0123]|[01]?[0-9])") +add_pattern("time", "([^0-9]?)%{_hour:hour:string}:%{_minute:minute:int}(?::%{_second:second:float})([^0-9]?)") + +grok_match_ok = grok(_, "%{DATE_US:date} %{time}") + +add_key(grok_match_ok) + +# result +{ + "date": "12/01/2021", + "hour": "21", + "message": "12/01/2021 21:13:14.123", + "minute": 13, + "second": 14.123 +} + +{ + "date": "12/01/2021", + "grok_match_ok": true, + "hour": "21", + "message": "12/01/2021 21:13:14.123", + "minute": 13, + "second": 14.123, + "status": "unknown", + "time": 1665994187473917724 +} +``` diff --git a/pkg/inimpl/guancecloud/ptinput/funcs/md/grok.md b/pkg/inimpl/guancecloud/ptinput/funcs/md/grok.md new file mode 100644 index 0000000..4667806 --- /dev/null +++ b/pkg/inimpl/guancecloud/ptinput/funcs/md/grok.md @@ -0,0 +1,52 @@ +### `grok()` {#fn-grok} + +函数原型：`fn grok(input: str, pattern: str, trim_space: bool = true) bool` + +函数说明：通过 `pattern` 提取文本串 `input` 中的内容，当 pattern 匹配 input 成功时返回 true 否则返回 false。 + +参数： + +- `input`：待提取文本，可以是原始文本（`_`）或经过初次提取之后的某个 `key` +- `pattern`: grok 表达式，表达式中支持指定 key 的数据类型：bool, float, int, string(对应 ppl 的 str，亦可写为 str)，默认为 string +- `trim_space`: 删除提取出的字符中的空白首尾字符，默认值为 true + +```python +grok(_, pattern) # 直接使用输入的文本作为原始数据 +grok(key, pattern) # 对之前已经提取出来的某个 key，做再次 grok +``` + +示例： + +```python +# 待处理数据："12/01/2021 21:13:14.123" + +# pipline 脚本 +add_pattern("_second", "(?:(?:[0-5]?[0-9]|60)(?:[:.,][0-9]+)?)") +add_pattern("_minute", "(?:[0-5][0-9])") +add_pattern("_hour", "(?:2[0123]|[01]?[0-9])") +add_pattern("time", "([^0-9]?)%{_hour:hour:string}:%{_minute:minute:int}(?::%{_second:second:float})([^0-9]?)") + +grok_match_ok = grok(_, "%{DATE_US:date} %{time}") + +add_key(grok_match_ok) + +# 处理结果 +{ + "date": "12/01/2021", + "hour": "21", + "message": "12/01/2021 21:13:14.123", + "minute": 13, + "second": 14.123 +} + +{ + "date": "12/01/2021", + "grok_match_ok": true, + "hour": "21", + "message": "12/01/2021 21:13:14.123", + "minute": 13, + "second": 14.123, + "status": "unknown", + "time": 1665994187473917724 +} +``` diff --git a/pkg/inimpl/guancecloud/ptinput/funcs/md/group_between.en.md b/pkg/inimpl/guancecloud/ptinput/funcs/md/group_between.en.md new file mode 100644 index 0000000..505f218 --- /dev/null +++ b/pkg/inimpl/guancecloud/ptinput/funcs/md/group_between.en.md @@ -0,0 +1,38 @@ +### `group_between()` {#fn-group-between} + +Function prototype: `fn group_between(key: int, between: list, new_value: int|float|bool|str|map|list|nil, new_key)` + +Function description: If the `key` value is within the specified range `between` (note: it can only be a single interval, such as `[0,100]`), a new field can be created and assigned a new value. If no new field is provided, the original field value will be overwritten + +Example 1: + +```python +# input data: {"http_status": 200, "code": "success"} + +json(_, http_status) + +# If the field http_status value is within the specified range, change its value to "OK" +group_between(http_status, [200, 300], "OK") + +# result +# { +# "http_status": "OK" +# } +``` + +Example 2: + +```python +# input data: {"http_status": 200, "code": "success"} + +json(_, http_status) + +# If the value of the field http_status is within the specified range, create a new status field with the value "OK" +group_between(http_status, [200, 300], "OK", status) + +# result +{ + "http_status": 200, + "status": "OK" +} +``` diff --git a/pkg/inimpl/guancecloud/ptinput/funcs/md/group_between.md b/pkg/inimpl/guancecloud/ptinput/funcs/md/group_between.md new file mode 100644 index 0000000..8302ca6 --- /dev/null +++ b/pkg/inimpl/guancecloud/ptinput/funcs/md/group_between.md @@ -0,0 +1,38 @@ +### `group_between()` {#fn-group-between} + +函数原型：`fn group_between(key: int, between: list, new_value: int|float|bool|str|map|list|nil, new_key)` + +函数说明：如果 `key` 值在指定范围 `between` 内（注意：只能是单个区间，如 `[0,100]`），则可创建一个新字段，并赋予新值。若不提供新字段，则覆盖原字段值 + +示例一： + +```python +# 待处理数据：{"http_status": 200, "code": "success"} + +json(_, http_status) + +# 如果字段 http_status 值在指定范围内，则将其值改为 "OK" +group_between(http_status, [200, 300], "OK") + +# 处理结果 +{ + "http_status": "OK" +} +``` + +示例二： + +```python +# 待处理数据：{"http_status": 200, "code": "success"} + +json(_, http_status) + +# 如果字段 http_status 值在指定范围内，则新建 status 字段，其值为 "OK" +group_between(http_status, [200, 300], "OK", status) + +# 处理结果 +{ + "http_status": 200, + "status": "OK" +} +``` diff --git a/pkg/inimpl/guancecloud/ptinput/funcs/md/group_in.en.md b/pkg/inimpl/guancecloud/ptinput/funcs/md/group_in.en.md new file mode 100644 index 0000000..e273df6 --- /dev/null +++ b/pkg/inimpl/guancecloud/ptinput/funcs/md/group_in.en.md @@ -0,0 +1,15 @@ +### `group_in()` {#fn-group-in} + +Function prototype: `fn group_in(key: int|float|bool|str, range: list, new_value: int|float|bool|str|map|list|nil, new-key = "")` + +Function description: If the `key` value is in the list `in`, a new field can be created and assigned the new value. If no new field is provided, the original field value will be overwritten + +Example: + +```python +# If the field log_level value is in the list, change its value to "OK" +group_in(log_level, ["info", "debug"], "OK") + +# If the field http_status value is in the specified list, create a new status field with the value "not-ok" +group_in(log_level, ["error", "panic"], "not-ok", status) +``` diff --git a/pkg/inimpl/guancecloud/ptinput/funcs/md/group_in.md b/pkg/inimpl/guancecloud/ptinput/funcs/md/group_in.md new file mode 100644 index 0000000..f30a513 --- /dev/null +++ b/pkg/inimpl/guancecloud/ptinput/funcs/md/group_in.md @@ -0,0 +1,15 @@ +### `group_in()` {#fn-group-in} + +函数原型：`fn group_in(key: int|float|bool|str, range: list, new_value: int|float|bool|str|map|list|nil, new-key = "")` + +函数说明：如果 `key` 值在列表 `in` 中，则可创建一个新字段，并赋予新值。若不提供新字段，则覆盖原字段值 + +示例： + +```python +# 如果字段 log_level 值在列表中，则将其值改为 "OK" +group_in(log_level, ["info", "debug"], "OK") + +# 如果字段 http_status 值在指定列表中，则新建 status 字段，其值为 "not-ok" +group_in(log_level, ["error", "panic"], "not-ok", status) +``` diff --git a/pkg/inimpl/guancecloud/ptinput/funcs/md/json.en.md b/pkg/inimpl/guancecloud/ptinput/funcs/md/json.en.md new file mode 100644 index 0000000..ac0db7b --- /dev/null +++ b/pkg/inimpl/guancecloud/ptinput/funcs/md/json.en.md @@ -0,0 +1,110 @@ +### `json()` {#fn-json} + +Function prototype: `fn json(input: str, json_path, newkey, trim_space: bool = true)` + +Function description: Extract the specified field in JSON and name it as a new field. + +Function parameters: + +- `input`: The JSON to be extracted can be the original text (`_`) or a `key` after the initial extraction +- `json_path`: JSON path information +- `newkey`：Write the data to the new key after extraction +- `trim_space`: Delete the leading and trailing blank characters in the extracted characters, the default value is true +- `delete_after_extract`: After extract delete the extracted info from input. Only map key and map value are deletable, list(array) are not supported. Default is `false'. + +```python +# Directly extract the x.y field in the original input json, and name it as a new field abc +json(_, x.y, abc) + +# For a `key` that has been extracted, extract `x.y` again, and the extracted field name is `x.y` +json(key, x.y) +``` + +Example 1: + +```python +# input data: +# {"info": {"age": 17, "name": "zhangsan", "height": 180}} + +# script: +json(_, info, "zhangsan") +json(zhangsan, name) +json(zhangsan, age, "age") + +# result: +{ + "age": 17, + "message": "{\"info\": {\"age\": 17, \"name\": \"zhangsan\", \"height\": 180}}", + "name": "zhangsan", + "zhangsan": "{\"age\":17,\"height\":180,\"name\":\"zhangsan\"}" +} +``` + +Example 2: + +```python +# input data: +# data = { +# "name": {"first": "Tom", "last": "Anderson"}, +# "age":37, +# "children": ["Sara","Alex","Jack"], +# "fav.movie": "Deer Hunter", +# "friends": [ +# {"first": "Dale", "last": "Murphy", "age": 44, "nets": ["ig", "fb", "tw"]}, +# {"first": "Roger", "last": "Craig", "age": 68, "nets": ["fb", "tw"]}, +# {"first": "Jane", "last": "Murphy", "age": 47, "nets": ["ig", "tw"]} +# ] +# } + +# script: +json(_, name) +json(name, first) +``` + +Example 3: + +```python +# input data: +# [ +# {"first": "Dale", "last": "Murphy", "age": 44, "nets": ["ig", "fb", "tw"]}, +# {"first": "Roger", "last": "Craig", "age": 68, "nets": ["fb", "tw"]}, +# {"first": "Jane", "last": "Murphy", "age": 47, "nets": ["ig", "tw"]} +# ] + +# script: +json(_, .[0].nets[-1]) +``` + +Example 4: + +```python +# input data: +{"item": " not_space ", "item2":{"item3": [123]}} + +# script: +json(_, item2.item3, item, delete_after_extract = true) + +# result: +{ + "item": "[123]", + "message": "{\"item\":\" not_space \",\"item2\":{}}", +} +``` + + +Example 5: + +```python +# input data: +{"item": " not_space ", "item2":{"item3": [123]}} + +# If you try to remove a list element it will fail the script check. +# Script: +json(_, item2.item3[0], item, delete_after_extract = true) + + +# test command: +# datakit pipeline -P j2.p -T '{"item": " not_space ", "item2":{"item3": [123]}}' +# report error: +# [E] j2.p:1:54: does not support deleting elements in the list +``` diff --git a/pkg/inimpl/guancecloud/ptinput/funcs/md/json.md b/pkg/inimpl/guancecloud/ptinput/funcs/md/json.md new file mode 100644 index 0000000..84aca8e --- /dev/null +++ b/pkg/inimpl/guancecloud/ptinput/funcs/md/json.md @@ -0,0 +1,109 @@ +### `json()` {#fn-json} + +函数原型：`fn json(input: str, json_path, newkey, trim_space: bool = true, delete_after_extract = false)` + +函数说明：提取 JSON 中的指定字段，并可将其命名成新的字段。 + +参数： + +- `input`: 待提取 JSON，可以是原始文本（`_`）或经过初次提取之后的某个 `key` +- `json_path`: JSON 路径信息 +- `newkey`：提取后数据写入新 key +- `trim_space`: 删除提取出的字符中的空白首尾字符，默认值为 `true` +- `delete_after_extract`: 在提取结束后删除当前对象，在重新序列化后回写待提取对象；只能应用于 map 的 key 与 value 的删除，不能用于删除 list 的元素；默认值为 `false`，不进行任何操作 + +```python +# 直接提取原始输入 JSON 中的 x.y 字段，并可将其命名成新字段 abc +json(_, x.y, abc) + +# 已提取出的某个 `key`，对其再提取一次 `x.y`，提取后字段名为 `x.y` +json(key, x.y) +``` + +示例一： + +```python +# 待处理数据： +# {"info": {"age": 17, "name": "zhangsan", "height": 180}} + +# 处理脚本： +json(_, info, "zhangsan") +json(zhangsan, name) +json(zhangsan, age, "age") + +# 处理结果： +{ + "age": 17, + "message": "{\"info\": {\"age\": 17, \"name\": \"zhangsan\", \"height\": 180}}", + "name": "zhangsan", + "zhangsan": "{\"age\":17,\"height\":180,\"name\":\"zhangsan\"}" +} +``` + +示例二： + +```python +# 待处理数据： +# data = { +# "name": {"first": "Tom", "last": "Anderson"}, +# "age":37, +# "children": ["Sara","Alex","Jack"], +# "fav.movie": "Deer Hunter", +# "friends": [ +# {"first": "Dale", "last": "Murphy", "age": 44, "nets": ["ig", "fb", "tw"]}, +# {"first": "Roger", "last": "Craig", "age": 68, "nets": ["fb", "tw"]}, +# {"first": "Jane", "last": "Murphy", "age": 47, "nets": ["ig", "tw"]} +# ] +# } + +# 处理脚本： +json(_, name) +json(name, first) +``` + +示例三： + +```python +# 待处理数据： +# [ +# {"first": "Dale", "last": "Murphy", "age": 44, "nets": ["ig", "fb", "tw"]}, +# {"first": "Roger", "last": "Craig", "age": 68, "nets": ["fb", "tw"]}, +# {"first": "Jane", "last": "Murphy", "age": 47, "nets": ["ig", "tw"]} +# ] + +# 处理脚本，json 数组处理： +json(_, .[0].nets[-1]) +``` + +示例四： + +```python +# 待处理数据： +{"item": " not_space ", "item2":{"item3": [123]}} + +# 处理脚本： +json(_, item2.item3, item, delete_after_extract = true) + +# 输出： +{ + "item": "[123]", + "message": "{\"item\":\" not_space \",\"item2\":{}}", +} +``` + + +示例五： + +```python +# 待处理数据： +{"item": " not_space ", "item2":{"item3": [123]}} + +# 处理脚本： +# 如果尝试删除列表元素将无法通过脚本检查 +json(_, item2.item3[0], item, true, true) + +# 本地测试命令： +# datakit pipeline -P j2.p -T '{"item": " not_space ", "item2":{"item3": [123]}}' +# 报错： +# [E] j2.p:1:37: does not support deleting elements in the list +``` diff --git a/pkg/inimpl/guancecloud/ptinput/funcs/md/kv_split.en.md b/pkg/inimpl/guancecloud/ptinput/funcs/md/kv_split.en.md new file mode 100644 index 0000000..b8426e6 --- /dev/null +++ b/pkg/inimpl/guancecloud/ptinput/funcs/md/kv_split.en.md @@ -0,0 +1,95 @@ +### `kv_split()` {#fn-kv_split} + +Function prototype: `fn kv_split(key, field_split_pattern = " ", value_split_pattern = "=", trim_key = "", trim_value = "", include_keys = [], prefix = "") -> bool` + +Function description: extract all key-value pairs from a string + +Function parameters: + +- `key`: key name +- `include_keys`: list of key names, only extract the keys in the list; **the default value is [], do not extract any key** +- `field_split_pattern`: string splitting, a regular expression used to extract all key-value pairs; the default value is " " +- `value_split_pattern`: used to split the key and value from the key-value pair string, non-recursive; the default value is "=" +- `trim_key`: delete all the specified characters leading and trailing the extracted key; the default value is "" +- `trim_value`: remove all leading and trailing characters from the extracted value; the default value is "" +- `prefix`: add prefix to all keys + +Example: + + +```python +# input: "a=1, b=2 c=3" +kv_split(_) + +'''output: +{ + "message": "a=1, b=2 c=3", + "status": "unknown", + "time": 1679558730846377132 +} +''' +``` + +```python +# input: "a=1, b=2 c=3" +kv_split(_, include_keys=["a", "c", "b"]) + +'''output: +{ + "a": "1,", + "b": "2", + "c": "3", + "message": "a=1 b=2 c=3", + "status": "unknown", + "time": 1678087119072769560 +} +''' +``` + +```python +# input: "a=1, b=2 c=3" +kv_split(_, trim_value=",", include_keys=["a", "c", "b"]) + +'''output: +{ + "a": "1", + "b": "2", + "c": "3", + "message": "a=1, b=2 c=3", + "status": "unknown", + "time": 1678087173651846101 +} +''' +``` + + +```python +# input: "a=1, b=2 c=3" +kv_split(_, trim_value=",", include_keys=["a", "c"]) + +'''output: +{ + "a": "1", + "c": "3", + "message": "a=1, b=2 c=3", + "status": "unknown", + "time": 1678087514906492912 +} +''' +``` + +```python +# input: "a::1,+b::2+c::3" +kv_split(_, field_split_pattern="\\+", value_split_pattern="[:]{2}", + prefix="with_prefix_",trim_value=",", trim_key="a", include_keys=["a", "b", "c"]) + +'''output: +{ + "message": "a::1,+b::2+c::3", + "status": "unknown", + "time": 1678087473255241547, + "with_prefix_b": "2", + "with_prefix_c": "3" +} +''' +``` diff --git a/pkg/inimpl/guancecloud/ptinput/funcs/md/kv_split.md b/pkg/inimpl/guancecloud/ptinput/funcs/md/kv_split.md new file mode 100644 index 0000000..c1e2e33 --- /dev/null +++ b/pkg/inimpl/guancecloud/ptinput/funcs/md/kv_split.md @@ -0,0 +1,95 @@ +### `kv_split()` {#fn-kv_split} + +函数原型：`fn kv_split(key, field_split_pattern = " ", value_split_pattern = "=", trim_key = "", trim_value = "", include_keys = [], prefix = "") -> bool` + +函数说明：从字符串中提取出所有的键值对 + +参数： + +- `key`: key 名称 +- `include_keys`: 包含的 key 名称列表，仅提取在该列表内的 key；**默认值为 []，不提取任何 key** +- `field_split_pattern`: 字符串分割，用于提取出所有键值对的正则表达式；默认值为 `" "` +- `value_split_pattern`: 用于从键值对字符串分割出键和值，非递归；默认值为 `"="` +- `trim_key`: 删除提取出的 key 的前导和尾随的所有指定的字符；默认值为 `""` +- `trim_value`: 删除提取出的 value 的前导和尾随的所有指定的字符；默认值为 `""` +- `prefix`: 给所有的 key 添加前缀字符串 + +示例： + + +```python +# input: "a=1, b=2 c=3" +kv_split(_) + +'''output: +{ + "message": "a=1, b=2 c=3", + "status": "unknown", + "time": 1679558730846377132 +} +''' +``` + +```python +# input: "a=1, b=2 c=3" +kv_split(_, include_keys=["a", "c", "b"]) + +'''output: +{ + "a": "1,", + "b": "2", + "c": "3", + "message": "a=1 b=2 c=3", + "status": "unknown", + "time": 1678087119072769560 +} +''' +``` + +```python +# input: "a=1, b=2 c=3" +kv_split(_, trim_value=",", include_keys=["a", "c", "b"]) + +'''output: +{ + "a": "1", + "b": "2", + "c": "3", + "message": "a=1, b=2 c=3", + "status": "unknown", + "time": 1678087173651846101 +} +''' +``` + + +```python +# input: "a=1, b=2 c=3" +kv_split(_, trim_value=",", include_keys=["a", "c"]) + +'''output: +{ + "a": "1", + "c": "3", + "message": "a=1, b=2 c=3", + "status": "unknown", + "time": 1678087514906492912 +} +''' +``` + +```python +# input: "a::1,+b::2+c::3" +kv_split(_, field_split_pattern="\\+", value_split_pattern="[:]{2}", + prefix="with_prefix_",trim_value=",", trim_key="a", include_keys=["a", "b", "c"]) + +'''output: +{ + "message": "a::1,+b::2+c::3", + "status": "unknown", + "time": 1678087473255241547, + "with_prefix_b": "2", + "with_prefix_c": "3" +} +''' +``` diff --git a/pkg/inimpl/guancecloud/ptinput/funcs/md/len.en.md b/pkg/inimpl/guancecloud/ptinput/funcs/md/len.en.md new file mode 100644 index 0000000..6c3a43b --- /dev/null +++ b/pkg/inimpl/guancecloud/ptinput/funcs/md/len.en.md @@ -0,0 +1,27 @@ +### `len()` {#fn-len} + +Function prototype: `fn len(val: str|map|list) int` + +Function description: Calculate the number of bytes in string, the number of elements in map and list. + +Function parameters: + +- `val`: Can be map, list or string + +Example: + +```python +# example 1 +add_key(abc, len("abc")) +# result +{ + "abc": 3, +} + +# example 2 +add_key(abc, len(["abc"])) +# result +{ + "abc": 1, +} +``` diff --git a/pkg/inimpl/guancecloud/ptinput/funcs/md/len.md b/pkg/inimpl/guancecloud/ptinput/funcs/md/len.md new file mode 100644 index 0000000..25db755 --- /dev/null +++ b/pkg/inimpl/guancecloud/ptinput/funcs/md/len.md @@ -0,0 +1,27 @@ +### `len()` {#fn-len} + +函数原型：`fn len(val: str|map|list) int` + +函数说明：计算 string 字节数，map 和 list 的元素个数。 + +参数： + +- `val`: 可以是 map、list 或 string + +示例： + +```python +# 例 1 +add_key(abc, len("abc")) +# 输出 +{ + "abc": 3, +} + +# 例 2 +add_key(abc, len(["abc"])) +#处理结果 +{ + "abc": 1, +} +``` diff --git a/pkg/inimpl/guancecloud/ptinput/funcs/md/load_json.en.md b/pkg/inimpl/guancecloud/ptinput/funcs/md/load_json.en.md new file mode 100644 index 0000000..a66813a --- /dev/null +++ b/pkg/inimpl/guancecloud/ptinput/funcs/md/load_json.en.md @@ -0,0 +1,27 @@ +### `load_json()` {#fn-load-json} + +Function prototype: `fn load_json(val: str) nil|bool|float|map|list` + +Function description: Convert the JSON string to one of map, list, nil, bool, float, and the value can be obtained and modified through the index expression.If deserialization fails, it also returns nil instead of terminating the script run. + +Function parameters: + +- `val`: Requires data of type string. + +Example: + +```python +# _: {"a":{"first": [2.2, 1.1], "ff": "[2.2, 1.1]","second":2,"third":"aBC","forth":true},"age":47} +abc = load_json(_) + +add_key(abc, abc["a"]["first"][-1]) + +abc["a"]["first"][-1] = 11 + +# Need to synchronize the data on the stack to point +add_key(abc, abc["a"]["first"][-1]) + +add_key(len_abc, len(abc)) + +add_key(len_abc, len(load_json(abc["a"]["ff"]))) +``` diff --git a/pkg/inimpl/guancecloud/ptinput/funcs/md/load_json.md b/pkg/inimpl/guancecloud/ptinput/funcs/md/load_json.md new file mode 100644 index 0000000..63b60af --- /dev/null +++ b/pkg/inimpl/guancecloud/ptinput/funcs/md/load_json.md @@ -0,0 +1,27 @@ +### `load_json()` {#fn-load-json} + +函数原型：`fn load_json(val: str) nil|bool|float|map|list` + +函数说明：将 JSON 字符串转换成 map、list、nil、bool、float 的其中一种，可通过 index 表达式取值及修改值。若反序列化失败，也返回 nil，而不是终止脚本运行。 + +参数： + +- `val`: 要求是 string 类型的数据。 + +示例： + +```python +# _: {"a":{"first": [2.2, 1.1], "ff": "[2.2, 1.1]","second":2,"third":"aBC","forth":true},"age":47} +abc = load_json(_) + +add_key(abc, abc["a"]["first"][-1]) + +abc["a"]["first"][-1] = 11 + +# 需要将堆栈上的数据同步到 point 中 +add_key(abc, abc["a"]["first"][-1]) + +add_key(len_abc, len(abc)) + +add_key(len_abc, len(load_json(abc["a"]["ff"]))) +``` diff --git a/pkg/inimpl/guancecloud/ptinput/funcs/md/lowercase.en.md b/pkg/inimpl/guancecloud/ptinput/funcs/md/lowercase.en.md new file mode 100644 index 0000000..6ecee25 --- /dev/null +++ b/pkg/inimpl/guancecloud/ptinput/funcs/md/lowercase.en.md @@ -0,0 +1,23 @@ +### `lowercase()` {#fn-lowercase} + +Function prototype: `fn lowercase(key: str)` + +Function description: Convert the content of the extracted key to lowercase + +Function parameters: + +- `key`: Specify the extracted field name to be converted + +Example: + +```python +# input data: {"first": "HeLLo","second":2,"third":"aBC","forth":true} + +# script +json(_, first) lowercase(first) + +# result +{ + "first": "hello" +} +``` diff --git a/pkg/inimpl/guancecloud/ptinput/funcs/md/lowercase.md b/pkg/inimpl/guancecloud/ptinput/funcs/md/lowercase.md new file mode 100644 index 0000000..38b0af4 --- /dev/null +++ b/pkg/inimpl/guancecloud/ptinput/funcs/md/lowercase.md @@ -0,0 +1,24 @@ +### `lowercase()` {#fn-lowercase} + +函数原型：`fn lowercase(key: str)` + +函数说明：将已提取 key 中内容转换成小写 + +函数参数 + +- `key`: 指定已提取的待转换字段名 + +示例： + +```python +# 待处理数据：{"first": "HeLLo","second":2,"third":"aBC","forth":true} + +# 处理脚本 +json(_, first) lowercase(first) + +# 处理结果 +{ + "first": "hello" +} +``` + diff --git a/pkg/inimpl/guancecloud/ptinput/funcs/md/match.en.md b/pkg/inimpl/guancecloud/ptinput/funcs/md/match.en.md new file mode 100644 index 0000000..c82287f --- /dev/null +++ b/pkg/inimpl/guancecloud/ptinput/funcs/md/match.en.md @@ -0,0 +1,28 @@ +### `match()` {#fn-match} + +Function prototype: `fn match(pattern: str, s: str) bool` + +Function description: Use the specified regular expression to match the string, return true if the match is successful, otherwise return false + +Function parameters: + +- `pattern`: regular expression +- `s`: string to match + +Example: + +```python +# script +test_1 = "pattern 1,a" +test_2 = "pattern -1," + +add_key(match_1, match('''\w+\s[,\w]+''', test_1)) + +add_key(match_2, match('''\w+\s[,\w]+''', test_2)) + +# result +{ + "match_1": true, + "match_2": false +} +``` diff --git a/pkg/inimpl/guancecloud/ptinput/funcs/md/match.md b/pkg/inimpl/guancecloud/ptinput/funcs/md/match.md new file mode 100644 index 0000000..ef98fa0 --- /dev/null +++ b/pkg/inimpl/guancecloud/ptinput/funcs/md/match.md @@ -0,0 +1,28 @@ +### `match()` {#fn-match} + +函数原型：`fn match(pattern: str, s: str) bool` + +函数说明：使用指定的正则表达式匹配字符串，匹配成功返回 true，否则返回 false + +参数： + +- `pattern`: 正则表达式 +- `s`: 待匹配的字符串 + +示例： + +```python +# 脚本 +test_1 = "pattern 1,a" +test_2 = "pattern -1," + +add_key(match_1, match('''\w+\s[,\w]+''', test_1)) + +add_key(match_2, match('''\w+\s[,\w]+''', test_2)) + +# 处理结果 +{ + "match_1": true, + "match_2": false +} +``` diff --git a/pkg/inimpl/guancecloud/ptinput/funcs/md/mquery_refer_table.en.md b/pkg/inimpl/guancecloud/ptinput/funcs/md/mquery_refer_table.en.md new file mode 100644 index 0000000..d8a51b9 --- /dev/null +++ b/pkg/inimpl/guancecloud/ptinput/funcs/md/mquery_refer_table.en.md @@ -0,0 +1,38 @@ +### `mquery_refer_table()` {#fn-mquery-refer-table} + +Function prototype: `fn mquery_refer_table(table_name: str, keys: list, values: list)` + +Function description: Query the external reference table by specifying multiple keys, and append all columns of the first row of the query result to field. + +Function parameters: + +- `table_name`: the name of the table to be looked up +- `keys`: a list of multiple column names +- `values`: the values corresponding to each column + +Example: + +```python +json(_, table) +json(_, key) +json(_, value) + +# Query and append the data of the current column, which is added to the data as a field by default +mquery_refer_table(table, values=[value, false], keys=[key, "col4"]) + +# result + +# { +# "col": "ab", +# "col2": 1234, +# "col3": 1235, +# "col4": false, +# "key": "col2", +# "message": "{\"table\": \"table_abc\", \"key\": \"col2\", \"value\": 1234.0}", +# "status": "unknown", +# "table": "table_abc", +# "time": "2022-08-16T16:23:31.940600281+08:00", +# "value": 1234 +# } + +``` diff --git a/pkg/inimpl/guancecloud/ptinput/funcs/md/mquery_refer_table.md b/pkg/inimpl/guancecloud/ptinput/funcs/md/mquery_refer_table.md new file mode 100644 index 0000000..34b7981 --- /dev/null +++ b/pkg/inimpl/guancecloud/ptinput/funcs/md/mquery_refer_table.md @@ -0,0 +1,40 @@ +### `mquery_refer_table()` {#fn-mquery-refer-table} + +函数原型：`fn mquery_refer_table(table_name: str, keys: list, values: list)` + +函数说明：通过指定多个 key 查询外部引用表，并将查询结果的首行的所有列追加到 field 中。 + +参数： + +- `table_name`: 待查找的表名 +- `keys`: 多个列名构成的列表 +- `values`: 每个列对应的值 + +示例： + +```python +json(_, table) +json(_, key) +json(_, value) + +# 查询并追加当前列的数据，默认作为 field 添加到数据中 +mquery_refer_table(table, values=[value, false], keys=[key, "col4"]) +``` + +示例结果： + +```json +{ + "col": "ab", + "col2": 1234, + "col3": 1235, + "col4": false, + "key": "col2", + "message": "{\"table\": \"table_abc\", \"key\": \"col2\", \"value\": 1234.0}", + "status": "unknown", + "table": "table_abc", + "time": "2022-08-16T16:23:31.940600281+08:00", + "value": 1234 +} + +``` diff --git a/pkg/inimpl/guancecloud/ptinput/funcs/md/nullif.en.md b/pkg/inimpl/guancecloud/ptinput/funcs/md/nullif.en.md new file mode 100644 index 0000000..cd23a1e --- /dev/null +++ b/pkg/inimpl/guancecloud/ptinput/funcs/md/nullif.en.md @@ -0,0 +1,34 @@ +### `nullif()` {#fn-nullif} + +Function prototype: `fn nullif(key, value)` + +Function description: If the content of the field specified by the extracted `key` is equal to the value of `value`, delete this field + +Function parameters: + + +- `key`: specified field +- `value`: target value + +Example: + +```python +# input data: {"first": 1,"second":2,"third":"aBC","forth":true} + +# script +json(_, first) json(_, second) nullif(first, "1") + +# result +{ + "second":2 +} +``` + +> Note: This feature can be implemented with `if/else` semantics: + +```python +if first == "1" { + drop_key(first) +} +``` + diff --git a/pkg/inimpl/guancecloud/ptinput/funcs/md/nullif.md b/pkg/inimpl/guancecloud/ptinput/funcs/md/nullif.md new file mode 100644 index 0000000..e32194d --- /dev/null +++ b/pkg/inimpl/guancecloud/ptinput/funcs/md/nullif.md @@ -0,0 +1,33 @@ +### `nullif()` {#fn-nullif} + +函数原型：`fn nullif(key, value)` + +函数说明：若已提取 `key` 指定的字段内容等于 `value` 值，则删除此字段 + +函数参数 + +- `key`: 指定字段 +- `value`: 目标值 + +示例： + +```python +# 待处理数据：{"first": 1,"second":2,"third":"aBC","forth":true} + +# 处理脚本 +json(_, first) json(_, second) nullif(first, "1") + +# 处理结果 +{ + "second":2 +} +``` + +> 注：该功能可通过 `if/else` 语义来实现： + +```python +if first == "1" { + drop_key(first) +} +``` + diff --git a/pkg/inimpl/guancecloud/ptinput/funcs/md/parse_date.en.md b/pkg/inimpl/guancecloud/ptinput/funcs/md/parse_date.en.md new file mode 100644 index 0000000..3667ae1 --- /dev/null +++ b/pkg/inimpl/guancecloud/ptinput/funcs/md/parse_date.en.md @@ -0,0 +1,31 @@ +### `parse_date()` {#fn-parse-date} + +Function prototype: `fn parse_date(key: str, yy: str, MM: str, dd: str, hh: str, mm: str, ss: str, ms: str, zone: str)` + +Function description: Convert the value of each part of the incoming date field into a timestamp + +Function parameters: + +- `key`: newly inserted field +- `yy` : Year numeric string, supports four or two digit strings, if it is an empty string, the current year will be used when processing +- `MM`: month string, supports numbers, English, English abbreviation +- `dd`: day string +- `hh`: hour string +- `mm`: minute string +- `ss`: seconds string +- `ms`: milliseconds string +- `us`: microseconds string +- `ns`: string of nanoseconds +- `zone`: time zone string, in the form of "+8" or \"Asia/Shanghai\" + +Example: + +```python +parse_date(aa, "2021", "May", "12", "10", "10", "34", "", "Asia/Shanghai") # Result aa=1620785434000000000 + +parse_date(aa, "2021", "12", "12", "10", "10", "34", "", "Asia/Shanghai") # result aa=1639275034000000000 + +parse_date(aa, "2021", "12", "12", "10", "10", "34", "100", "Asia/Shanghai") # Result aa=1639275034000000100 + +parse_date(aa, "20", "February", "12", "10", "10", "34", "", "+8") result aa=1581473434000000000 +``` diff --git a/pkg/inimpl/guancecloud/ptinput/funcs/md/parse_date.md b/pkg/inimpl/guancecloud/ptinput/funcs/md/parse_date.md new file mode 100644 index 0000000..8c9eb46 --- /dev/null +++ b/pkg/inimpl/guancecloud/ptinput/funcs/md/parse_date.md @@ -0,0 +1,31 @@ +### `parse_date()` {#fn-parse-date} + +函数原型：`fn parse_date(key: str, yy: str, MM: str, dd: str, hh: str, mm: str, ss: str, ms: str, zone: str)` + +函数说明：将传入的日期字段各部分的值转化为时间戳 + +函数参数 + +- `key`: 新插入的字段 +- `yy` : 年份数字字符串，支持四位或两位数字字符串，为空字符串，则处理时取当前年份 +- `MM`: 月份字符串，支持数字，英文，英文缩写 +- `dd`: 日字符串 +- `hh`: 小时字符串 +- `mm`: 分钟字符串 +- `ss`: 秒字符串 +- `ms`: 毫秒字符串 +- `us`: 微秒字符串 +- `ns`: 纳秒字符串 +- `zone`: 时区字符串，“+8”或\"Asia/Shanghai\"形式 + +示例： + +```python +parse_date(aa, "2021", "May", "12", "10", "10", "34", "", "Asia/Shanghai") # 结果 aa=1620785434000000000 + +parse_date(aa, "2021", "12", "12", "10", "10", "34", "", "Asia/Shanghai") # 结果 aa=1639275034000000000 + +parse_date(aa, "2021", "12", "12", "10", "10", "34", "100", "Asia/Shanghai") # 结果 aa=1639275034000000100 + +parse_date(aa, "20", "February", "12", "10", "10", "34", "", "+8") 结果 aa=1581473434000000000 +``` diff --git a/pkg/inimpl/guancecloud/ptinput/funcs/md/parse_duration.en.md b/pkg/inimpl/guancecloud/ptinput/funcs/md/parse_duration.en.md new file mode 100644 index 0000000..6dbee26 --- /dev/null +++ b/pkg/inimpl/guancecloud/ptinput/funcs/md/parse_duration.en.md @@ -0,0 +1,31 @@ +### `parse_duration()` {#fn-parse-duration} + +Function prototype: `fn parse_duration(key: str)` + +Function description: If the value of `key` is a golang duration string (such as `123ms`), then `key` will be automatically parsed into an integer in nanoseconds + +The current duration units in golang are as follows: + +- `ns` nanoseconds +- `us/µs` microseconds +- `ms` milliseconds +- `s` seconds +- `m` minutes +- `h` hours + +Function parameters: + +- `key`: the field to be parsed + +Example: + +```python +# assume abc = "3.5s" +parse_duration(abc) # result abc = 3500000000 + +# Support negative numbers: abc = "-3.5s" +parse_duration(abc) # result abc = -3500000000 + +# support floating point: abc = "-2.3s" +parse_duration(abc) # result abc = -2300000000 +``` diff --git a/pkg/inimpl/guancecloud/ptinput/funcs/md/parse_duration.md b/pkg/inimpl/guancecloud/ptinput/funcs/md/parse_duration.md new file mode 100644 index 0000000..8598e3a --- /dev/null +++ b/pkg/inimpl/guancecloud/ptinput/funcs/md/parse_duration.md @@ -0,0 +1,33 @@ +### `parse_duration()` {#fn-parse-duration} + +函数原型：`fn parse_duration(key: str)` + +函数说明：如果 `key` 的值是一个 golang 的 duration 字符串（如 `123ms`），则自动将 `key` 解析成纳秒为单位的整数 + +目前 golang 中的 duration 单位如下： + +- `ns` 纳秒 +- `us/µs` 微秒 +- `ms` 毫秒 +- `s` 秒 +- `m` 分钟 +- `h` 小时 + +函数参数 + +- `key`: 待解析的字段 + +示例： + +```python +# 假定 abc = "3.5s" +parse_duration(abc) # 结果 abc = 3500000000 + +# 支持负数：abc = "-3.5s" +parse_duration(abc) # 结果 abc = -3500000000 + +# 支持浮点：abc = "-2.3s" +parse_duration(abc) # 结果 abc = -2300000000 + +``` + diff --git a/pkg/inimpl/guancecloud/ptinput/funcs/md/parse_int.en.md b/pkg/inimpl/guancecloud/ptinput/funcs/md/parse_int.en.md new file mode 100644 index 0000000..0eb6a49 --- /dev/null +++ b/pkg/inimpl/guancecloud/ptinput/funcs/md/parse_int.en.md @@ -0,0 +1,69 @@ +### `parse_int()` {#fn-parse-int} + +Function prototype: `fn parse_int(val: int, base: int) str` + +Function description: Converts the string representation of a numeric value to a numeric value. + +Function parameters: + +- `val`: The string to be converted. +- `base`: Base, the range is 0, or 2 to 36; when the value is 0, the base is judged according to the string prefix. + +Example: + +```python +# script0 +a = "7665324064912355185" +b = format_int(parse_int(a, 10), 16) +if b != "6a60b39fd95aaf71" { + add_key(abc, b) +} else { + add_key(abc, "ok") +} + +# result +''' +{ + "abc": "ok" +} +''' + +# script1 +a = "6a60b39fd95aaf71" +b = parse_int(a, 16) # base 16 +if b != 7665324064912355185 { + add_key(abc, b) +} else { + add_key(abc, "ok") +} + +# result +''' +{ + "abc": "ok" +} +''' + + +# script2 +a = "0x6a60b39fd95aaf71" +b = parse_int(a, 0) # the true base is implied by the string's +if b != 7665324064912355185 { + add_key(abc, b) +} else { + c = format_int(b, 16) + if "0x"+c != a { + add_key(abc, c) + } else { + add_key(abc, "ok") + } +} + + +# result +''' +{ + "abc": "ok" +} +''' +``` diff --git a/pkg/inimpl/guancecloud/ptinput/funcs/md/parse_int.md b/pkg/inimpl/guancecloud/ptinput/funcs/md/parse_int.md new file mode 100644 index 0000000..aaf5ba1 --- /dev/null +++ b/pkg/inimpl/guancecloud/ptinput/funcs/md/parse_int.md @@ -0,0 +1,69 @@ +### `parse_int()` {#fn-parse-int} + +函数原型：`fn parse_int(val: int, base: int) str` + +函数说明：将数值的字符串表示转换为数值。 + +参数： + +- `val`: 待转换的字符串 +- `base`: 进制，范围 0，或 2 到 36；值为 0 时根据字符串前缀判断进制。 + +示例： + +```python +# script0 +a = "7665324064912355185" +b = format_int(parse_int(a, 10), 16) +if b != "6a60b39fd95aaf71" { + add_key(abc, b) +} else { + add_key(abc, "ok") +} + +# result +''' +{ + "abc": "ok" +} +''' + +# script1 +a = "6a60b39fd95aaf71" +b = parse_int(a, 16) # base 16 +if b != 7665324064912355185 { + add_key(abc, b) +} else { + add_key(abc, "ok") +} + +# result +''' +{ + "abc": "ok" +} +''' + + +# script2 +a = "0x6a60b39fd95aaf71" +b = parse_int(a, 0) # the true base is implied by the string's +if b != 7665324064912355185 { + add_key(abc, b) +} else { + c = format_int(b, 16) + if "0x"+c != a { + add_key(abc, c) + } else { + add_key(abc, "ok") + } +} + + +# result +''' +{ + "abc": "ok" +} +''' +``` diff --git a/pkg/inimpl/guancecloud/ptinput/funcs/md/query_refer_table.en.md b/pkg/inimpl/guancecloud/ptinput/funcs/md/query_refer_table.en.md new file mode 100644 index 0000000..7caa8e2 --- /dev/null +++ b/pkg/inimpl/guancecloud/ptinput/funcs/md/query_refer_table.en.md @@ -0,0 +1,41 @@ +### `query_refer_table()` {#fn-query-refer-table} + +Function prototype: `fn query_refer_table(table_name: str, key: str, value)` + +Function description: Query the external reference table through the specified key, and append all the columns of the first row of the query result to field. + +Function parameters: + +- `table_name`: the name of the table to be looked up +- `key`: column name +- `value`: the value corresponding to the column + +Example: + +```python +# extract table name, column name, column value from input +json(_, table) +json(_, key) +json(_, value) + +# Query and append the data of the current column, which is added to the data as a field by default +query_refer_table(table, key, value) + +``` + +Result: + +```json +{ + "col": "ab", + "col2": 1234, + "col3": 123, + "col4": true, + "key": "col2", + "message": "{\"table\": \"table_abc\", \"key\": \"col2\", \"value\": 1234.0}", + "status": "unknown", + "table": "table_abc", + "time": "2022-08-16T15:02:14.158452592+08:00", + "value": 1234 +} +``` diff --git a/pkg/inimpl/guancecloud/ptinput/funcs/md/query_refer_table.md b/pkg/inimpl/guancecloud/ptinput/funcs/md/query_refer_table.md new file mode 100644 index 0000000..a31c0be --- /dev/null +++ b/pkg/inimpl/guancecloud/ptinput/funcs/md/query_refer_table.md @@ -0,0 +1,41 @@ +### `query_refer_table()` {#fn-query-refer-table} + +函数原型：`fn query_refer_table(table_name: str, key: str, value)` + +函数说明：通过指定的 key 查询外部引用表，并将查询结果的首行的所有列追加到 field 中。 + +参数： + +- `table_name`: 待查找的表名 +- `key`: 列名 +- `value`: 列对应的值 + +示例： + +```python +# 从输入中提取 表名，列名，列值 +json(_, table) +json(_, key) +json(_, value) + +# 查询并追加当前列的数据，默认作为 field 添加到数据中 +query_refer_table(table, key, value) + +``` + +示例结果： + +```json +{ + "col": "ab", + "col2": 1234, + "col3": 123, + "col4": true, + "key": "col2", + "message": "{\"table\": \"table_abc\", \"key\": \"col2\", \"value\": 1234.0}", + "status": "unknown", + "table": "table_abc", + "time": "2022-08-16T15:02:14.158452592+08:00", + "value": 1234 +} +``` diff --git a/pkg/inimpl/guancecloud/ptinput/funcs/md/rename.en.md b/pkg/inimpl/guancecloud/ptinput/funcs/md/rename.en.md new file mode 100644 index 0000000..df542f9 --- /dev/null +++ b/pkg/inimpl/guancecloud/ptinput/funcs/md/rename.en.md @@ -0,0 +1,38 @@ +### `rename()` {#fn-rename} + +Function prototype: `fn rename(new_key, old_key)` + +Function description: Rename the extracted fields + +Function parameters: + +- `new_key`: new field name +- `old_key`: the extracted field name + +Example: + +```python +# Rename the extracted abc field to abc1 +rename('abc1', abc) + +# or + +rename(abc1, abc) +``` + +```python +# Data to be processed: {"info": {"age": 17, "name": "zhangsan", "height": 180}} + +# process script +json(_, info.name, "name") + +# process result +{ + "message": "{\"info\": {\"age\": 17, \"name\": \"zhangsan\", \"height\": 180}}", + "zhangsan": { + "age": 17, + "height": 180, + "Name": "zhangsan" + } +} +``` diff --git a/pkg/inimpl/guancecloud/ptinput/funcs/md/rename.md b/pkg/inimpl/guancecloud/ptinput/funcs/md/rename.md new file mode 100644 index 0000000..2c93549 --- /dev/null +++ b/pkg/inimpl/guancecloud/ptinput/funcs/md/rename.md @@ -0,0 +1,39 @@ +### `rename()` {#fn-rename} + +函数原型：`fn rename(new_key, old_key)` + +函数说明：将已提取的字段重新命名 + +参数： + +- `new_key`: 新字段名 +- `old_key`: 已提取的字段名 + +示例： + +```python +# 把已提取的 abc 字段重新命名为 abc1 +rename('abc1', abc) + +# or + +rename(abc1, abc) +``` + +```python +# 待处理数据：{"info": {"age": 17, "name": "zhangsan", "height": 180}} + +# 处理脚本 +json(_, info.name, "姓名") + +# 处理结果 +{ + "message": "{\"info\": {\"age\": 17, \"name\": \"zhangsan\", \"height\": 180}}", + "zhangsan": { + "age": 17, + "height": 180, + "姓名": "zhangsan" + } +} +``` + diff --git a/pkg/inimpl/guancecloud/ptinput/funcs/md/replace.en.md b/pkg/inimpl/guancecloud/ptinput/funcs/md/replace.en.md new file mode 100644 index 0000000..af571a5 --- /dev/null +++ b/pkg/inimpl/guancecloud/ptinput/funcs/md/replace.en.md @@ -0,0 +1,31 @@ +### `replace()` {#fn-replace} + +Function prototype: `fn replace(key: str, regex: str, replace_str: str)` + +Function description: Replace the string data obtained on the specified field according to regular rules + +Function parameters: + +- `key`: the field to be extracted +- `regex`: regular expression +- `replace_str`: string to replace + +Example: + +```python +# Phone number: {"str_abc": "13789123014"} +json(_, str_abc) +replace(str_abc, "(1[0-9]{2})[0-9]{4}([0-9]{4})", "$1****$2") + +# English name {"str_abc": "zhang san"} +json(_, str_abc) +replace(str_abc, "([a-z]*) \\w*", "$1 ***") + +# ID number {"str_abc": "362201200005302565"} +json(_, str_abc) +replace(str_abc, "([1-9]{4})[0-9]{10}([0-9]{4})", "$1**********$2") + +# Chinese name {"str_abc": "Little Aka"} +json(_, str_abc) +replace(str_abc, '([\u4e00-\u9fa5])[\u4e00-\u9fa5]([\u4e00-\u9fa5])', "$1＊$2") +``` diff --git a/pkg/inimpl/guancecloud/ptinput/funcs/md/replace.md b/pkg/inimpl/guancecloud/ptinput/funcs/md/replace.md new file mode 100644 index 0000000..2748f66 --- /dev/null +++ b/pkg/inimpl/guancecloud/ptinput/funcs/md/replace.md @@ -0,0 +1,32 @@ +### `replace()` {#fn-replace} + +函数原型：`fn replace(key: str, regex: str, replace_str: str)` + +函数说明：对指定字段上获取的字符串数据按正则进行替换 + +函数参数 + +- `key`: 待提取字段 +- `regex`: 正则表达式 +- `replace_str`: 替换的字符串 + +示例： + +```python +# 电话号码：{"str_abc": "13789123014"} +json(_, str_abc) +replace(str_abc, "(1[0-9]{2})[0-9]{4}([0-9]{4})", "$1****$2") + +# 英文名 {"str_abc": "zhang san"} +json(_, str_abc) +replace(str_abc, "([a-z]*) \\w*", "$1 ***") + +# 身份证号 {"str_abc": "362201200005302565"} +json(_, str_abc) +replace(str_abc, "([1-9]{4})[0-9]{10}([0-9]{4})", "$1**********$2") + +# 中文名 {"str_abc": "小阿卡"} +json(_, str_abc) +replace(str_abc, '([\u4e00-\u9fa5])[\u4e00-\u9fa5]([\u4e00-\u9fa5])', "$1＊$2") +``` + diff --git a/pkg/inimpl/guancecloud/ptinput/funcs/md/sample.en.md b/pkg/inimpl/guancecloud/ptinput/funcs/md/sample.en.md new file mode 100644 index 0000000..e460af7 --- /dev/null +++ b/pkg/inimpl/guancecloud/ptinput/funcs/md/sample.en.md @@ -0,0 +1,18 @@ +### `sample()` {#fn-sample} + +Function prototype: `fn sample(p)` + +Function description: Choose to collect/discard data with probability p. + +Function parameters: +- `p`: the probability that the sample function returns true, the value range is [0, 1] + +Example: + +```python +# process script +if !sample(0.3) { # sample(0.3) indicates that the sampling rate is 30%, that is, it returns true with a 30% probability, and 70% of the data will be discarded here + drop() # mark the data to be discarded + exit() # Exit the follow-up processing process +} +``` diff --git a/pkg/inimpl/guancecloud/ptinput/funcs/md/sample.md b/pkg/inimpl/guancecloud/ptinput/funcs/md/sample.md new file mode 100644 index 0000000..5935f63 --- /dev/null +++ b/pkg/inimpl/guancecloud/ptinput/funcs/md/sample.md @@ -0,0 +1,19 @@ +### `sample()` {#fn-sample} + +函数原型：`fn sample(p)` + +函数说明：以概率 p 选择采集/丢弃数据。 + +函数参数： + +- `p`: sample 函数返回 true 的概率，取值范围为[0, 1] + +示例： + +```python +# 处理脚本 +if !sample(0.3) { # sample(0.3) 表示采样率为 30%，即以 30% 概率返回真，此处将丢弃 70% 的数据 + drop() # 标记该数据丢弃 + exit() # 退出后续处理流程 +} +``` diff --git a/pkg/inimpl/guancecloud/ptinput/funcs/md/set_measurement.en.md b/pkg/inimpl/guancecloud/ptinput/funcs/md/set_measurement.en.md new file mode 100644 index 0000000..f487b76 --- /dev/null +++ b/pkg/inimpl/guancecloud/ptinput/funcs/md/set_measurement.en.md @@ -0,0 +1,25 @@ +### `set_measurement()` {#fn-set-measurement} + +Function prototype: `fn set_measurement(name: str, delete_key: bool = false)` + +Function description: change the name of the line protocol + +Function parameters: + +- `name`: The value is used as the measurement name, which can be passed in as a string constant or variable +- `delete_key`: If there is a tag or field with the same name as the variable in point, delete it + +The field mapping relationship between row protocol name and various types of data storage or other purposes: + +| category | field name | other usage | +| - | - | - | +| custom_object | class | - | +| keyevent | - | - | +| logging | source | - | +| metric | - | metric set name | +| network | source | - | +| object | class | - | +| profiling | source | - | +| rum | source | - | +| security | rule | - | +| tracing | source | - | diff --git a/pkg/inimpl/guancecloud/ptinput/funcs/md/set_measurement.md b/pkg/inimpl/guancecloud/ptinput/funcs/md/set_measurement.md new file mode 100644 index 0000000..bb8c550 --- /dev/null +++ b/pkg/inimpl/guancecloud/ptinput/funcs/md/set_measurement.md @@ -0,0 +1,24 @@ +### `set_measurement()` {#fn-set-measurement} + +函数原型：`fn set_measurement(name: str, delete_key: bool = false)` + +函数说明：改变行协议的 name +函数参数： + +- `name`: 值作为 measurement name，可传入字符串常量或变量 +- `delete_key`: 如果在 point 中存在与变量同名的 tag 或 field 则删除它 + +行协议 name 与各个类型数据存储时的字段映射关系或其他用途： + +| 类别 | 字段名 | 其他用途 | +| - | - | - | +| custom_object | class | - | +| keyevent | - | - | +| logging | source | - | +| metric | - | 指标集名 | +| network | source | - | +| object | class | - | +| profiling | source | - | +| rum | source | - | +| security | rule | - | +| tracing | source | - | diff --git a/pkg/inimpl/guancecloud/ptinput/funcs/md/set_tag.en.md b/pkg/inimpl/guancecloud/ptinput/funcs/md/set_tag.en.md new file mode 100644 index 0000000..fe8bad1 --- /dev/null +++ b/pkg/inimpl/guancecloud/ptinput/funcs/md/set_tag.en.md @@ -0,0 +1,44 @@ +### `set_tag()` {#fn-set-tag} + +Function prototype: `fn set_tag(key, value: str)` + +Function description: mark the specified field as tag output, after setting as tag, other functions can still operate on the variable. If the key set as a tag is a field that has been cut out, it will not appear in the field, so as to avoid the same name of the cut out field key as the tag key on the existing data + +Function parameters: + +- `key`: the field to be tagged +- `value`: can be a string literal or a variable + +```python +# in << {"str": "13789123014"} +set_tag(str) +json(_, str) # str == "13789123014" +replace(str, "(1[0-9]{2})[0-9]{4}([0-9]{4})", "$1****$2") +# Extracted data(drop: false, cost: 49.248µs): +# { +# "message": "{\"str\": \"13789123014\", \"str_b\": \"3\"}", +# "str#": "137****3014" +# } +# * The character `#` is only the tag whose field is tag when datakit --pl --txt output display + +# in << {"str_a": "2", "str_b": "3"} +json(_, str_a) +set_tag(str_a, "3") # str_a == 3 +# Extracted data(drop: false, cost: 30.069µs): +# { +# "message": "{\"str_a\": \"2\", \"str_b\": \"3\"}", +# "str_a#": "3" +# } + + +# in << {"str_a": "2", "str_b": "3"} +json(_, str_a) +json(_, str_b) +set_tag(str_a, str_b) # str_a == str_b == "3" +# Extracted data(drop: false, cost: 32.903µs): +# { +# "message": "{\"str_a\": \"2\", \"str_b\": \"3\"}", +# "str_a#": "3", +# "str_b": "3" +# } +``` diff --git a/pkg/inimpl/guancecloud/ptinput/funcs/md/set_tag.md b/pkg/inimpl/guancecloud/ptinput/funcs/md/set_tag.md new file mode 100644 index 0000000..3eb600c --- /dev/null +++ b/pkg/inimpl/guancecloud/ptinput/funcs/md/set_tag.md @@ -0,0 +1,44 @@ +### `set_tag()` {#fn-set-tag} + +函数原型：`fn set_tag(key, value: str)` + +函数说明：对指定字段标记为 tag 输出，设置为 tag 后，其他函数仍可对该变量操作。如果被置为 tag 的 key 是已经切割出来的 field，那么它将不会在 field 中出现，这样可以避免切割出来的 field key 跟已有数据上的 tag key 重名 + +函数参数 + +- `key`: 待标记为 tag 的字段 +- `value`: 可以为字符串字面量或者变量 + +```python +# in << {"str": "13789123014"} +set_tag(str) +json(_, str) # str == "13789123014" +replace(str, "(1[0-9]{2})[0-9]{4}([0-9]{4})", "$1****$2") +# Extracted data(drop: false, cost: 49.248µs): +# { +# "message": "{\"str\": \"13789123014\", \"str_b\": \"3\"}", +# "str#": "137****3014" +# } +# * 字符 `#` 仅为 datakit --pl --txt 输出展示时字段为 tag 的标记 + +# in << {"str_a": "2", "str_b": "3"} +json(_, str_a) +set_tag(str_a, "3") # str_a == 3 +# Extracted data(drop: false, cost: 30.069µs): +# { +# "message": "{\"str_a\": \"2\", \"str_b\": \"3\"}", +# "str_a#": "3" +# } + + +# in << {"str_a": "2", "str_b": "3"} +json(_, str_a) +json(_, str_b) +set_tag(str_a, str_b) # str_a == str_b == "3" +# Extracted data(drop: false, cost: 32.903µs): +# { +# "message": "{\"str_a\": \"2\", \"str_b\": \"3\"}", +# "str_a#": "3", +# "str_b": "3" +# } +``` diff --git a/pkg/inimpl/guancecloud/ptinput/funcs/md/sql_cover.en.md b/pkg/inimpl/guancecloud/ptinput/funcs/md/sql_cover.en.md new file mode 100644 index 0000000..73d2391 --- /dev/null +++ b/pkg/inimpl/guancecloud/ptinput/funcs/md/sql_cover.en.md @@ -0,0 +1,17 @@ +### `sql_cover()` {#fn-sql-cover} + +Function prototype: `fn sql_cover(sql_test: str)` + +Function description: desensitized SQL statement + +Example: + +```python +# in << {"select abc from def where x > 3 and y < 5"} +sql_cover(_) + +# Extracted data(drop: false, cost: 33.279µs): +# { +# "message": "select abc from def where x > ? and y < ?" +# } +``` diff --git a/pkg/inimpl/guancecloud/ptinput/funcs/md/sql_cover.md b/pkg/inimpl/guancecloud/ptinput/funcs/md/sql_cover.md new file mode 100644 index 0000000..ff3d9df --- /dev/null +++ b/pkg/inimpl/guancecloud/ptinput/funcs/md/sql_cover.md @@ -0,0 +1,15 @@ +### `sql_cover()` {#fn-sql-cover} + +函数原型：`fn sql_cover(sql_test: str)` + +函数说明：脱敏 SQL 语句 + +```python +# in << {"select abc from def where x > 3 and y < 5"} +sql_cover(_) + +# Extracted data(drop: false, cost: 33.279µs): +# { +# "message": "select abc from def where x > ? and y < ?" +# } +``` diff --git a/pkg/inimpl/guancecloud/ptinput/funcs/md/strfmt.en.md b/pkg/inimpl/guancecloud/ptinput/funcs/md/strfmt.en.md new file mode 100644 index 0000000..c997a94 --- /dev/null +++ b/pkg/inimpl/guancecloud/ptinput/funcs/md/strfmt.en.md @@ -0,0 +1,24 @@ +### `strfmt()` {#fn-strfmt} + +Function prototype: `fn strfmt(key, fmt: str, args ...: int|float|bool|str|list|map|nil)` + +Function description: Format the content of the field specified by the extracted `arg1, arg2, ...` according to `fmt`, and write the formatted content into the `key` field + +Function parameters: + +- `key`: Specify the field name of the formatted data to be written +- `fmt`: format string template +- `args`: Variable Function parameters:, which can be multiple extracted field names to be formatted + +Example: + +```python +# Data to be processed: {"a":{"first":2.3,"second":2,"third":"abc","forth":true},"age":47} + +# process script +json(_, a.second) +json(_, a.thrid) +cast(a. second, "int") +json(_, a.forth) +strfmt(bb, "%v %s %v", a.second, a.thrid, a.forth) +``` diff --git a/pkg/inimpl/guancecloud/ptinput/funcs/md/strfmt.md b/pkg/inimpl/guancecloud/ptinput/funcs/md/strfmt.md new file mode 100644 index 0000000..9647b88 --- /dev/null +++ b/pkg/inimpl/guancecloud/ptinput/funcs/md/strfmt.md @@ -0,0 +1,24 @@ +### `strfmt()` {#fn-strfmt} + +函数原型：`fn strfmt(key, fmt: str, args ...: int|float|bool|str|list|map|nil)` + +函数说明：对已提取 `arg1, arg2, ...` 指定的字段内容根据 `fmt` 进行格式化，并把格式化后的内容写入 `key` 字段中 + +函数参数 + +- `key`: 指定格式化后数据写入字段名 +- `fmt`: 格式化字符串模板 +- `args`：可变参数，可以是多个已提取的待格式化字段名 + +示例： + +```python +# 待处理数据：{"a":{"first":2.3,"second":2,"third":"abc","forth":true},"age":47} + +# 处理脚本 +json(_, a.second) +json(_, a.thrid) +cast(a.second, "int") +json(_, a.forth) +strfmt(bb, "%v %s %v", a.second, a.thrid, a.forth) +``` diff --git a/pkg/inimpl/guancecloud/ptinput/funcs/md/timestamp.en.md b/pkg/inimpl/guancecloud/ptinput/funcs/md/timestamp.en.md new file mode 100644 index 0000000..e094718 --- /dev/null +++ b/pkg/inimpl/guancecloud/ptinput/funcs/md/timestamp.en.md @@ -0,0 +1,54 @@ +### `timestamp()` {#fn-timestamp} + +Function prototype: `fn timestamp(precision: str = "ns") -> int` + +Function description: 返回当前 Unix 时间戳，默认精度为 ns + +Function parameters: + +- `precision`: 时间戳精度，取值范围为 "ns", "us", "ns", "s", 默认值 "ns"。 + +Example: + + +```python +# process script +add_key(time_now_record, timestamp()) + +datetime(time_now_record, "ns", + "%Y-%m-%d %H:%M:%S", "UTC") + + +# process result +{ + "time_now_record": "2023-03-07 10:41:12" +} + +``` + + +```python +# process script +add_key(time_now_record, timestamp()) + +datetime(time_now_record, "ns", + "%Y-%m-%d %H:%M:%S", "Asia/Shanghai") + + +# process result +{ + "time_now_record": "2023-03-07 18:41:49" +} +``` + + +```python +# process script +add_key(time_now_record, timestamp("ms")) + + +# process result +{ + "time_now_record": 1678185980578 +} +``` diff --git a/pkg/inimpl/guancecloud/ptinput/funcs/md/timestamp.md b/pkg/inimpl/guancecloud/ptinput/funcs/md/timestamp.md new file mode 100644 index 0000000..2095a8d --- /dev/null +++ b/pkg/inimpl/guancecloud/ptinput/funcs/md/timestamp.md @@ -0,0 +1,51 @@ +### `timestamp()` {#fn-timestamp} + +函数原型：`fn timestamp(precision: str = "ns") -> int` + +函数说明：返回当前 Unix 时间戳，默认精度为 ns + +函数参数： + +- `precision`: 时间戳精度，取值范围为 "ns", "us", "ns", "s", 默认值 "ns"。 + +示例： + +```python +# 处理脚本 +add_key(time_now_record, timestamp()) + +datetime(time_now_record, "ns", + "%Y-%m-%d %H:%M:%S", "UTC") + + +# 处理结果 +{ + "time_now_record": "2023-03-07 10:41:12" +} + +``` + +```python +# 处理脚本 +add_key(time_now_record, timestamp()) + +datetime(time_now_record, "ns", + "%Y-%m-%d %H:%M:%S", "Asia/Shanghai") + + +# 处理结果 +{ + "time_now_record": "2023-03-07 18:41:49" +} +``` + +```python +# 处理脚本 +add_key(time_now_record, timestamp("ms")) + + +# 处理结果 +{ + "time_now_record": 1678185980578 +} +``` diff --git a/pkg/inimpl/guancecloud/ptinput/funcs/md/trim.en.md b/pkg/inimpl/guancecloud/ptinput/funcs/md/trim.en.md new file mode 100644 index 0000000..34fbfaf --- /dev/null +++ b/pkg/inimpl/guancecloud/ptinput/funcs/md/trim.en.md @@ -0,0 +1,25 @@ +### `trim()` {#fn-trim} + +Function prototype: `fn trim(key, cutset: str = "")` + +Function description: delete the characters specified at the beginning and end of the key, and delete all blank characters by default when the `cutset` is an empty string + +Function parameters: + +- `key`: a field that has been extracted, string type +- `cutset`: Delete the first and last characters in the `cutset` string in the key + +Example: + +```python +# Data to be processed: "trim(key, cutset)" + +# process script +add_key(test_data, "ACCAA_test_DataA_ACBA") +trim(test_data, "ABC_") + +# process result +{ + "test_data": "test_Data" +} +``` diff --git a/pkg/inimpl/guancecloud/ptinput/funcs/md/trim.md b/pkg/inimpl/guancecloud/ptinput/funcs/md/trim.md new file mode 100644 index 0000000..378eab5 --- /dev/null +++ b/pkg/inimpl/guancecloud/ptinput/funcs/md/trim.md @@ -0,0 +1,25 @@ +### `trim()` {#fn-trim} + +函数原型：`fn trim(key, cutset: str = "")` + +函数说明：删除 `key` 中首尾中指定的字符，`cutset` 为空字符串时默认删除所有空白符 + +函数参数： + +- `key`: 已提取的某字段，字符串类型 +- `cutset`: 删除 `key` 中出现在 `cutset` 字符串的中首尾字符 + +示例： + +```python +# 待处理数据："trim(key, cutset)" + +# 处理脚本 +add_key(test_data, "ACCAA_test_DataA_ACBA") +trim(test_data, "ABC_") + +# 处理结果 +{ + "test_data": "test_Data" +} +``` diff --git a/pkg/inimpl/guancecloud/ptinput/funcs/md/uppercase.en.md b/pkg/inimpl/guancecloud/ptinput/funcs/md/uppercase.en.md new file mode 100644 index 0000000..898d432 --- /dev/null +++ b/pkg/inimpl/guancecloud/ptinput/funcs/md/uppercase.en.md @@ -0,0 +1,23 @@ +### `uppercase()` {#fn-uppercase} + +Function prototype: `fn uppercase(key: str)` + +Function description: Convert the content in the extracted key to uppercase + +Function parameters: + +- `key`: Specify the extracted field name to be converted, and convert the content of `key` to uppercase + +Example: + +```python +# Data to be processed: {"first": "hello","second":2,"third":"aBC","forth":true} + +# process script +json(_, first) uppercase(first) + +# process result +{ + "first": "HELLO" +} +``` diff --git a/pkg/inimpl/guancecloud/ptinput/funcs/md/uppercase.md b/pkg/inimpl/guancecloud/ptinput/funcs/md/uppercase.md new file mode 100644 index 0000000..0835cd2 --- /dev/null +++ b/pkg/inimpl/guancecloud/ptinput/funcs/md/uppercase.md @@ -0,0 +1,24 @@ +### `uppercase()` {#fn-uppercase} + +函数原型：`fn uppercase(key: str)` + +函数说明：将已提取 key 中内容转换成大写 + +函数参数 + +- `key`: 指定已提取的待转换字段名，将 `key` 内容转成大写 + +示例： + +```python +# 待处理数据：{"first": "hello","second":2,"third":"aBC","forth":true} + +# 处理脚本 +json(_, first) uppercase(first) + +# 处理结果 +{ + "first": "HELLO" +} +``` + diff --git a/pkg/inimpl/guancecloud/ptinput/funcs/md/url_decode.en.md b/pkg/inimpl/guancecloud/ptinput/funcs/md/url_decode.en.md new file mode 100644 index 0000000..b116ce1 --- /dev/null +++ b/pkg/inimpl/guancecloud/ptinput/funcs/md/url_decode.en.md @@ -0,0 +1,24 @@ +### `url_decode()` {#fn-url-decode} + +Function prototype: `fn url_decode(key: str)` + +Function description: parse the URL in the extracted `key` into plain text + +Function parameters: + +- `key`: a `key` that has been extracted + +Example: + +```python +# Data to be processed: {"url":"http%3a%2f%2fwww.baidu.com%2fs%3fwd%3d%e6%b5%8b%e8%af%95"} + +# process script +json(_, url) url_decode(url) + +# process result +{ + "message": "{"url":"http%3a%2f%2fwww.baidu.com%2fs%3fwd%3d%e6%b5%8b%e8%af%95"}", + "url": "http://www.baidu.com/s?wd=test" +} +``` diff --git a/pkg/inimpl/guancecloud/ptinput/funcs/md/url_decode.md b/pkg/inimpl/guancecloud/ptinput/funcs/md/url_decode.md new file mode 100644 index 0000000..65cdf9d --- /dev/null +++ b/pkg/inimpl/guancecloud/ptinput/funcs/md/url_decode.md @@ -0,0 +1,24 @@ +### `url_decode()` {#fn-url-decode} + +函数原型：`fn url_decode(key: str)` + +函数说明：将已提取 `key` 中的 URL 解析成明文 + +参数： + +- `key`: 已经提取的某个 `key` + +示例： + +```python +# 待处理数据：{"url":"http%3a%2f%2fwww.baidu.com%2fs%3fwd%3d%e6%b5%8b%e8%af%95"} + +# 处理脚本 +json(_, url) url_decode(url) + +# 处理结果 +{ + "message": "{"url":"http%3a%2f%2fwww.baidu.com%2fs%3fwd%3d%e6%b5%8b%e8%af%95"}", + "url": "http://www.baidu.com/s?wd=测试" +} +``` diff --git a/pkg/inimpl/guancecloud/ptinput/funcs/md/url_parse.en.md b/pkg/inimpl/guancecloud/ptinput/funcs/md/url_parse.en.md new file mode 100644 index 0000000..09a0204 --- /dev/null +++ b/pkg/inimpl/guancecloud/ptinput/funcs/md/url_parse.en.md @@ -0,0 +1,47 @@ +### `url_parse()` {#fn-url-parse} + +Function prototype: `fn url_parse(key)` + +Function description: parse the url whose field name is key. + +Function parameters: + +- `key`: field name of the url to parse. + +Example: + +```python +# Data to be processed: {"url": "https://www.baidu.com"} + +# process script +json(_, url) +m = url_parse(url) +add_key(scheme, m["scheme"]) + +# process result +{ + "url": "https://www.baidu.com", + "scheme": "https" +} +``` + +The above example extracts its scheme from the url. In addition, it can also extract information such as host, port, path, and Function parameters: carried in the url from the url, as shown in the following example: + +```python +# Data to be processed: {"url": "https://www.google.com/search?q=abc&sclient=gws-wiz"} + +# process script +json(_, url) +m = url_parse(url) +add_key(sclient, m["params"]["sclient"]) # The Function parameters: carried in the url are saved under the params field +add_key(h, m["host"]) +add_key(path, m["path"]) + +# process result +{ + "url": "https://www.google.com/search?q=abc&sclient=gws-wiz", + "h": "www.google.com", + "path": "/search", + "sclient": "gws-wiz" +} +``` diff --git a/pkg/inimpl/guancecloud/ptinput/funcs/md/url_parse.md b/pkg/inimpl/guancecloud/ptinput/funcs/md/url_parse.md new file mode 100644 index 0000000..d75b8ef --- /dev/null +++ b/pkg/inimpl/guancecloud/ptinput/funcs/md/url_parse.md @@ -0,0 +1,47 @@ +### `url_parse()` {#fn-url-parse} + +函数原型：`fn url_parse(key)` + +函数说明：解析字段名称为 key 的 url。 + +函数参数 + +- `key`: 要解析的 url 的字段名称。 + +示例： + +```python +# 待处理数据：{"url": "https://www.baidu.com"} + +# 处理脚本 +json(_, url) +m = url_parse(url) +add_key(scheme, m["scheme"]) + +# 处理结果 +{ + "url": "https://www.baidu.com", + "scheme": "https" +} +``` + +上述示例从 url 提取了其 scheme，除此以外，还能从 url 提取出 host, port, path, 以及 url 中携带的参数等信息，如下例子所示： + +```python +# 待处理数据：{"url": "https://www.google.com/search?q=abc&sclient=gws-wiz"} + +# 处理脚本 +json(_, url) +m = url_parse(url) +add_key(sclient, m["params"]["sclient"]) # url 中携带的参数被保存在 params 字段下 +add_key(h, m["host"]) +add_key(path, m["path"]) + +# 处理结果 +{ + "url": "https://www.google.com/search?q=abc&sclient=gws-wiz", + "h": "www.google.com", + "path": "/search", + "sclient": "gws-wiz" +} +``` diff --git a/pkg/inimpl/guancecloud/ptinput/funcs/md/use.en.md b/pkg/inimpl/guancecloud/ptinput/funcs/md/use.en.md new file mode 100644 index 0000000..65b196a --- /dev/null +++ b/pkg/inimpl/guancecloud/ptinput/funcs/md/use.en.md @@ -0,0 +1,32 @@ +### `use()` {#fn-use} + +Function prototype: `fn use(name: str)` + +Parameter: + +- `name`: script name, such as abp.p + +Function description: call other scripts, all current data can be accessed in the called script + +Example: + +```python +# Data to be processed: {"ip":"1.2.3.4"} + +# Process script a.p +use(\"b.p\") + +# Process script b.p +json(_, ip) +geoip (ip) + +# Execute the processing result of script a.p +{ + "city" : "Brisbane", + "country" : "AU", + "ip" : "1.2.3.4", + "province" : "Queensland", + "isp" : "unknown" + "message" : "{\"ip\": \"1.2.3.4\"}", +} +``` diff --git a/pkg/inimpl/guancecloud/ptinput/funcs/md/use.md b/pkg/inimpl/guancecloud/ptinput/funcs/md/use.md new file mode 100644 index 0000000..28e4ae9 --- /dev/null +++ b/pkg/inimpl/guancecloud/ptinput/funcs/md/use.md @@ -0,0 +1,31 @@ +### `use()` {#fn-use} + +函数原型：`fn use(name: str)` + +参数： + +- `name`: 脚本名，如 abp.p + +函数说明：调用其他脚本，可在被调用的脚本访问当前的所有数据 +示例： + +```python +# 待处理数据：{"ip":"1.2.3.4"} + +# 处理脚本 a.p +use(\"b.p\") + +# 处理脚本 b.p +json(_, ip) +geoip(ip) + +# 执行脚本 a.p 的处理结果 +{ + "city" : "Brisbane", + "country" : "AU", + "ip" : "1.2.3.4", + "province" : "Queensland", + "isp" : "unknown" + "message" : "{\"ip\": \"1.2.3.4\"}", +} +``` diff --git a/pkg/inimpl/guancecloud/ptinput/funcs/md/user_agent.en.md b/pkg/inimpl/guancecloud/ptinput/funcs/md/user_agent.en.md new file mode 100644 index 0000000..8d9cc26 --- /dev/null +++ b/pkg/inimpl/guancecloud/ptinput/funcs/md/user_agent.en.md @@ -0,0 +1,28 @@ +### `user_agent()` {#fn-user-agent} + +Function prototype: `fn user_agent(key: str)` + +Function description: Obtain client information on the specified field + +Function parameters: + +- `key`: the field to be extracted + +`user_agent()` will generate multiple fields, such as: + +- `os`: operating system +- `browser`: browser + +Example: + +```python +# data to be processed +# { +# "userAgent" : "Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/36.0.1985.125 Safari/537.36", +# "second" : 2, +# "third" : "abc", +# "forth" : true +# } + +json(_, userAgent) user_agent(userAgent) +``` diff --git a/pkg/inimpl/guancecloud/ptinput/funcs/md/user_agent.md b/pkg/inimpl/guancecloud/ptinput/funcs/md/user_agent.md new file mode 100644 index 0000000..4ccd8f2 --- /dev/null +++ b/pkg/inimpl/guancecloud/ptinput/funcs/md/user_agent.md @@ -0,0 +1,28 @@ +### `user_agent()` {#fn-user-agent} + +函数原型：`fn user_agent(key: str)` + +函数说明：对指定字段上获取客户端信息 + +函数参数 + +- `key`: 待提取字段 + +`user_agent()` 会生产多个字段，如： + +- `os`: 操作系统 +- `browser`: 浏览器 + +示例： + +```python +# 待处理数据 +# { +# "userAgent" : "Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/36.0.1985.125 Safari/537.36", +# "second" : 2, +# "third" : "abc", +# "forth" : true +# } + +json(_, userAgent) user_agent(userAgent) +``` diff --git a/pkg/inimpl/guancecloud/ptinput/funcs/md/valid_json.en.md b/pkg/inimpl/guancecloud/ptinput/funcs/md/valid_json.en.md new file mode 100644 index 0000000..0988bea --- /dev/null +++ b/pkg/inimpl/guancecloud/ptinput/funcs/md/valid_json.en.md @@ -0,0 +1,48 @@ +### `valid_json()` {#fn-valid-json} + +Function prototype: `fn valid_json(val: str) bool` + +Function description: Determine if it is a valid JSON string. + +Function parameters: + +- `val`: Requires data of type string. + +Example: + +```python +a = "null" +if valid_json(a) { # true + if load_json(a) == nil { + add_key("a", "nil") + } +} + +b = "[1, 2, 3]" +if valid_json(b) { # true + add_key("b", load_json(b)) +} + +c = "{\"a\": 1}" +if valid_json(c) { # true + add_key("c", load_json(c)) +} + +d = "???{\"d\": 1}" +if valid_json(d) { # true + add_key("d", load_json(c)) +} else { + add_key("d", "invalid json") +} +``` + +Result: + +```json +{ + "a": "nil", + "b": "[1,2,3]", + "c": "{\"a\":1}", + "d": "invalid json", +} +``` diff --git a/pkg/inimpl/guancecloud/ptinput/funcs/md/valid_json.md b/pkg/inimpl/guancecloud/ptinput/funcs/md/valid_json.md new file mode 100644 index 0000000..7adcb7c --- /dev/null +++ b/pkg/inimpl/guancecloud/ptinput/funcs/md/valid_json.md @@ -0,0 +1,48 @@ +### `valid_json()` {#fn-valid-json} + +函数原型：`fn valid_json(val: str) bool` + +函数说明：判断是否为一个有效的 JSON 字符串。 + +参数： + +- `val`: 要求是 string 类型的数据。 + +示例： + +```python +a = "null" +if valid_json(a) { # true + if load_json(a) == nil { + add_key("a", "nil") + } +} + +b = "[1, 2, 3]" +if valid_json(b) { # true + add_key("b", load_json(b)) +} + +c = "{\"a\": 1}" +if valid_json(c) { # true + add_key("c", load_json(c)) +} + +d = "???{\"d\": 1}" +if valid_json(d) { # true + add_key("d", load_json(c)) +} else { + add_key("d", "invalid json") +} +``` + +结果： + +```json +{ + "a": "nil", + "b": "[1,2,3]", + "c": "{\"a\":1}", + "d": "invalid json", +} +``` diff --git a/pkg/inimpl/guancecloud/ptinput/funcs/md/value_type.en.md b/pkg/inimpl/guancecloud/ptinput/funcs/md/value_type.en.md new file mode 100644 index 0000000..58ea3fe --- /dev/null +++ b/pkg/inimpl/guancecloud/ptinput/funcs/md/value_type.en.md @@ -0,0 +1,38 @@ +### `value_type()` {#fn-value-type} + +Function prototype: `fn value_type(val) str` + +Function description: Obtain the type of the variable's value and return the value range ["int", "float", "bool", "str", "list", "map", "]. If the value is nil, return an empty string. + +Function parameters: + +- `val`: The value of the type to be determined. + +Example: + + +Input: + +```json +{"a":{"first": [2.2, 1.1], "ff": "[2.2, 1.1]","second":2,"third":"aBC","forth":true},"age":47} +``` + +Script: + +```python +d = load_json(_) + +if value_type(d) == "map" && "a" in d { + add_key("val_type", value_type(d["a"])) +} +``` + +Output: + +```json +// Fields +{ + "message": "{\"a\":{\"first\": [2.2, 1.1], \"ff\": \"[2.2, 1.1]\",\"second\":2,\"third\":\"aBC\",\"forth\":true},\"age\":47}", + "val_type": "map" +} +``` diff --git a/pkg/inimpl/guancecloud/ptinput/funcs/md/value_type.md b/pkg/inimpl/guancecloud/ptinput/funcs/md/value_type.md new file mode 100644 index 0000000..9e236da --- /dev/null +++ b/pkg/inimpl/guancecloud/ptinput/funcs/md/value_type.md @@ -0,0 +1,37 @@ +### `value_type()` {#fn-value-type} + +函数原型：`fn value_type(val) str` + +函数说明：获取变量的值的类型，返回值范围 ["int", "float", "bool", "str", "list", "map", ""], 若值为 nil，则返回空字符串 + +参数： + +- `val`: 待判断类型的值 + +示例： + +输入： + +```json +{"a":{"first": [2.2, 1.1], "ff": "[2.2, 1.1]","second":2,"third":"aBC","forth":true},"age":47} +``` + +脚本： + +```python +d = load_json(_) + +if value_type(d) == "map" && "a" in d { + add_key("val_type", value_type(d["a"])) +} +``` + +输出： + +```json +// Fields +{ + "message": "{\"a\":{\"first\": [2.2, 1.1], \"ff\": \"[2.2, 1.1]\",\"second\":2,\"third\":\"aBC\",\"forth\":true},\"age\":47}", + "val_type": "map" +} +``` diff --git a/pkg/inimpl/guancecloud/ptinput/funcs/md/xml.en.md b/pkg/inimpl/guancecloud/ptinput/funcs/md/xml.en.md new file mode 100644 index 0000000..ec9ded3 --- /dev/null +++ b/pkg/inimpl/guancecloud/ptinput/funcs/md/xml.en.md @@ -0,0 +1,60 @@ +### `xml()` {#fn-xml} + +Function prototype: `fn xml(input: str, xpath_expr: str, key_name)` + +Function description: Extract fields from XML through xpath expressions. + +Function parameters: + +- input: XML to extract +- xpath_expr: xpath expression +- key_name: The extracted data is written to a new key + +Example one: + +```python +# data to be processed + + valuex + ... + ... + + element_a_1 + element_a_2 + + + +# process script +xml(_, '/entry/fieldarray//fielda[1]/text()', field_a_1) + +# process result +{ + "field_a_1": "element_a_1", # extracted element_a_1 + "message": "\t\t\u003centry\u003e\n \u003cfieldx\u003evaluex\u003c/fieldx\u003e\n \u003cfieldy\u003e...\u003c/fieldy\u003e\n \u003cfieldz\u003e...\ u003c/fieldz\u003e\n \u003cfieldarray\u003e\n \u003cfielda\u003eelement_a_1\u003c/fielda\u003e\n \u003cfielda\u003eelement_a_2\u003c/fielda\u003e\n \u003c/fieldarray\n\c\u003 u003e", + "status": "unknown", + "time": 1655522989104916000 +} +``` + +Example two: + +```python +# data to be processed + + ORD12345 + V11111 + + +# process script +xml(_, '/OrderEvent/@actionCode', action_code) +xml(_, '/OrderEvent/OrderNumber/text()', OrderNumber) + +# process result +{ + "OrderNumber": "ORD12345", + "action_code": "5", + "message": "\u003cOrderEvent actionCode = \"5\"\u003e\n \u003cOrderNumber\u003eORD12345\u003c/OrderNumber\u003e\n \u003cVendorNumber\u003eV11111\u003c/VendorNumber\n\u003e\u003e" + "status": "unknown", + "time": 1655523193632471000 +} +``` diff --git a/pkg/inimpl/guancecloud/ptinput/funcs/md/xml.md b/pkg/inimpl/guancecloud/ptinput/funcs/md/xml.md new file mode 100644 index 0000000..afcb869 --- /dev/null +++ b/pkg/inimpl/guancecloud/ptinput/funcs/md/xml.md @@ -0,0 +1,60 @@ +### `xml()` {#fn-xml} + +函数原型：`fn xml(input: str, xpath_expr: str, key_name)` + +函数说明：通过 xpath 表达式，从 XML 中提取字段。 + +参数： + +- input: 待提取的 XML +- xpath_expr: xpath 表达式 +- key_name: 提取后数据写入新 key + +示例一： + +```python +# 待处理数据 + + valuex + ... + ... + + element_a_1 + element_a_2 + + + +# 处理脚本 +xml(_, '/entry/fieldarray//fielda[1]/text()', field_a_1) + +# 处理结果 +{ + "field_a_1": "element_a_1", # 提取了 element_a_1 + "message": "\t\t\u003centry\u003e\n \u003cfieldx\u003evaluex\u003c/fieldx\u003e\n \u003cfieldy\u003e...\u003c/fieldy\u003e\n \u003cfieldz\u003e...\u003c/fieldz\u003e\n \u003cfieldarray\u003e\n \u003cfielda\u003eelement_a_1\u003c/fielda\u003e\n \u003cfielda\u003eelement_a_2\u003c/fielda\u003e\n \u003c/fieldarray\u003e\n \u003c/entry\u003e", + "status": "unknown", + "time": 1655522989104916000 +} +``` + +示例二： + +```python +# 待处理数据 + + ORD12345 + V11111 + + +# 处理脚本 +xml(_, '/OrderEvent/@actionCode', action_code) +xml(_, '/OrderEvent/OrderNumber/text()', OrderNumber) + +# 处理结果 +{ + "OrderNumber": "ORD12345", + "action_code": "5", + "message": "\u003cOrderEvent actionCode = \"5\"\u003e\n \u003cOrderNumber\u003eORD12345\u003c/OrderNumber\u003e\n \u003cVendorNumber\u003eV11111\u003c/VendorNumber\u003e\n\u003c/OrderEvent\u003e", + "status": "unknown", + "time": 1655523193632471000 +} +``` diff --git a/pkg/inimpl/guancecloud/funcs/utils.go b/pkg/inimpl/guancecloud/ptinput/funcs/utils.go similarity index 55% rename from pkg/inimpl/guancecloud/funcs/utils.go rename to pkg/inimpl/guancecloud/ptinput/funcs/utils.go index da78239..3689b2b 100644 --- a/pkg/inimpl/guancecloud/funcs/utils.go +++ b/pkg/inimpl/guancecloud/ptinput/funcs/utils.go @@ -8,12 +8,12 @@ package funcs import ( "fmt" "math" + "strconv" "strings" "time" "github.com/GuanceCloud/platypus/pkg/ast" - "github.com/GuanceCloud/platypus/pkg/inimpl/guancecloud/input" - + "github.com/GuanceCloud/platypus/pkg/inimpl/guancecloud/ptinput" conv "github.com/spf13/cast" ) @@ -34,6 +34,18 @@ func getKeyName(node *ast.Node) (string, error) { return key, nil } +func isPlVarbOrFunc(node *ast.Node) bool { + if node == nil { + return false + } + switch node.NodeType { //nolint:exhaustive + case ast.TypeIdentifier, ast.TypeAttrExpr, ast.TypeCallExpr: + return true + default: + return false + } +} + const ( InvalidInt = math.MinInt32 DefaultInt = int64(0xdeadbeef) @@ -43,134 +55,105 @@ const ( DefaultFloat = float64(0.0) ) -func getPoint(in any) (*input.Point, error) { - if in == nil { - return nil, fmt.Errorf("nil ptr: input") +func fixYear(now time.Time, x int64) (int, error) { + if x == DefaultInt { + return now.Year(), nil } - pt, ok := in.(*input.Point) - - if !ok { - return nil, fmt.Errorf("typeof input is not Point") + if x < 0 { + return -1, fmt.Errorf("year should larger than 0") } - if pt == nil { - return nil, fmt.Errorf("nil ptr: input") + // year like 02 -> 2002, 21 -> 2021 + if x < int64(100) { //nolint:gomnd + x += int64(now.Year() / 100 * 100) //nolint:gomnd + return int(x), nil } - return pt, nil -} -func getPtKey(in any, key string) (any, ast.DType, error) { - pt, err := getPoint(in) - if err != nil { - return nil, ast.Invalid, err - } + return int(x), nil +} - if key == "_" { - key = input.Originkey +func fixMonth(now time.Time, x string) (time.Month, error) { + if x == DefaultStr { + return now.Month(), nil + } else { + if v, err := strconv.ParseInt(x, 10, 64); err == nil { // digital month: i.e., 01, 11 + if v < 1 || v > 12 { + return time.Month(-1), fmt.Errorf("month should between [1,12], got %x,", x) + } + return time.Month(v), nil + } else { // month like aug/august, january/jan + v, ok := monthMaps[strings.ToLower(x)] + if !ok { + return InvalidInt, fmt.Errorf("unknown month %s", x) + } + return v, nil + } } - return pt.Get(key) } -func deletePtKey(in any, key string) { - pt, err := getPoint(in) - if err != nil { - return +func fixDay(now time.Time, x int64) (int, error) { + if x == DefaultInt { + return now.Day(), nil } - if key == "_" { - key = input.Originkey + if x < 1 || x > 31 { + return -1, fmt.Errorf("month day should between [1,31], got %d", x) } - pt.Delete(key) + return int(x), nil } -func pointTime(in any) time.Time { - pt, err := getPoint(in) - if err != nil { - return time.Now() +func fixHour(now time.Time, x int64) (int, error) { + if x == DefaultInt { + return now.Hour(), nil } - return pt.Time -} -func setPointTime(in any, tn time.Time) error { - pt, err := getPoint(in) - if err != nil { - return err + if x < 0 || x > 23 { + return -1, fmt.Errorf("hour should between [0,24], got %d", x) } - pt.Time = tn - return nil + + return int(x), nil } -func addKey2PtWithVal(in any, key string, value any, dtype ast.DType, kind input.KeyKind) error { - pt, err := getPoint(in) - if err != nil { - return err +func fixMinute(now time.Time, x int64) (int, error) { + if x == DefaultInt { + return now.Minute(), nil } - if key == "_" { - key = input.Originkey + if x < 0 || x > 59 { + return -1, fmt.Errorf("minute should between [0,59], got %d", x) } - switch kind { - case input.KindPtTag: - return pt.SetTag(key, value, dtype) - default: - return pt.Set(key, value, dtype) - } + return int(x), nil } -func renamePtKey(in any, to, from string) error { - if to == "_" { - to = input.Originkey - } - - if from == "_" { - from = input.Originkey - } - - if to == from { - return nil +func fixSecond(x int64) (int, error) { + if x == DefaultInt { + return 0, nil } - if in == nil { - return fmt.Errorf("nil ptr: input") + if x < 0 || x > 59 { + return -1, fmt.Errorf("second should between [0,59], got %d", x) } - pt, err := getPoint(in) + return int(x), nil +} +func tz(s string) (z *time.Location, err error) { + z, err = time.LoadLocation(s) if err != nil { - return err - } - - v, ok := pt.Meta[from] - if !ok { - return fmt.Errorf("key(from) %s not found", from) - } - - switch v.PtFlag { //nolint:exhaustive - case input.PtField: - if v, ok := pt.Fields[from]; ok { - pt.Fields[to] = v + if _, ok := timezoneList[s]; !ok { + return nil, fmt.Errorf("unknown timezone %s", s) } - delete(pt.Fields, from) - case input.PtTag: - if v, ok := pt.Tags[from]; ok { - pt.Tags[to] = v + + z, err = time.LoadLocation(timezoneList[s]) + if err != nil { + return nil, err } - delete(pt.Tags, from) } - return nil -} -func setMeasurement(in any, val string) error { - pt, err := getPoint(in) - - if err != nil { - return err - } - pt.Measurement = val - return nil + return z, nil } func doCast(result interface{}, tInfo string) (interface{}, ast.DType) { @@ -191,7 +174,7 @@ func doCast(result interface{}, tInfo string) (interface{}, ast.DType) { return nil, ast.Nil } -func reIndexFuncArgs(fnStmt *ast.CallExpr, keyList []string, reqParm int) error { +func reindexFuncArgs(fnStmt *ast.CallExpr, keyList []string, reqParm int) error { // reqParm >= 1, if < 0, no optional args args := fnStmt.Param @@ -243,3 +226,121 @@ func reIndexFuncArgs(fnStmt *ast.CallExpr, keyList []string, reqParm int) error fnStmt.Param = ret return nil } + +func getPoint(in any) (ptinput.PlInputPt, error) { + if in == nil { + return nil, fmt.Errorf("nil ptr: input") + } + + pt, ok := in.(ptinput.PlInputPt) + + if !ok { + return nil, fmt.Errorf("typeof input is not Point") + } + + if pt == nil { + return nil, fmt.Errorf("nil ptr: input") + } + return pt, nil +} + +func getPtKey(in any, key string) (any, ast.DType, error) { + pt, err := getPoint(in) + if err != nil { + return nil, ast.Invalid, err + } + + if key == "_" { + key = ptinput.Originkey + } + return pt.Get(key) +} + +func deletePtKey(in any, key string) { + pt, err := getPoint(in) + if err != nil { + return + } + + if key == "_" { + key = ptinput.Originkey + } + + pt.Delete(key) +} + +func pointTime(in any) int64 { + pt, ok := in.(ptinput.PlInputPt) + if !ok { + return time.Now().UnixNano() + } + t := pt.PtTime() + if t.IsZero() { + return time.Now().UnixNano() + } else { + return t.UnixNano() + } +} + +func addKey2PtWithVal(in any, key string, value any, dtype ast.DType, kind ptinput.KeyKind) error { + pt, err := getPoint(in) + if err != nil { + return err + } + + if key == "_" { + key = ptinput.Originkey + } + + switch kind { //nolint:exhaustive + case ptinput.KindPtTag: + return pt.SetTag(key, value, dtype) + default: + return pt.Set(key, value, dtype) + } +} + +func renamePtKey(in any, to, from string) error { + if to == "_" { + to = ptinput.Originkey + } + + if from == "_" { + from = ptinput.Originkey + } + + if to == from { + return nil + } + + if in == nil { + return fmt.Errorf("nil ptr: input") + } + + pt, err := getPoint(in) + if err != nil { + return err + } + + return pt.RenameKey(from, to) +} + +func setMeasurement(in any, val string) error { + pt, err := getPoint(in) + if err != nil { + return err + } + pt.SetPtName(val) + return nil +} + +func markPtDrop(in any) error { + pt, err := getPoint(in) + if err != nil { + return err + } + + pt.MarkDrop(true) + + return nil +} diff --git a/pkg/inimpl/guancecloud/funcs/utilts_test.go b/pkg/inimpl/guancecloud/ptinput/funcs/utilts_test.go similarity index 98% rename from pkg/inimpl/guancecloud/funcs/utilts_test.go rename to pkg/inimpl/guancecloud/ptinput/funcs/utilts_test.go index 9a19c5f..7430814 100644 --- a/pkg/inimpl/guancecloud/funcs/utilts_test.go +++ b/pkg/inimpl/guancecloud/ptinput/funcs/utilts_test.go @@ -188,7 +188,7 @@ func TestReIndexFuncArgs(t *testing.T) { for _, v := range cases { t.Run(v.name, func(t *testing.T) { - err := reIndexFuncArgs(v.fnArgs, v.keyList, v.reqParm) + err := reindexFuncArgs(v.fnArgs, v.keyList, v.reqParm) if err != nil { if !v.fail { t.Error(err) diff --git a/pkg/inimpl/guancecloud/ptinput/ip2isp/bit.go b/pkg/inimpl/guancecloud/ptinput/ip2isp/bit.go new file mode 100644 index 0000000..1ef42cc --- /dev/null +++ b/pkg/inimpl/guancecloud/ptinput/ip2isp/bit.go @@ -0,0 +1,265 @@ +// Unless explicitly stated otherwise all files in this repository are licensed +// under the MIT License. +// This product includes software developed at Guance Cloud (https://www.guance.com/). +// Copyright 2021-present Guance, Inc. + +package ip2isp + +var BitConvTemplate = []string{ + "00000000", + "00000001", + "00000010", + "00000011", + "00000100", + "00000101", + "00000110", + "00000111", + "00001000", + "00001001", + "00001010", + "00001011", + "00001100", + "00001101", + "00001110", + "00001111", + "00010000", + "00010001", + "00010010", + "00010011", + "00010100", + "00010101", + "00010110", + "00010111", + "00011000", + "00011001", + "00011010", + "00011011", + "00011100", + "00011101", + "00011110", + "00011111", + "00100000", + "00100001", + "00100010", + "00100011", + "00100100", + "00100101", + "00100110", + "00100111", + "00101000", + "00101001", + "00101010", + "00101011", + "00101100", + "00101101", + "00101110", + "00101111", + "00110000", + "00110001", + "00110010", + "00110011", + "00110100", + "00110101", + "00110110", + "00110111", + "00111000", + "00111001", + "00111010", + "00111011", + "00111100", + "00111101", + "00111110", + "00111111", + "01000000", + "01000001", + "01000010", + "01000011", + "01000100", + "01000101", + "01000110", + "01000111", + "01001000", + "01001001", + "01001010", + "01001011", + "01001100", + "01001101", + "01001110", + "01001111", + "01010000", + "01010001", + "01010010", + "01010011", + "01010100", + "01010101", + "01010110", + "01010111", + "01011000", + "01011001", + "01011010", + "01011011", + "01011100", + "01011101", + "01011110", + "01011111", + "01100000", + "01100001", + "01100010", + "01100011", + "01100100", + "01100101", + "01100110", + "01100111", + "01101000", + "01101001", + "01101010", + "01101011", + "01101100", + "01101101", + "01101110", + "01101111", + "01110000", + "01110001", + "01110010", + "01110011", + "01110100", + "01110101", + "01110110", + "01110111", + "01111000", + "01111001", + "01111010", + "01111011", + "01111100", + "01111101", + "01111110", + "01111111", + "10000000", + "10000001", + "10000010", + "10000011", + "10000100", + "10000101", + "10000110", + "10000111", + "10001000", + "10001001", + "10001010", + "10001011", + "10001100", + "10001101", + "10001110", + "10001111", + "10010000", + "10010001", + "10010010", + "10010011", + "10010100", + "10010101", + "10010110", + "10010111", + "10011000", + "10011001", + "10011010", + "10011011", + "10011100", + "10011101", + "10011110", + "10011111", + "10100000", + "10100001", + "10100010", + "10100011", + "10100100", + "10100101", + "10100110", + "10100111", + "10101000", + "10101001", + "10101010", + "10101011", + "10101100", + "10101101", + "10101110", + "10101111", + "10110000", + "10110001", + "10110010", + "10110011", + "10110100", + "10110101", + "10110110", + "10110111", + "10111000", + "10111001", + "10111010", + "10111011", + "10111100", + "10111101", + "10111110", + "10111111", + "11000000", + "11000001", + "11000010", + "11000011", + "11000100", + "11000101", + "11000110", + "11000111", + "11001000", + "11001001", + "11001010", + "11001011", + "11001100", + "11001101", + "11001110", + "11001111", + "11010000", + "11010001", + "11010010", + "11010011", + "11010100", + "11010101", + "11010110", + "11010111", + "11011000", + "11011001", + "11011010", + "11011011", + "11011100", + "11011101", + "11011110", + "11011111", + "11100000", + "11100001", + "11100010", + "11100011", + "11100100", + "11100101", + "11100110", + "11100111", + "11101000", + "11101001", + "11101010", + "11101011", + "11101100", + "11101101", + "11101110", + "11101111", + "11110000", + "11110001", + "11110010", + "11110011", + "11110100", + "11110101", + "11110110", + "11110111", + "11111000", + "11111001", + "11111010", + "11111011", + "11111100", + "11111101", + "11111110", + "11111111", +} diff --git a/pkg/inimpl/guancecloud/ptinput/ip2isp/isp.go b/pkg/inimpl/guancecloud/ptinput/ip2isp/isp.go new file mode 100644 index 0000000..6aac64a --- /dev/null +++ b/pkg/inimpl/guancecloud/ptinput/ip2isp/isp.go @@ -0,0 +1,137 @@ +// Unless explicitly stated otherwise all files in this repository are licensed +// under the MIT License. +// This product includes software developed at Guance Cloud (https://www.guance.com/). +// Copyright 2021-present Guance, Inc. + +// Package ip2isp append ISP info to IP address +package ip2isp + +import ( + "bufio" + "encoding/csv" + "fmt" + "os" + "path/filepath" + "strings" + + "github.com/GuanceCloud/platypus/pkg/inimpl/guancecloud/ptinput/ipdb" + yaml "gopkg.in/yaml.v2" +) + +const ( + IPV4Len = 4 + FileSeparator = " " +) + +var ipdbInstance ipdb.IPdb + +var IspValid = map[string]string{ + "chinanet": "中国电信", + "cmcc": "中国移动", + "unicom": "中国联通", + "tietong": "中国铁通", + "cernet": "教育网", + "cstnet": "科技网", + "drpeng": "鹏博士", + "googlecn": "谷歌中国", +} + +func InitIPDB(instance ipdb.IPdb) { + ipdbInstance = instance +} + +func SearchISP(ip string) string { + if ipdbInstance != nil { + return ipdbInstance.SearchIsp(ip) + } + return "unknown" +} + +func MergeISP(from, to string) error { + files, err := os.ReadDir(from) + if err != nil { + return err + } + + var content []string + + for _, f := range files { + file := f.Name() + + // 去掉统计信息文件 + if !strings.HasSuffix(file, ".txt") { + continue + } + + // 去掉ipv6文件 + if strings.HasSuffix(file, "6.txt") { + continue + } + + isp := strings.TrimSuffix(file, ".txt") + if _, ok := IspValid[isp]; !ok { + continue + } + + fd, err := os.Open(filepath.Clean(filepath.Join(from, file))) + if err != nil { + return err + } + defer fd.Close() //nolint:errcheck,gosec + + scanner := bufio.NewScanner(fd) + for scanner.Scan() { + c := fmt.Sprintf("%v%v%v", scanner.Text(), FileSeparator, isp) + content = append(content, c) + } + } + + return os.WriteFile(to, []byte(strings.Join(content, "\n")), os.ModePerm) +} + +func BuildContryCity(csvFile, outputFile string) error { + d := make(map[string]map[string][]string) + found := make(map[string]uint8) + + f, err := os.Open(filepath.Clean(csvFile)) + if err != nil { + return err + } + defer f.Close() //nolint:errcheck,gosec + + w := csv.NewReader(f) + data, err := w.ReadAll() + if err != nil { + return err + } + + for _, ip := range data { + contry := ip[3] + province := ip[4] + city := ip[5] + if contry == "-" || city == "-" { + continue + } + + uniKey := fmt.Sprintf("%v%v%v", contry, province, city) + if _, ok := found[uniKey]; ok { + continue + } + + c, ok := d[contry] + if !ok { + c = make(map[string][]string) + d[contry] = c + } + + c[province] = append(c[province], city) + found[uniKey] = 0 + } + + r, err := yaml.Marshal(d) + if err != nil { + return err + } + + return os.WriteFile(outputFile, r, os.ModePerm) +} diff --git a/pkg/inimpl/guancecloud/ptinput/ipdb/bit.go b/pkg/inimpl/guancecloud/ptinput/ipdb/bit.go new file mode 100644 index 0000000..ec93689 --- /dev/null +++ b/pkg/inimpl/guancecloud/ptinput/ipdb/bit.go @@ -0,0 +1,265 @@ +// Unless explicitly stated otherwise all files in this repository are licensed +// under the MIT License. +// This product includes software developed at Guance Cloud (https://www.guance.com/). +// Copyright 2021-present Guance, Inc. + +package ipdb + +var BitConvTemplate = []string{ + "00000000", + "00000001", + "00000010", + "00000011", + "00000100", + "00000101", + "00000110", + "00000111", + "00001000", + "00001001", + "00001010", + "00001011", + "00001100", + "00001101", + "00001110", + "00001111", + "00010000", + "00010001", + "00010010", + "00010011", + "00010100", + "00010101", + "00010110", + "00010111", + "00011000", + "00011001", + "00011010", + "00011011", + "00011100", + "00011101", + "00011110", + "00011111", + "00100000", + "00100001", + "00100010", + "00100011", + "00100100", + "00100101", + "00100110", + "00100111", + "00101000", + "00101001", + "00101010", + "00101011", + "00101100", + "00101101", + "00101110", + "00101111", + "00110000", + "00110001", + "00110010", + "00110011", + "00110100", + "00110101", + "00110110", + "00110111", + "00111000", + "00111001", + "00111010", + "00111011", + "00111100", + "00111101", + "00111110", + "00111111", + "01000000", + "01000001", + "01000010", + "01000011", + "01000100", + "01000101", + "01000110", + "01000111", + "01001000", + "01001001", + "01001010", + "01001011", + "01001100", + "01001101", + "01001110", + "01001111", + "01010000", + "01010001", + "01010010", + "01010011", + "01010100", + "01010101", + "01010110", + "01010111", + "01011000", + "01011001", + "01011010", + "01011011", + "01011100", + "01011101", + "01011110", + "01011111", + "01100000", + "01100001", + "01100010", + "01100011", + "01100100", + "01100101", + "01100110", + "01100111", + "01101000", + "01101001", + "01101010", + "01101011", + "01101100", + "01101101", + "01101110", + "01101111", + "01110000", + "01110001", + "01110010", + "01110011", + "01110100", + "01110101", + "01110110", + "01110111", + "01111000", + "01111001", + "01111010", + "01111011", + "01111100", + "01111101", + "01111110", + "01111111", + "10000000", + "10000001", + "10000010", + "10000011", + "10000100", + "10000101", + "10000110", + "10000111", + "10001000", + "10001001", + "10001010", + "10001011", + "10001100", + "10001101", + "10001110", + "10001111", + "10010000", + "10010001", + "10010010", + "10010011", + "10010100", + "10010101", + "10010110", + "10010111", + "10011000", + "10011001", + "10011010", + "10011011", + "10011100", + "10011101", + "10011110", + "10011111", + "10100000", + "10100001", + "10100010", + "10100011", + "10100100", + "10100101", + "10100110", + "10100111", + "10101000", + "10101001", + "10101010", + "10101011", + "10101100", + "10101101", + "10101110", + "10101111", + "10110000", + "10110001", + "10110010", + "10110011", + "10110100", + "10110101", + "10110110", + "10110111", + "10111000", + "10111001", + "10111010", + "10111011", + "10111100", + "10111101", + "10111110", + "10111111", + "11000000", + "11000001", + "11000010", + "11000011", + "11000100", + "11000101", + "11000110", + "11000111", + "11001000", + "11001001", + "11001010", + "11001011", + "11001100", + "11001101", + "11001110", + "11001111", + "11010000", + "11010001", + "11010010", + "11010011", + "11010100", + "11010101", + "11010110", + "11010111", + "11011000", + "11011001", + "11011010", + "11011011", + "11011100", + "11011101", + "11011110", + "11011111", + "11100000", + "11100001", + "11100010", + "11100011", + "11100100", + "11100101", + "11100110", + "11100111", + "11101000", + "11101001", + "11101010", + "11101011", + "11101100", + "11101101", + "11101110", + "11101111", + "11110000", + "11110001", + "11110010", + "11110011", + "11110100", + "11110101", + "11110110", + "11110111", + "11111000", + "11111001", + "11111010", + "11111011", + "11111100", + "11111101", + "11111110", + "11111111", +} diff --git a/pkg/inimpl/guancecloud/ptinput/ipdb/geoip/geolite2.go b/pkg/inimpl/guancecloud/ptinput/ipdb/geoip/geolite2.go new file mode 100644 index 0000000..51051e2 --- /dev/null +++ b/pkg/inimpl/guancecloud/ptinput/ipdb/geoip/geolite2.go @@ -0,0 +1,114 @@ +// Unless explicitly stated otherwise all files in this repository are licensed +// under the MIT License. +// This product includes software developed at Guance Cloud (https://www.guance.com/). +// Copyright 2021-present Guance, Inc. + +// Package geoip implement ipdb. +package geoip + +import ( + "fmt" + "net" + "path/filepath" + + "github.com/GuanceCloud/cliutils/logger" + "github.com/GuanceCloud/platypus/pkg/inimpl/guancecloud/ptinput/ipdb" + "github.com/GuanceCloud/platypus/pkg/inimpl/guancecloud/ptinput/utils" + "github.com/oschwald/geoip2-golang" +) + +const INVALIDIP = "Invalid IP address" + +var l = logger.DefaultSLogger("geolite2") + +var openDB = func(f string) (*geoip2.Reader, error) { + db, err := geoip2.Open(f) + return db, err +} + +type Geoip struct { + db *geoip2.Reader +} + +func (g *Geoip) loadIPLib(f string) error { + if !utils.FileExist(f) { + l.Warnf("%v not found", f) + return nil + } + + db, err := openDB(f) + if err != nil { + return err + } else { + g.db = db + } + + return nil +} + +func (g *Geoip) Init(dataDir string, config map[string]string) { + l = logger.SLogger("geoip") + l.Debug("use geolite2 db") + ipdbDir := filepath.Join(dataDir, "ipdb", "geolite2", "GeoLite2-City_20220617") + ipdbFile := "GeoLite2-City.mmdb" + + if file, ok := config["geoip_file"]; ok { + if len(file) > 0 { + ipdbFile = file + } + } + + if err := g.loadIPLib(filepath.Join(ipdbDir, ipdbFile)); err != nil { + l.Warnf("geolite2 load ip lib error: %s", err.Error()) + } +} + +func (g *Geoip) Geo(ip string) (*ipdb.IPdbRecord, error) { + record := &ipdb.IPdbRecord{} + if g.db == nil { + return record, nil + } + + r, err := g.get(ip) + if err != nil { + return record, err + } + // ip invalid + if r == nil { + record.City = INVALIDIP + record.Timezone = INVALIDIP + record.Region = INVALIDIP + record.Country = INVALIDIP + return record, nil + } + + record.City = r.City.Names["en"] + record.Timezone = r.Location.TimeZone + if len(r.Subdivisions) != 0 { + record.Region = r.Subdivisions[0].Names["en"] + } + record.Country = r.Country.Names["en"] + + return record.CheckData(), err +} + +func (g *Geoip) get(ip string) (*geoip2.City, error) { + if g.db == nil { + return nil, fmt.Errorf("GEO DB not set") + } + ipParse := net.ParseIP(ip) + // ip invalid + if ipParse == nil { + return nil, nil + } + r, err := g.db.City(ipParse) + if err != nil { + return nil, err + } + + return r, nil +} + +func (g *Geoip) SearchIsp(ip string) string { + return "" +} diff --git a/pkg/inimpl/guancecloud/ptinput/ipdb/ipdb.go b/pkg/inimpl/guancecloud/ptinput/ipdb/ipdb.go new file mode 100644 index 0000000..2b727aa --- /dev/null +++ b/pkg/inimpl/guancecloud/ptinput/ipdb/ipdb.go @@ -0,0 +1,83 @@ +// Unless explicitly stated otherwise all files in this repository are licensed +// under the MIT License. +// This product includes software developed at Guance Cloud (https://www.guance.com/). +// Copyright 2021-present Guance, Inc. + +// Package ipdb define the interface for ip location or isp. +package ipdb + +import ( + "fmt" + "strconv" + "strings" +) + +type IPdb interface { + Init(dataDir string, config map[string]string) + Geo(ip string) (*IPdbRecord, error) + SearchIsp(ip string) string +} + +type IPdbRecord struct { + Country string + Region string + City string + Isp string + Latitude float32 + Longitude float32 + Timezone string + Areacode string +} + +func (record *IPdbRecord) CheckData() *IPdbRecord { + switch record.Country { // #issue 354 + case "TW": + record.Country = "CN" + record.Region = "Taiwan" + case "MO": + record.Country = "CN" + record.Region = "Macao" + case "HK": + record.Country = "CN" + record.Region = "Hong Kong" + } + return record +} + +func ParseIPCIDR(ipCidr string) (string, error) { + var err error + var cidrLen int64 = 32 + + ipCidrs := strings.Split(ipCidr, "/") + if len(ipCidrs) == 2 { + cidrLen, err = strconv.ParseInt(ipCidrs[1], 10, 8) + if err != nil { + return "", err + } + } + + ipBytes := strings.Split(ipCidrs[0], ".") + if len(ipBytes) != 4 { + return "", fmt.Errorf("invalid ip address") + } + ipBitStr := "" + for _, ipByteStr := range ipBytes { + ip, err := strconv.ParseInt(ipByteStr, 10, 16) + if err != nil { + return "", err + } + if ip >= int64(len(BitConvTemplate)) { + return "", fmt.Errorf("invalid ip number: %s", ipByteStr) + } + if cidrLen >= 8 { + ipBitStr += BitConvTemplate[ip] + } else { + ipBitStr += BitConvTemplate[ip][0:cidrLen] + } + cidrLen -= 8 + if cidrLen <= 0 { + break + } + } + return ipBitStr, nil +} diff --git a/pkg/inimpl/guancecloud/ptinput/ipdb/ipdb_test.go b/pkg/inimpl/guancecloud/ptinput/ipdb/ipdb_test.go new file mode 100644 index 0000000..a316b00 --- /dev/null +++ b/pkg/inimpl/guancecloud/ptinput/ipdb/ipdb_test.go @@ -0,0 +1,67 @@ +// Unless explicitly stated otherwise all files in this repository are licensed +// under the MIT License. +// This product includes software developed at Guance Cloud (https://www.guance.com/). +// Copyright 2021-present Guance, Inc. + +package ipdb + +import ( + "testing" + + "github.com/stretchr/testify/assert" +) + +func TestParseIPCIDR(t *testing.T) { + cases := []struct { + title string + cidr string + expected string + isValidCidr bool + }{ + { + title: "correct cidr", + cidr: "1.0.0.0/24", + expected: "000000010000000000000000", + isValidCidr: true, + }, + { + title: "correct cidr", + cidr: "1.0.0.0/26", + expected: "00000001000000000000000000", + isValidCidr: true, + }, + { + title: "invalid ip length", + cidr: "1.0.0.0.0/24", + expected: "000000010000000000000000", + isValidCidr: false, + }, + { + title: "invalid cidr", + cidr: "256.0.0.0/24", + expected: "", + isValidCidr: false, + }, + { + title: "invalid ip", + cidr: "a.0.0.0/24", + expected: "", + isValidCidr: false, + }, + { + title: "invalid mask", + cidr: "1.0.0.0/b", + expected: "", + isValidCidr: false, + }, + } + + for _, item := range cases { + ip, err := ParseIPCIDR(item.cidr) + if !item.isValidCidr { + assert.Error(t, err) + } else { + assert.Equal(t, ip, item.expected) + } + } +} diff --git a/pkg/inimpl/guancecloud/ptinput/ipdb/iploc/iploc.go b/pkg/inimpl/guancecloud/ptinput/ipdb/iploc/iploc.go new file mode 100644 index 0000000..a50720c --- /dev/null +++ b/pkg/inimpl/guancecloud/ptinput/ipdb/iploc/iploc.go @@ -0,0 +1,190 @@ +// Unless explicitly stated otherwise all files in this repository are licensed +// under the MIT License. +// This product includes software developed at Guance Cloud (https://www.guance.com/). +// Copyright 2021-present Guance, Inc. + +// Package iploc implement ipdb. +package iploc + +import ( + "bufio" + "fmt" + "os" + "path/filepath" + "strings" + + "github.com/GuanceCloud/cliutils/logger" + "github.com/GuanceCloud/platypus/pkg/inimpl/guancecloud/ptinput/ipdb" + "github.com/GuanceCloud/platypus/pkg/inimpl/guancecloud/ptinput/utils" + "github.com/ip2location/ip2location-go" +) + +var l = logger.DefaultSLogger("iploc") + +type DB interface { + Get_all(ipaddress string) (ip2location.IP2Locationrecord, error) +} + +var openDB = func(f string) (DB, error) { + db, err := ip2location.OpenDB(f) + return db, err +} + +const ( + FileSeparator = " " + + ErrInvalidIP = "Invalid IP address" + ErrInvalidDB = "Invalid database file" + + geoDefaultVal = "unknown" +) + +type IPloc struct { + db DB + ispDB map[string]string +} + +func (iploc *IPloc) Init(dataDir string, config map[string]string) { + l = logger.SLogger("iploc") + ipdbDir := filepath.Join(dataDir, "ipdb", "iploc") + iplocFile := "iploc.bin" + ispFile := "ip2isp.txt" + + if file, ok := config["iploc_file"]; ok { + if len(file) > 0 { + iplocFile = file + } + } + + if file, ok := config["isp_file"]; ok { + if len(file) > 0 { + ispFile = file + } + } + + if err := iploc.loadIPLib(filepath.Join(ipdbDir, iplocFile)); err != nil { + l.Warnf("iploc load ip lib error: %s", err.Error()) + } + + iploc.ispDB = map[string]string{} + + if err := iploc.loadISP(filepath.Join(ipdbDir, ispFile)); err != nil { + l.Warnf("isp file load error: %s", err.Error()) + } +} + +func (iploc *IPloc) loadIPLib(f string) error { + if !utils.FileExist(f) { + l.Warnf("%v not found", f) + return nil + } + + db, err := openDB(f) + if err != nil { + return err + } else { + iploc.db = db + } + + return nil +} + +func (iploc *IPloc) loadISP(f string) error { + m := make(map[string]string) + + if !utils.FileExist(f) { + l.Warnf("%v not found", f) + return nil + } + + fd, err := os.Open(filepath.Clean(f)) + if err != nil { + return err + } + + defer fd.Close() //nolint:errcheck,gosec + + scanner := bufio.NewScanner(fd) + for scanner.Scan() { + contents := strings.Split(scanner.Text(), FileSeparator) + if len(contents) != 2 { + continue + } + + ipBitStr, err := ipdb.ParseIPCIDR(contents[0]) + if err != nil { + continue + } + m[ipBitStr] = contents[1] + } + + if len(m) != 0 { + iploc.ispDB = m + l.Infof("found new %d rules", len(m)) + } else { + l.Infof("no rules founded") + } + + return nil +} + +func (iploc *IPloc) SearchIsp(ip string) string { + if len(iploc.ispDB) == 0 { + return "unknown" + } + + for i := 32; i > 0; i-- { + ipCidr := fmt.Sprintf("%s/%v", ip, i) + ipBitStr, _ := ipdb.ParseIPCIDR(ipCidr) + if v, ok := iploc.ispDB[ipBitStr]; ok { + return v + } + } + return "unknown" +} + +func (iploc *IPloc) Geo(ip string) (*ipdb.IPdbRecord, error) { + record := &ipdb.IPdbRecord{} + if iploc.db == nil { + return record, nil + } + + r, err := iploc.get(ip) + if err != nil { + return nil, err + } + + switch r.City { + case ErrInvalidIP, ErrInvalidDB: + r.City = geoDefaultVal + } + + switch r.Region { + case ErrInvalidIP, ErrInvalidDB: + r.Region = geoDefaultVal + } + + switch r.Country_short { + case ErrInvalidIP, ErrInvalidDB: + r.Country_short = geoDefaultVal + } + + record.City = r.City + record.Region = r.Region + record.Country = r.Country_short + + return record.CheckData(), err +} + +func (iploc *IPloc) get(ip string) (*ip2location.IP2Locationrecord, error) { + if iploc.db == nil { + return nil, fmt.Errorf("GEO DB not set") + } + + r, err := iploc.db.Get_all(ip) + if err != nil { + return nil, err + } + + return &r, nil +} diff --git a/pkg/inimpl/guancecloud/ptinput/ipdb/iploc/iploc_test.go b/pkg/inimpl/guancecloud/ptinput/ipdb/iploc/iploc_test.go new file mode 100644 index 0000000..e4d27ed --- /dev/null +++ b/pkg/inimpl/guancecloud/ptinput/ipdb/iploc/iploc_test.go @@ -0,0 +1,79 @@ +// Unless explicitly stated otherwise all files in this repository are licensed +// under the MIT License. +// This product includes software developed at Guance Cloud (https://www.guance.com/). +// Copyright 2021-present Guance, Inc. + +package iploc + +import ( + "io/fs" + "os" + "path/filepath" + "testing" + + "github.com/ip2location/ip2location-go" + "github.com/stretchr/testify/assert" +) + +var iplocationMockResult ip2location.IP2Locationrecord + +type mockDB struct{} + +func (db *mockDB) Get_all(ip string) (ip2location.IP2Locationrecord, error) { //nolint:stylecheck + return iplocationMockResult, nil +} + +var mockOpenDB = func(f string) (DB, error) { + return &mockDB{}, nil +} + +func TestIPloc(t *testing.T) { + openDB = mockOpenDB + iplocInstance := &IPloc{} + datadir, err := os.MkdirTemp(os.TempDir(), "iploc") + if err != nil { + t.Fatalf("tmp dir create error") + } + defer os.RemoveAll(datadir) //nolint:errcheck + iplocFile := "ip.bin" + ispFile := "isp.txt" + + iplocDir := filepath.Join(datadir, "ipdb/iploc") + err = os.MkdirAll(iplocDir, os.ModePerm) + if err != nil { + l.Fatal(err) + } + + err = os.WriteFile(filepath.Join(iplocDir, iplocFile), []byte(""), fs.ModePerm) + if err != nil { + t.Fatalf("iploc file create errror") + } + + err = os.WriteFile(filepath.Join(iplocDir, ispFile), []byte("221.0.0.0/13 unicom"), fs.ModePerm) + if err != nil { + t.Fatalf("isp file create errror") + } + + iplocInstance.Init(datadir, map[string]string{"isp_file": ispFile, "iploc_file": iplocFile}) + + t.Run("SearchIsp", func(t *testing.T) { + assert.Equal(t, "unicom", iplocInstance.SearchIsp("221.0.0.0")) + assert.Equal(t, "unknown", iplocInstance.SearchIsp("0.0.0.0")) + }) + + t.Run("Geo", func(t *testing.T) { + iplocationMockResult = ip2location.IP2Locationrecord{Country_short: "CN", City: ErrInvalidIP, Region: ErrInvalidIP} + ipInfo, err := iplocInstance.Geo("0.0.0.0") + assert.NoError(t, err) + assert.Equal(t, "CN", ipInfo.Country) + assert.Equal(t, "unknown", ipInfo.City) + + backDB := iplocInstance.db + iplocInstance.db = nil + _, err = iplocInstance.Geo("0.0.0.0") + + assert.NoError(t, err) + + iplocInstance.db = backDB + }) +} diff --git a/pkg/inimpl/guancecloud/ptinput/plmap/aggmap.go b/pkg/inimpl/guancecloud/ptinput/plmap/aggmap.go new file mode 100644 index 0000000..930f021 --- /dev/null +++ b/pkg/inimpl/guancecloud/ptinput/plmap/aggmap.go @@ -0,0 +1,456 @@ +// Unless explicitly stated otherwise all files in this repository are licensed +// under the MIT License. +// This product includes software developed at Guance Cloud (https://www.guance.com/). +// Copyright 2021-present Guance, Inc. + +package plmap + +import ( + "fmt" + "runtime" + "sync" + "time" + + "github.com/GuanceCloud/cliutils/logger" + "github.com/GuanceCloud/cliutils/point" + "github.com/GuanceCloud/platypus/pkg/hash" + "github.com/GuanceCloud/platypus/pkg/inimpl/guancecloud/ptinput/utils" + "github.com/spf13/cast" +) + +var l = logger.DefaultSLogger("pl-map") + +func InitPlMap() { + l = logger.SLogger("pl-map") +} + +type AggBuckets struct { + uploadDataFn UploadFunc + // key: [hash(name), hash(...tagName)] + data map[point.Category]map[string]*bucket + + gTags map[string]string + + sync.RWMutex +} + +func NewAggBuks(upFn UploadFunc, globalTags map[string]string) *AggBuckets { + return &AggBuckets{ + uploadDataFn: upFn, + data: map[point.Category]map[string]*bucket{}, + gTags: globalTags, + } +} + +func (a *AggBuckets) CreateBucket(cat point.Category, name string, interval time.Duration, + count int, keepValue bool, constTags map[string]string, +) { + a.Lock() + defer a.Unlock() + + if a.data == nil { + a.data = map[point.Category]map[string]*bucket{} + } + + catBuk, ok := a.data[cat] + if !ok { + catBuk = map[string]*bucket{} + a.data[cat] = catBuk + } + + buk, ok := catBuk[name] + if !ok { + buk = newBucket(cat, name, interval, count, + keepValue, a.uploadDataFn, a.gTags) + catBuk[name] = buk + + buk.startScan() + } + + buk.setExtraTag(constTags) +} + +func (a *AggBuckets) SetUploadFunc(fn UploadFunc) { + a.Lock() + defer a.Unlock() + a.uploadDataFn = fn +} + +func (a *AggBuckets) StopAllBukScanner() { + a.Lock() + defer a.Unlock() + + for _, catBuk := range a.data { + for name, b := range catBuk { + b.stopScan() + delete(catBuk, name) + } + } +} + +func (a *AggBuckets) GetBucket(cat point.Category, name string) (*bucket, bool) { + a.RLock() + defer a.RUnlock() + + if a.data == nil { + return nil, false + } + if buks, ok := a.data[cat]; !ok { + return nil, false + } else { + v, ok := buks[name] + return v, ok + } +} + +type aggFields struct { + tags []string + fields map[string]aggMetric +} + +type ptsGroup struct { + timeline map[uint64]*aggFields + + countLimit int +} + +func (g *ptsGroup) addMetric(tagsValue []string, name, action string, value any) bool { + if g.timeline == nil { + g.timeline = map[uint64]*aggFields{} + } + + tagsHash := hash.Fnv1aHash(tagsValue) + + agg, ok := g.timeline[tagsHash] + if !ok { + agg = &aggFields{ + tags: tagsValue, + fields: map[string]aggMetric{}, + } + g.timeline[tagsHash] = agg + } + + m, ok := agg.fields[name] + if !ok { + m, ok = NewAggMetric(name, action) + if !ok { + return false + } + agg.fields[name] = m + } + m.Append(value) + + return true +} + +type bucket struct { + bukName string + + category point.Category + + interval time.Duration + keepValue bool + countLimit int + curCount int + + // tagsNameHash: tagsName + by map[uint64][]string + // tagsNameHash: pts + group map[uint64]*ptsGroup + + extraTags map[string]string + globalTags map[string]string + + stop chan struct{} + + uploadFn UploadFunc + + sync.Mutex +} + +func (buk *bucket) startScan() { + if buk.stop != nil || buk.interval <= 0 { + return + } + + stop := make(chan struct{}) + buk.stop = stop + + go func() { + ticker := time.NewTicker(buk.interval) + defer ticker.Stop() + + defer func() { + if r := recover(); r != nil { + buf := make([]byte, 4096) //nolint:gomnd + buf = buf[:runtime.Stack(buf, false)] + + if e, ok := r.(error); ok { + buf = append([]byte(fmt.Sprintf("%s\n", e.Error())), buf...) + } + l.Error("%s", buf) + } + }() + + for { + select { + case <-ticker.C: + buk.Lock() + pts := endAgg(buk) + if len(pts) > 0 && buk.uploadFn != nil { + _ = buk.uploadFn(buk.category, buk.bukName, pts) + } + buk.Unlock() + case <-stop: + return + } + } + }() +} + +func (buk *bucket) stopScan() { + buk.Lock() + defer buk.Unlock() + + if buk.stop == nil { + return + } + close(buk.stop) + buk.stop = nil + + if buk.uploadFn != nil { + pts := endAgg(buk) + _ = buk.uploadFn(buk.category, buk.bukName, pts) + } +} + +func (buk *bucket) setExtraTag(extra map[string]string) { + buk.Lock() + defer buk.Unlock() + + buk.extraTags = extra +} + +func (buk *bucket) AddMetric(fieldName, action string, tagsName, + tagsValue []string, aggField any, +) bool { + tagNameHash := hash.Fnv1aHash(tagsName) + + buk.Lock() + defer buk.Unlock() + + if buk.by == nil { + buk.by = map[uint64][]string{} + } + + if buk.group == nil { + buk.group = map[uint64]*ptsGroup{} + } + + if _, ok := buk.by[tagNameHash]; !ok { + t := make([]string, len(tagsValue)) + copy(t, tagsName) + buk.by[tagNameHash] = t + } + + group, ok := buk.group[tagNameHash] + if !ok { + group = &ptsGroup{ + countLimit: buk.countLimit, + } + buk.group[tagNameHash] = group + } + + if ok := group.addMetric(tagsValue, fieldName, action, aggField); ok { + if buk.countLimit > 0 { + buk.curCount++ + if buk.curCount >= buk.countLimit { + if buk.uploadFn != nil { + pts := endAgg(buk) + _ = buk.uploadFn(buk.category, buk.bukName, pts) + } + buk.curCount = 0 + } + } + return true + } + + return false +} + +func newBucket(cat point.Category, name string, interval time.Duration, + count int, keepValue bool, uploadFn UploadFunc, gTags map[string]string, +) *bucket { + return &bucket{ + bukName: name, + interval: interval, + keepValue: keepValue, + countLimit: count, + by: map[uint64][]string{}, + group: map[uint64]*ptsGroup{}, + uploadFn: uploadFn, + category: cat, + globalTags: gTags, + } +} + +func conv2Pt(b *bucket, tagsName []string, aggTF *aggFields) (*point.Point, bool) { + if len(tagsName) != len(aggTF.tags) { + return nil, false + } + tags := map[string]string{} + + for k, v := range b.globalTags { + tags[k] = v + } + + for k, v := range b.extraTags { + tags[k] = v + } + + for i := 0; i < len(tagsName); i++ { + tags[tagsName[i]] = aggTF.tags[i] + } + + fields := map[string]any{} + for k, v := range aggTF.fields { + if v != nil { + fields[k] = v.Value() + } + } + + fieldsKV := point.NewTags(tags) + fieldsKV = append(fieldsKV, point.NewKVs(fields)...) + + var opt []point.Option + if len(b.globalTags) > 0 { + opt = append(opt, point.WithExtraTags(b.globalTags)) + } + opt = append(opt, utils.PtCatOption(b.category)...) + + pt := point.NewPointV2(b.bukName, fieldsKV, opt...) + return pt, true +} + +// 结束聚合. +func endAgg(b *bucket) []*point.Point { + pts := []*point.Point{} + + for tagNameHash, group := range b.group { + if group == nil { + continue + } + tagsName, ok := b.by[tagNameHash] + if !ok { + continue + } + for _, tl := range group.timeline { + if pt, ok := conv2Pt(b, tagsName, tl); ok { + pts = append(pts, pt) + } + } + } + + if !b.keepValue { + b.by = map[uint64][]string{} + b.group = map[uint64]*ptsGroup{} + } + + return pts +} + +type aggMetric interface { + Append(any) + Value() any +} + +func NewAggMetric(name, action string) (aggMetric, bool) { + switch action { + case "avg": + return &avgMetric{}, true + case "sum": + return &sumMetric{}, true + case "min": + return &minMetric{}, true + case "max": + return &maxMetric{}, true + case "set": + return &setMetric{}, true + default: + return nil, false + } +} + +type avgMetric struct { + sum float64 + count float64 +} + +func (f *avgMetric) Append(v any) { + f.sum += cast.ToFloat64(v) + f.count++ +} + +func (f *avgMetric) Value() any { + return f.sum / f.count +} + +type sumMetric struct { + sum float64 +} + +func (f *sumMetric) Append(v any) { + f.sum += cast.ToFloat64(v) +} + +func (f *sumMetric) Value() any { + return f.sum +} + +type minMetric struct { + inserted bool + min float64 +} + +func (f *minMetric) Append(v any) { + min := cast.ToFloat64(v) + + if f.inserted { + if f.min > min { + f.min = min + } + } else { + f.min = min + f.inserted = false + } +} + +func (f *minMetric) Value() any { + return f.min +} + +type maxMetric struct { + max float64 +} + +func (f *maxMetric) Append(v any) { + if max := cast.ToFloat64(v); f.max < max { + f.max = max + } +} + +func (f *maxMetric) Value() any { + return f.max +} + +type setMetric struct { + set float64 +} + +func (f *setMetric) Append(v any) { + f.set = cast.ToFloat64(v) +} + +func (f *setMetric) Value() any { + return f.set +} diff --git a/pkg/inimpl/guancecloud/ptinput/plmap/aggmap_test.go b/pkg/inimpl/guancecloud/ptinput/plmap/aggmap_test.go new file mode 100644 index 0000000..2afe82f --- /dev/null +++ b/pkg/inimpl/guancecloud/ptinput/plmap/aggmap_test.go @@ -0,0 +1,146 @@ +// Unless explicitly stated otherwise all files in this repository are licensed +// under the MIT License. +// This product includes software developed at Guance Cloud (https://www.guance.com/). +// Copyright 2021-present Guance, Inc. + +package plmap + +import ( + "testing" + "time" + + "github.com/GuanceCloud/cliutils/point" + "github.com/stretchr/testify/assert" +) + +func TestAggBuckets(t *testing.T) { + ptsLi := map[string][]*point.Point{} + var fn UploadFunc = func(cat point.Category, n string, d any) error { + ptsLi[n] = append(ptsLi[n], d.([]*point.Point)...) + return nil + } + + buks := NewAggBuks(fn, nil) + buks.CreateBucket(point.Metric, "bucket_a", time.Second*5, 0, false, nil) + buks.CreateBucket(point.Metric, "bucket_a", time.Second, 0, false, nil) + buks.CreateBucket(point.Metric, "bucket_a", time.Second, 0, false, nil) + buks.CreateBucket(point.Metric, "bucket_b", time.Second, 0, false, nil) + + v, ok := buks.GetBucket(point.Metric, "bucket_a") + assert.NotEqual(t, nil, v) + assert.Equal(t, true, ok) + + if v, ok := buks.GetBucket(point.Metric, "bucket_b"); ok { + assert.NotEqual(t, nil, v) + v.stopScan() + } else { + assert.Equal(t, true, ok) + } + v.AddMetric("f1", "avg", []string{"t1"}, []string{"t1_val"}, 1) + v.AddMetric("f1", "avg", []string{"t1"}, []string{"t1_val"}, 2) + v.AddMetric("f1", "avg", []string{"t1"}, []string{"t1_val"}, 3) + + buks.StopAllBukScanner() + time.Sleep(time.Millisecond * 10) +} + +func TestAggBuckets2(t *testing.T) { + ptsLi := map[point.Category]map[string][]*point.Point{} + var fn UploadFunc = func(cat point.Category, n string, d any) error { + if ptsLi[cat] == nil { + ptsLi[cat] = map[string][]*point.Point{} + } + + ptsLi[cat][n] = append(ptsLi[cat][n], d.([]*point.Point)...) + return nil + } + + buks := NewAggBuks(fn, nil) + buks.CreateBucket(point.Metric, "bucket_a", time.Second*5, 0, false, nil) + buks.CreateBucket(point.Metric, "bucket_a", 0, 2, false, nil) + buks.CreateBucket(point.Metric, "bucket_a", 0, 2, false, nil) + buks.CreateBucket(point.Metric, "bucket_b", 0, 2, false, nil) + buks.CreateBucket(point.Logging, "bucket_b", 0, 2, false, nil) + + v, ok := buks.GetBucket(point.Metric, "bucket_a") + assert.NotEqual(t, nil, v) + assert.Equal(t, true, ok) + + v.AddMetric("f1", "avg", []string{"t1"}, []string{"t1_val"}, 1) + v.AddMetric("f1", "avg", []string{"t1"}, []string{"t1_val"}, 2) + v.AddMetric("f1", "avg", []string{"t1"}, []string{"t1_val"}, 3) + + if v, ok := buks.GetBucket(point.Metric, "bucket_b"); ok { + assert.NotEqual(t, nil, v) + + v.AddMetric("f1", "avg", []string{"t1"}, []string{"t1_val"}, 1) + v.AddMetric("f1", "avg", []string{"t1"}, []string{"t1_val"}, 2) + v.AddMetric("f1", "avg", []string{"t1"}, []string{"t1_val"}, 3) + v.stopScan() + } else { + assert.Equal(t, true, ok) + } + + if v, ok := buks.GetBucket(point.Logging, "bucket_b"); ok { + assert.NotEqual(t, nil, v) + + v.AddMetric("f1", "avg", []string{"t1"}, []string{"t1_val"}, 1) + v.AddMetric("f1", "avg", []string{"t1"}, []string{"t1_val"}, 2) + v.AddMetric("f1", "avg", []string{"t1"}, []string{"t1_val"}, 3) + v.stopScan() + } else { + assert.Equal(t, true, ok) + } + + buks.StopAllBukScanner() + + time.Sleep(time.Millisecond * 100) +} + +func TestAggMetric(t *testing.T) { + cases := []struct { + action string + d []any + o any + failed bool + }{ + { + action: "avg", + d: []any{1, 2, 1, 2}, + o: 1.5, + }, + { + action: "sum", + d: []any{1, 2, 1, 2}, + o: 6.0, + }, + { + action: "min", + d: []any{1, 2, 3, 5, 1, 2, 1}, + o: 1.0, + }, + { + action: "max", + d: []any{1, 2, 1, 2}, + o: 2.0, + }, + { + action: "set", + d: []any{1, 2, 1, 2, -1}, + o: -1.0, + }, + } + + for _, tc := range cases { + t.Run(tc.action, func(t *testing.T) { + v, ok := NewAggMetric("", tc.action) + assert.Equal(t, !tc.failed, ok) + if v != nil { + for _, d := range tc.d { + v.Append(d) + } + } + assert.Equal(t, tc.o, v.Value()) + }) + } +} diff --git a/pkg/inimpl/guancecloud/ptinput/plmap/manager.go b/pkg/inimpl/guancecloud/ptinput/plmap/manager.go new file mode 100644 index 0000000..027f346 --- /dev/null +++ b/pkg/inimpl/guancecloud/ptinput/plmap/manager.go @@ -0,0 +1,19 @@ +// Unless explicitly stated otherwise all files in this repository are licensed +// under the MIT License. +// This product includes software developed at Guance Cloud (https://www.guance.com/). +// Copyright 2021-present Guance, Inc. + +// Package plmap used to store data during pl script execution +package plmap + +import "github.com/GuanceCloud/cliutils/point" + +// type PtSource [3]string // source,sub_source,tmp_source + +const FeedName = "pl_agg" + +type UploadFunc func(cat point.Category, ptID string, elem any) error + +type PlMapIface interface { + SetUploadFunc(UploadFunc) +} diff --git a/pkg/inimpl/guancecloud/ptinput/point.go b/pkg/inimpl/guancecloud/ptinput/point.go new file mode 100644 index 0000000..821746a --- /dev/null +++ b/pkg/inimpl/guancecloud/ptinput/point.go @@ -0,0 +1,319 @@ +// Unless explicitly stated otherwise all files in this repository are licensed +// under the MIT License. +// This product includes software developed at Guance Cloud (https://www.guance.com/). +// Copyright 2021-present Guance, Inc. + +// Package ptinput impl ppl input interface +package ptinput + +import ( + "fmt" + "time" + + "github.com/GuanceCloud/cliutils/point" + "github.com/GuanceCloud/platypus/pkg/ast" + plruntime "github.com/GuanceCloud/platypus/pkg/engine/runtime" + "github.com/GuanceCloud/platypus/pkg/inimpl/guancecloud/ptinput/plmap" + "github.com/GuanceCloud/platypus/pkg/inimpl/guancecloud/ptinput/utils" + + "github.com/spf13/cast" +) + +type ( + KeyKind uint + PtFlag uint +) + +var _ PlInputPt = (*PlPoint)(nil) + +type PlInputPt interface { + GetPtName() string + SetPtName(m string) + + Get(key string) (any, ast.DType, error) + GetWithIsTag(key string) (any, bool, bool) + Set(key string, value any, dtype ast.DType) error + Delete(key string) + RenameKey(from, to string) error + + SetTag(key string, value any, dtype ast.DType) error + + PtTime() time.Time + + GetAggBuckets() *plmap.AggBuckets + SetAggBuckets(*plmap.AggBuckets) + + AppendSubPoint(PlInputPt) + GetSubPoint() []PlInputPt + Category() point.Category + + KeyTime2Time() + + MarkDrop(bool) + Dropped() bool + Point() *point.Point + + Tags() map[string]string + Fields() map[string]any +} + +const ( + PtMeasurement PtFlag = iota + PtTag + PtField + PtTFDefaulutOrKeep + PtTime +) + +const ( + KindPtDefault KeyKind = iota + KindPtTag +) + +const Originkey = "message" + +type InputWithVarbMapRW interface { + Get(key string) (any, ast.DType, bool) + Set(key string, value any, dtype ast.DType) bool + Delete(key string) bool +} + +type InputWithVarbMapR interface { + Get(key string) (any, ast.DType, bool) +} + +type InputWithoutVarbMap interface{} + +type DoFeedCache func(name, category string, pt *point.Point) error + +var DoFeedNOP = func(name, category string, pt *point.Point) error { return nil } + +type PlmapManager interface { + // createPtCaheMap(category string, source PtSource) (*fucs.PtCacheMap, bool) +} + +type PlPoint struct { + name string + tags map[string]string + fields map[string]any // int, float, bool, string, map, slice, array + time time.Time + + aggBuckets *plmap.AggBuckets + + subPlpt []PlInputPt + + category point.Category + drop bool +} + +func NewPlPoint(category point.Category, name string, + tags map[string]string, fields map[string]any, ptTime time.Time, +) PlInputPt { + if tags == nil { + tags = map[string]string{} + } + + if fields == nil { + fields = map[string]any{} + } + + dPt := &PlPoint{ + name: name, + tags: tags, + fields: fields, + time: ptTime, + category: category, + } + return dPt +} + +func valueDtype(v any) (any, ast.DType) { + switch v := v.(type) { + case int32, int8, int16, int, + uint, uint16, uint32, uint64, uint8: + return cast.ToInt64(v), ast.Int + case int64: + return v, ast.Int + case float32: + return cast.ToFloat64(v), ast.Float + case float64: + return v, ast.Float + case bool: + return v, ast.Bool + case []byte: + return string(v), ast.String + case string: + return v, ast.String + } + + // ignore unknown type + return nil, ast.Nil +} + +func (pt *PlPoint) GetPtName() string { + return pt.name +} + +func (pt *PlPoint) SetPtName(m string) { + pt.name = m +} + +func (pt *PlPoint) AppendSubPoint(plpt PlInputPt) { + pt.subPlpt = append(pt.subPlpt, plpt) +} + +func (pt *PlPoint) GetSubPoint() []PlInputPt { + return pt.subPlpt +} + +func (pt *PlPoint) Category() point.Category { + return pt.category +} + +func (pt *PlPoint) Get(key string) (any, ast.DType, error) { + if v, ok := pt.tags[key]; ok { + return v, ast.String, nil + } + + if v, ok := pt.fields[key]; ok { + v, dtype := valueDtype(v) + return v, dtype, nil + } + return nil, ast.Invalid, fmt.Errorf("unsupported pt key type") +} + +func (pt *PlPoint) GetWithIsTag(key string) (any, bool, bool) { + if v, ok := pt.tags[key]; ok { + return v, true, true + } + + if v, ok := pt.fields[key]; ok { + v, _ := valueDtype(v) + return v, false, true + } + return nil, false, false +} + +func (pt *PlPoint) Set(key string, value any, dtype ast.DType) error { + if _, ok := pt.tags[key]; ok { // is tag + if dtype == ast.Void || dtype == ast.Invalid { + delete(pt.tags, key) + return nil + } + if v, err := plruntime.Conv2String(value, dtype); err == nil { + pt.tags[key] = v + return nil + } else { + return err + } + } else { // is field + switch dtype { //nolint:exhaustive + case ast.Nil, ast.Void, ast.Invalid: + pt.fields[key] = nil + return nil + case ast.List, ast.Map: + if v, err := plruntime.Conv2String(value, dtype); err == nil { + pt.fields[key] = v + } else { + pt.fields[key] = nil + return nil + } + default: + pt.fields[key] = value + } + } + return nil +} + +func (pt *PlPoint) Delete(key string) { + if _, ok := pt.tags[key]; ok { + delete(pt.tags, key) + } else { + delete(pt.fields, key) + } +} + +func (pt *PlPoint) RenameKey(from, to string) error { + if v, ok := pt.fields[from]; ok { + pt.fields[to] = v + delete(pt.fields, from) + } else if v, ok := pt.tags[from]; ok { + pt.tags[to] = v + delete(pt.tags, from) + } else { + return fmt.Errorf("key(from) %s not found", from) + } + return nil +} + +func (pt *PlPoint) SetTag(key string, value any, dtype ast.DType) error { + delete(pt.fields, key) + + if str, err := plruntime.Conv2String(value, dtype); err == nil { + pt.tags[key] = str + return nil + } else { + pt.tags[key] = "" + return err + } +} + +func (pt *PlPoint) PtTime() time.Time { + return pt.time +} + +func (pt *PlPoint) GetAggBuckets() *plmap.AggBuckets { + return pt.aggBuckets +} + +func (pt *PlPoint) SetAggBuckets(buks *plmap.AggBuckets) { + pt.aggBuckets = buks +} + +func (pt *PlPoint) KeyTime2Time() { + if v, _, err := pt.Get("time"); err == nil { + if nanots, ok := v.(int64); ok { + t := time.Unix(nanots/int64(time.Second), + nanots%int64(time.Second)) + if !t.IsZero() { + pt.time = t + } + } + pt.Delete("time") + } +} + +func (pt *PlPoint) MarkDrop(drop bool) { + pt.drop = drop +} + +func (pt *PlPoint) Dropped() bool { + return pt.drop +} + +func (pt *PlPoint) Tags() map[string]string { + return pt.tags +} + +func (pt *PlPoint) Fields() map[string]any { + return pt.fields +} + +func (pt *PlPoint) Point() *point.Point { + opt := utils.PtCatOption(pt.category) + + fieldsKVS := point.NewTags(pt.tags) + fieldsKVS = append(fieldsKVS, point.NewKVs(pt.fields)...) + return point.NewPointV2(pt.name, fieldsKVS, opt...) +} + +func WrapPoint(cat point.Category, pt *point.Point) PlInputPt { + if pt == nil { + return nil + } + + tags := pt.InfluxTags() + fields := pt.InfluxFields() + + return NewPlPoint(cat, pt.Name(), tags, fields, pt.Time()) +} diff --git a/pkg/inimpl/guancecloud/ptinput/point_test.go b/pkg/inimpl/guancecloud/ptinput/point_test.go new file mode 100644 index 0000000..2620157 --- /dev/null +++ b/pkg/inimpl/guancecloud/ptinput/point_test.go @@ -0,0 +1,120 @@ +// Unless explicitly stated otherwise all files in this repository are licensed +// under the MIT License. +// This product includes software developed at Guance Cloud (https://www.guance.com/). +// Copyright 2021-present Guance, Inc. + +package ptinput + +import ( + "testing" + "time" + + "github.com/GuanceCloud/cliutils/point" + "github.com/GuanceCloud/platypus/pkg/ast" +) + +func TestPt(t *testing.T) { + pt := NewPlPoint(point.Logging, "t", nil, nil, time.Now()) + if _, _, err := pt.Get("a"); err == nil { + t.Fatal("err == nil") + } + + if _, _, ok := pt.GetWithIsTag("a"); ok { + t.Fatal("ok") + } + + if err := pt.Set("a", 1, ast.Int); err != nil { + t.Fatal(err) + } + + if err := pt.Set("a1", []any{1}, ast.List); err != nil { + t.Fatal(err) + } + + if err := pt.Set("xx2", []any{1}, ast.List); err != nil { + t.Fatal(err) + } + + if err := pt.Set("xx2", 1.2, ast.Float); err != nil { + t.Fatal(err) + } + + if _, _, err := pt.Get("xx2"); err != nil { + t.Fatal(err) + } + + if err := pt.RenameKey("xx2", "xxb"); err != nil { + t.Fatal(err) + } + + if err := pt.SetTag("a", 1., ast.Float); err != nil { + t.Fatal(err) + } + + if err := pt.Set("a", 1, ast.Int); err != nil { + t.Fatal(err) + } + + if _, ok := pt.Fields()["a"]; ok { + t.Fatal("a in fields") + } + + if err := pt.RenameKey("a", "b"); err != nil { + t.Fatal(err) + } + + if pt.PtTime().UnixNano() == 0 { + t.Fatal("time == 0") + } + + pt.GetAggBuckets() + pt.SetAggBuckets(nil) + + pt.Set("time", 1, ast.Int) + pt.KeyTime2Time() + if pt.PtTime().UnixNano() != 1 { + t.Fatal("time != 1") + } + + pt.MarkDrop(true) + if !pt.Dropped() { + t.Fatal("!dropped") + } + + dpt := pt.Point() + + pt = WrapPoint(point.Logging, dpt) + + if _, _, err := pt.Get("b"); err != nil { + t.Fatal(err.Error()) + } + + if _, ok := pt.Tags()["b"]; !ok { + t.Fatal("b not in tags") + } + + if _, istag, ok := pt.GetWithIsTag("b"); !ok || !istag { + t.Fatal("not tag") + } + + if err := pt.Set("b", []any{}, ast.List); err != nil { + t.Fatal(err) + } + + if _, istag, ok := pt.GetWithIsTag("a1"); !ok || istag { + t.Fatal("is tag") + } + + if _, ok := pt.Fields()["xxb"]; !ok { + t.Fatal("xxb not in field") + } + + if pt.GetPtName() != "t" { + t.Fatal("name != \"t\"") + } + + pt.SetPtName("t2") + if pt.GetPtName() != "t2" { + t.Fatal("name != \"t2\"") + } +} diff --git a/pkg/inimpl/guancecloud/ptinput/refertable/cli.go b/pkg/inimpl/guancecloud/ptinput/refertable/cli.go new file mode 100644 index 0000000..2dce30a --- /dev/null +++ b/pkg/inimpl/guancecloud/ptinput/refertable/cli.go @@ -0,0 +1,51 @@ +// Unless explicitly stated otherwise all files in this repository are licensed +// under the MIT License. +// This product includes software developed at Guance Cloud (https://www.guance.com/). +// Copyright 2021-present Guance, Inc. + +package refertable + +import ( + "net/http" + "time" + + "github.com/hashicorp/go-retryablehttp" +) + +type cliLogger struct{} + +func (*cliLogger) Error(msg string, kv ...any) { + l.Error(msg, kv) +} + +func (*cliLogger) Warn(msg string, kv ...any) { + l.Warn(msg, kv) +} + +func (*cliLogger) Info(msg string, kv ...any) { + l.Info(msg, kv) +} + +func (*cliLogger) Debug(msg string, kv ...any) { + l.Debug(msg, kv) +} + +func newRetryCli(client *http.Client, timeout time.Duration) *retryablehttp.Client { + cli := retryablehttp.NewClient() + + cli.RetryMax = 3 + cli.RetryWaitMin = time.Second + cli.RetryWaitMax = time.Second * 5 + + cli.RequestLogHook = func(_ retryablehttp.Logger, r *http.Request, n int) { + if n > 0 { + l.Warnf("retry %d time on API %s", n, r.URL.Path) + } + } + + cli.HTTPClient = client + cli.HTTPClient.Timeout = timeout + cli.Logger = &cliLogger{} + + return cli +} diff --git a/pkg/inimpl/guancecloud/ptinput/refertable/query.go b/pkg/inimpl/guancecloud/ptinput/refertable/query.go new file mode 100644 index 0000000..dfea202 --- /dev/null +++ b/pkg/inimpl/guancecloud/ptinput/refertable/query.go @@ -0,0 +1,111 @@ +// Unless explicitly stated otherwise all files in this repository are licensed +// under the MIT License. +// This product includes software developed at Guance Cloud (https://www.guance.com/). +// Copyright 2021-present Guance, Inc. + +package refertable + +import ( + "math" +) + +func query(index map[string]map[any][]int, + keys []string, values []any, count int, +) ([]int, bool) { + if len(keys) != len(values) { + return nil, false + } + + // index 数据索引递增排序 + tmp := [][]int{} + + // key 中结果数量最少的 + var startDataIdxs []int + + headerDataIdx := 0 + tailDataIdx := math.MaxInt + + for kIdx, key := range keys { + // value map + m, ok := index[key] + if !ok { + return nil, false + } + // value -> index + dIdxs := m[values[kIdx]] + lenDIdxs := len(dIdxs) + switch lenDIdxs { + case 0: + return nil, false + default: + // max header + if dIdxs[0] > headerDataIdx { + headerDataIdx = dIdxs[0] + } + // min tail + if dIdxs[lenDIdxs-1] < tailDataIdx { + tailDataIdx = dIdxs[lenDIdxs-1] + } + + if len(startDataIdxs) == 0 { + startDataIdxs = dIdxs + break + } + if len(dIdxs) < len(startDataIdxs) { + tmp = append(tmp, startDataIdxs) + startDataIdxs = dIdxs + } else { + tmp = append(tmp, dIdxs) + } + } + } + + var ret []int + for _, idx := range startDataIdxs { + // 取相交段 + if idx < headerDataIdx { + continue + } + if idx > tailDataIdx { + break + } + + flag := true + // 取相交项 + for _, idxs := range tmp { + if _, ok := binSearch(idxs, idx); !ok { + flag = false + break + } + } + if flag { + ret = append(ret, idx) + // count = 0, 取全部结果 + if len(ret) == count { + break + } + } + } + if len(ret) > 0 { + return ret, true + } + return nil, false +} + +func binSearch(li []int, s int) (int, bool) { + start := 0 + end := len(li) - 1 + for start <= end { + mid := (start + end) / 2 + v := li[mid] + switch { + case s == v: + return mid, true + case s > v: + start = mid + 1 + case s < v: + end = mid - 1 + } + } + return 0, false +} diff --git a/pkg/inimpl/guancecloud/ptinput/refertable/query_test.go b/pkg/inimpl/guancecloud/ptinput/refertable/query_test.go new file mode 100644 index 0000000..de51b5d --- /dev/null +++ b/pkg/inimpl/guancecloud/ptinput/refertable/query_test.go @@ -0,0 +1,196 @@ +// Unless explicitly stated otherwise all files in this repository are licensed +// under the MIT License. +// This product includes software developed at Guance Cloud (https://www.guance.com/). +// Copyright 2021-present Guance, Inc. + +package refertable + +import ( + "fmt" + "testing" + + "github.com/stretchr/testify/assert" +) + +func TestBinSearch(t *testing.T) { + cases := []struct { + li []int + v int + ok bool + idx int + }{ + { + []int{1, 2, 5, 6}, + 2, + true, + 1, + }, + { + []int{1, 2, 5, 6}, + 5, + true, + 2, + }, + { + []int{1, 2, 5, 6}, + 1, + true, + 0, + }, + { + []int{1, 2, 5, 6}, + 6, + true, + 3, + }, + { + []int{1, 2, 5, 6, 11}, + 2, + true, + 1, + }, + { + []int{1, 2, 5, 6, 11}, + 11, + true, + 4, + }, + { + []int{1}, + 6, + false, + 0, + }, + { + []int{1}, + 1, + true, + 0, + }, + } + + for _, v := range cases { + if idx, ok := binSearch(v.li, v.v); ok { + assert.Equal(t, v.ok, ok) + assert.Equal(t, v.idx, idx) + } else { + assert.Equal(t, v.ok, ok) + } + } +} + +func TestQuery(t *testing.T) { + cases := []struct { + index map[string]map[any][]int + keys []string + values []any + count int + ok bool + ret []int + }{ + { + index: map[string]map[any][]int{ + "key1": { + "1": {1, 3, 5, 11, 18}, + "3": {2, 4, 6, 8, 12}, + }, + "key2": { + 1: {1, 2, 5, 8, 12, 17}, + 2: {3, 4, 6, 11}, + }, + }, + keys: []string{"key2", "key1"}, + values: []any{1, "1"}, + count: 0, + ok: true, + ret: []int{1, 5}, + }, + { + index: map[string]map[any][]int{ + "key1": { + "1": {3, 5, 11, 18}, + "3": {2, 4, 6, 8, 12}, + }, + "key2": { + 1: {1, 2, 3, 5, 8, 12, 17}, + 2: {4, 6, 11}, + }, + "key3": { + 1.1: {2, 3, 5}, + 2.2: {1, 4, 6, 7}, + }, + }, + keys: []string{"key3", "key2", "key1"}, + values: []any{1.1, 1, "1"}, + count: 1, + ok: true, + ret: []int{3}, + }, + { + index: map[string]map[any][]int{}, + keys: []string{"key5", "key2", "key1"}, + values: []any{1.1, 1, "1"}, + count: 1, + ok: false, + ret: []int{3}, + }, + { + index: map[string]map[any][]int{ + "key1": {}, + "key3": { + 1.1: {2, 3, 5}, + 2.2: {1, 4, 6, 7}, + }, + }, + keys: []string{"key3", "key1"}, + values: []any{1.1, "1"}, + count: 1, + ok: false, + ret: []int{3}, + }, + { + index: map[string]map[any][]int{ + "key1": { + "1": {1, 3, 5, 11, 18}, + "3": {2, 4, 6, 8, 12}, + }, + "key2": { + 1: {2, 8, 12, 17}, + 2: {3, 4, 6, 11}, + }, + }, + keys: []string{"key2", "key1"}, + values: []any{1, "1"}, + count: 0, + ok: false, + ret: []int{1, 5}, + }, + { + index: map[string]map[any][]int{ + "key1": { + "1": {1, 3, 5, 11, 18}, + "3": {2, 4, 6, 8, 12}, + }, + "key2": { + 1: {2, 8, 12, 17}, + 2: {3, 4, 6, 11}, + }, + }, + keys: []string{"key2", "key1"}, + values: []any{1}, + count: 0, + ok: false, + ret: []int{1, 5}, + }, + } + for idx, v := range cases { + t.Run(fmt.Sprint("index_", idx), func(t *testing.T) { + if ret, ok := query(v.index, v.keys, v.values, v.count); ok { + assert.Equal(t, v.ok, ok) + assert.Equal(t, v.ret, ret) + } else { + assert.Equal(t, v.ok, ok) + } + }) + } +} diff --git a/pkg/inimpl/guancecloud/ptinput/refertable/runner.go b/pkg/inimpl/guancecloud/ptinput/refertable/runner.go new file mode 100644 index 0000000..7a4d6eb --- /dev/null +++ b/pkg/inimpl/guancecloud/ptinput/refertable/runner.go @@ -0,0 +1,232 @@ +// Unless explicitly stated otherwise all files in this repository are licensed +// under the MIT License. +// This product includes software developed at Guance Cloud (https://www.guance.com/). +// Copyright 2021-present Guance, Inc. + +package refertable + +import ( + "context" + "database/sql" + "fmt" + "io" + "net" + "net/http" + "net/url" + "strings" + "time" + + "github.com/GuanceCloud/cliutils/logger" + "github.com/hashicorp/go-retryablehttp" +) + +var ( + _plReferTables PlReferTables + _runner = &Runner{ + initFinished: make(chan struct{}), + } + + l = logger.DefaultSLogger("refer-table") +) + +const ( + SchemeHTTP = "http" + SchemeHTTPS = "https" +) + +func QueryReferTable(tableName string, colName []string, colValue []any, + selected []string, +) (map[string]any, bool) { + defer func() { + if err := recover(); err != nil { + l.Error(fmt.Errorf("run pl: %s", err)) + } + }() + + if _plReferTables == nil { + return nil, false + } + + return _plReferTables.query(tableName, colName, colValue, selected) +} + +func Stats() *ReferTableStats { + if _plReferTables == nil { + return nil + } + return _plReferTables.stats() +} + +func InitFinished(interval time.Duration) bool { + return _runner.InitFinished(interval) +} + +type InConfig struct { + URL string `toml:"url"` + Interval time.Duration `toml:"interval"` +} + +type Runner struct { + inConfig InConfig + + cli *retryablehttp.Client + + initFinished chan struct{} +} + +func (r *Runner) InitFinished(interval time.Duration) bool { + ticker := time.NewTicker(interval) + + if r.initFinished == nil { + return false + } + + select { + case <-r.initFinished: + return true + case <-ticker.C: + return false + } +} + +func InitLog() { + l = logger.SLogger("refer-table") +} + +func InitReferTableRunner(ctx context.Context, tableURL, dbPath string, interval time.Duration, useSQLite bool, sqliteMemMode bool) error { + if useSQLite { + if sqliteMemMode { + l.Infof("using in-memory SQLite for refer-table") + d, err := sql.Open("sqlite", ":memory:") + if err != nil { + return fmt.Errorf("open in-memory SQLite failed: %w", err) + } + _plReferTables = &PlReferTablesSqlite{db: d} + } else { + l.Infof("using on-disk SQLite for refer-table") + d, err := sql.Open("sqlite", dbPath) + if err != nil { + return fmt.Errorf("open SQLite at %s failed: %w", dbPath, err) + } + _plReferTables = &PlReferTablesSqlite{db: d} + } + } else { + l.Infof("using memory mode for refer-table") + _plReferTables = &PlReferTablesInMemory{} + } + return initReferTableRunner(ctx, _runner, _plReferTables, tableURL, interval) +} + +func initReferTableRunner(ctx context.Context, runner *Runner, plRefTables PlReferTables, tableURL string, interval time.Duration) error { + if tableURL == "" { + return nil + } + if runner == nil { + return fmt.Errorf("runner == nil") + } + + if interval < time.Second*10 { + interval = time.Second * 10 + } + + runner.inConfig.Interval = interval + + scheme, err := checkURL(tableURL) + if err != nil { + l.Error(err) + return err + } + + runner.inConfig.URL = tableURL + + switch scheme { + case SchemeHTTP, SchemeHTTPS: + cli := http.Client{ + Transport: &http.Transport{ + DialContext: (&net.Dialer{ + Timeout: time.Second * 30, + KeepAlive: time.Second * 90, + }).DialContext, + MaxIdleConns: 100, + MaxConnsPerHost: 64, + IdleConnTimeout: time.Second * 90, + TLSHandshakeTimeout: time.Second * 10, + ExpectContinueTimeout: time.Second, + }, + } + runner.cli = newRetryCli(&cli, time.Minute) + go httpGetWkr(ctx, plRefTables, runner) + } + + return nil +} + +func checkURL(tableURL string) (string, error) { + u, err := url.Parse(tableURL) + if err != nil { + return "", fmt.Errorf("parse url: %s, error: %w", + tableURL, err) + } + scheme := strings.ToLower(u.Scheme) + switch scheme { + case SchemeHTTP, SchemeHTTPS: + default: + return "", fmt.Errorf("url: %s, unsupported scheme %s", + tableURL, scheme) + } + return scheme, nil +} + +func httpGetWkr(ctx context.Context, plRefTables PlReferTables, runner *Runner) { + ticker := time.NewTicker(runner.inConfig.Interval) + for { + getAndUpdate(plRefTables, runner) + select { + case <-ticker.C: + case <-ctx.Done(): + return + } + } +} + +func getAndUpdate(plRefTables PlReferTables, runner *Runner) { + if tables, err := httpGet(runner.cli, runner.inConfig.URL); err != nil { + l.Errorf("get table data from URL: %v", err) + } else { + if err := plRefTables.updateAll(tables); err != nil { + l.Errorf("failed to update tables: %v", err) + } + } + + select { + case <-runner.initFinished: + default: + if runner.initFinished != nil { + close(runner.initFinished) + } + } +} + +func httpGet(cli *retryablehttp.Client, url string) ([]referTable, error) { + resp, err := cli.Get(url) + defer func() { _ = resp.Body.Close() }() + if err != nil { + return nil, err + } + + if resp.StatusCode != http.StatusOK { + return nil, fmt.Errorf("url: %s, status: %s", url, resp.Status) + } + + data, err := io.ReadAll(resp.Body) + if err != nil { + return nil, err + } + + tables, err := decodeJSONData(data) + if err != nil { + return nil, err + } + + return tables, nil +} diff --git a/pkg/inimpl/guancecloud/ptinput/refertable/runner_test.go b/pkg/inimpl/guancecloud/ptinput/refertable/runner_test.go new file mode 100644 index 0000000..e7f1b48 --- /dev/null +++ b/pkg/inimpl/guancecloud/ptinput/refertable/runner_test.go @@ -0,0 +1,118 @@ +// Unless explicitly stated otherwise all files in this repository are licensed +// under the MIT License. +// This product includes software developed at Guance Cloud (https://www.guance.com/). +// Copyright 2021-present Guance, Inc. + +package refertable + +import ( + "context" + "net/http" + "net/http/httptest" + "testing" + "time" + + "github.com/stretchr/testify/assert" +) + +func TestCheckUrl(t *testing.T) { + cases := []struct { + url string + scheme string + fail bool + }{ + { + url: "ht\tp//localss(", + scheme: "http", + fail: true, + }, + { + url: "httpS://localss(", + scheme: "https", + fail: true, + }, + { + url: "https://localhost/aa?a", + scheme: "https", + }, + { + url: "http://localhost/aa?a", + scheme: "http", + }, + { + url: "oss://localhost/aa?a", + scheme: "oss", + fail: true, + }, + } + + for _, v := range cases { + scheme, err := checkURL(v.url) + if err != nil { + if !v.fail { + t.Error(err) + } + continue + } + assert.Equal(t, v.scheme, scheme) + } +} + +func TestRunner(t *testing.T) { + files := map[string]string{ + "a.json": testTableData, + } + server := newJSONDataServer(files) + defer server.Close() + + url := server.URL + + plReferTable := &PlReferTablesInMemory{} + runner := &Runner{ + initFinished: make(chan struct{}), + } + + ok := runner.InitFinished(time.Nanosecond) + if ok { + t.Errorf("func InitFinished exp: false, act %v", ok) + } + + ctx := context.Background() + ctx, cancel := context.WithCancel(ctx) + defer cancel() + initReferTableRunner(ctx, runner, plReferTable, url+"?name=a.json", time.Second*5) + + ok = runner.InitFinished(time.Second) + if !ok { + t.Fatal("init not finishd") + } + + v, ok := plReferTable.query("table1", []string{"key1"}, []any{"a"}, nil) + if !ok || len(v) == 0 { + t.Error("!ok") + } + + stats := plReferTable.stats() + assert.Equal(t, stats.Name[0], "table1") + assert.Equal(t, stats.Name[1], "table2") + assert.Equal(t, stats.Row[0], 3) + assert.Equal(t, stats.Row[1], 2) +} + +func newJSONDataServer(files map[string]string) *httptest.Server { + server := httptest.NewServer(http.HandlerFunc( + func(w http.ResponseWriter, r *http.Request) { + switch r.Method { + case http.MethodGet: + default: + w.WriteHeader(http.StatusBadRequest) + return + } + name := r.FormValue("name") + data := files[name] + w.Write([]byte(data)) + w.WriteHeader(http.StatusOK) + }, + )) + return server +} diff --git a/pkg/inimpl/guancecloud/ptinput/refertable/table.go b/pkg/inimpl/guancecloud/ptinput/refertable/table.go new file mode 100644 index 0000000..04d9115 --- /dev/null +++ b/pkg/inimpl/guancecloud/ptinput/refertable/table.go @@ -0,0 +1,236 @@ +// Unless explicitly stated otherwise all files in this repository are licensed +// under the MIT License. +// This product includes software developed at Guance Cloud (https://www.guance.com/). +// Copyright 2021-present Guance, Inc. + +// Package refertable for saving external data +package refertable + +import ( + "encoding/json" + "fmt" + "strings" + "sync" + + "github.com/spf13/cast" +) + +const ( + columnTypeStr = "string" + columnTypeInt = "int" + columnTypeFloat = "float" + columnTypeBool = "bool" +) + +type PlReferTables interface { + query(tableName string, colName []string, colValue []any, kGet []string) (map[string]any, bool) + updateAll(tables []referTable) (retErr error) + stats() *ReferTableStats +} + +type PlReferTablesInMemory struct { + tables map[string]*referTable + tablesName []string + updateMutex sync.Mutex + queryRWmutex sync.RWMutex +} + +type ReferTableStats struct { + Name []string + Row []int +} + +func (plrefer *PlReferTablesInMemory) query(tableName string, colName []string, colValue []any, + kGet []string, +) (map[string]any, bool) { + plrefer.queryRWmutex.RLock() + defer plrefer.queryRWmutex.RUnlock() + + var table *referTable + + if tableName == "" { + return nil, false + } + + table = plrefer.tables[tableName] + if table == nil { + return nil, false + } + + var row []any + if allRow, ok := query(table.index, colName, colValue, 1); ok { + row = table.RowData[allRow[0]] + } else { + return nil, false + } + + result := map[string]any{} + + if len(kGet) != 0 { + for _, key := range kGet { + cIdx, ok := table.colIndex[key] + if !ok { + continue + } + result[key] = row[cIdx] + } + } else { + for k, v := range table.colIndex { + result[k] = row[v] + } + } + + return result, true +} + +func (plrefer *PlReferTablesInMemory) updateAll(tables []referTable) (retErr error) { + defer func() { + if err := recover(); err != nil { + retErr = fmt.Errorf("run pl: %s", err) + } + }() + + plrefer.updateMutex.Lock() + defer plrefer.updateMutex.Unlock() + + refTableMap := map[string]*referTable{} + tablesName := []string{} + for idx := range tables { + table := tables[idx] + if err := table.buildTableIndex(); err != nil { + return err + } + if _, ok := refTableMap[table.TableName]; !ok { + refTableMap[table.TableName] = &table + tablesName = append(tablesName, table.TableName) + } + } + + plrefer.queryRWmutex.Lock() + defer plrefer.queryRWmutex.Unlock() + plrefer.tables = refTableMap + plrefer.tablesName = tablesName + return nil +} + +func (plrefer *PlReferTablesInMemory) stats() *ReferTableStats { + plrefer.queryRWmutex.RLock() + defer plrefer.queryRWmutex.RUnlock() + + tableStats := ReferTableStats{} + for _, name := range plrefer.tablesName { + tableStats.Name = append(tableStats.Name, name) + + tableStats.Row = append(tableStats.Row, + len(plrefer.tables[name].RowData)) + } + + return &tableStats +} + +type referTable struct { + TableName string `json:"table_name"` + ColumnName []string `json:"column_name"` + ColumnType []string `json:"column_type"` + RowData [][]any `json:"row_data"` + + // 要求 []int 中的行号递增 + index map[string]map[any][]int + + colIndex map[string]int +} + +func (table *referTable) check() error { + if table.TableName == "" { + return fmt.Errorf("table: \"%s\", error: empty table name", table.TableName) + } + if len(table.ColumnName) != len(table.ColumnType) { + return fmt.Errorf("table: %s, error: len(table.ColumnName) != len(table.ColumnType)", + table.TableName) + } + + for idx, row := range table.RowData { + if len(table.ColumnName) != len(row) { + return fmt.Errorf("table: %s, col: %d, error: len(table.ColumnName) != len(table.RowData[%d])", + table.TableName, idx, idx) + } + } + + for idx, columnName := range table.ColumnName { + if columnName == "" { + return fmt.Errorf("table: %s, column: %v, index: %d, value: \"\"", + table.TableName, table.ColumnName, idx) + } + } + + for idx, columnType := range table.ColumnType { + switch columnType { + case columnTypeInt, columnTypeFloat, + columnTypeBool, columnTypeStr: + default: + return fmt.Errorf("table: %s, unsupported column type: %s -> %s", + table.TableName, table.ColumnName[idx], columnType) + } + } + + return nil +} + +func (table *referTable) buildTableIndex() error { + if err := table.check(); err != nil { + return err + } + + table.index = map[string]map[any][]int{} + table.colIndex = map[string]int{} + + // 遍历行 + for rowIdx, row := range table.RowData { + // 遍历列，建立索引: colName -> colValue -> []rowIndex + for colIdx := 0; colIdx < len(table.ColumnName); colIdx++ { + colName := table.ColumnName[colIdx] + + if _, ok := table.index[colName]; !ok { + table.colIndex[colName] = colIdx + table.index[colName] = map[any][]int{} + } + + // 列数据转换为指定类型 + v, err := conv(row[colIdx], table.ColumnType[colIdx]) + if err != nil { + return fmt.Errorf("table: %s, row: %d, col: %d, cast error: %w", + table.TableName, rowIdx, colIdx, err) + } + row[colIdx] = v + // column 反向索引: colValue -> []rowIndex + table.index[colName][v] = append(table.index[colName][v], + rowIdx) + } + } + + return nil +} + +func conv(col any, dtype string) (any, error) { + switch strings.ToLower(dtype) { + case columnTypeBool: + return cast.ToBoolE(col) + case columnTypeInt: + return cast.ToInt64E(col) + case columnTypeFloat: + return cast.ToFloat64E(col) + case columnTypeStr: + return cast.ToStringE(col) + default: + return nil, fmt.Errorf("unsupported type: %s", dtype) + } +} + +func decodeJSONData(data []byte) ([]referTable, error) { + var tables []referTable + if err := json.Unmarshal(data, &tables); err != nil { + return nil, err + } else { + return tables, nil + } +} diff --git a/pkg/inimpl/guancecloud/ptinput/refertable/table_sqlite_other.go b/pkg/inimpl/guancecloud/ptinput/refertable/table_sqlite_other.go new file mode 100644 index 0000000..8cefb9a --- /dev/null +++ b/pkg/inimpl/guancecloud/ptinput/refertable/table_sqlite_other.go @@ -0,0 +1,227 @@ +// Unless explicitly stated otherwise all files in this repository are licensed +// under the MIT License. +// This product includes software developed at Guance Cloud (https://www.guance.com/). +// Copyright 2021-present Guance, Inc. + +//go:build !(windows && 386) +// +build !windows !386 + +package refertable + +import ( + "database/sql" + "errors" + "fmt" + "strings" + + _ "modernc.org/sqlite" +) + +type PlReferTablesSqlite struct { + tableNames []string + db *sql.DB +} + +func (p *PlReferTablesSqlite) query(tableName string, colName []string, colValue []any, kGet []string) (map[string]any, bool) { + if p.db == nil { + return nil, false + } + query := buildSelectStmt(tableName, colName, kGet) + l.Debugf("got SQL statement '%s' with params %v", query, colValue) + + result, err := p.db.Query(query, colValue...) + if err != nil { + l.Errorf("Query returned: %v", err) + return nil, false + } + defer result.Close() //nolint:errcheck + var cols []string + if len(kGet) == 0 { + // Get all columns. + cols, err = result.Columns() + if err != nil { + l.Errorf("failed to get column names: %v", err) + return nil, false + } + } else { + cols = kGet + } + nCol := len(cols) + + its := make([]interface{}, nCol) + itAddrs := make([]interface{}, nCol) + for i := 0; i < nCol; i++ { + itAddrs[i] = &its[i] + } + // Scan only one row. + if result.Next() { + if err := result.Scan(itAddrs...); err != nil { + l.Errorf("failed to scan query result: %v", err) + return nil, false + } + } + + ret := make(map[string]any) + for i := 0; i < nCol; i++ { + ret[cols[i]] = its[i] + } + return ret, true +} + +func (p *PlReferTablesSqlite) updateAll(tables []referTable) (retErr error) { + if p.db == nil { + return errors.New("PlReferTablesSqlite is not initialized") + } + for _, table := range tables { + if err := table.check(); err != nil { + return err + } + } + + tx, err := p.db.Begin() + if err != nil { + return fmt.Errorf("failed to start a TX: %w", err) + } + defer func() { + if retErr != nil { + if err := tx.Rollback(); err != nil { + l.Errorf("failed to rollback TX: %v", err) + } + } else { + if err := tx.Commit(); err != nil { + l.Errorf("failed to commit TX: %v", err) + } + } + }() + + // Drop deprecated tables. + for _, t := range p.tableNames { + dropStmt := fmt.Sprintf("DROP TABLE IF EXISTS %s", t) + if _, err := tx.Exec(dropStmt); err != nil { + return fmt.Errorf("failed to execute '%s': %w", dropStmt, err) + } + } + for _, t := range tables { + dropStmt := fmt.Sprintf("DROP TABLE IF EXISTS %s", t.TableName) + if _, err := tx.Exec(dropStmt); err != nil { + return fmt.Errorf("failed to execute '%s': %w", dropStmt, err) + } + } + + // Create new tables. + for i := range tables { + createStmt := buildCreateTableStmt(&tables[i]) + if _, err := tx.Exec(createStmt); err != nil { + return fmt.Errorf("failed to execute '%s': %w", createStmt, err) + } + } + + // Insert tuples into these tables. + for i := range tables { + insertStmt := buildInsertIntoStmts(&tables[i]) + for j := 0; j < len(tables[i].RowData); j++ { + if _, err := tx.Exec(insertStmt, tables[i].RowData[j]...); err != nil { + return fmt.Errorf("failed to execute '%s' with params %v: %w", insertStmt, tables[i].RowData[j], err) + } + } + } + + // Update table list. + p.tableNames = []string{} + for _, t := range tables { + p.tableNames = append(p.tableNames, t.TableName) + } + + return nil +} + +func (p *PlReferTablesSqlite) stats() *ReferTableStats { + if p.db == nil { + return nil + } + var ( + res ReferTableStats + numRow int + ) + for _, tableName := range p.tableNames { + if err := p.db.QueryRow(fmt.Sprintf("SELECT COUNT(*) FROM %s", tableName)).Scan(&numRow); err != nil { + l.Errorf("Query retuned: %v", err) + return nil + } + res.Row = append(res.Row, numRow) + } + return &res +} + +func buildSelectStmt(tableName string, colName []string, kGet []string) string { + var query, items string + + if len(kGet) == 0 { + items = "*" + } else { + items = strings.Join(kGet, ", ") + } + + if len(colName) > 0 { + var conStmt strings.Builder + for i, c := range colName { + if i != 0 { + conStmt.WriteString(" AND ") + } + conStmt.WriteString(c + " = ?") + } + query = fmt.Sprintf("SELECT %s FROM %s WHERE %s", items, tableName, conStmt.String()) + } else { + query = fmt.Sprintf("SELECT %s FROM %s", items, tableName) + } + return query +} + +func buildCreateTableStmt(table *referTable) string { + var res strings.Builder + res.WriteString("CREATE TABLE " + table.TableName + " (") + for i, colName := range table.ColumnName { + if i != 0 { + res.WriteString(", ") + } + res.WriteString(colName) + res.WriteString(" " + ColType2SqliteType(table.ColumnType[i])) + } + res.WriteString(")") + return res.String() +} + +func buildInsertIntoStmts(table *referTable) string { + var sb strings.Builder + sb.WriteString("INSERT INTO " + table.TableName + " (") + for i, colName := range table.ColumnName { + if i != 0 { + sb.WriteString(", ") + } + sb.WriteString(colName) + } + sb.WriteString(") VALUES (") + for i := range table.ColumnName { + if i != 0 { + sb.WriteString(", ") + } + sb.WriteByte('?') + } + sb.WriteByte(')') + return sb.String() +} + +func ColType2SqliteType(typeName string) string { + switch typeName { + case columnTypeStr: + return "TEXT" + case columnTypeFloat: + return "REAL" + case columnTypeInt: + return "INTEGER" + case columnTypeBool: + return "NUMERIC" + default: + return "" + } +} diff --git a/pkg/inimpl/guancecloud/ptinput/refertable/table_sqlite_test.go b/pkg/inimpl/guancecloud/ptinput/refertable/table_sqlite_test.go new file mode 100644 index 0000000..d44e946 --- /dev/null +++ b/pkg/inimpl/guancecloud/ptinput/refertable/table_sqlite_test.go @@ -0,0 +1,211 @@ +// Unless explicitly stated otherwise all files in this repository are licensed +// under the MIT License. +// This product includes software developed at Guance Cloud (https://www.guance.com/). +// Copyright 2021-present Guance, Inc. + +//go:build !(windows && 386) +// +build !windows !386 + +package refertable + +import ( + "database/sql" + "reflect" + "testing" + + _ "modernc.org/sqlite" +) + +func Test_buildCreateTableStmt(t *testing.T) { + type args struct { + table referTable + } + tests := []struct { + name string + in args + want string + }{ + { + name: "normal", + in: args{referTable{ + TableName: "test_table", + ColumnName: []string{"c1", "c2"}, + ColumnType: []string{columnTypeStr, columnTypeInt}, + }}, + want: "CREATE TABLE test_table (c1 TEXT, c2 INTEGER)", + }, + { + name: "normal", + in: args{referTable{ + TableName: "employee", + ColumnName: []string{"c1", "c2"}, + ColumnType: []string{columnTypeFloat, columnTypeBool}, + }}, + want: "CREATE TABLE employee (c1 REAL, c2 NUMERIC)", + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + if got := buildCreateTableStmt(&tt.in.table); !reflect.DeepEqual(got, tt.want) { + t.Errorf("buildCreateTableStmt() = %v, want %v", got, tt.want) + } + }) + } +} + +func Test_buildInsertIntoStmts(t *testing.T) { + type args struct { + table referTable + } + tests := []struct { + name string + in args + want string + }{ + { + name: "normal", + in: args{referTable{ + TableName: "test_table", + ColumnName: []string{"c1", "c2"}, + ColumnType: []string{columnTypeStr, columnTypeInt}, + RowData: [][]any{{"fiona", 20}, {"michael", 25}}, + }}, + want: "INSERT INTO test_table (c1, c2) VALUES (?, ?)", + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + if got := buildInsertIntoStmts(&tt.in.table); !reflect.DeepEqual(got, tt.want) { + t.Errorf("buildInsertIntoStmts() = %v, want %v", got, tt.want) + } + }) + } +} + +func Test_buildSelectStmt(t *testing.T) { + type args struct { + tableName string + colName []string + colValue []any + kGet []string + } + tests := []struct { + name string + in args + want string + }{ + { + name: "normal", + in: args{ + tableName: "test_table", + colName: []string{}, + colValue: []any{}, + kGet: []string{"id"}, + }, + want: "SELECT id FROM test_table", + }, + { + name: "normal", + in: args{ + tableName: "test_table", + colName: []string{"weight"}, + colValue: []any{50.5}, + kGet: []string{"id"}, + }, + want: "SELECT id FROM test_table WHERE weight = ?", + }, + { + name: "normal", + in: args{ + tableName: "test_table", + colName: []string{"name", "age"}, + colValue: []any{"fiona", 21}, + kGet: []string{"id", "name", "age"}, + }, + want: "SELECT id, name, age FROM test_table WHERE name = ? AND age = ?", + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + if got := buildSelectStmt(tt.in.tableName, tt.in.colName, tt.in.kGet); !reflect.DeepEqual(got, tt.want) { + t.Errorf("buildSelectStmt() = %v, want %v", got, tt.want) + } + }) + } +} + +func TestPlReferTablesSqlite_query(t *testing.T) { + p := &PlReferTablesSqlite{} + d, err := sql.Open("sqlite", ":memory:") + p.db = d + if err != nil { + t.Fatal(err) + } + if err := p.db.Ping(); err != nil { + t.Fatal(err) + } + r := p.db.QueryRow("select sqlite_version()") + var ver string + r.Scan(&ver) + t.Logf("sqlite version: %s", ver) + if _, err := p.db.Exec("CREATE TABLE test(id INTEGER, name TEXT, age INTEGER, alive NUMERIC, grade REAL)"); err != nil { + t.Fatal(err) + } + if _, err := p.db.Exec("INSERT INTO test(id, name, age, alive, grade) VALUES(23, \"Michael\", 25, true, 99.5)"); err != nil { + t.Fatal(err) + } + if _, err := p.db.Exec("INSERT INTO test(id, name) VALUES(23, \"Jimmy\")"); err != nil { + t.Fatal(err) + } + if _, err := p.db.Exec("INSERT INTO test(id, name, age, alive, grade) VALUES(8, \"Kobe\", 40, false, 99.0)"); err != nil { + t.Fatal(err) + } + + type args struct { + tableName string + colName []string + colValue []any + kGet []string + } + tests := []struct { + name string + args args + want map[string]any + want1 bool + }{ + { + name: "select by id", + args: args{ + tableName: "test", + colName: []string{"id"}, + colValue: []any{8}, + kGet: []string{"id", "name"}, + }, + want: map[string]any{"id": int64(8), "name": "Kobe"}, + want1: true, + }, + { + name: "select by id and name", + args: args{ + tableName: "test", + colName: []string{"id", "name"}, + colValue: []any{23, "Michael"}, + kGet: []string{}, + }, + want: map[string]any{"id": int64(23), "name": "Michael", "age": int64(25), "alive": int64(1), "grade": 99.5}, + want1: true, + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + got, got1 := p.query(tt.args.tableName, tt.args.colName, tt.args.colValue, tt.args.kGet) + if !reflect.DeepEqual(got, tt.want) { + t.Errorf("PlReferTablesSqlite.query() got = %v, want %v", got, tt.want) + } + if got1 != tt.want1 { + t.Errorf("PlReferTablesSqlite.query() got1 = %v, want %v", got1, tt.want1) + } + }) + } +} diff --git a/pkg/inimpl/guancecloud/ptinput/refertable/table_sqlite_windows_386.go b/pkg/inimpl/guancecloud/ptinput/refertable/table_sqlite_windows_386.go new file mode 100644 index 0000000..9b92735 --- /dev/null +++ b/pkg/inimpl/guancecloud/ptinput/refertable/table_sqlite_windows_386.go @@ -0,0 +1,34 @@ +// Unless explicitly stated otherwise all files in this repository are licensed +// under the MIT License. +// This product includes software developed at Guance Cloud (https://www.guance.com/). +// Copyright 2021-present Guance, Inc. + +//go:build windows && 386 +// +build windows,386 + +package refertable + +import ( + "database/sql" +) + +//nolint:unused +type PlReferTablesSqlite struct { + tableNames []string + db *sql.DB +} + +func (p *PlReferTablesSqlite) query(tableName string, colName []string, colValue []any, kGet []string) (map[string]any, bool) { + l.Errorf("windows-386 does not support query using SQLite") + return nil, false +} + +func (p *PlReferTablesSqlite) updateAll(tables []referTable) (retErr error) { + l.Errorf("windows-386 does not support query using SQLite") + return nil +} + +func (p *PlReferTablesSqlite) stats() *ReferTableStats { + l.Errorf("windows-386 does not support query using SQLite") + return nil +} diff --git a/pkg/inimpl/guancecloud/ptinput/refertable/table_test.go b/pkg/inimpl/guancecloud/ptinput/refertable/table_test.go new file mode 100644 index 0000000..fb7bd4d --- /dev/null +++ b/pkg/inimpl/guancecloud/ptinput/refertable/table_test.go @@ -0,0 +1,122 @@ +// Unless explicitly stated otherwise all files in this repository are licensed +// under the MIT License. +// This product includes software developed at Guance Cloud (https://www.guance.com/). +// Copyright 2021-present Guance, Inc. + +package refertable + +import ( + "testing" +) + +func TestTable(t *testing.T) { + tables, err := decodeJSONData([]byte(testTableData)) + if err != nil { + t.Fatal(err) + } + + plRefTable := PlReferTablesInMemory{} + plRefTable.updateAll(tables) + + if v, ok := plRefTable.query("table1", []string{"key1"}, []any{"ab"}, nil); ok { + t.Log(v) + } else { + t.Error(ok) + } + + if _, ok := plRefTable.query("table1", []string{"key1", "key2"}, []any{"ab"}, nil); ok { + t.Errorf("exp: false, act: %v", ok) + } +} + +func BenchmarkTableQueyr(b *testing.B) { + tables, err := decodeJSONData([]byte(testTableData)) + if err != nil { + b.Fatal(err) + } + + plRefTable := PlReferTablesInMemory{} + plRefTable.updateAll(tables) + + for i := 0; i < b.N; i++ { + if v, ok := plRefTable.query("table1", []string{"key1"}, []any{"ab"}, nil); ok { + b.Log(v) + } else { + b.Error(ok) + } + } +} + +var testTableData = ` +[ + { + "table_name": "table1", + "column_name": [ + "key1", + "key2", + "f1", + "f2" + ], + "column_type": [ + "string", + "float", + "int", + "bool" + ], + "row_data": [ + [ + "a", + 123, + "123", + "true" + ], + [ + "ab", + "1234", + "123", + "true" + ], + [ + "ab", + "1234", + "123", + "true" + ] + ] + }, + { + "table_name": "table2", + "primary_key": [ + "key1", + "key2" + ], + "column_name": [ + "key1", + "key2", + "f1", + "f2" + ], + "column_type": [ + "string", + "float", + "int", + "bool" + ], + "row_data": [ + [ + "a", + 123, + "123", + "true" + ], + [ + "a", + "1234", + "123", + "true" + ] + ] + } +] + +` diff --git a/pkg/inimpl/guancecloud/ptinput/utils/utils.go b/pkg/inimpl/guancecloud/ptinput/utils/utils.go new file mode 100644 index 0000000..308ad01 --- /dev/null +++ b/pkg/inimpl/guancecloud/ptinput/utils/utils.go @@ -0,0 +1,34 @@ +package utils + +import ( + "os" + + "github.com/GuanceCloud/cliutils/point" +) + +func PtCatOption(cat point.Category) []point.Option { + var opt []point.Option + switch cat { + case point.Logging: + opt = point.DefaultLoggingOptions() + case point.Tracing, + point.Network, + point.KeyEvent, + point.RUM, + point.Security, + point.Profiling: + opt = point.CommonLoggingOptions() + case point.Object, + point.CustomObject: + opt = point.DefaultObjectOptions() + case point.Metric: + opt = point.DefaultMetricOptions() + default: + } + return opt +} + +func FileExist(filename string) bool { + _, err := os.Stat(filename) + return err == nil || os.IsExist(err) +}