From 7addaa6bcdbb0828f21e98ae93fd8af8b6adad03 Mon Sep 17 00:00:00 2001 From: Kaleb Saw <49179770+ItsKaleb@users.noreply.github.com> Date: Mon, 21 Aug 2023 02:04:08 +0000 Subject: [PATCH 1/9] feat(http-log): add allow list for logged headers For some use cases, only a set of known, allowed headers should be included in traffic logs. This updates the http-log plugin to add an optional behaviour to only include a set of allowed request/response headers in the generated traffic logs. --- kong/plugins/http-log/handler.lua | 26 ++++++++++++ kong/plugins/http-log/schema.lua | 37 +++++++++++++++++ spec/03-plugins/03-http-log/01-log_spec.lua | 45 +++++++++++++++++++++ 3 files changed, 108 insertions(+) diff --git a/kong/plugins/http-log/handler.lua b/kong/plugins/http-log/handler.lua index 8bd382f926c0..e9e0216d522b 100644 --- a/kong/plugins/http-log/handler.lua +++ b/kong/plugins/http-log/handler.lua @@ -181,6 +181,7 @@ end function HttpLogHandler:log(conf) + -- do custom fields if conf.custom_fields_by_lua then local set_serialize_value = kong.log.set_serialize_value for key, expression in pairs(conf.custom_fields_by_lua) do @@ -188,6 +189,31 @@ function HttpLogHandler:log(conf) end end + -- do logged header allow list if enabled + if conf.enable_logged_header_allow_list then + local set_serialize_value = kong.log.set_serialize_value + + -- clear all request and response headers + set_serialize_value("request.headers", nil) + set_serialize_value("response.headers", nil) + + -- restore allowed headers + local request_get_header = kong.request.get_header + local response_get_header = kong.response.get_header + for _, header_name in ipairs(conf.logged_headers_allow_list) do + local request_header_val = request_get_header(header_name) + local response_header_val = response_get_header(header_name) + + if request_header_val then + set_serialize_value("request.headers." .. header_name, request_header_val) + end + + if response_header_val then + set_serialize_value("response.headers." .. header_name, response_header_val) + end + end + end + local queue_conf = Queue.get_plugin_params("http-log", conf, make_queue_name(conf)) kong.log.debug("Queue name automatically configured based on configuration parameters to: ", queue_conf.name) diff --git a/kong/plugins/http-log/schema.lua b/kong/plugins/http-log/schema.lua index ef2dfdcdebc7..9ac72b3fb0ee 100644 --- a/kong/plugins/http-log/schema.lua +++ b/kong/plugins/http-log/schema.lua @@ -42,6 +42,43 @@ return { }}, { queue = typedefs.queue }, { custom_fields_by_lua = typedefs.lua_code }, + { enable_logged_headers_allow_list = { + type = "boolean", + default = false, + description = "When enabled, will ensure only headers from the logged_headers_allow_list are included in log payloads. All headers are logged if this is disabled.", + }}, + { logged_headers_allow_list = { + type = "array", + elements = typedefs.header_name, + default = { + "accept", + "accept-encoding", + "accept-language", + "cache-control", + "content-length", + "content-type", + "date", + "expires", + "host", + "ipremoteaddr", + "last-modified", + "ratelimit-limit", + "ratelimit-remaining", + "ratelimit-reset", + "referer", + "server", + "user-agent", + "via", + "x-forwarded-for", + "x-forwarded-host", + "x-forwarded-proto", + "x-original-for", + "x-original-host", + "x-original-proto", + "x-request-start", + }, + description = "An array of header names that will be included in logged payloads (if present in the request and/or response headers). Only effective when enable_logged_header_allow_list is set to true." + }}, }, entity_checks = { diff --git a/spec/03-plugins/03-http-log/01-log_spec.lua b/spec/03-plugins/03-http-log/01-log_spec.lua index 508933487351..b5cc9adebf65 100644 --- a/spec/03-plugins/03-http-log/01-log_spec.lua +++ b/spec/03-plugins/03-http-log/01-log_spec.lua @@ -322,6 +322,32 @@ for _, strategy in helpers.each_strategy() do } } + local service11 = bp.services:insert{ + protocol = "http", + host = helpers.mock_upstream_host, + port = helpers.mock_upstream_port, + } + + local route11 = bp.routes:insert { + hosts = { "log_allowed_headers.test" }, + service = service9 + } + + bp.plugins:insert { + route = { id = route11.id }, + name = "http-log", + config = { + http_endpoint = "http://" .. helpers.mock_upstream_host + .. ":" + .. helpers.mock_upstream_port + .. "/post_log/custom_http", + enable_logged_header_allow_list = true, + logged_headers_allow_list = { + "allowed_header", + }, + } + } + helpers.setenv(vault_env_name, vault_env_value) assert(helpers.start_kong({ @@ -420,6 +446,25 @@ for _, strategy in helpers.each_strategy() do end) end) + describe("logged headers allow list", function() + it("only log allowed headers", function() + reset_log("allowed_headers") + local res = proxy_client:get("/status/200", { + headers = { + ["Host"] = "log_allowed_headers.test", + ["allowed_header"] = "123", + ["unallowed_header"] = "123" + } + }) + assert.res_status(200, res) + + local entries = get_log("allowed_headers", 1) + assert.same("127.0.0.1", entries[1].client_ip) + assert.same(123, entries[1].request.headers.allowed_header) + assert.same(nil, entries[1].request.headers.unallowed_header) + end) + end) + it("logs to HTTP (#grpc)", function() reset_log("grpc") -- Making the request From 8c2401c8f1ccb1885f684fc7d5c23fae33334b6c Mon Sep 17 00:00:00 2001 From: Kaleb Saw <49179770+ItsKaleb@users.noreply.github.com> Date: Wed, 23 Aug 2023 12:09:03 +0800 Subject: [PATCH 2/9] Reorder local field declaration MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Co-authored-by: Hans Hübner --- kong/plugins/http-log/handler.lua | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/kong/plugins/http-log/handler.lua b/kong/plugins/http-log/handler.lua index e9e0216d522b..fd2149f0ba90 100644 --- a/kong/plugins/http-log/handler.lua +++ b/kong/plugins/http-log/handler.lua @@ -202,12 +202,11 @@ function HttpLogHandler:log(conf) local response_get_header = kong.response.get_header for _, header_name in ipairs(conf.logged_headers_allow_list) do local request_header_val = request_get_header(header_name) - local response_header_val = response_get_header(header_name) - if request_header_val then set_serialize_value("request.headers." .. header_name, request_header_val) end + local response_header_val = response_get_header(header_name) if response_header_val then set_serialize_value("response.headers." .. header_name, response_header_val) end From 7307edc033a2424ff150275fa1bbb4331133f303 Mon Sep 17 00:00:00 2001 From: Kaleb Saw <49179770+ItsKaleb@users.noreply.github.com> Date: Mon, 28 Aug 2023 20:55:15 +0800 Subject: [PATCH 3/9] feat(http-log): add logged header allow list Update removed_fields to include new config values for version 3.5.x --- kong/clustering/compat/removed_fields.lua | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/kong/clustering/compat/removed_fields.lua b/kong/clustering/compat/removed_fields.lua index d41af4bf0f63..de4b1656a826 100644 --- a/kong/clustering/compat/removed_fields.lua +++ b/kong/clustering/compat/removed_fields.lua @@ -96,4 +96,12 @@ return { "response_headers", }, }, + + -- Any dataplane older than 3.5.0 + [3005000000] = { + http_log = { + "enable_logged_headers_allow_list", + "logged_headers_allow_list", + } + } } From 3188bef1d83445ca41f39135ee3b2dcebce9d315 Mon Sep 17 00:00:00 2001 From: Douglas-Lee Date: Tue, 29 Aug 2023 22:36:21 +0800 Subject: [PATCH 4/9] Add changelog file --- CHANGELOG/unreleased/kong/11427.yaml | 5 +++++ 1 file changed, 5 insertions(+) create mode 100644 CHANGELOG/unreleased/kong/11427.yaml diff --git a/CHANGELOG/unreleased/kong/11427.yaml b/CHANGELOG/unreleased/kong/11427.yaml new file mode 100644 index 000000000000..1106e703cdb2 --- /dev/null +++ b/CHANGELOG/unreleased/kong/11427.yaml @@ -0,0 +1,5 @@ +message: "**HTTP-Log**: add `enable_logged_headers_allow_list` and `logged_headers_allow_list` fields in HTTP Log plugin." +type: feature +scope: Plugin +prs: + - 11427 From 2011bf22619f3fcc5d43289de227aac06805189a Mon Sep 17 00:00:00 2001 From: Kaleb Saw <49179770+ItsKaleb@users.noreply.github.com> Date: Mon, 4 Sep 2023 14:19:43 +0800 Subject: [PATCH 5/9] Address typos in config field names --- kong/plugins/http-log/handler.lua | 4 ++-- spec/03-plugins/03-http-log/01-log_spec.lua | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/kong/plugins/http-log/handler.lua b/kong/plugins/http-log/handler.lua index fd2149f0ba90..dee25ceb2284 100644 --- a/kong/plugins/http-log/handler.lua +++ b/kong/plugins/http-log/handler.lua @@ -189,8 +189,8 @@ function HttpLogHandler:log(conf) end end - -- do logged header allow list if enabled - if conf.enable_logged_header_allow_list then + -- do logged headers allow list if enabled + if conf.enable_logged_headers_allow_list then local set_serialize_value = kong.log.set_serialize_value -- clear all request and response headers diff --git a/spec/03-plugins/03-http-log/01-log_spec.lua b/spec/03-plugins/03-http-log/01-log_spec.lua index b5cc9adebf65..e7fadcf66acd 100644 --- a/spec/03-plugins/03-http-log/01-log_spec.lua +++ b/spec/03-plugins/03-http-log/01-log_spec.lua @@ -341,7 +341,7 @@ for _, strategy in helpers.each_strategy() do .. ":" .. helpers.mock_upstream_port .. "/post_log/custom_http", - enable_logged_header_allow_list = true, + enable_logged_headers_allow_list = true, logged_headers_allow_list = { "allowed_header", }, From cfe4e0e85d3eb824f20b847f25c554bb834de387 Mon Sep 17 00:00:00 2001 From: Kaleb Saw <49179770+ItsKaleb@users.noreply.github.com> Date: Mon, 4 Sep 2023 14:25:15 +0800 Subject: [PATCH 6/9] Change logged_headers_allow_list to empty by default --- kong/plugins/http-log/schema.lua | 28 +--------------------------- 1 file changed, 1 insertion(+), 27 deletions(-) diff --git a/kong/plugins/http-log/schema.lua b/kong/plugins/http-log/schema.lua index 9ac72b3fb0ee..b12798847b42 100644 --- a/kong/plugins/http-log/schema.lua +++ b/kong/plugins/http-log/schema.lua @@ -50,33 +50,7 @@ return { { logged_headers_allow_list = { type = "array", elements = typedefs.header_name, - default = { - "accept", - "accept-encoding", - "accept-language", - "cache-control", - "content-length", - "content-type", - "date", - "expires", - "host", - "ipremoteaddr", - "last-modified", - "ratelimit-limit", - "ratelimit-remaining", - "ratelimit-reset", - "referer", - "server", - "user-agent", - "via", - "x-forwarded-for", - "x-forwarded-host", - "x-forwarded-proto", - "x-original-for", - "x-original-host", - "x-original-proto", - "x-request-start", - }, + default = {}, description = "An array of header names that will be included in logged payloads (if present in the request and/or response headers). Only effective when enable_logged_header_allow_list is set to true." }}, }, From eca41c65f7ba3a226a6f0ea8739980a5d3d3510c Mon Sep 17 00:00:00 2001 From: Kaleb Saw <49179770+ItsKaleb@users.noreply.github.com> Date: Mon, 4 Sep 2023 14:28:52 +0800 Subject: [PATCH 7/9] Fix typo in new http-log test spec --- spec/03-plugins/03-http-log/01-log_spec.lua | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/spec/03-plugins/03-http-log/01-log_spec.lua b/spec/03-plugins/03-http-log/01-log_spec.lua index e7fadcf66acd..56153e9935e4 100644 --- a/spec/03-plugins/03-http-log/01-log_spec.lua +++ b/spec/03-plugins/03-http-log/01-log_spec.lua @@ -330,7 +330,7 @@ for _, strategy in helpers.each_strategy() do local route11 = bp.routes:insert { hosts = { "log_allowed_headers.test" }, - service = service9 + service = service11 } bp.plugins:insert { From d797fed47182de20344ab145633c13d046845376 Mon Sep 17 00:00:00 2001 From: Kaleb Saw <49179770+ItsKaleb@users.noreply.github.com> Date: Wed, 6 Sep 2023 08:40:56 +0800 Subject: [PATCH 8/9] Update log target for allowed headers test --- spec/03-plugins/03-http-log/01-log_spec.lua | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/spec/03-plugins/03-http-log/01-log_spec.lua b/spec/03-plugins/03-http-log/01-log_spec.lua index 56153e9935e4..3647e42e832e 100644 --- a/spec/03-plugins/03-http-log/01-log_spec.lua +++ b/spec/03-plugins/03-http-log/01-log_spec.lua @@ -340,7 +340,7 @@ for _, strategy in helpers.each_strategy() do http_endpoint = "http://" .. helpers.mock_upstream_host .. ":" .. helpers.mock_upstream_port - .. "/post_log/custom_http", + .. "/post_log/allowed_headers", enable_logged_headers_allow_list = true, logged_headers_allow_list = { "allowed_header", From a27597ec8b3e85952ff590d5b4944d4e5f69a624 Mon Sep 17 00:00:00 2001 From: Kaleb Saw <49179770+ItsKaleb@users.noreply.github.com> Date: Wed, 6 Sep 2023 16:25:39 +0800 Subject: [PATCH 9/9] Fix expected value type for new test --- spec/03-plugins/03-http-log/01-log_spec.lua | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/spec/03-plugins/03-http-log/01-log_spec.lua b/spec/03-plugins/03-http-log/01-log_spec.lua index 3647e42e832e..5c12af7594ac 100644 --- a/spec/03-plugins/03-http-log/01-log_spec.lua +++ b/spec/03-plugins/03-http-log/01-log_spec.lua @@ -460,7 +460,7 @@ for _, strategy in helpers.each_strategy() do local entries = get_log("allowed_headers", 1) assert.same("127.0.0.1", entries[1].client_ip) - assert.same(123, entries[1].request.headers.allowed_header) + assert.same("123", entries[1].request.headers.allowed_header) assert.same(nil, entries[1].request.headers.unallowed_header) end) end)