From 7015392bafe38546bf08fbe1cc5c1c1745521012 Mon Sep 17 00:00:00 2001 From: ms2008 Date: Wed, 15 Nov 2023 15:06:28 +0800 Subject: [PATCH 1/7] feat(pdk): increase the precision of JSON number encoding from 14 to 16 decimals With this, we can safely store integers up to 2^53, with no loss of precision. Before the change, cJSON started generating scientific notation output at a much smaller value than 2^53. --- .../pdk-json-encoding-numbers-precision.yml | 3 +++ kong/db/strategies/postgres/init.lua | 6 +++++ kong/pdk/response.lua | 1 + .../05-proxy/02-router_spec.lua | 16 +++++++++++++ t/01-pdk/08-response/11-exit.t | 24 +++++++++++++++++++ 5 files changed, 50 insertions(+) create mode 100644 changelog/unreleased/kong/pdk-json-encoding-numbers-precision.yml diff --git a/changelog/unreleased/kong/pdk-json-encoding-numbers-precision.yml b/changelog/unreleased/kong/pdk-json-encoding-numbers-precision.yml new file mode 100644 index 000000000000..0560d8b68150 --- /dev/null +++ b/changelog/unreleased/kong/pdk-json-encoding-numbers-precision.yml @@ -0,0 +1,3 @@ +message: Increase the precision of JSON number encoding from 14 to 16 decimals +type: feature +scope: PDK diff --git a/kong/db/strategies/postgres/init.lua b/kong/db/strategies/postgres/init.lua index 74da93465aa6..e76af557e293 100644 --- a/kong/db/strategies/postgres/init.lua +++ b/kong/db/strategies/postgres/init.lua @@ -6,6 +6,8 @@ local utils = require "kong.tools.utils" local new_tab = require "table.new" local clear_tab = require "table.clear" +cjson.encode_number_precision(16) +cjson_safe.encode_number_precision(16) local kong = kong local ngx = ngx @@ -180,6 +182,10 @@ local function escape_literal(connector, literal, field) return concat { "TO_TIMESTAMP(", connector:escape_literal(tonumber(fmt("%.3f", literal))), ") AT TIME ZONE 'UTC'" } end + if field.type == "integer" then + return fmt("%16.f", literal) + end + if field.type == "array" or field.type == "set" then if not literal[1] then return connector:escape_literal("{}") diff --git a/kong/pdk/response.lua b/kong/pdk/response.lua index 37a0c67d11f4..e196067cd311 100644 --- a/kong/pdk/response.lua +++ b/kong/pdk/response.lua @@ -20,6 +20,7 @@ local utils = require "kong.tools.utils" local request_id = require "kong.tracing.request_id" local constants = require "kong.constants" +cjson.encode_number_precision(16) local ngx = ngx local arg = ngx.arg diff --git a/spec/02-integration/05-proxy/02-router_spec.lua b/spec/02-integration/05-proxy/02-router_spec.lua index 26ba41a46176..bcd4a008ce87 100644 --- a/spec/02-integration/05-proxy/02-router_spec.lua +++ b/spec/02-integration/05-proxy/02-router_spec.lua @@ -2486,6 +2486,7 @@ do for _, strategy in helpers.each_strategy() do describe("Router [#" .. strategy .. ", flavor = " .. flavor .. "]", function() local proxy_client + local route_with_max_safe_integer_priority reload_router(flavor) @@ -2520,6 +2521,13 @@ do service = service, } + route_with_max_safe_integer_priority = bp.routes:insert { + protocols = { "http" }, + expression = [[http.path == "/max"]], + priority = 9007199254740992, + service = service, + } + assert(helpers.start_kong({ router_flavor = flavor, database = strategy, @@ -2596,6 +2604,14 @@ do assert.res_status(404, res) end) + it("the maximum safe integer can be accurately represented as a decimal number", function() + local admin_client = helpers.admin_client() + + local res = admin_client:get("/routes/" .. route_with_max_safe_integer_priority.id) + assert.res_status(200, res) + assert.match_re(res:read_body(), "9007199254740992") + end) + end) end -- strategy diff --git a/t/01-pdk/08-response/11-exit.t b/t/01-pdk/08-response/11-exit.t index f45564eed560..8048b91cdbe7 100644 --- a/t/01-pdk/08-response/11-exit.t +++ b/t/01-pdk/08-response/11-exit.t @@ -1155,3 +1155,27 @@ X-test: test manually setting Transfer-Encoding. Ignored. + +=== TEST 45: response.exit() json encoding of numbers with a precision of 16 decimals +--- http_config eval: $t::Util::HttpConfig +--- config + location = /t { + default_type 'text/test'; + access_by_lua_block { + local PDK = require "kong.pdk" + local pdk = PDK.new() + + pdk.response.exit(200, { n = 9007199254740992 }) + } + } +--- request +GET /t +--- error_code: 200 +--- response_headers_like +Content-Type: application/json; charset=utf-8 +--- response_body chop +{"n":9007199254740992} +--- no_error_log +[error] + + From 2bd55b2bc02f5ecbfade8ce6a6191d72b4a88479 Mon Sep 17 00:00:00 2001 From: ms2008 Date: Wed, 15 Nov 2023 21:49:54 +0800 Subject: [PATCH 2/7] tests(*): fix failed tests --- .../04-admin_api/25-max_safe_integer_spec.lua | 110 ++++++++++++++++++ .../05-proxy/02-router_spec.lua | 16 --- 2 files changed, 110 insertions(+), 16 deletions(-) create mode 100644 spec/02-integration/04-admin_api/25-max_safe_integer_spec.lua diff --git a/spec/02-integration/04-admin_api/25-max_safe_integer_spec.lua b/spec/02-integration/04-admin_api/25-max_safe_integer_spec.lua new file mode 100644 index 000000000000..a54ff9452258 --- /dev/null +++ b/spec/02-integration/04-admin_api/25-max_safe_integer_spec.lua @@ -0,0 +1,110 @@ +local helpers = require "spec.helpers" + +local LMDB_MAP_SIZE = "10m" + +for _, strategy in helpers.each_strategy() do + if strategy ~= "off" then + describe("Admin API #" .. strategy, function() + local bp + local client, route + + lazy_setup(function() + bp = helpers.get_db_utils(strategy, { + "routes", + "services", + }) + + route = bp.routes:insert({ + paths = { "/route_with_max_safe_integer_priority"}, + regex_priority = 9007199254740992, + }) + + assert(helpers.start_kong({ + database = strategy, + })) + end) + + lazy_teardown(function() + helpers.stop_kong(nil, true) + end) + + before_each(function() + client = assert(helpers.admin_client()) + end) + + after_each(function() + if client then + client:close() + end + end) + + it("the maximum safe integer can be accurately represented as a decimal number", function() + local res = assert(client:send { + method = "GET", + path = "/routes/" .. route.id + }) + assert.res_status(200, res) + assert.match_re(res:read_body(), "9007199254740992") + end) + end) + end + + if strategy == "off" then + describe("Admin API #off", function() + local client + + lazy_setup(function() + assert(helpers.start_kong({ + database = "off", + lmdb_map_size = LMDB_MAP_SIZE, + stream_listen = "127.0.0.1:9011", + nginx_conf = "spec/fixtures/custom_nginx.template", + })) + end) + + lazy_teardown(function() + helpers.stop_kong(nil, true) + end) + + before_each(function() + client = assert(helpers.admin_client()) + end) + + after_each(function() + if client then + client:close() + end + end) + + it("the maximum safe integer can be accurately represented as a decimal number", function() + local res = assert(client:send { + method = "POST", + path = "/config", + body = { + config = [[ + _format_version: "1.1" + services: + - name: my-service + id: 0855b320-0dd2-547d-891d-601e9b38647f + url: https://localhost + routes: + - name: my-route + id: 481a9539-f49c-51b6-b2e2-fe99ee68866c + paths: + - / + regex_priority: 9007199254740992 + ]], + }, + headers = { + ["Content-Type"] = "application/json" + } + }) + + assert.response(res).has.status(201) + local res = client:get("/routes/481a9539-f49c-51b6-b2e2-fe99ee68866c") + assert.res_status(200, res) + assert.match_re(res:read_body(), "9007199254740992") + end) + end) + end +end diff --git a/spec/02-integration/05-proxy/02-router_spec.lua b/spec/02-integration/05-proxy/02-router_spec.lua index bcd4a008ce87..26ba41a46176 100644 --- a/spec/02-integration/05-proxy/02-router_spec.lua +++ b/spec/02-integration/05-proxy/02-router_spec.lua @@ -2486,7 +2486,6 @@ do for _, strategy in helpers.each_strategy() do describe("Router [#" .. strategy .. ", flavor = " .. flavor .. "]", function() local proxy_client - local route_with_max_safe_integer_priority reload_router(flavor) @@ -2521,13 +2520,6 @@ do service = service, } - route_with_max_safe_integer_priority = bp.routes:insert { - protocols = { "http" }, - expression = [[http.path == "/max"]], - priority = 9007199254740992, - service = service, - } - assert(helpers.start_kong({ router_flavor = flavor, database = strategy, @@ -2604,14 +2596,6 @@ do assert.res_status(404, res) end) - it("the maximum safe integer can be accurately represented as a decimal number", function() - local admin_client = helpers.admin_client() - - local res = admin_client:get("/routes/" .. route_with_max_safe_integer_priority.id) - assert.res_status(200, res) - assert.match_re(res:read_body(), "9007199254740992") - end) - end) end -- strategy From 0a2d5aa34285ff18ec14806efe6cba7a8fd5370c Mon Sep 17 00:00:00 2001 From: ms2008 Date: Thu, 28 Dec 2023 23:04:58 +0800 Subject: [PATCH 3/7] feat(cjson): ensure `cjson.new` encode number with a precision of 16 decimals --- kong/db/strategies/postgres/init.lua | 2 - kong/globalpatches.lua | 21 +++++++ kong/pdk/response.lua | 1 - kong/plugins/zipkin/reporter.lua | 1 - .../05-proxy/34-max_safe_integer_spec.lua | 60 +++++++++++++++++++ t/01-pdk/08-response/11-exit.t | 1 + 6 files changed, 82 insertions(+), 4 deletions(-) create mode 100644 spec/02-integration/05-proxy/34-max_safe_integer_spec.lua diff --git a/kong/db/strategies/postgres/init.lua b/kong/db/strategies/postgres/init.lua index e76af557e293..8f8c70ff12bc 100644 --- a/kong/db/strategies/postgres/init.lua +++ b/kong/db/strategies/postgres/init.lua @@ -6,8 +6,6 @@ local utils = require "kong.tools.utils" local new_tab = require "table.new" local clear_tab = require "table.clear" -cjson.encode_number_precision(16) -cjson_safe.encode_number_precision(16) local kong = kong local ngx = ngx diff --git a/kong/globalpatches.lua b/kong/globalpatches.lua index c3782f0c8a0f..876c8234603f 100644 --- a/kong/globalpatches.lua +++ b/kong/globalpatches.lua @@ -14,6 +14,27 @@ return function(options) options = options or {} local meta = require "kong.meta" + do -- make cjson encode numbers with a precision of up to 16 decimals + local cjson = require "cjson" + local cjson_safe = require "cjson.safe" + local native_cjson_new = cjson.new + local native_cjson_safe_new = cjson_safe.new + + cjson.encode_number_precision(16) + cjson_safe.encode_number_precision(16) + + cjson.new = function() + local json = native_cjson_new() + json.encode_number_precision(16) + return json + end + + cjson_safe.new = function() + local json = native_cjson_safe_new() + json.encode_number_precision(16) + return json + end + end local cjson = require("cjson.safe") cjson.encode_sparse_array(nil, nil, 2^15) diff --git a/kong/pdk/response.lua b/kong/pdk/response.lua index e196067cd311..37a0c67d11f4 100644 --- a/kong/pdk/response.lua +++ b/kong/pdk/response.lua @@ -20,7 +20,6 @@ local utils = require "kong.tools.utils" local request_id = require "kong.tracing.request_id" local constants = require "kong.constants" -cjson.encode_number_precision(16) local ngx = ngx local arg = ngx.arg diff --git a/kong/plugins/zipkin/reporter.lua b/kong/plugins/zipkin/reporter.lua index 73c1111a81de..d01c3d605bc6 100644 --- a/kong/plugins/zipkin/reporter.lua +++ b/kong/plugins/zipkin/reporter.lua @@ -3,7 +3,6 @@ local to_hex = require "resty.string".to_hex local cjson = require "cjson".new() local Queue = require "kong.tools.queue" -cjson.encode_number_precision(16) local zipkin_reporter_methods = {} local zipkin_reporter_mt = { diff --git a/spec/02-integration/05-proxy/34-max_safe_integer_spec.lua b/spec/02-integration/05-proxy/34-max_safe_integer_spec.lua new file mode 100644 index 000000000000..ed216c8bfc5b --- /dev/null +++ b/spec/02-integration/05-proxy/34-max_safe_integer_spec.lua @@ -0,0 +1,60 @@ +local helpers = require "spec.helpers" + +for _, strategy in helpers.each_strategy() do + describe("cjson.new encode number with a precision of 16 decimals [#" .. strategy .. "]", function() + local proxy_client + + lazy_setup(function() + local bp = helpers.get_db_utils(strategy, { + "routes", + "services", + "plugins", + }, { "pre-function" }) + + local route = bp.routes:insert({ + paths = { "/route_with_max_safe_integer_priority"}, + }) + + bp.plugins:insert { + route = { id = route.id }, + name = "pre-function", + config = { + access = { + [[ + local cjson = require("cjson").new() + ngx.say(cjson.encode({ n = 9007199254740992 })) + ]] + }, + } + } + + assert(helpers.start_kong({ + database = strategy, + untrusted_lua = "on", + plugins = "bundled", + nginx_conf = "spec/fixtures/custom_nginx.template", + nginx_worker_processes = 1 + })) + + proxy_client = helpers.proxy_client() + end) + + lazy_teardown(function() + if proxy_client then + proxy_client:close() + end + + helpers.stop_kong() + end) + + it("the maximum safe integer can be accurately represented as a decimal number", function() + local res = assert(proxy_client:send { + method = "GET", + path = "/route_with_max_safe_integer_priority" + }) + + assert.res_status(200, res) + assert.match_re(res:read_body(), "9007199254740992") + end) + end) +end diff --git a/t/01-pdk/08-response/11-exit.t b/t/01-pdk/08-response/11-exit.t index 8048b91cdbe7..4a6f7a624c92 100644 --- a/t/01-pdk/08-response/11-exit.t +++ b/t/01-pdk/08-response/11-exit.t @@ -1162,6 +1162,7 @@ manually setting Transfer-Encoding. Ignored. location = /t { default_type 'text/test'; access_by_lua_block { + require("kong.globalpatches")() local PDK = require "kong.pdk" local pdk = PDK.new() From 018b400bf87fca44c4c7b657925fc986ac83c6ba Mon Sep 17 00:00:00 2001 From: ms2008 Date: Thu, 4 Jan 2024 13:55:10 +0800 Subject: [PATCH 4/7] Revert "feat(cjson): ensure `cjson.new` encode number with a precision of 16" This reverts commit dad99d098a77f2c0a3693a767f87a3cb954b5e81. --- kong/db/strategies/postgres/init.lua | 2 + kong/globalpatches.lua | 21 ------- kong/pdk/response.lua | 1 + kong/plugins/zipkin/reporter.lua | 1 + .../05-proxy/34-max_safe_integer_spec.lua | 60 ------------------- t/01-pdk/08-response/11-exit.t | 1 - 6 files changed, 4 insertions(+), 82 deletions(-) delete mode 100644 spec/02-integration/05-proxy/34-max_safe_integer_spec.lua diff --git a/kong/db/strategies/postgres/init.lua b/kong/db/strategies/postgres/init.lua index 8f8c70ff12bc..e76af557e293 100644 --- a/kong/db/strategies/postgres/init.lua +++ b/kong/db/strategies/postgres/init.lua @@ -6,6 +6,8 @@ local utils = require "kong.tools.utils" local new_tab = require "table.new" local clear_tab = require "table.clear" +cjson.encode_number_precision(16) +cjson_safe.encode_number_precision(16) local kong = kong local ngx = ngx diff --git a/kong/globalpatches.lua b/kong/globalpatches.lua index 876c8234603f..c3782f0c8a0f 100644 --- a/kong/globalpatches.lua +++ b/kong/globalpatches.lua @@ -14,27 +14,6 @@ return function(options) options = options or {} local meta = require "kong.meta" - do -- make cjson encode numbers with a precision of up to 16 decimals - local cjson = require "cjson" - local cjson_safe = require "cjson.safe" - local native_cjson_new = cjson.new - local native_cjson_safe_new = cjson_safe.new - - cjson.encode_number_precision(16) - cjson_safe.encode_number_precision(16) - - cjson.new = function() - local json = native_cjson_new() - json.encode_number_precision(16) - return json - end - - cjson_safe.new = function() - local json = native_cjson_safe_new() - json.encode_number_precision(16) - return json - end - end local cjson = require("cjson.safe") cjson.encode_sparse_array(nil, nil, 2^15) diff --git a/kong/pdk/response.lua b/kong/pdk/response.lua index 37a0c67d11f4..e196067cd311 100644 --- a/kong/pdk/response.lua +++ b/kong/pdk/response.lua @@ -20,6 +20,7 @@ local utils = require "kong.tools.utils" local request_id = require "kong.tracing.request_id" local constants = require "kong.constants" +cjson.encode_number_precision(16) local ngx = ngx local arg = ngx.arg diff --git a/kong/plugins/zipkin/reporter.lua b/kong/plugins/zipkin/reporter.lua index d01c3d605bc6..73c1111a81de 100644 --- a/kong/plugins/zipkin/reporter.lua +++ b/kong/plugins/zipkin/reporter.lua @@ -3,6 +3,7 @@ local to_hex = require "resty.string".to_hex local cjson = require "cjson".new() local Queue = require "kong.tools.queue" +cjson.encode_number_precision(16) local zipkin_reporter_methods = {} local zipkin_reporter_mt = { diff --git a/spec/02-integration/05-proxy/34-max_safe_integer_spec.lua b/spec/02-integration/05-proxy/34-max_safe_integer_spec.lua deleted file mode 100644 index ed216c8bfc5b..000000000000 --- a/spec/02-integration/05-proxy/34-max_safe_integer_spec.lua +++ /dev/null @@ -1,60 +0,0 @@ -local helpers = require "spec.helpers" - -for _, strategy in helpers.each_strategy() do - describe("cjson.new encode number with a precision of 16 decimals [#" .. strategy .. "]", function() - local proxy_client - - lazy_setup(function() - local bp = helpers.get_db_utils(strategy, { - "routes", - "services", - "plugins", - }, { "pre-function" }) - - local route = bp.routes:insert({ - paths = { "/route_with_max_safe_integer_priority"}, - }) - - bp.plugins:insert { - route = { id = route.id }, - name = "pre-function", - config = { - access = { - [[ - local cjson = require("cjson").new() - ngx.say(cjson.encode({ n = 9007199254740992 })) - ]] - }, - } - } - - assert(helpers.start_kong({ - database = strategy, - untrusted_lua = "on", - plugins = "bundled", - nginx_conf = "spec/fixtures/custom_nginx.template", - nginx_worker_processes = 1 - })) - - proxy_client = helpers.proxy_client() - end) - - lazy_teardown(function() - if proxy_client then - proxy_client:close() - end - - helpers.stop_kong() - end) - - it("the maximum safe integer can be accurately represented as a decimal number", function() - local res = assert(proxy_client:send { - method = "GET", - path = "/route_with_max_safe_integer_priority" - }) - - assert.res_status(200, res) - assert.match_re(res:read_body(), "9007199254740992") - end) - end) -end diff --git a/t/01-pdk/08-response/11-exit.t b/t/01-pdk/08-response/11-exit.t index 4a6f7a624c92..8048b91cdbe7 100644 --- a/t/01-pdk/08-response/11-exit.t +++ b/t/01-pdk/08-response/11-exit.t @@ -1162,7 +1162,6 @@ manually setting Transfer-Encoding. Ignored. location = /t { default_type 'text/test'; access_by_lua_block { - require("kong.globalpatches")() local PDK = require "kong.pdk" local pdk = PDK.new() From 57ff5df39b3cb7bf1df3f4a6362c2cd5e41028c4 Mon Sep 17 00:00:00 2001 From: ms2008 Date: Thu, 4 Jan 2024 22:50:09 +0800 Subject: [PATCH 5/7] feat(PDK): ensure that the PDK uses a separate cjson instance and has settings with the encoding of numbers with a precision of 16 decimals --- kong/pdk/request.lua | 2 ++ kong/pdk/response.lua | 3 ++- kong/pdk/service/request.lua | 5 ++++- kong/pdk/service/response.lua | 2 ++ 4 files changed, 10 insertions(+), 2 deletions(-) diff --git a/kong/pdk/request.lua b/kong/pdk/request.lua index 06fb846a2ae6..c3e31c2e3eb7 100644 --- a/kong/pdk/request.lua +++ b/kong/pdk/request.lua @@ -45,6 +45,8 @@ local PHASES = phase_checker.phases cjson.decode_array_with_array_mt(true) +cjson.encode_sparse_array(nil, nil, 2^15) +cjson.encode_number_precision(16) local function new(self) diff --git a/kong/pdk/response.lua b/kong/pdk/response.lua index e196067cd311..3611421233d2 100644 --- a/kong/pdk/response.lua +++ b/kong/pdk/response.lua @@ -13,13 +13,14 @@ local buffer = require "string.buffer" -local cjson = require "cjson.safe" +local cjson = require "cjson.safe".new() local checks = require "kong.pdk.private.checks" local phase_checker = require "kong.pdk.private.phases" local utils = require "kong.tools.utils" local request_id = require "kong.tracing.request_id" local constants = require "kong.constants" +cjson.encode_sparse_array(nil, nil, 2^15) cjson.encode_number_precision(16) local ngx = ngx diff --git a/kong/pdk/service/request.lua b/kong/pdk/service/request.lua index 495dbf0febcf..5003e5a261b6 100644 --- a/kong/pdk/service/request.lua +++ b/kong/pdk/service/request.lua @@ -2,7 +2,7 @@ -- Module for manipulating the request sent to the Service. -- @module kong.service.request -local cjson = require "cjson.safe" +local cjson = require "cjson.safe".new() local buffer = require "string.buffer" local checks = require "kong.pdk.private.checks" local phase_checker = require "kong.pdk.private.phases" @@ -24,6 +24,9 @@ local validate_headers = checks.validate_headers local check_phase = phase_checker.check local escape = require("kong.tools.uri").escape +cjson.encode_sparse_array(nil, nil, 2^15) +cjson.encode_number_precision(16) + local PHASES = phase_checker.phases diff --git a/kong/pdk/service/response.lua b/kong/pdk/service/response.lua index ec51fe4fac08..d57df3546eb0 100644 --- a/kong/pdk/service/response.lua +++ b/kong/pdk/service/response.lua @@ -24,6 +24,8 @@ local check_phase = phase_checker.check cjson.decode_array_with_array_mt(true) +cjson.encode_sparse_array(nil, nil, 2^15) +cjson.encode_number_precision(16) local replace_dashes = string_tools.replace_dashes From 1601bc828f1ec32e4287dffc5df609e232e98dc4 Mon Sep 17 00:00:00 2001 From: ms2008 Date: Fri, 5 Jan 2024 16:01:02 +0800 Subject: [PATCH 6/7] chore(*): applying the suggestions from reviewer --- kong-3.6.0-0.rockspec | 1 + kong/constants.lua | 1 + kong/db/strategies/postgres/init.lua | 2 -- kong/globalpatches.lua | 8 +++++++- kong/pdk/request.lua | 8 ++------ kong/pdk/response.lua | 4 +--- kong/pdk/service/request.lua | 5 +---- kong/pdk/service/response.lua | 8 ++------ kong/tools/cjson.lua | 21 +++++++++++++++++++++ t/01-pdk/08-response/11-exit.t | 1 + 10 files changed, 37 insertions(+), 22 deletions(-) create mode 100644 kong/tools/cjson.lua diff --git a/kong-3.6.0-0.rockspec b/kong-3.6.0-0.rockspec index 3b0e10e449db..5e9ec6846656 100644 --- a/kong-3.6.0-0.rockspec +++ b/kong-3.6.0-0.rockspec @@ -180,6 +180,7 @@ build = { ["kong.tools.module"] = "kong/tools/module.lua", ["kong.tools.ip"] = "kong/tools/ip.lua", ["kong.tools.http"] = "kong/tools/http.lua", + ["kong.tools.cjson"] = "kong/tools/cjson.lua", ["kong.runloop.handler"] = "kong/runloop/handler.lua", ["kong.runloop.events"] = "kong/runloop/events.lua", diff --git a/kong/constants.lua b/kong/constants.lua index fc3b8a18a3b2..649a4380d6e1 100644 --- a/kong/constants.lua +++ b/kong/constants.lua @@ -92,6 +92,7 @@ for k in pairs(key_formats_map) do end local constants = { + CJSON_MAX_PRECISION = 16, BUNDLED_PLUGINS = plugin_map, DEPRECATED_PLUGINS = deprecated_plugin_map, BUNDLED_VAULTS = vault_map, diff --git a/kong/db/strategies/postgres/init.lua b/kong/db/strategies/postgres/init.lua index e76af557e293..8f8c70ff12bc 100644 --- a/kong/db/strategies/postgres/init.lua +++ b/kong/db/strategies/postgres/init.lua @@ -6,8 +6,6 @@ local utils = require "kong.tools.utils" local new_tab = require "table.new" local clear_tab = require "table.clear" -cjson.encode_number_precision(16) -cjson_safe.encode_number_precision(16) local kong = kong local ngx = ngx diff --git a/kong/globalpatches.lua b/kong/globalpatches.lua index c3782f0c8a0f..ab728fcd4e03 100644 --- a/kong/globalpatches.lua +++ b/kong/globalpatches.lua @@ -1,3 +1,5 @@ +local constants = require "kong.constants" + local ran_before @@ -15,8 +17,12 @@ return function(options) local meta = require "kong.meta" - local cjson = require("cjson.safe") + local cjson = require("cjson") + local cjson_safe = require("cjson.safe") cjson.encode_sparse_array(nil, nil, 2^15) + cjson.encode_number_precision(constants.CJSON_MAX_PRECISION) + cjson_safe.encode_sparse_array(nil, nil, 2^15) + cjson_safe.encode_number_precision(constants.CJSON_MAX_PRECISION) local pb = require "pb" diff --git a/kong/pdk/request.lua b/kong/pdk/request.lua index c3e31c2e3eb7..10bb08dfe5df 100644 --- a/kong/pdk/request.lua +++ b/kong/pdk/request.lua @@ -6,7 +6,7 @@ -- @module kong.request -local cjson = require "cjson.safe".new() +local cjson = require "kong.tools.cjson" local multipart = require "multipart" local phase_checker = require "kong.pdk.private.phases" local normalize = require("kong.tools.uri").normalize @@ -44,10 +44,6 @@ local decode_args = ngx.decode_args local PHASES = phase_checker.phases -cjson.decode_array_with_array_mt(true) -cjson.encode_sparse_array(nil, nil, 2^15) -cjson.encode_number_precision(16) - local function new(self) local _REQUEST = {} @@ -834,7 +830,7 @@ local function new(self) return nil, err, CONTENT_TYPE_JSON end - local json = cjson.decode(body) + local json = cjson.decode_with_array_mt(body) if type(json) ~= "table" then return nil, "invalid json body", CONTENT_TYPE_JSON end diff --git a/kong/pdk/response.lua b/kong/pdk/response.lua index 3611421233d2..37a0c67d11f4 100644 --- a/kong/pdk/response.lua +++ b/kong/pdk/response.lua @@ -13,15 +13,13 @@ local buffer = require "string.buffer" -local cjson = require "cjson.safe".new() +local cjson = require "cjson.safe" local checks = require "kong.pdk.private.checks" local phase_checker = require "kong.pdk.private.phases" local utils = require "kong.tools.utils" local request_id = require "kong.tracing.request_id" local constants = require "kong.constants" -cjson.encode_sparse_array(nil, nil, 2^15) -cjson.encode_number_precision(16) local ngx = ngx local arg = ngx.arg diff --git a/kong/pdk/service/request.lua b/kong/pdk/service/request.lua index 5003e5a261b6..495dbf0febcf 100644 --- a/kong/pdk/service/request.lua +++ b/kong/pdk/service/request.lua @@ -2,7 +2,7 @@ -- Module for manipulating the request sent to the Service. -- @module kong.service.request -local cjson = require "cjson.safe".new() +local cjson = require "cjson.safe" local buffer = require "string.buffer" local checks = require "kong.pdk.private.checks" local phase_checker = require "kong.pdk.private.phases" @@ -24,9 +24,6 @@ local validate_headers = checks.validate_headers local check_phase = phase_checker.check local escape = require("kong.tools.uri").escape -cjson.encode_sparse_array(nil, nil, 2^15) -cjson.encode_number_precision(16) - local PHASES = phase_checker.phases diff --git a/kong/pdk/service/response.lua b/kong/pdk/service/response.lua index d57df3546eb0..5a6621abf543 100644 --- a/kong/pdk/service/response.lua +++ b/kong/pdk/service/response.lua @@ -3,7 +3,7 @@ -- @module kong.service.response -local cjson = require "cjson.safe".new() +local cjson = require "kong.tools.cjson" local multipart = require "multipart" local phase_checker = require "kong.pdk.private.phases" local string_tools = require "kong.tools.string" @@ -23,10 +23,6 @@ local setmetatable = setmetatable local check_phase = phase_checker.check -cjson.decode_array_with_array_mt(true) -cjson.encode_sparse_array(nil, nil, 2^15) -cjson.encode_number_precision(16) - local replace_dashes = string_tools.replace_dashes local replace_dashes_lower = string_tools.replace_dashes_lower @@ -358,7 +354,7 @@ local function new(pdk, major_version) elseif find(content_type_lower, CONTENT_TYPE_JSON, 1, true) == 1 then local body = response.get_raw_body() - local json = cjson.decode(body) + local json = cjson.decode_with_array_mt(body) if type(json) ~= "table" then return nil, "invalid json body", CONTENT_TYPE_JSON end diff --git a/kong/tools/cjson.lua b/kong/tools/cjson.lua new file mode 100644 index 000000000000..ea668be90178 --- /dev/null +++ b/kong/tools/cjson.lua @@ -0,0 +1,21 @@ +local cjson = require "cjson.safe".new() +local constants = require "kong.constants" + +cjson.decode_array_with_array_mt(true) +cjson.encode_sparse_array(nil, nil, 2^15) +cjson.encode_number_precision(constants.CJSON_MAX_PRECISION) + +local _M = {} + + +function _M.encode(json_text) + return cjson.encode(json_text) +end + +function _M.decode_with_array_mt(json_text) + return cjson.decode(json_text) +end + +_M.array_mt = cjson.array_mt + +return _M diff --git a/t/01-pdk/08-response/11-exit.t b/t/01-pdk/08-response/11-exit.t index 8048b91cdbe7..4a6f7a624c92 100644 --- a/t/01-pdk/08-response/11-exit.t +++ b/t/01-pdk/08-response/11-exit.t @@ -1162,6 +1162,7 @@ manually setting Transfer-Encoding. Ignored. location = /t { default_type 'text/test'; access_by_lua_block { + require("kong.globalpatches")() local PDK = require "kong.pdk" local pdk = PDK.new() From 9deee708395ba714a2813bcdba35638b9bc86bea Mon Sep 17 00:00:00 2001 From: ms2008 Date: Tue, 9 Jan 2024 12:44:14 +0800 Subject: [PATCH 7/7] chore(*): remove the default global cjson instance setting --- kong/db/strategies/postgres/init.lua | 5 ++--- kong/db/strategies/postgres/tags.lua | 3 +-- kong/globalpatches.lua | 5 +---- 3 files changed, 4 insertions(+), 9 deletions(-) diff --git a/kong/db/strategies/postgres/init.lua b/kong/db/strategies/postgres/init.lua index 8f8c70ff12bc..c09bf9ed5878 100644 --- a/kong/db/strategies/postgres/init.lua +++ b/kong/db/strategies/postgres/init.lua @@ -1,6 +1,5 @@ local arrays = require "pgmoon.arrays" local json = require "pgmoon.json" -local cjson = require "cjson" local cjson_safe = require "cjson.safe" local utils = require "kong.tools.utils" local new_tab = require "table.new" @@ -215,7 +214,7 @@ local function escape_literal(connector, literal, field) elseif et == "map" or et == "record" or et == "json" then local jsons = {} for i, v in ipairs(literal) do - jsons[i] = cjson.encode(v) + jsons[i] = cjson_safe.encode(v) end return encode_array(jsons) .. '::JSONB[]' @@ -526,7 +525,7 @@ local function page(self, size, token, foreign_key, foreign_entity_name, options insert(offset, row[field_name]) end - offset = cjson.encode(offset) + offset = cjson_safe.encode(offset) offset = encode_base64(offset, true) return rows, nil, offset diff --git a/kong/db/strategies/postgres/tags.lua b/kong/db/strategies/postgres/tags.lua index 48341af94f63..f9b8bb884458 100644 --- a/kong/db/strategies/postgres/tags.lua +++ b/kong/db/strategies/postgres/tags.lua @@ -1,4 +1,3 @@ -local cjson = require "cjson" local cjson_safe = require "cjson.safe" @@ -118,7 +117,7 @@ local function page(self, size, token, options, tag) last_ordinality } - offset = cjson.encode(offset) + offset = cjson_safe.encode(offset) offset = encode_base64(offset, true) return rows, nil, offset diff --git a/kong/globalpatches.lua b/kong/globalpatches.lua index ab728fcd4e03..332e07db5903 100644 --- a/kong/globalpatches.lua +++ b/kong/globalpatches.lua @@ -17,10 +17,7 @@ return function(options) local meta = require "kong.meta" - local cjson = require("cjson") local cjson_safe = require("cjson.safe") - cjson.encode_sparse_array(nil, nil, 2^15) - cjson.encode_number_precision(constants.CJSON_MAX_PRECISION) cjson_safe.encode_sparse_array(nil, nil, 2^15) cjson_safe.encode_number_precision(constants.CJSON_MAX_PRECISION) @@ -29,7 +26,7 @@ return function(options) -- let pb decode arrays to table cjson.empty_array_mt metatable -- so empty arrays are encoded as `[]` instead of `nil` or `{}` by cjson. pb.option("decode_default_array") - pb.defaults("*array", cjson.empty_array_mt) + pb.defaults("*array", cjson_safe.empty_array_mt) if options.cli then -- disable the _G write guard alert log introduced in OpenResty 1.15.8.1