feat(http-log): add allow list for logged headers

kong/clustering/compat/removed_fields.lua

@@ -96,4 +96,12 @@ return {
       "response_headers",
     },
   },
+
+  -- Any dataplane older than 3.5.0
+  [3005000000] = {
+    http_log = {
+      "enable_logged_headers_allow_list",
+      "logged_headers_allow_list",
+    }
+  }
 }

kong/plugins/http-log/handler.lua

@@ -181,13 +181,38 @@ end
 
 
 function HttpLogHandler:log(conf)
+  -- do custom fields
   if conf.custom_fields_by_lua then
     local set_serialize_value = kong.log.set_serialize_value
     for key, expression in pairs(conf.custom_fields_by_lua) do
       set_serialize_value(key, sandbox(expression, sandbox_opts)())
     end
   end
 
+  -- do logged header allow list if enabled
+  if conf.enable_logged_header_allow_list then
+    local set_serialize_value = kong.log.set_serialize_value
+
+    -- clear all request and response headers
+    set_serialize_value("request.headers", nil)
+    set_serialize_value("response.headers", nil)
+
+    -- restore allowed headers
+    local request_get_header = kong.request.get_header
+    local response_get_header = kong.response.get_header
+    for _, header_name in ipairs(conf.logged_headers_allow_list) do
+      local request_header_val = request_get_header(header_name)
+      if request_header_val then
+        set_serialize_value("request.headers." .. header_name, request_header_val)
+      end
+
+      local response_header_val = response_get_header(header_name)
+      if response_header_val then
+        set_serialize_value("response.headers." .. header_name, response_header_val)
+      end
+    end
+  end
+
   local queue_conf = Queue.get_plugin_params("http-log", conf, make_queue_name(conf))
   kong.log.debug("Queue name automatically configured based on configuration parameters to: ", queue_conf.name)
 

kong/plugins/http-log/schema.lua

@@ -42,6 +42,43 @@ return {
           }},
           { queue = typedefs.queue },
           { custom_fields_by_lua = typedefs.lua_code },
+          { enable_logged_headers_allow_list = {
+            type = "boolean",
+            default = false,
+            description = "When enabled, will ensure only headers from the logged_headers_allow_list are included in log payloads. All headers are logged if this is disabled.",
+          }},
+          { logged_headers_allow_list = {
+            type = "array",
+            elements = typedefs.header_name,
+            default = {
+              "accept",
+              "accept-encoding",
+              "accept-language",
+              "cache-control",
+              "content-length",
+              "content-type",
+              "date",
+              "expires",
+              "host",
+              "ipremoteaddr",
+              "last-modified",
+              "ratelimit-limit",
+              "ratelimit-remaining",
+              "ratelimit-reset",
+              "referer",
+              "server",
+              "user-agent",
+              "via",
+              "x-forwarded-for",
+              "x-forwarded-host",
+              "x-forwarded-proto",
+              "x-original-for",
+              "x-original-host",
+              "x-original-proto",
+              "x-request-start",
+            },
+            description = "An array of header names that will be included in logged payloads (if present in the request and/or response headers). Only effective when enable_logged_header_allow_list is set to true."
+          }},
         },
 
         entity_checks = {

spec/03-plugins/03-http-log/01-log_spec.lua

@@ -322,6 +322,32 @@ for _, strategy in helpers.each_strategy() do
         }
       }
 
+      local service11 = bp.services:insert{
+        protocol = "http",
+        host     = helpers.mock_upstream_host,
+        port     = helpers.mock_upstream_port,
+      }
+
+      local route11 = bp.routes:insert {
+        hosts   = { "log_allowed_headers.test" },
+        service = service9
+      }
+
+      bp.plugins:insert {
+        route = { id = route11.id },
+        name     = "http-log",
+        config   = {
+          http_endpoint = "http://" .. helpers.mock_upstream_host
+                                    .. ":"
+                                    .. helpers.mock_upstream_port
+                                    .. "/post_log/custom_http",
+          enable_logged_header_allow_list = true,
+          logged_headers_allow_list = {
+            "allowed_header",
+          },
+        }
+      }
+
       helpers.setenv(vault_env_name, vault_env_value)
 
       assert(helpers.start_kong({
@@ -420,6 +446,25 @@ for _, strategy in helpers.each_strategy() do
       end)
     end)
 
+    describe("logged headers allow list", function()
+      it("only log allowed headers", function()
+        reset_log("allowed_headers")
+        local res = proxy_client:get("/status/200", {
+          headers = {
+            ["Host"] = "log_allowed_headers.test",
+            ["allowed_header"] = "123",
+            ["unallowed_header"] = "123"
+          }
+        })
+        assert.res_status(200, res)
+
+        local entries = get_log("allowed_headers", 1)
+        assert.same("127.0.0.1", entries[1].client_ip)
+        assert.same(123, entries[1].request.headers.allowed_header)
+        assert.same(nil, entries[1].request.headers.unallowed_header)
+      end)
+    end)
+
     it("logs to HTTP (#grpc)", function()
       reset_log("grpc")
       -- Making the request