conf, router: Make the listen(2) backlog configurable

@oopsoop2 on GitHub reported a performance issue related to the default
listen(2) backlog size of 511 on nginx. They found that increasing it
helped, nginx has a config option to configure this.

They would like to be able to do the same on Unit (which also defaults
to 511 on some systems). This seems reasonable.

NOTE: On Linux before commit 97c15fa ("socket: Use a default listen
backlog of -1 on Linux") we defaulted to 511. Since that commit we
default to the Kernels default, which before 5.4 is 128 and after is
4096.

This adds a new per-listener 'backlog' config option, e.g

  {
      "listeners": {
          "[::1]:8080": {
              "pass": "routes",
              "backlog": 1024
          },
      }

      ...
  }

This doesn't effect the control socket.

Closes: #1384
Reported-by: <https://github.com/oopsoop2>
Signed-off-by: Andrew Clayton <[email protected]>

src/nxt_conf_validation.c

@@ -176,6 +176,8 @@ static nxt_int_t nxt_conf_vldt_app_name(nxt_conf_validation_t *vldt,
     nxt_conf_value_t *value, void *data);
 static nxt_int_t nxt_conf_vldt_forwarded(nxt_conf_validation_t *vldt,
     nxt_conf_value_t *value, void *data);
+static nxt_int_t nxt_conf_vldt_listen_backlog(nxt_conf_validation_t *vldt,
+    nxt_conf_value_t *value, void *data);
 static nxt_int_t nxt_conf_vldt_app(nxt_conf_validation_t *vldt,
     nxt_str_t *name, nxt_conf_value_t *value);
 static nxt_int_t nxt_conf_vldt_object(nxt_conf_validation_t *vldt,
@@ -424,6 +426,10 @@ static nxt_conf_vldt_object_t  nxt_conf_vldt_listener_members[] = {
         .type       = NXT_CONF_VLDT_OBJECT,
         .validator  = nxt_conf_vldt_object,
         .u.members  = nxt_conf_vldt_client_ip_members
+    }, {
+        .name       = nxt_string("backlog"),
+        .type       = NXT_CONF_VLDT_NUMBER,
+        .validator  = nxt_conf_vldt_listen_backlog,
     },
 
 #if (NXT_TLS)
@@ -2677,6 +2683,32 @@ nxt_conf_vldt_forwarded(nxt_conf_validation_t *vldt, nxt_conf_value_t *value,
 }
 
 
+static nxt_int_t
+nxt_conf_vldt_listen_backlog(nxt_conf_validation_t *vldt,
+    nxt_conf_value_t *value, void *data)
+{
+    int64_t  backlog;
+
+    backlog = nxt_conf_get_number(value);
+
+    /*
+     * POSIX allows this to be 0 and some systems use -1 to
+     * indicate to use the OS's default value.
+     */
+    if (backlog < -1) {
+        return nxt_conf_vldt_error(vldt, "The \"backlog\" number must be "
+                                   "equal to or greater than -1.");
+    }
+
+    if (backlog > NXT_INT32_T_MAX) {
+        return nxt_conf_vldt_error(vldt, "The \"backlog\" number must "
+                                   "not exceed %d.", NXT_INT32_T_MAX);
+    }
+
+    return NXT_OK;
+}
+
+
 static nxt_int_t
 nxt_conf_vldt_app(nxt_conf_validation_t *vldt, nxt_str_t *name,
     nxt_conf_value_t *value)

src/nxt_router.c

@@ -40,6 +40,7 @@ typedef struct {
 typedef struct {
     nxt_str_t         pass;
     nxt_str_t         application;
+    int               backlog;
 } nxt_router_listener_conf_t;
 
 
@@ -166,7 +167,7 @@ static void nxt_router_app_prefork_ready(nxt_task_t *task,
 static void nxt_router_app_prefork_error(nxt_task_t *task,
     nxt_port_recv_msg_t *msg, void *data);
 static nxt_socket_conf_t *nxt_router_socket_conf(nxt_task_t *task,
-    nxt_router_temp_conf_t *tmcf, nxt_str_t *name);
+    nxt_router_temp_conf_t *tmcf, nxt_str_t *name, int backlog);
 static nxt_int_t nxt_router_listen_socket_find(nxt_router_temp_conf_t *tmcf,
     nxt_socket_conf_t *nskcf, nxt_sockaddr_t *sa);
 
@@ -1494,6 +1495,12 @@ static nxt_conf_map_t  nxt_router_listener_conf[] = {
         NXT_CONF_MAP_STR_COPY,
         offsetof(nxt_router_listener_conf_t, application),
     },
+
+    {
+        nxt_string("backlog"),
+        NXT_CONF_MAP_INT32,
+        offsetof(nxt_router_listener_conf_t, backlog),
+    },
 };
 
 
@@ -1964,13 +1971,10 @@ nxt_router_conf_create(nxt_task_t *task, nxt_router_temp_conf_t *tmcf,
                 break;
             }
 
-            skcf = nxt_router_socket_conf(task, tmcf, &name);
-            if (skcf == NULL) {
-                goto fail;
-            }
-
             nxt_memzero(&lscf, sizeof(lscf));
 
+            lscf.backlog = -1;
+
             ret = nxt_conf_map_object(mp, listener, nxt_router_listener_conf,
                                       nxt_nitems(nxt_router_listener_conf),
                                       &lscf);
@@ -1981,6 +1985,11 @@ nxt_router_conf_create(nxt_task_t *task, nxt_router_temp_conf_t *tmcf,
 
             nxt_debug(task, "application: %V", &lscf.application);
 
+            skcf = nxt_router_socket_conf(task, tmcf, &name, lscf.backlog);
+            if (skcf == NULL) {
+                goto fail;
+            }
+
             // STUB, default values if http block is not defined.
             skcf->header_buffer_size = 2048;
             skcf->large_header_buffer_size = 8192;
@@ -2684,7 +2693,7 @@ nxt_router_application_init(nxt_router_conf_t *rtcf, nxt_str_t *name,
 
 static nxt_socket_conf_t *
 nxt_router_socket_conf(nxt_task_t *task, nxt_router_temp_conf_t *tmcf,
-    nxt_str_t *name)
+    nxt_str_t *name, int backlog)
 {
     size_t               size;
     nxt_int_t            ret;
@@ -2728,7 +2737,7 @@ nxt_router_socket_conf(nxt_task_t *task, nxt_router_temp_conf_t *tmcf,
         nxt_listen_socket_remote_size(ls);
 
         ls->socket = -1;
-        ls->backlog = NXT_LISTEN_BACKLOG;
+        ls->backlog = backlog > -1 ? backlog : NXT_LISTEN_BACKLOG;
         ls->flags = NXT_NONBLOCK;
         ls->read_after_accept = 1;
     }
@@ -2875,7 +2884,7 @@ nxt_router_listen_socket_ready(nxt_task_t *task, nxt_port_recv_msg_t *msg,
 
     nxt_socket_defer_accept(task, s, rpc->socket_conf->listen->sockaddr);
 
-    ret = nxt_listen_socket(task, s, NXT_LISTEN_BACKLOG);
+    ret = nxt_listen_socket(task, s, rpc->socket_conf->listen->backlog);
     if (nxt_slow_path(ret != NXT_OK)) {
         goto fail;
     }