From b2f041c0246262f3e37beeff474df2b4a0ad5431 Mon Sep 17 00:00:00 2001
From: William Yang <mscame@gmail.com>
Date: Mon, 9 Dec 2024 14:17:20 +0100
Subject: [PATCH 01/12] feat: support cust spawn opts

---
 src/quicer_conn_acceptor_sup.erl | 16 +++++++++++++---
 src/quicer_connection.erl        | 15 ++++++++++++++-
 2 files changed, 27 insertions(+), 4 deletions(-)

diff --git a/src/quicer_conn_acceptor_sup.erl b/src/quicer_conn_acceptor_sup.erl
index 8953d265..6e8aa46f 100644
--- a/src/quicer_conn_acceptor_sup.erl
+++ b/src/quicer_conn_acceptor_sup.erl
@@ -63,22 +63,32 @@ start_link(ListenerH, ConnOpts) ->
     {ok, {SupFlags :: supervisor:sup_flags(), [ChildSpec :: supervisor:child_spec()]}}
     | ignore.
 init([ListenerH, Opts]) ->
+    OptsTab = init_opts_tab(Opts),
     SupFlags = #{
         strategy => simple_one_for_one,
         intensity => 1,
         period => 5
     },
-
     OneChild = #{
         id => ignored,
-        start => {quicer_connection, start_link, [undefined, ListenerH, Opts]},
+        start => {quicer_connection, start_acceptor, [ListenerH, OptsTab]},
         restart => temporary,
         shutdown => 5000,
         type => worker
     },
-
     {ok, {SupFlags, [OneChild]}}.
 
 %%%===================================================================
 %%% Internal functions
 %%%===================================================================
+init_opts_tab({LOpts, COpts, SOpts}) ->
+    SharedTab = ets:new(quicer_listener_tab, [set, {keypos, 1}, {read_concurrency, true}]),
+    true = store_config(SharedTab, l_opts, LOpts),
+    true = store_config(SharedTab, c_opts, COpts),
+    true = store_config(SharedTab, s_opts, SOpts),
+    SharedTab.
+
+store_config(Tab, K, V) when is_list(V) ->
+    store_config(Tab, K, maps:from_list(V));
+store_config(Tab, K, V) ->
+    true = ets:insert(Tab, {K, V}).
diff --git a/src/quicer_connection.erl b/src/quicer_connection.erl
index ff144f02..cc0108d8 100644
--- a/src/quicer_connection.erl
+++ b/src/quicer_connection.erl
@@ -153,6 +153,7 @@
 -export([
     start_link/3,
     %% for server
+    start_acceptor/3,
     start_link/4,
     get_cb_state/1,
     merge_cb_state/2,
@@ -216,6 +217,13 @@ start_link(undefined, Listener, {_LOpts, COpts, _SOpts} = Opts, Sup) when is_map
 start_link(CallbackModule, Listener, Opts, Sup) ->
     gen_server:start_link(?MODULE, [CallbackModule, Listener, Opts, Sup], []).
 
+-define(DEFAULT_SPAWN_OPTS, [{spawn_opt, [link]}]).
+%% @doc start acceptor with shared Listener confs
+start_acceptor(ListenHandle, Tab, Sup) when is_pid(Sup) ->
+    [{c_opts, #{conn_callback := CallbackModule} = Conf}] = ets:lookup(Tab, c_opts),
+    StartOpts = maps:get(spawn_opts, Conf, ?DEFAULT_SPAWN_OPTS),
+    gen_server:start(?MODULE, [CallbackModule, ListenHandle, Tab, Sup], StartOpts).
+
 -spec get_cb_state(ConnPid :: pid()) -> cb_state() | {error, any()}.
 get_cb_state(ConnPid) ->
     gen_server:call(ConnPid, get_cb_state, infinity).
@@ -313,7 +321,12 @@ init([CallbackModule, Listener, {_LOpts, COpts, SOpts}, Sup]) when CallbackModul
         %% ignore, {stop, Reason} ...
         Other ->
             Other
-    end.
+    end;
+init([CallbackModule, Listener, ConfTab, Sup]) ->
+    [{l_opts, LOpts}] = ets:lookup(ConfTab, l_opts),
+    [{c_opts, COpts}] = ets:lookup(ConfTab, c_opts),
+    [{s_opts, SOpts}] = ets:lookup(ConfTab, s_opts),
+    init([CallbackModule, Listener, {LOpts, COpts, SOpts}, Sup]).
 
 %%--------------------------------------------------------------------
 %% @private

From 642404645378446c47e869d4618616a8f955f973 Mon Sep 17 00:00:00 2001
From: William Yang <mscame@gmail.com>
Date: Mon, 9 Dec 2024 15:57:08 +0100
Subject: [PATCH 02/12] feat: support get/set listener opts

---
 src/quicer_conn_acceptor_sup.erl | 14 +--------
 src/quicer_listener.erl          | 51 ++++++++++++++++++++++++++------
 test/quicer_listener_SUITE.erl   | 20 ++++++++-----
 3 files changed, 56 insertions(+), 29 deletions(-)

diff --git a/src/quicer_conn_acceptor_sup.erl b/src/quicer_conn_acceptor_sup.erl
index 6e8aa46f..81a82451 100644
--- a/src/quicer_conn_acceptor_sup.erl
+++ b/src/quicer_conn_acceptor_sup.erl
@@ -62,8 +62,7 @@ start_link(ListenerH, ConnOpts) ->
 -spec init(Args :: term()) ->
     {ok, {SupFlags :: supervisor:sup_flags(), [ChildSpec :: supervisor:child_spec()]}}
     | ignore.
-init([ListenerH, Opts]) ->
-    OptsTab = init_opts_tab(Opts),
+init([ListenerH, OptsTab]) ->
     SupFlags = #{
         strategy => simple_one_for_one,
         intensity => 1,
@@ -81,14 +80,3 @@ init([ListenerH, Opts]) ->
 %%%===================================================================
 %%% Internal functions
 %%%===================================================================
-init_opts_tab({LOpts, COpts, SOpts}) ->
-    SharedTab = ets:new(quicer_listener_tab, [set, {keypos, 1}, {read_concurrency, true}]),
-    true = store_config(SharedTab, l_opts, LOpts),
-    true = store_config(SharedTab, c_opts, COpts),
-    true = store_config(SharedTab, s_opts, SOpts),
-    SharedTab.
-
-store_config(Tab, K, V) when is_list(V) ->
-    store_config(Tab, K, maps:from_list(V));
-store_config(Tab, K, V) ->
-    true = ets:insert(Tab, {K, V}).
diff --git a/src/quicer_listener.erl b/src/quicer_listener.erl
index 55dcc828..0c6a6c6a 100644
--- a/src/quicer_listener.erl
+++ b/src/quicer_listener.erl
@@ -26,6 +26,7 @@
     unlock/2,
     reload/2,
     reload/3,
+    get_conf/2,
     get_handle/2,
     count_conns/1
 ]).
@@ -45,7 +46,7 @@
     listener :: quicer:listener_handle(),
     conn_sup :: pid(),
     alpn :: [string()],
-    opts :: quicer:listener_opts()
+    opts_tab :: ets:tid()
 }).
 
 -export_type([listener_name/0]).
@@ -113,6 +114,11 @@ get_handle(Pid, Timeout) ->
 count_conns(Pid) ->
     gen_server:call(Pid, count_conns, infinity).
 
+%% @doc get the listener configuration
+-spec get_conf(pid(), timeout()) -> {map(), map(), map()}.
+get_conf(Pid, Timeout) ->
+    gen_server:call(Pid, get_conf, Timeout).
+
 %%%===================================================================
 %%% gen_server callbacks
 %%%===================================================================
@@ -134,15 +140,16 @@ init([Name, ListenOn, {LOpts, COpts, SOpts}]) when is_list(LOpts) ->
     init([Name, ListenOn, {maps:from_list(LOpts), COpts, SOpts}]);
 init([Name, ListenOn, {#{conn_acceptors := N, alpn := Alpn} = LOpts, _COpts, _SOpts} = Opts]) ->
     process_flag(trap_exit, true),
+    OptsTab = init_opts_tab(Opts),
     {ok, L} = quicer:listen(ListenOn, maps:without([conn_acceptors], LOpts)),
-    {ok, ConnSup} = supervisor:start_link(quicer_conn_acceptor_sup, [L, Opts]),
+    {ok, ConnSup} = supervisor:start_link(quicer_conn_acceptor_sup, [L, OptsTab]),
     _ = [{ok, _} = supervisor:start_child(ConnSup, [ConnSup]) || _ <- lists:seq(1, N)],
     {ok, #state{
         name = Name,
         listen_on = ListenOn,
         listener = L,
         conn_sup = ConnSup,
-        opts = LOpts,
+        opts_tab = OptsTab,
         alpn = Alpn
     }}.
 
@@ -167,10 +174,11 @@ handle_call(lock, _From, State) ->
     Res = quicer:stop_listener(State#state.listener),
     {reply, Res, State};
 handle_call(unlock, _From, State) ->
+    LOpts = ets:lookup_element(State#state.opts_tab, l_opts, 2),
     Res = quicer:start_listener(
         State#state.listener,
         State#state.listen_on,
-        State#state.opts
+        LOpts
     ),
     {reply, Res, State};
 handle_call({reload, NewConf}, _From, State) ->
@@ -184,12 +192,21 @@ handle_call(
     _From,
     #state{
         conn_sup = ConnSup,
-        opts = #{conn_acceptors := NoAcceptors}
+        opts_tab = OptsTab
     } =
         State
 ) ->
-    ConnPids = supervisor:which_children(ConnSup),
-    {reply, length(ConnPids) - NoAcceptors, State};
+    #{conn_acceptors := NoAcceptors} = ets:lookup_element(OptsTab, l_opts, 2),
+    {active, ActiveCnt} = lists:nth(2, supervisor:count_children(ConnSup)),
+    {reply, ActiveCnt - NoAcceptors, State};
+handle_call(get_conf, _From, #state{opts_tab = OptsTab} = State) ->
+    {reply,
+        {
+            ets:lookup_element(OptsTab, l_opts, 2),
+            ets:lookup_element(OptsTab, c_opts, 2),
+            ets:lookup_element(OptsTab, s_opts, 2)
+        },
+        State};
 handle_call(Request, _From, State) ->
     Reply = {error, {unimpl, Request}},
     {reply, Reply, State}.
@@ -245,7 +262,7 @@ terminate(_Reason, #state{listener = L}) ->
     ok.
 
 -spec do_reload(quicer:listen_on(), map(), #state{}) -> {ok | {error, any()}, #state{}}.
-do_reload(ListenOn, NewConf, State) ->
+do_reload(ListenOn, NewConf, #state{opts_tab = OptsTab} = State) ->
     _ = quicer:stop_listener(State#state.listener),
     Res = quicer:start_listener(
         State#state.listener,
@@ -254,7 +271,23 @@ do_reload(ListenOn, NewConf, State) ->
     ),
     case Res of
         ok ->
-            {ok, State#state{listen_on = ListenOn, opts = NewConf}};
+            true = ets:insert(OptsTab, {l_opts, to_map(NewConf)}),
+            {ok, State#state{listen_on = ListenOn}};
         Error ->
             {Error, State}
     end.
+
+init_opts_tab({LOpts, COpts, SOpts}) ->
+    Tab = ets:new(quicer_listener_tab, [set, {keypos, 1}, {read_concurrency, true}]),
+    %% @NOTE: Be careful with the lifecyle of the table.
+    true = ets:insert(Tab, [
+        {l_opts, to_map(LOpts)},
+        {c_opts, to_map(COpts)},
+        {s_opts, to_map(SOpts)}
+    ]),
+    Tab.
+
+to_map(Opts) when is_list(Opts) ->
+    maps:from_list(Opts);
+to_map(Opts) when is_map(Opts) ->
+    Opts.
diff --git a/test/quicer_listener_SUITE.erl b/test/quicer_listener_SUITE.erl
index e19eee88..1a762aaf 100644
--- a/test/quicer_listener_SUITE.erl
+++ b/test/quicer_listener_SUITE.erl
@@ -521,17 +521,23 @@ tc_listener_conf_reload(Config) ->
     %% WHEN: the listener is reloaded with new listener opts (New cert, key and cacert).
     ok = quicer_listener:lock(QuicApp, infinity),
     ok = quicer_listener:unlock(QuicApp, infinity),
-    NewListenerOpts =
-        ListenerOpts ++
-            [
-                {certfile, filename:join(DataDir, "other-server.pem")},
-                {keyfile, filename:join(DataDir, "other-server.key")},
-                {cacertfile, filename:join(DataDir, "other-ca.pem")}
-            ],
+
+    NewCerts = [
+        {certfile, filename:join(DataDir, "other-server.pem")},
+        {keyfile, filename:join(DataDir, "other-server.key")},
+        {cacertfile, filename:join(DataDir, "other-ca.pem")}
+    ],
+    NewListenerOpts = ListenerOpts ++ NewCerts,
     ok = quicer_listener:reload(QuicApp, NewListenerOpts),
     %% THEN: the listener handle is unchanged
     ?assertEqual({ok, LHandle}, quicer_listener:get_handle(QuicApp, 5000)),
 
+    %% THEN: the listener conf is changed
+    {LMap, CMap, SMap} = quicer_listener:get_conf(QuicApp, 5000),
+    ?assertEqual(LMap, maps:from_list(NewListenerOpts)),
+    ?assertEqual(CMap, maps:from_list(ConnectionOpts)),
+    ?assertEqual(SMap, maps:from_list(StreamOpts)),
+
     %% THEN: start new connection with old cacert must fail
     ?assertMatch(
         {error, transport_down, #{error := _, status := Status}} when

From 77ed6d874ce2131ec35ffc5f627cfa91df6d3c7a Mon Sep 17 00:00:00 2001
From: William Yang <mscame@gmail.com>
Date: Mon, 9 Dec 2024 21:25:15 +0100
Subject: [PATCH 03/12] feat: defer conn setting to handshake

- remove QUIC_SETTINGS from acceptor
---
 c_src/quicer_connection.c | 88 +++++++++++++++++++--------------------
 c_src/quicer_connection.h |  2 +-
 c_src/quicer_nif.c        |  3 +-
 c_src/quicer_queue.h      |  3 --
 src/quicer.erl            | 36 ++++++++++++++--
 src/quicer_connection.erl | 57 +++++++++++++++++++------
 src/quicer_listener.erl   | 24 +++++++----
 src/quicer_nif.erl        |  5 +++
 test/quicer_snb_SUITE.erl |  2 +-
 9 files changed, 146 insertions(+), 74 deletions(-)

diff --git a/c_src/quicer_connection.c b/c_src/quicer_connection.c
index e0ccd46f..006ef84c 100644
--- a/c_src/quicer_connection.c
+++ b/c_src/quicer_connection.c
@@ -899,18 +899,13 @@ async_accept2(ErlNifEnv *env,
               const ERL_NIF_TERM argv[])
 {
   ERL_NIF_TERM listener = argv[0];
-  ERL_NIF_TERM conn_opts = argv[1];
+  // @NOTE: since 0.2, we ignore argv[1]
   QuicerListenerCTX *l_ctx = NULL;
-  ERL_NIF_TERM active_val = ATOM_TRUE;
   if (!enif_get_resource(env, listener, ctx_listener_t, (void **)&l_ctx))
     {
       return ERROR_TUPLE_2(ATOM_BADARG);
     }
 
-  // Set parm active is optional
-  enif_get_map_value(
-      env, conn_opts, ATOM_QUIC_STREAM_OPTS_ACTIVE, &active_val);
-
   ACCEPTOR *acceptor = AcceptorAlloc();
   if (!acceptor)
     {
@@ -923,24 +918,11 @@ async_accept2(ErlNifEnv *env,
       return ERROR_TUPLE_2(ATOM_BAD_PID);
     }
 
-  if (!set_owner_recv_mode(acceptor, env, active_val))
-    {
-      AcceptorDestroy(acceptor);
-      return ERROR_TUPLE_2(ATOM_BADARG);
-    }
-
-  if (!create_settings(env, &conn_opts, &acceptor->Settings))
-    {
-      AcceptorDestroy(acceptor);
-      return ERROR_TUPLE_2(ATOM_PARAM_ERROR);
-    }
-
   AcceptorEnqueue(l_ctx->acceptor_queue, acceptor);
 
   assert(enif_is_process_alive(env, &(acceptor->Pid)));
 
-  ERL_NIF_TERM listenHandle = enif_make_resource(env, l_ctx);
-  return SUCCESS(listenHandle);
+  return SUCCESS(listener);
 }
 
 ERL_NIF_TERM
@@ -1084,40 +1066,28 @@ continue_connection_handshake(QuicerConnCTX *c_ctx)
 {
   QUIC_STATUS Status = QUIC_STATUS_SUCCESS;
 
-  if (!c_ctx)
-    {
-      return QUIC_STATUS_INTERNAL_ERROR;
-    }
-
-  if (!c_ctx->Connection)
-    {
-      return QUIC_STATUS_INVALID_STATE;
-    }
-
-  if (QUIC_FAILED(Status = MsQuic->ConnectionSetConfiguration(
-                      c_ctx->Connection, c_ctx->config_ctx->Configuration)))
-    {
-      return Status;
-    }
+  CXPLAT_FRE_ASSERT(c_ctx);
+  CXPLAT_FRE_ASSERT(c_ctx->Connection);
 
-  // Apply connection owners' option overrides
-  Status = MsQuic->SetParam(c_ctx->Connection,
-                            QUIC_PARAM_CONN_SETTINGS,
-                            sizeof(QUIC_SETTINGS),
-                            &c_ctx->owner->Settings);
+  Status = MsQuic->ConnectionSetConfiguration(
+      c_ctx->Connection, c_ctx->config_ctx->Configuration);
   return Status;
 }
 
 ERL_NIF_TERM
-async_handshake_1(ErlNifEnv *env, int argc, const ERL_NIF_TERM argv[])
+async_handshake_X(ErlNifEnv *env, int argc, const ERL_NIF_TERM argv[])
 
 {
   QuicerConnCTX *c_ctx;
   QUIC_STATUS Status = QUIC_STATUS_SUCCESS;
   ERL_NIF_TERM res = ATOM_OK;
-  CXPLAT_FRE_ASSERT(argc == 1);
+  CXPLAT_FRE_ASSERT(argc == 1 || argc == 2);
+  ERL_NIF_TERM econn = argv[0];
 
-  if (!enif_get_resource(env, argv[0], ctx_connection_t, (void **)&c_ctx))
+  QUIC_SETTINGS Settings = { 0 };
+  ERL_NIF_TERM active_val = ATOM_TRUE;
+
+  if (!enif_get_resource(env, econn, ctx_connection_t, (void **)&c_ctx))
     {
       return ERROR_TUPLE_2(ATOM_BADARG);
     }
@@ -1129,11 +1099,41 @@ async_handshake_1(ErlNifEnv *env, int argc, const ERL_NIF_TERM argv[])
       return ERROR_TUPLE_2(ATOM_CLOSED);
     }
 
+  if (argc > 1)
+    {
+      ERL_NIF_TERM econn_opts = argv[1];
+      // Set parm active is optional
+      enif_get_map_value(
+          env, econn_opts, ATOM_QUIC_STREAM_OPTS_ACTIVE, &active_val);
+
+      if (!create_settings(env, &econn_opts, &Settings))
+        {
+          res = ERROR_TUPLE_2(ATOM_PARAM_ERROR);
+          goto exit;
+        }
+
+      if (!set_owner_recv_mode(c_ctx->owner, env, active_val))
+        {
+          res = ERROR_TUPLE_2(ATOM_BADARG);
+          goto exit;
+        }
+
+      // Apply connection owners' option overrides
+      if (QUIC_FAILED(Status = MsQuic->SetParam(c_ctx->Connection,
+                                                QUIC_PARAM_CONN_SETTINGS,
+                                                sizeof(QUIC_SETTINGS),
+                                                &Settings)))
+        {
+          res = ERROR_TUPLE_2(ATOM_STATUS(Status));
+          goto exit;
+        }
+    }
+
   if (QUIC_FAILED(Status = continue_connection_handshake(c_ctx)))
     {
       res = ERROR_TUPLE_2(ATOM_STATUS(Status));
     }
-
+exit:
   put_conn_handle(c_ctx);
   return res;
 }
diff --git a/c_src/quicer_connection.h b/c_src/quicer_connection.h
index 09323417..8052770b 100644
--- a/c_src/quicer_connection.h
+++ b/c_src/quicer_connection.h
@@ -48,7 +48,7 @@ ERL_NIF_TERM
 get_conn_rid1(ErlNifEnv *env, int argc, const ERL_NIF_TERM argv[]);
 
 ERL_NIF_TERM
-async_handshake_1(ErlNifEnv *env, int argc, const ERL_NIF_TERM argv[]);
+async_handshake_X(ErlNifEnv *env, int argc, const ERL_NIF_TERM argv[]);
 
 QUIC_STATUS continue_connection_handshake(QuicerConnCTX *c_ctx);
 
diff --git a/c_src/quicer_nif.c b/c_src/quicer_nif.c
index c130f286..ac2caf31 100644
--- a/c_src/quicer_nif.c
+++ b/c_src/quicer_nif.c
@@ -1746,7 +1746,8 @@ static ErlNifFunc nif_funcs[] = {
   { "open_connection", 1, open_connectionX, 0},
   { "async_connect", 3, async_connect3, 0},
   { "async_accept", 2, async_accept2, 0},
-  { "async_handshake", 1, async_handshake_1, 0},
+  { "async_handshake", 1, async_handshake_X, 0},
+  { "async_handshake", 2, async_handshake_X, 0},
   { "async_shutdown_connection", 3, shutdown_connection3, 0},
   { "async_accept_stream", 2, async_accept_stream2, 0},
   { "start_stream", 2, async_start_stream2, 0},
diff --git a/c_src/quicer_queue.h b/c_src/quicer_queue.h
index 96324a7c..2a4f87db 100644
--- a/c_src/quicer_queue.h
+++ b/c_src/quicer_queue.h
@@ -55,10 +55,7 @@ typedef struct ACCEPTOR
   ErlNifPid Pid;
   ACCEPTOR_RECV_MODE active;
   uint16_t active_count; /* counter for active_n */
-  QUIC_SETTINGS Settings;
   void *reserved1;
-  void *reserved2;
-  void *reserved3;
 } ACCEPTOR;
 
 typedef struct AcceptorsQueue
diff --git a/src/quicer.erl b/src/quicer.erl
index 95e35fa3..e32ae74d 100644
--- a/src/quicer.erl
+++ b/src/quicer.erl
@@ -45,7 +45,9 @@
     async_connect/3,
     handshake/1,
     handshake/2,
+    handshake/3,
     async_handshake/1,
+    async_handshake/2,
     accept/2,
     accept/3,
     async_accept/2,
@@ -447,14 +449,29 @@ async_connect(Host, Port, Opts) when is_map(Opts) ->
 handshake(Conn) ->
     handshake(Conn, 5000).
 
+-spec handshake(connection_handle(), timeout()) ->
+    {ok, connection_handle()} | {error, any()}.
+handshake(Conn, Timeout) ->
+    case async_handshake(Conn) of
+        {error, _} = E ->
+            E;
+        ok ->
+            receive
+                {quic, connected, Conn, _} -> {ok, Conn};
+                {quic, closed, Conn, _Flags} -> {error, closed}
+            after Timeout ->
+                {error, timeout}
+            end
+    end.
+
 %% @doc Complete TLS handshake after accepted a Connection
 %% @see handshake/2
 %% @see async_handshake/1
--spec handshake(connection_handle(), timeout()) ->
+-spec handshake(connection_handle(), conn_opts(), timeout()) ->
     {ok, connection_handle()}
     | {error, any()}.
-handshake(Conn, Timeout) ->
-    case async_handshake(Conn) of
+handshake(Conn, ConnOpts, Timeout) ->
+    case async_handshake(Conn, ConnOpts) of
         {error, _} = E ->
             E;
         ok ->
@@ -467,13 +484,24 @@ handshake(Conn, Timeout) ->
     end.
 
 %% @doc Complete TLS handshake after accepted a Connection.
-%% Caller should expect to receive ```{quic, connected, connection_handle()}'''
 %%
 %% @see handshake/2
+%% @see async_handshake/2
 -spec async_handshake(connection_handle()) -> ok | {error, any()}.
 async_handshake(Conn) ->
     quicer_nif:async_handshake(Conn).
 
+%% @doc Complete TLS handshake after accepted a Connection.
+%%      also set connection options which override the default listener options.
+%%
+%% @see handshake/2
+%% @see async_handshake/1
+-spec async_handshake(connection_handle(), conn_opts()) -> ok | {error, any()}.
+async_handshake(Conn, ConnOpts) when is_list(ConnOpts) ->
+    async_handshake(Conn, maps:from_list(ConnOpts));
+async_handshake(Conn, ConnOpts) ->
+    quicer_nif:async_handshake(Conn, ConnOpts).
+
 %% @doc Accept new Connection (Server)
 %%
 %% Accept new connection from listener_handle().
diff --git a/src/quicer_connection.erl b/src/quicer_connection.erl
index cc0108d8..98951273 100644
--- a/src/quicer_connection.erl
+++ b/src/quicer_connection.erl
@@ -217,12 +217,13 @@ start_link(undefined, Listener, {_LOpts, COpts, _SOpts} = Opts, Sup) when is_map
 start_link(CallbackModule, Listener, Opts, Sup) ->
     gen_server:start_link(?MODULE, [CallbackModule, Listener, Opts, Sup], []).
 
--define(DEFAULT_SPAWN_OPTS, [{spawn_opt, [link]}]).
+-define(DEFAULT_ACCEPTOR_START_OPTS, [{spawn_opt, [link]}]).
 %% @doc start acceptor with shared Listener confs
+-spec start_acceptor(listener_handle(), ets:tid(), pid()) -> {ok, pid()} | {error, any()}.
 start_acceptor(ListenHandle, Tab, Sup) when is_pid(Sup) ->
     [{c_opts, #{conn_callback := CallbackModule} = Conf}] = ets:lookup(Tab, c_opts),
-    StartOpts = maps:get(spawn_opts, Conf, ?DEFAULT_SPAWN_OPTS),
-    gen_server:start(?MODULE, [CallbackModule, ListenHandle, Tab, Sup], StartOpts).
+    StartOpts = maps:get(proc_start_opts, Conf, ?DEFAULT_ACCEPTOR_START_OPTS),
+    gen_server:start_link(?MODULE, [CallbackModule, ListenHandle, Tab, Sup], StartOpts).
 
 -spec get_cb_state(ConnPid :: pid()) -> cb_state() | {error, any()}.
 get_cb_state(ConnPid) ->
@@ -285,7 +286,8 @@ init([CallbackModule, {Host, Port}, {COpts, SOpts}]) when
         callback => CallbackModule,
         conn_opts => COpts,
         stream_opts => SOpts,
-        sup => undefined
+        sup => undefined,
+        is_server => false
     },
     {ok, Conn} = quicer:async_connect(Host, Port, COpts),
     State1 = State0#{conn := Conn},
@@ -298,7 +300,7 @@ init([CallbackModule, {Host, Port}, {COpts, SOpts}]) when
         Other ->
             Other
     end;
-%% For Server
+%% For Server, deprecating since v0.2
 init([CallbackModule, Listener, {LOpts, COpts, SOpts}, Sup]) when is_list(COpts) ->
     init([CallbackModule, Listener, {LOpts, maps:from_list(COpts), SOpts}, Sup]);
 init([CallbackModule, Listener, {_LOpts, COpts, SOpts}, Sup]) when CallbackModule =/= undefined ->
@@ -309,7 +311,8 @@ init([CallbackModule, Listener, {_LOpts, COpts, SOpts}, Sup]) when CallbackModul
         callback => CallbackModule,
         conn_opts => COpts,
         stream_opts => SOpts,
-        sup => Sup
+        sup => Sup,
+        is_server => true
     },
     %% Async Acceptor
     {ok, Listener} = quicer_nif:async_accept(Listener, COpts),
@@ -322,11 +325,26 @@ init([CallbackModule, Listener, {_LOpts, COpts, SOpts}, Sup]) when CallbackModul
         Other ->
             Other
     end;
-init([CallbackModule, Listener, ConfTab, Sup]) ->
-    [{l_opts, LOpts}] = ets:lookup(ConfTab, l_opts),
-    [{c_opts, COpts}] = ets:lookup(ConfTab, c_opts),
-    [{s_opts, SOpts}] = ets:lookup(ConfTab, s_opts),
-    init([CallbackModule, Listener, {LOpts, COpts, SOpts}, Sup]).
+%% For Acceptor since v0.2
+init([_, _, _ConfTab, _] = Args) ->
+    acceptor_init(Args).
+acceptor_init([CallbackModule, Listener, ConfTab, Sup]) ->
+    process_flag(trap_exit, true),
+    State0 = #{
+        listener => Listener,
+        callback => CallbackModule,
+        conf_tab => ConfTab,
+        sup => Sup,
+        is_server => true,
+        conn => undefined,
+        conn_opts => undefined,
+        stream_opts => undefined,
+        callback_state => undefined
+    },
+    %% Async Acceptor
+    NOOP = #{},
+    {ok, Listener} = quicer_nif:async_accept(Listener, NOOP),
+    {ok, State0}.
 
 %%--------------------------------------------------------------------
 %% @private
@@ -400,16 +418,31 @@ handle_cast(_Request, State) ->
     | {noreply, NewState :: term(), Timeout :: timeout()}
     | {noreply, NewState :: term(), hibernate}
     | {stop, Reason :: normal | term(), NewState :: term()}.
+%% deprecating this clause
 handle_info(
     {quic, new_conn, C, Props},
     #{callback := M, sup := Sup, callback_state := CBState} = State
-) ->
+) when is_map(CBState) ->
     ?tp_ignore_side_effects_in_prod(debug, #{
         module => ?MODULE, conn => C, props => Props, event => new_conn
     }),
     %% I become the connection owner, I should start an new acceptor.
     Sup =/= undefined andalso (catch supervisor:start_child(Sup, [Sup])),
     default_cb_ret(M:new_conn(C, Props, CBState), State#{conn := C});
+handle_info(
+    {quic, new_conn, C, Props},
+    #{callback := M, sup := Sup, callback_state := undefined, conf_tab := ConfTab} = State
+) ->
+    %% deferred init
+    COpts = ets:lookup_element(ConfTab, c_opts, 2),
+    SOpts = ets:lookup_element(ConfTab, s_opts, 2),
+    Sup =/= undefined andalso (catch supervisor:start_child(Sup, [Sup])),
+    case M:init(COpts#{stream_opts => SOpts}) of
+        {ok, CBState0} ->
+            default_cb_ret(M:new_conn(C, Props, CBState0), State#{conn := C});
+        Other ->
+            Other
+    end;
 handle_info(
     {quic, connected, C, #{is_resumed := IsResumed} = Props},
     #{
diff --git a/src/quicer_listener.erl b/src/quicer_listener.erl
index 0c6a6c6a..ef9c89f3 100644
--- a/src/quicer_listener.erl
+++ b/src/quicer_listener.erl
@@ -101,7 +101,8 @@ reload(Pid, NewConf) ->
 %% @NOTE: the acceptor opts and stream opts are not reloaded.
 %%%       if you want to reload them, you should restart the listener (terminate and spawn).
 %% @end
--spec reload(pid(), quicer:listen_on(), NewConf :: map()) -> ok | {error, _}.
+-spec reload(pid(), quicer:listen_on(), NewConf :: map() | {map(), map(), map()}) ->
+    ok | {error, _}.
 reload(Pid, ListenOn, NewConf) ->
     gen_server:call(Pid, {reload, ListenOn, NewConf}, infinity).
 
@@ -261,7 +262,8 @@ terminate(_Reason, #state{listener = L}) ->
     _ = quicer:close_listener(L),
     ok.
 
--spec do_reload(quicer:listen_on(), map(), #state{}) -> {ok | {error, any()}, #state{}}.
+-spec do_reload(quicer:listen_on(), map() | {map(), map(), map()}, #state{}) ->
+    {ok | {error, any()}, #state{}}.
 do_reload(ListenOn, NewConf, #state{opts_tab = OptsTab} = State) ->
     _ = quicer:stop_listener(State#state.listener),
     Res = quicer:start_listener(
@@ -271,21 +273,27 @@ do_reload(ListenOn, NewConf, #state{opts_tab = OptsTab} = State) ->
     ),
     case Res of
         ok ->
-            true = ets:insert(OptsTab, {l_opts, to_map(NewConf)}),
+            true = conf_tab_refresh(OptsTab, NewConf),
             {ok, State#state{listen_on = ListenOn}};
         Error ->
             {Error, State}
     end.
 
-init_opts_tab({LOpts, COpts, SOpts}) ->
+init_opts_tab({_LOpts, _COpts, _SOpts} = Opts) ->
     Tab = ets:new(quicer_listener_tab, [set, {keypos, 1}, {read_concurrency, true}]),
-    %% @NOTE: Be careful with the lifecyle of the table.
-    true = ets:insert(Tab, [
+    %% @NOTE: Be careful with the lifecyle of the items in this table.
+    %%        handles in this table cannot be released until table is gone.
+    true = conf_tab_refresh(Tab, Opts),
+    Tab.
+
+conf_tab_refresh(Tab, {LOpts, COpts, SOpts}) ->
+    ets:insert(Tab, [
         {l_opts, to_map(LOpts)},
         {c_opts, to_map(COpts)},
         {s_opts, to_map(SOpts)}
-    ]),
-    Tab.
+    ]);
+conf_tab_refresh(Tab, LOpts) ->
+    ets:insert(Tab, {l_opts, to_map(LOpts)}).
 
 to_map(Opts) when is_list(Opts) ->
     maps:from_list(Opts);
diff --git a/src/quicer_nif.erl b/src/quicer_nif.erl
index 37bf8ad5..be95e7b6 100644
--- a/src/quicer_nif.erl
+++ b/src/quicer_nif.erl
@@ -33,6 +33,7 @@
     async_connect/3,
     async_accept/2,
     async_handshake/1,
+    async_handshake/2,
     async_shutdown_connection/3,
     async_accept_stream/2,
     start_stream/2,
@@ -276,6 +277,10 @@ async_accept(_Listener, _Opts) ->
     ok | {error, badarg | atom_reason()}.
 async_handshake(_Connection) ->
     erlang:nif_error(nif_library_not_loaded).
+-spec async_handshake(connection_handle(), conn_opts()) ->
+    ok | {error, badarg | atom_reason()}.
+async_handshake(_Connection, _ConnOpts) ->
+    erlang:nif_error(nif_library_not_loaded).
 
 -spec async_shutdown_connection(connection_handle(), conn_shutdown_flag(), app_errno()) ->
     ok | {error, badarg}.
diff --git a/test/quicer_snb_SUITE.erl b/test/quicer_snb_SUITE.erl
index 4eebf781..59bc1938 100644
--- a/test/quicer_snb_SUITE.erl
+++ b/test/quicer_snb_SUITE.erl
@@ -353,7 +353,7 @@ tc_slow_conn(Config) ->
                     },
                     #{
                         ?snk_kind := debug,
-                        function := "async_handshake_1",
+                        function := "async_handshake_X",
                         tag := "start",
                         mark := 0,
                         resource_id := _Rid

From a5851beed98d981acf9bbbd04ed6f7107b5fc363 Mon Sep 17 00:00:00 2001
From: William Yang <mscame@gmail.com>
Date: Tue, 10 Dec 2024 13:15:18 +0100
Subject: [PATCH 04/12] test: add prop test for async_handshake_2

---
 test/prop_quic_types.hrl           | 36 ++++++++++++++++++++++++++++++
 test/prop_stateful_server_conn.erl |  4 ++++
 test/quicer_prop_gen.erl           | 23 +++++++++++++++++++
 3 files changed, 63 insertions(+)

diff --git a/test/prop_quic_types.hrl b/test/prop_quic_types.hrl
index 15323167..66edc253 100644
--- a/test/prop_quic_types.hrl
+++ b/test/prop_quic_types.hrl
@@ -61,6 +61,7 @@
     | {conn_acceptors, non_neg_integer()}
     | {settings, [quicer_setting()]}.
 
+-define(UINT32_MAX, 4294967295).
 -type quicer_setting() ::
     {max_bytes_per_key, uint64()}
     | {handshake_idle_timeout_ms, uint64()}
@@ -95,6 +96,41 @@
     | {max_binding_stateless_operations, uint16()}
     | {stateless_operation_expiration_ms, uint16()}.
 
+%% happy quicer_settings that msquic won't return invalid_param
+-type quicer_setting_with_range() ::
+    {max_bytes_per_key, 0..(4 bsl 34 - 1)}
+    | {handshake_idle_timeout_ms, 0..(1 bsl 62 - 1)}
+    | {idle_timeout_ms, 0..(1 bsl 62 - 1)}
+    | {tls_client_max_send_buffer, uint32()}
+    | {tls_server_max_send_buffer, uint32()}
+    | {stream_recv_window_default, 1..?UINT32_MAX}
+    | {stream_recv_buffer_default, 4096..?UINT32_MAX}
+    | {conn_flow_control_window, uint32()}
+    | {max_stateless_operations, uint32()}
+    | {initial_window_packets, uint32()}
+    | {send_idle_timeout_ms, uint32()}
+    | {initial_rtt_ms, 1..?UINT32_MAX}
+    | {max_ack_delay_ms, 1..(1 bsl 14 - 1)}
+    | {disconnect_timeout_ms, 1..600000}
+    | {keep_alive_interval_ms, uint32()}
+    | {congestion_control_algorithm, 0 | 1}
+    | {peer_bidi_stream_count, uint16()}
+    | {peer_unidi_stream_count, uint16()}
+    | {retry_memory_limit, uint16()}
+    | {load_balancing_mode, 0..2}
+    | {max_operations_per_drain, uint8()}
+    | {send_buffering_enabled, 0 | 1}
+    | {pacing_enabled, 0 | 1}
+    | {migration_enabled, 0 | 1}
+    | {datagram_receive_enabled, 0 | 1}
+    | {server_resumption_level, 0 | 1 | 2}
+    | {minimum_mtu, uint16()}
+    | {maximum_mtu, uint16()}
+    | {mtu_discovery_search_complete_timeout_us, uint64()}
+    | {mtu_discovery_missing_probe_count, uint8()}
+    | {max_binding_stateless_operations, uint16()}
+    | {stateless_operation_expiration_ms, uint16()}.
+
 -type quicer_conn_opts() :: [conn_opt()].
 -type conn_opt() ::
     {alpn, [string()]}
diff --git a/test/prop_stateful_server_conn.erl b/test/prop_stateful_server_conn.erl
index a5952ecc..ef292b12 100644
--- a/test/prop_stateful_server_conn.erl
+++ b/test/prop_stateful_server_conn.erl
@@ -95,6 +95,7 @@ initial_state() ->
 command(#{handle := Handle}) ->
     frequency([
         {200, {call, quicer, handshake, [Handle, 1000]}},
+        {100, {call, quicer, handshake, [Handle, valid_quicer_settings(), 1000]}},
         {100, {call, quicer, getopt, [Handle, ?LET({Opt, _}, conn_opt(), Opt)]}},
         {100,
             {call, quicer, async_accept_stream, [Handle, ?LET(Opts, quicer_acceptor_opts(), Opts)]}},
@@ -327,6 +328,9 @@ step_calls(#{calls := Calls} = S) ->
     S#{calls := Calls + 1}.
 %%% Generators
 
+valid_quicer_settings() ->
+    quicer_prop_gen:valid_quicer_settings().
+
 %%%%%%%%%%%%%%%%%%%%%%%
 %%% Listener helper %%%
 %%%%%%%%%%%%%%%%%%%%%%%
diff --git a/test/quicer_prop_gen.erl b/test/quicer_prop_gen.erl
index c7e47930..a1750d7e 100644
--- a/test/quicer_prop_gen.erl
+++ b/test/quicer_prop_gen.erl
@@ -32,6 +32,7 @@
     valid_reg_handle/0,
     valid_handle/0,
     valid_csend_stream_opts/0,
+    valid_quicer_settings/0,
     pid/0,
     data/0,
     quicer_send_flags/0,
@@ -290,6 +291,20 @@ valid_csend_stream_opts() ->
         )
     ).
 
+%% @see msquic/src/core/settings.c
+valid_quicer_settings() ->
+    ?SUCHTHAT(
+        Opts,
+        ?LET(Q, list(quicer_setting_with_range()), Q),
+        %% Conds below from msquic/src/core/settings.c
+        quicer_setting_val_is_power_2(stream_recv_window_default, Opts) andalso
+            quicer_setting_val_is_power_2(stream_recv_window_bidi_local_default, Opts) andalso
+            quicer_setting_val_is_power_2(stream_recv_window_bidi_remote_default, Opts) andalso
+            quicer_setting_val_is_power_2(stream_recv_window_unidi_default, Opts) andalso
+            (proplists:get_value(maximum_mtu, Opts, 1500) >
+                proplists:get_value(minimum_mtu, Opts, 1248))
+    ).
+
 -spec ensure_dummy_listener(non_neg_integer()) -> _.
 ensure_dummy_listener(Port) ->
     case is_pid(whereis(?dummy_listener)) of
@@ -335,3 +350,11 @@ acceptor_loop(L) ->
         _ ->
             acceptor_loop(L)
     end.
+
+-spec quicer_setting_val_is_power_2(atom(), proplists:proplist()) -> boolean().
+quicer_setting_val_is_power_2(Key, Opts) ->
+    is_pow_2(maps:get(Key, maps:from_list(Opts), 2)).
+is_pow_2(N) when is_integer(N), N > 0 ->
+    (N band (N - 1)) == 0;
+is_pow_2(_) ->
+    false.

From 2ad3d033976aa373a8ccae463cfe33009f96abb6 Mon Sep 17 00:00:00 2001
From: William Yang <mscame@gmail.com>
Date: Wed, 11 Dec 2024 10:04:15 +0100
Subject: [PATCH 05/12] test: update range

---
 test/prop_quic_types.hrl | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/test/prop_quic_types.hrl b/test/prop_quic_types.hrl
index 66edc253..149c29f3 100644
--- a/test/prop_quic_types.hrl
+++ b/test/prop_quic_types.hrl
@@ -106,7 +106,7 @@
     | {stream_recv_window_default, 1..?UINT32_MAX}
     | {stream_recv_buffer_default, 4096..?UINT32_MAX}
     | {conn_flow_control_window, uint32()}
-    | {max_stateless_operations, uint32()}
+    | {max_stateless_operations, 1..16}
     | {initial_window_packets, uint32()}
     | {send_idle_timeout_ms, uint32()}
     | {initial_rtt_ms, 1..?UINT32_MAX}
@@ -129,7 +129,7 @@
     | {mtu_discovery_search_complete_timeout_us, uint64()}
     | {mtu_discovery_missing_probe_count, uint8()}
     | {max_binding_stateless_operations, uint16()}
-    | {stateless_operation_expiration_ms, uint16()}.
+    | {stateless_operation_expiration_ms, 10..(1 bsl 16 -1)}.
 
 -type quicer_conn_opts() :: [conn_opt()].
 -type conn_opt() ::

From 5eade4d1bc2f1189ef822e2bc622fe6aa6b5b582 Mon Sep 17 00:00:00 2001
From: William Yang <mscame@gmail.com>
Date: Wed, 11 Dec 2024 11:25:55 +0100
Subject: [PATCH 06/12] test(test): disable ct log in proper test

---
 test/example_client_connection.erl |  6 +++---
 test/example_client_stream.erl     | 18 +++++++++++++-----
 test/example_server_connection.erl |  3 ++-
 test/example_server_stream.erl     | 11 ++++++-----
 test/prop_quic_types.hrl           |  2 +-
 test/quicer_connection_SUITE.erl   |  2 ++
 test/quicer_ct.hrl                 | 22 ++++++++++++++++++++++
 7 files changed, 49 insertions(+), 15 deletions(-)
 create mode 100644 test/quicer_ct.hrl

diff --git a/test/example_client_connection.erl b/test/example_client_connection.erl
index 01bf2bc9..cc83de5c 100644
--- a/test/example_client_connection.erl
+++ b/test/example_client_connection.erl
@@ -20,7 +20,7 @@
 
 -include_lib("snabbkaffe/include/snabbkaffe.hrl").
 -include("quicer_types.hrl").
-
+-include("quicer_ct.hrl").
 %% API
 -export([start_link/3]).
 
@@ -83,7 +83,7 @@ new_conn(_Conn, #{version := _Vsn}, #{stream_opts := _SOpts} = S) ->
 
 connected(Conn, Flags, #{conn := Conn} = S) ->
     ?tp(debug, #{module => ?MODULE, conn => Conn, flags => Flags, event => connected}),
-    ct:pal("~p connected and expecting NST within 100ms", [?MODULE]),
+    ?LOG("~p connected and expecting NST within 100ms", [?MODULE]),
     {100, maps:merge(S, Flags)}.
 
 resumed(Conn, Data, #{resumed_callback := ResumeFun} = S) when
@@ -116,7 +116,7 @@ new_stream(
                     {ok, CBState#{streams := [{E, Stream} | Streams]}}
             end;
         Other ->
-            ct:pal("Start accepting remote stream error ~p", [Other]),
+            ?LOG("Start accepting remote stream error ~p", [Other]),
             {ok, CBState#{streams := [{start_error, Stream} | Streams]}}
     end.
 
diff --git a/test/example_client_stream.erl b/test/example_client_stream.erl
index 3323eb9e..8d28b650 100644
--- a/test/example_client_stream.erl
+++ b/test/example_client_stream.erl
@@ -37,6 +37,7 @@
 -export([handle_stream_data/4]).
 
 -include("quicer.hrl").
+-include("quicer_ct.hrl").
 -include_lib("snabbkaffe/include/snabbkaffe.hrl").
 
 init_handoff(Stream, _StreamOpts, Conn, #{flags := Flags}) ->
@@ -47,7 +48,7 @@ init_handoff(Stream, _StreamOpts, Conn, #{flags := Flags}) ->
         is_local => false,
         is_unidir => quicer:is_unidirectional(Flags)
     },
-    ct:pal("init_handoff ~p", [{InitState, _StreamOpts}]),
+    ?LOG("init_handoff ~p", [{InitState, _StreamOpts}]),
     {ok, InitState}.
 
 post_handoff(Stream, _PostData, State) ->
@@ -94,11 +95,11 @@ peer_send_shutdown(Stream, _Flags, S) ->
 send_complete(_Stream, false, S) ->
     {ok, S};
 send_complete(_Stream, true = _IsCanceled, S) ->
-    ct:pal("~p : send is canceled", [?FUNCTION_NAME]),
+    ?LOG("~p : send is canceled", [?FUNCTION_NAME]),
     {ok, S}.
 
 send_shutdown_complete(_Stream, _Flags, S) ->
-    ct:pal("~p : stream send is complete", [?FUNCTION_NAME]),
+    ?LOG("~p : stream send is complete", [?FUNCTION_NAME]),
     {ok, S}.
 
 start_completed(Stream, #{status := success, stream_id := StreamId} = P, S) ->
@@ -114,14 +115,21 @@ start_completed(_Stream, #{status := Other}, S) ->
 %% Local stream, Unidir
 handle_stream_data(Stream, Bin, _Flags, #{is_local := true, is_unidir := false} = State) ->
     ?tp(debug, #{stream => Stream, data => Bin, module => ?MODULE, dir => local_bidir}),
-    ct:pal("Client recv: ~p from ~p", [Bin, Stream]),
+    ?LOG("Client recv: ~p from ~p", [Bin, Stream]),
     {ok, State};
 %% Remote stream
 handle_stream_data(
     Stream, Bin, _Flags, #{is_local := false, is_unidir := true, conn := _Conn} = State
 ) ->
     ?tp(debug, #{stream => Stream, data => Bin, module => ?MODULE, dir => remote_unidir}),
-    ct:pal("Client recv: ~p from ~p", [Bin, Stream]),
+    ?LOG("Client recv: ~p from ~p", [Bin, Stream]),
+    {ok, State};
+handle_stream_data(
+    Stream, Bin, _Flags, #{is_local := false, is_unidir := false, conn := _Conn} = State
+) ->
+    %% for proper test
+    ?tp(debug, #{stream => Stream, data => Bin, module => ?MODULE, dir => remote_unidir}),
+    ?LOG("Client recv: ~p from ~p", [Bin, Stream]),
     {ok, State};
 handle_stream_data(
     _Stream,
diff --git a/test/example_server_connection.erl b/test/example_server_connection.erl
index d63a338d..61c24be0 100644
--- a/test/example_server_connection.erl
+++ b/test/example_server_connection.erl
@@ -29,6 +29,7 @@
 
 -include_lib("snabbkaffe/include/snabbkaffe.hrl").
 -include("quicer_types.hrl").
+-include("quicer_ct.hrl").
 
 %% Callback init
 -export([init/1]).
@@ -149,7 +150,7 @@ dgram_recv(C, Bin, _Flag, S) ->
     case quicer:send_dgram(C, Bin) of
         {ok, _} -> ok;
         %% for testing when peer disable the receiving
-        Error -> ct:pal("send dgram error: ~p~n", [Error])
+        Error -> ?LOG("send dgram error: ~p~n", [Error])
     end,
     {ok, S}.
 
diff --git a/test/example_server_stream.erl b/test/example_server_stream.erl
index 11b5e6be..6a3549cf 100644
--- a/test/example_server_stream.erl
+++ b/test/example_server_stream.erl
@@ -39,6 +39,7 @@
 
 -include("quicer.hrl").
 -include_lib("snabbkaffe/include/snabbkaffe.hrl").
+-include("quicer_ct.hrl").
 
 init_handoff(Stream, _StreamOpts, Conn, #{flags := Flags}) ->
     InitState = #{
@@ -48,7 +49,7 @@ init_handoff(Stream, _StreamOpts, Conn, #{flags := Flags}) ->
         is_local => false,
         is_unidir => quicer:is_unidirectional(Flags)
     },
-    ct:pal("init_handoff ~p", [{InitState, _StreamOpts}]),
+    ?LOG("init_handoff ~p", [{InitState, _StreamOpts}]),
     {ok, InitState}.
 
 post_handoff(Stream, _PostData, State) ->
@@ -99,11 +100,11 @@ peer_send_shutdown(Stream, _Flags, S) ->
 send_complete(_Stream, false, S) ->
     {ok, S};
 send_complete(_Stream, true = _IsCanceled, S) ->
-    ct:pal("~p : send is canceled", [?FUNCTION_NAME]),
+    ?LOG("~p : send is canceled", [?FUNCTION_NAME]),
     {ok, S}.
 
 send_shutdown_complete(_Stream, _Flags, S) ->
-    ct:pal("~p : stream send is complete", [?FUNCTION_NAME]),
+    ?LOG("~p : stream send is complete", [?FUNCTION_NAME]),
     {ok, S}.
 
 start_completed(_Stream, #{status := success, stream_id := StreamId}, S) ->
@@ -121,7 +122,7 @@ handle_stream_data(
 handle_stream_data(Stream, Bin, _Flags, #{is_unidir := false} = State) ->
     %% for bidir stream, we just echo in place.
     ?tp(debug, #{stream => Stream, data => Bin, module => ?MODULE, dir => bidir}),
-    ct:pal("Server recv: ~p from ~p", [Bin, Stream]),
+    ?LOG("Server recv: ~p from ~p", [Bin, Stream]),
     case quicer:send(Stream, Bin) of
         {ok, _} ->
             {ok, State};
@@ -132,7 +133,7 @@ handle_stream_data(
     Stream, Bin, _Flags, #{is_unidir := true, peer_stream := PeerStream, conn := Conn} = State
 ) ->
     ?tp(debug, #{stream => Stream, data => Bin, module => ?MODULE, dir => unidir}),
-    ct:pal("Server recv: ~p from ~p", [Bin, Stream]),
+    ?LOG("Server recv: ~p from ~p", [Bin, Stream]),
 
     case PeerStream of
         undefined ->
diff --git a/test/prop_quic_types.hrl b/test/prop_quic_types.hrl
index 149c29f3..744429f2 100644
--- a/test/prop_quic_types.hrl
+++ b/test/prop_quic_types.hrl
@@ -129,7 +129,7 @@
     | {mtu_discovery_search_complete_timeout_us, uint64()}
     | {mtu_discovery_missing_probe_count, uint8()}
     | {max_binding_stateless_operations, uint16()}
-    | {stateless_operation_expiration_ms, 10..(1 bsl 16 -1)}.
+    | {stateless_operation_expiration_ms, 10..(1 bsl 16 - 1)}.
 
 -type quicer_conn_opts() :: [conn_opt()].
 -type conn_opt() ::
diff --git a/test/quicer_connection_SUITE.erl b/test/quicer_connection_SUITE.erl
index edb74f39..8dfa9c80 100644
--- a/test/quicer_connection_SUITE.erl
+++ b/test/quicer_connection_SUITE.erl
@@ -549,6 +549,7 @@ tc_datagram_disallowed(Config) ->
     %% THEN: It get an error
     ?assertEqual({error, dgram_send_error, invalid_state}, quicer:send_dgram(Conn, <<"dg_ping">>)),
     quicer:shutdown_connection(Conn),
+    ok = quicer:terminate_listener(mqtt),
     ok.
 
 tc_datagram_peer_allowed(Config) ->
@@ -585,6 +586,7 @@ tc_datagram_peer_allowed(Config) ->
         ok
     end,
     quicer:shutdown_connection(Conn),
+    ok = quicer:terminate_listener(mqtt),
     ok.
 
 tc_datagram_local_peer_allowed(Config) ->
diff --git a/test/quicer_ct.hrl b/test/quicer_ct.hrl
new file mode 100644
index 00000000..036c54e9
--- /dev/null
+++ b/test/quicer_ct.hrl
@@ -0,0 +1,22 @@
+%%--------------------------------------------------------------------
+%% Copyright (c) 2024 EMQ Technologies Co., Ltd. All Rights Reserved.
+%%
+%% Licensed under the Apache License, Version 2.0 (the "License");
+%% you may not use this file except in compliance with the License.
+%% You may obtain a copy of the License at
+%%
+%%     http://www.apache.org/licenses/LICENSE-2.0
+%%
+%% Unless required by applicable law or agreed to in writing, software
+%% distributed under the License is distributed on an "AS IS" BASIS,
+%% WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+%% See the License for the specific language governing permissions and
+%% limitations under the License.
+%%--------------------------------------------------------------------
+-ifndef(QUICER_CT_HRL).
+-define(QUICER_CT_HRL, true).
+
+-define(LOG(_Arg1, _Arg2), logger:debug(_Arg1, _Arg2)).
+
+%% QUICER_CT_HRL
+-endif.

From 3c5c6cab7d7ddcc9fa9f00738d95beddd4569946 Mon Sep 17 00:00:00 2001
From: William Yang <mscame@gmail.com>
Date: Wed, 11 Dec 2024 11:27:06 +0100
Subject: [PATCH 07/12] ci(cover): proper cover -noshrink

---
 Makefile | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/Makefile b/Makefile
index 13007141..fd267ab7 100644
--- a/Makefile
+++ b/Makefile
@@ -47,7 +47,7 @@ proper:
 .PHONY: proper-cover
 proper-cover:
 	mkdir -p coverage
-	QUICER_TEST_COVER=1 $(REBAR) as test proper -c -n 1000
+	QUICER_TEST_COVER=1 $(REBAR) as test proper -c -n 1000 --noshrink
 	lcov -c  --directory c_build/CMakeFiles/quicer_nif.dir/c_src/ \
 	--exclude "${PWD}/msquic/src/inc/*" \
 	--output-file ./coverage/proper-lcov.info

From 9238cddede0341b43b303b679cccb32c5b089432 Mon Sep 17 00:00:00 2001
From: William Yang <mscame@gmail.com>
Date: Wed, 11 Dec 2024 18:41:45 +0100
Subject: [PATCH 08/12] test(proper): further limit testing range

---
 test/prop_quic_types.hrl | 2 +-
 test/quicer_prop_gen.erl | 4 ++--
 2 files changed, 3 insertions(+), 3 deletions(-)

diff --git a/test/prop_quic_types.hrl b/test/prop_quic_types.hrl
index 744429f2..3f76a522 100644
--- a/test/prop_quic_types.hrl
+++ b/test/prop_quic_types.hrl
@@ -118,7 +118,7 @@
     | {peer_unidi_stream_count, uint16()}
     | {retry_memory_limit, uint16()}
     | {load_balancing_mode, 0..2}
-    | {max_operations_per_drain, uint8()}
+    | {max_operations_per_drain, 1..(1 bsl 8 - 1)}
     | {send_buffering_enabled, 0 | 1}
     | {pacing_enabled, 0 | 1}
     | {migration_enabled, 0 | 1}
diff --git a/test/quicer_prop_gen.erl b/test/quicer_prop_gen.erl
index a1750d7e..dd0c3d24 100644
--- a/test/quicer_prop_gen.erl
+++ b/test/quicer_prop_gen.erl
@@ -301,8 +301,8 @@ valid_quicer_settings() ->
             quicer_setting_val_is_power_2(stream_recv_window_bidi_local_default, Opts) andalso
             quicer_setting_val_is_power_2(stream_recv_window_bidi_remote_default, Opts) andalso
             quicer_setting_val_is_power_2(stream_recv_window_unidi_default, Opts) andalso
-            (proplists:get_value(maximum_mtu, Opts, 1500) >
-                proplists:get_value(minimum_mtu, Opts, 1248))
+            (proplists:get_value(maximum_mtu, lists:reverse(Opts), 1500) >
+                proplists:get_value(minimum_mtu, lists:reverse(Opts), 1248))
     ).
 
 -spec ensure_dummy_listener(non_neg_integer()) -> _.

From 63f93fac00fb70673a25215b85d9ed9b06d7304a Mon Sep 17 00:00:00 2001
From: William Yang <mscame@gmail.com>
Date: Mon, 14 Oct 2024 22:15:04 +0200
Subject: [PATCH 09/12] feat(dgram): improve send err handling

---
 src/quicer.erl        | 43 ++++++++++++++++++++++++-----------------
 src/quicer_lib.erl    | 45 ++++++++++++++++++++++++++++++++++++++++++-
 test/quicer_SUITE.erl | 12 +++++++++++-
 3 files changed, 80 insertions(+), 20 deletions(-)

diff --git a/src/quicer.erl b/src/quicer.erl
index e32ae74d..d3193086 100644
--- a/src/quicer.erl
+++ b/src/quicer.erl
@@ -71,6 +71,7 @@
     async_send/2,
     async_send/3,
     recv/2,
+    async_send_dgram/2,
     send_dgram/2,
     shutdown_stream/1,
     shutdown_stream/2,
@@ -844,33 +845,39 @@ do_recv(Stream, Count, Buff) ->
             E
     end.
 
-%% @doc Sending Unreliable Datagram
+%% @doc Sending Unreliable Datagram.
+%% Caller should handle the async signals for the send results
 %%
-%% ref: [https://datatracker.ietf.org/doc/html/draft-ietf-quic-datagram]
-%% @see send/2
+%% ref: [https://datatracker.ietf.org/doc/html/rfc9221]
+%% @see send/2 send_dgram/2
+-spec async_send_dgram(connection_handle(), binary()) ->
+    {ok, non_neg_integer()}
+    | {error, badarg | not_enough_mem | closed}
+    | {error, dgram_send_error, atom_reason()}.
+async_send_dgram(Conn, Data) ->
+    quicer_nif:send_dgram(Conn, Data, _IsSyncRel = 1).
+
+%% @doc Sending Unreliable Datagram, returns the end state.
+%%
+%% %% ref: [https://datatracker.ietf.org/doc/html/rfc9221]
+%% @see send/2, async_send_dgram
 -spec send_dgram(connection_handle(), binary()) ->
-    {ok, BytesSent :: pos_integer()}
+    {ok, BytesSent :: non_neg_integer()}
     | {error, badarg | not_enough_mem | closed}
     | {error, dgram_send_error, atom_reason()}.
 send_dgram(Conn, Data) ->
     case quicer_nif:send_dgram(Conn, Data, _IsSync = 1) of
-        %% @todo we need find tuned event mask
         {ok, _Len} = OK ->
-            receive
-                {quic, dgram_send_state, Conn, #{state := ?QUIC_DATAGRAM_SEND_SENT}} ->
-                    receive
-                        {quic, dgram_send_state, Conn, #{state := ?QUIC_DATAGRAM_SEND_ACKNOWLEDGED}} ->
-                            OK;
-                        {quic, dgram_send_state, Conn, #{state := Other}} ->
-                            {error, dgram_send_error, Other}
-                    end;
-                {quic, dgram_send_state, Conn, #{state := ?QUIC_DATAGRAM_SEND_ACKNOWLEDGED}} ->
+            case quicer_lib:handle_dgram_send_states(Conn) of
+                ok ->
                     OK;
-                {quic, dgram_send_state, Conn, #{state := Other}} ->
-                    {error, dgram_send_error, Other}
+                {error, E} ->
+                    {error, dgram_send_error, E}
             end;
-        E ->
-            E
+        {error, _, _} = E ->
+            E;
+        {error, E} ->
+            {error, dgram_send_error, E}
     end.
 
 %% @doc Shutdown stream gracefully, with infinity timeout
diff --git a/src/quicer_lib.erl b/src/quicer_lib.erl
index 90cdbade..42fe63d1 100644
--- a/src/quicer_lib.erl
+++ b/src/quicer_lib.erl
@@ -41,7 +41,10 @@
 
 -type action() :: hibernate | timeout() | {continue, Continue :: term()}.
 
--export([default_cb_ret/2]).
+-export([
+    default_cb_ret/2,
+    handle_dgram_send_states/1
+]).
 
 -spec default_cb_ret(cb_ret(), State :: term()) ->
     {reply, NewState :: term()}
@@ -69,3 +72,43 @@ default_cb_ret({reply, Reply, NewCBState, Action}, State) ->
     {reply, Reply, State#{callback_state := NewCBState}, Action};
 default_cb_ret({reply, Reply, NewCBState}, State) ->
     {reply, Reply, State#{callback_state := NewCBState}}.
+
+-spec handle_dgram_send_states(connection_handle()) ->
+    ok
+    | {error,
+        dgram_send_canceled
+        | dgram_send_unknown
+        | dgram_send_lost_discarded}.
+handle_dgram_send_states(Conn) ->
+    handle_dgram_send_states(init, Conn).
+handle_dgram_send_states(init, Conn) ->
+    receive
+        {quic, dgram_send_state, Conn, #{state := ?QUIC_DATAGRAM_SEND_SENT}} ->
+            handle_dgram_send_states(sent, Conn);
+        {quic, dgram_send_state, Conn, #{state := ?QUIC_DATAGRAM_SEND_ACKNOWLEDGED}} ->
+            %% @TODO unsure if it will hit here
+            ok;
+        {quic, dgram_send_state, Conn, #{state := E}} ->
+            {error, E}
+    end;
+handle_dgram_send_states(sent, Conn) ->
+    receive
+        {quic, dgram_send_state, Conn, #{state := ?QUIC_DATAGRAM_SEND_ACKNOWLEDGED}} ->
+            %% Happy Track
+            ok;
+        {quic, dgram_send_state, Conn, #{state := ?QUIC_DATAGRAM_SEND_LOST_SUSPECT}} ->
+            %% Lost suspected
+            receive
+                {quic, dgram_send_state, Conn, #{
+                    state := ?QUIC_DATAGRAM_SEND_ACKNOWLEDGED_SPURIOUS
+                }} ->
+                    %% Lost recovered
+                    ok;
+                {quic, dgram_send_state, Conn, #{state := EState}} ->
+                    %% Unrecoverable Errors.
+                    {error, EState}
+            end;
+        {quic, dgram_send_state, Conn, #{state := EState}} ->
+            %% Unrecoverable Errors.
+            {error, EState}
+    end.
diff --git a/test/quicer_SUITE.erl b/test/quicer_SUITE.erl
index d7701a88..51861906 100644
--- a/test/quicer_SUITE.erl
+++ b/test/quicer_SUITE.erl
@@ -75,6 +75,7 @@
 
     tc_dgram_client_send/1,
     tc_dgram_client_send_iolist/1,
+    tc_dgram_client_send_fail/1,
 
     % , tc_getopt_raw/1
     tc_getopt/1,
@@ -859,7 +860,7 @@ tc_stream_controlling_process_demon(Config) ->
             ok = quicer:setopt(Stm, active, true),
             {ok, _Len} = quicer:send(Stm, <<"owner_changed">>),
             receive
-                {quic, <<"owner_changed">>, Stm, _} ->
+                {quic, <<"owner_changed">>, _Stm, _} ->
                     ok
             end,
             %% Set controlling_process again
@@ -883,6 +884,15 @@ tc_stream_controlling_process_demon(Config) ->
         ct:fail("timeout")
     end.
 
+tc_dgram_client_send_fail(_) ->
+    Opts = default_conn_opts() ++ [{datagram_receive_enabled, 1}],
+    {ok, Conn} = quicer:async_connect("localhost", 65535, Opts),
+    ?assertEqual(
+        {error, dgram_send_error, dgram_send_canceled},
+        quicer:send_dgram(Conn, <<"ping">>)
+    ),
+    ok.
+
 tc_dgram_client_send(Config) ->
     Port = select_port(),
     Owner = self(),

From ac85fa015e43bab361d80c55e4025bb8618f8777 Mon Sep 17 00:00:00 2001
From: William Yang <mscame@gmail.com>
Date: Thu, 17 Oct 2024 14:28:41 +0200
Subject: [PATCH 10/12] feat: conn probe with unreliable data gram

---
 include/quicer.hrl                 |  7 +++
 include/quicer_types.hrl           |  6 ++
 src/quicer.erl                     | 24 ++++++--
 src/quicer_lib.erl                 | 93 ++++++++++++++++++++++--------
 test/prop_stateful_client_conn.erl | 39 ++++++++++++-
 test/prop_stateful_server_conn.erl | 36 ++++++++++++
 test/quicer_SUITE.erl              |  3 +-
 test/quicer_connection_SUITE.erl   | 10 ++++
 8 files changed, 186 insertions(+), 32 deletions(-)

diff --git a/include/quicer.hrl b/include/quicer.hrl
index 9a7f969a..ef13ebc6 100644
--- a/include/quicer.hrl
+++ b/include/quicer.hrl
@@ -149,4 +149,11 @@
 -define(QUIC_CONGESTION_CONTROL_ALGORITHM_CUBIC, 0).
 -define(QUIC_CONGESTION_CONTROL_ALGORITHM_BBR, 1).
 
+-record(probe_state, {
+    final :: term() | undefined,
+    sent_at :: integer() | undefined,
+    suspect_lost_at :: integer() | undefined,
+    final_at :: integer() | undefined
+}).
+
 -endif. %% QUICER_HRL
diff --git a/include/quicer_types.hrl b/include/quicer_types.hrl
index 8a46ee82..6230522e 100644
--- a/include/quicer_types.hrl
+++ b/include/quicer_types.hrl
@@ -506,5 +506,11 @@
     dgram_max_len := uint64()
 }.
 
+-type probe_state() :: #probe_state{}.
+-type probe_res() ::
+    #probe_state{}
+    | {error, dgram_send_error, atom()}
+    | {error, atom()}.
+
 %% QUICER_TYPES_HRL
 -endif.
diff --git a/src/quicer.erl b/src/quicer.erl
index d3193086..01b4637f 100644
--- a/src/quicer.erl
+++ b/src/quicer.erl
@@ -61,6 +61,7 @@
     close_connection/4,
     async_close_connection/1,
     async_close_connection/3,
+    probe/2,
     accept_stream/2,
     accept_stream/3,
     async_accept_stream/2,
@@ -177,7 +178,10 @@
     quicer_addr/0,
 
     %% Registraion Profiles
-    registration_profile/0
+    registration_profile/0,
+
+    %% probes
+    probe_res/0
 ]).
 
 -type connection_opts() :: proplists:proplist() | conn_opts().
@@ -857,10 +861,13 @@ do_recv(Stream, Count, Buff) ->
 async_send_dgram(Conn, Data) ->
     quicer_nif:send_dgram(Conn, Data, _IsSyncRel = 1).
 
-%% @doc Sending Unreliable Datagram, returns the end state.
+%% @doc Sending Unreliable Datagram
+%%  return error only if sending could not be scheduled such as
+%%  not_enough_mem, connection is already closed or wrong args.
+%%  otherwise, it is fire and forget.
 %%
 %% %% ref: [https://datatracker.ietf.org/doc/html/rfc9221]
-%% @see send/2, async_send_dgram
+%% @see send/2, async_send_dgram/2
 -spec send_dgram(connection_handle(), binary()) ->
     {ok, BytesSent :: non_neg_integer()}
     | {error, badarg | not_enough_mem | closed}
@@ -874,12 +881,17 @@ send_dgram(Conn, Data) ->
                 {error, E} ->
                     {error, dgram_send_error, E}
             end;
-        {error, _, _} = E ->
-            E;
         {error, E} ->
-            {error, dgram_send_error, E}
+            {error, E};
+        E ->
+            E
     end.
 
+%% @doc Probe conn state with 0 len dgram.
+-spec probe(connection_handle(), timeout()) -> probe_res().
+probe(Conn, Timeout) ->
+    quicer_lib:probe(Conn, Timeout).
+
 %% @doc Shutdown stream gracefully, with infinity timeout
 %%
 %% @see shutdown_stream/1
diff --git a/src/quicer_lib.erl b/src/quicer_lib.erl
index 42fe63d1..0eb3ff4e 100644
--- a/src/quicer_lib.erl
+++ b/src/quicer_lib.erl
@@ -20,6 +20,7 @@
     cb_ret/0,
     cb_state/0
 ]).
+
 -type cb_ret() :: cb_ret_noreply() | cb_ret_reply().
 -type cb_state() :: term().
 
@@ -43,7 +44,9 @@
 
 -export([
     default_cb_ret/2,
-    handle_dgram_send_states/1
+    handle_dgram_send_states/1,
+    handle_dgram_send_states/3,
+    probe/2
 ]).
 
 -spec default_cb_ret(cb_ret(), State :: term()) ->
@@ -73,6 +76,17 @@ default_cb_ret({reply, Reply, NewCBState, Action}, State) ->
 default_cb_ret({reply, Reply, NewCBState}, State) ->
     {reply, Reply, State#{callback_state := NewCBState}}.
 
+-spec probe(connection_handle(), timeout()) -> probe_res().
+probe(Conn, Timeout) ->
+    case quicer_nif:send_dgram(Conn, <<>>, _IsSync = 1) of
+        {ok, _Len} ->
+            handle_dgram_send_states(Conn, probe_dgram_send_cb(), Timeout);
+        {error, E} ->
+            {error, dgram_send_error, E};
+        E ->
+            E
+    end.
+
 -spec handle_dgram_send_states(connection_handle()) ->
     ok
     | {error,
@@ -80,35 +94,68 @@ default_cb_ret({reply, Reply, NewCBState}, State) ->
         | dgram_send_unknown
         | dgram_send_lost_discarded}.
 handle_dgram_send_states(Conn) ->
-    handle_dgram_send_states(init, Conn).
-handle_dgram_send_states(init, Conn) ->
+    handle_dgram_send_states(init, Conn, default_dgram_suspect_lost_cb(), 5000).
+
+-type lost_suspect_callback() ::
+    {fun((connection_handle(), term(), term()) -> term()), term()}
+    | {atom(), term()}.
+-spec handle_dgram_send_states(connection_handle(), lost_suspect_callback(), timeout()) -> any().
+handle_dgram_send_states(Conn, {_CBFun, _CBState} = CB, Timeout) ->
+    handle_dgram_send_states(init, Conn, CB, Timeout).
+
+handle_dgram_send_states(init, Conn, {Fun, CallbackState}, Timeout) ->
     receive
         {quic, dgram_send_state, Conn, #{state := ?QUIC_DATAGRAM_SEND_SENT}} ->
-            handle_dgram_send_states(sent, Conn);
-        {quic, dgram_send_state, Conn, #{state := ?QUIC_DATAGRAM_SEND_ACKNOWLEDGED}} ->
-            %% @TODO unsure if it will hit here
-            ok;
-        {quic, dgram_send_state, Conn, #{state := E}} ->
-            {error, E}
+            NewCBState = Fun(Conn, ?QUIC_DATAGRAM_SEND_SENT, CallbackState),
+            handle_dgram_send_states(sent, Conn, {Fun, NewCBState}, Timeout);
+        {quic, dgram_send_state, Conn, #{state := Final}} ->
+            Fun(Conn, Final, CallbackState)
+    after 5000 ->
+        Fun(Conn, timeout, CallbackState)
     end;
-handle_dgram_send_states(sent, Conn) ->
+handle_dgram_send_states(sent, Conn, {Fun, CallbackState}, Timeout) ->
     receive
-        {quic, dgram_send_state, Conn, #{state := ?QUIC_DATAGRAM_SEND_ACKNOWLEDGED}} ->
-            %% Happy Track
-            ok;
+        %% {quic, dgram_send_state, Conn, #{state := ?QUIC_DATAGRAM_SEND_ACKNOWLEDGED}} ->
+        %%     %% Happy Track
+        %%     Fun(Conn, ?QUIC_DATAGRAM_SEND_ACKNOWLEDGED, CallbackState);
         {quic, dgram_send_state, Conn, #{state := ?QUIC_DATAGRAM_SEND_LOST_SUSPECT}} ->
-            %% Lost suspected
+            %% Lost suspected, call the callback for the return hits.
+            %% however, we still need to wait for the final state.
+            NewCBState = Fun(Conn, ?QUIC_DATAGRAM_SEND_LOST_SUSPECT, CallbackState),
             receive
-                {quic, dgram_send_state, Conn, #{
-                    state := ?QUIC_DATAGRAM_SEND_ACKNOWLEDGED_SPURIOUS
-                }} ->
-                    %% Lost recovered
-                    ok;
                 {quic, dgram_send_state, Conn, #{state := EState}} ->
-                    %% Unrecoverable Errors.
-                    {error, EState}
+                    Fun(Conn, EState, NewCBState)
+            after Timeout ->
+                Fun(Conn, timeout, CallbackState)
             end;
-        {quic, dgram_send_state, Conn, #{state := EState}} ->
+        {quic, dgram_send_state, Conn, #{state := Final}} ->
             %% Unrecoverable Errors.
-            {error, EState}
+            Fun(Conn, Final, CallbackState)
+    after Timeout ->
+        Fun(Conn, timeout, CallbackState)
     end.
+
+%% Default Callback for Datagram Send lost suspected
+default_dgram_suspect_lost_cb() ->
+    Fun = fun(_Conn, _, _CallbackState) ->
+        %% just return ok, even it is lost, we don't care.
+        ok
+    end,
+    {Fun, undefined}.
+
+probe_dgram_send_cb() ->
+    Fun = fun
+        (_Conn, ?QUIC_DATAGRAM_SEND_SENT, CallbackState) ->
+            CallbackState#probe_state{sent_at = ts_ms()};
+        (_Conn, ?QUIC_DATAGRAM_SEND_LOST_SUSPECT, CallbackState) ->
+            CallbackState#probe_state{suspect_lost_at = ts_ms()};
+        (_Conn, State, CallbackState) ->
+            CallbackState#probe_state{
+                final_at = ts_ms(),
+                final = State
+            }
+    end,
+    {Fun, #probe_state{}}.
+
+ts_ms() ->
+    erlang:monotonic_time(millisecond).
diff --git a/test/prop_stateful_client_conn.erl b/test/prop_stateful_client_conn.erl
index 27d7ccda..576b529f 100644
--- a/test/prop_stateful_client_conn.erl
+++ b/test/prop_stateful_client_conn.erl
@@ -59,6 +59,7 @@ prop_client_state_test() ->
 %%%%%%%%%%%%%
 %% @doc Initial model value at system start. Should be deterministic.
 initial_state() ->
+    net_kernel:start([?MODULE, shortnames]),
     {ok, H} = quicer:connect("localhost", 14568, default_conn_opts(), 10000),
     #{
         state => connected,
@@ -82,6 +83,8 @@ command(#{handle := Handle}) ->
             ]}},
         {100, {call, quicer, peername, [Handle]}},
         {50, {call, quicer, peercert, [Handle]}},
+        {50, {call, quicer, probe, [Handle, 5000]}},
+        {50, {call, quicer, send_dgram, [Handle, binary()]}},
         {10, {call, quicer, negotiated_protocol, [Handle]}},
         {10, {call, quicer, get_connections, []}},
         {10, {call, quicer, get_conn_owner, [Handle]}},
@@ -198,6 +201,36 @@ postcondition(
     {error, not_owner}
 ) ->
     Owner =/= self();
+postcondition(
+    #{state := ConnState},
+    {call, quicer, probe, [_, _]},
+    {error, dgram_send_error, _}
+) ->
+    ConnState =/= connected;
+postcondition(
+    #{state := _ConnState},
+    {call, quicer, probe, [_, _]},
+    #probe_state{final = FinalState, final_at = FinalTs}
+) ->
+    FinalState =/= undefined andalso FinalTs =/= undefined;
+postcondition(
+    #{state := _ConnState},
+    {call, quicer, send_dgram, [_, _]},
+    {ok, _}
+) ->
+    true;
+postcondition(
+    #{state := ConnState},
+    {call, quicer, send_dgram, [_, _]},
+    {error, _, _}
+) ->
+    ConnState =/= connected;
+postcondition(
+    #{state := ConnState},
+    {call, quicer, send_dgram, [_, _]},
+    {error, _}
+) ->
+    ConnState =/= connected;
 postcondition(
     #{owner := _, state := connected},
     {call, quicer, controlling_process, [_, NewOwner]},
@@ -275,7 +308,8 @@ default_listen_opts() ->
         {handshake_idle_timeout_ms, 10000},
         % QUIC_SERVER_RESUME_AND_ZERORTT
         {server_resumption_level, 2},
-        {peer_bidi_stream_count, 10}
+        {peer_bidi_stream_count, 10},
+        {datagram_receive_enabled, 1}
     ].
 
 default_conn_opts() ->
@@ -286,7 +320,8 @@ default_conn_opts() ->
         {idle_timeout_ms, 0},
         {cacertfile, "./msquic/submodules/openssl/test/certs/rootCA.pem"},
         {certfile, "./msquic/submodules/openssl/test/certs/servercert.pem"},
-        {keyfile, "./msquic/submodules/openssl/test/certs/serverkey.pem"}
+        {keyfile, "./msquic/submodules/openssl/test/certs/serverkey.pem"},
+        {datagram_receive_enabled, 1}
     ].
 
 %% Test helpers
diff --git a/test/prop_stateful_server_conn.erl b/test/prop_stateful_server_conn.erl
index ef292b12..728c4615 100644
--- a/test/prop_stateful_server_conn.erl
+++ b/test/prop_stateful_server_conn.erl
@@ -101,6 +101,8 @@ command(#{handle := Handle}) ->
             {call, quicer, async_accept_stream, [Handle, ?LET(Opts, quicer_acceptor_opts(), Opts)]}},
         {100, {call, quicer, peername, [Handle]}},
         {50, {call, quicer, peercert, [Handle]}},
+        {50, {call, quicer, probe, [Handle, 5000]}},
+        {50, {call, quicer, send_dgram, [Handle, binary()]}},
         {10, {call, quicer, negotiated_protocol, [Handle]}},
         {10, {call, quicer, get_connections, []}},
         {10, {call, quicer, get_conn_owner, [Handle]}},
@@ -295,6 +297,36 @@ postcondition(#{state := closed}, {call, _Mod, _Fun, _Args}, {error, closed}) ->
 postcondition(#{state := accepted}, {call, _Mod, _Fun, _Args}, {error, closed}) ->
     %% handshake didnt take place on time
     true;
+postcondition(
+    #{state := ConnState},
+    {call, quicer, probe, [_, _]},
+    {error, dgram_send_error, _}
+) ->
+    ConnState =/= connected;
+postcondition(
+    #{state := _ConnState},
+    {call, quicer, probe, [_, _]},
+    #probe_state{final = FinalState, final_at = FinalTs}
+) ->
+    FinalState =/= undefined andalso FinalTs =/= undefined;
+postcondition(
+    #{state := _ConnState},
+    {call, quicer, send_dgram, [_, _]},
+    {ok, _}
+) ->
+    true;
+postcondition(
+    #{state := ConnState},
+    {call, quicer, send_dgram, [_, _]},
+    {error, _, _}
+) ->
+    ConnState =/= connected;
+postcondition(
+    #{state := ConnState},
+    {call, quicer, send_dgram, [_, _]},
+    {error, _}
+) ->
+    ConnState =/= connected;
 postcondition(_State, {call, _Mod, _Fun, _Args} = _Call, _Res) ->
     false.
 
@@ -321,6 +353,10 @@ do_next_state(
     #{state := _} = State, ok, {call, quicer, controlling_process, [_, Owner]}
 ) ->
     State#{owner := Owner};
+do_next_state(
+    #{state := _} = State, {error, closed}, {call, _M, _F, _A}
+) ->
+    State#{state := closed};
 do_next_state(State, _Res, {call, _Mod, _Fun, _Args}) ->
     State.
 
diff --git a/test/quicer_SUITE.erl b/test/quicer_SUITE.erl
index 51861906..fd3d6293 100644
--- a/test/quicer_SUITE.erl
+++ b/test/quicer_SUITE.erl
@@ -888,7 +888,8 @@ tc_dgram_client_send_fail(_) ->
     Opts = default_conn_opts() ++ [{datagram_receive_enabled, 1}],
     {ok, Conn} = quicer:async_connect("localhost", 65535, Opts),
     ?assertEqual(
-        {error, dgram_send_error, dgram_send_canceled},
+        %% fire and forget
+        {ok, 4},
         quicer:send_dgram(Conn, <<"ping">>)
     ),
     ok.
diff --git a/test/quicer_connection_SUITE.erl b/test/quicer_connection_SUITE.erl
index 8dfa9c80..301766a5 100644
--- a/test/quicer_connection_SUITE.erl
+++ b/test/quicer_connection_SUITE.erl
@@ -993,6 +993,16 @@ tc_closed_conn_reg(_Config) ->
     Opts = default_conn_opts() ++ [{quic_registration, ThisReg}],
     ?assertEqual({error, quic_registration}, quicer:connect("localhost", 443, Opts, 5000)).
 
+tc_conn_probe(_) ->
+    Opts = default_conn_opts() ++ [{datagram_receive_enabled, 1}],
+    {ok, Conn} = quicer:async_connect("localhost", 65535, Opts),
+    ?assertMatch(
+        #probe_state{final_at = TS, final = ?QUIC_DATAGRAM_SEND_CANCELED} when
+            TS =/= undefined,
+        quicer:probe(Conn, 5000)
+    ),
+    ok.
+
 %%%
 %%% Helpers
 %%%

From 6a0b5473820b31761eb4d5a4941d0a0980d2d6a3 Mon Sep 17 00:00:00 2001
From: William Yang <mscame@gmail.com>
Date: Thu, 17 Oct 2024 15:46:12 +0200
Subject: [PATCH 11/12] test: send_dgram exceed max MTU

---
 src/quicer.erl                     |  4 ++--
 src/quicer_nif.erl                 |  2 +-
 test/prop_stateful_server_conn.erl | 18 +++++++++-------
 test/quicer_SUITE.erl              | 33 ++++++++++++++++++++++++++++++
 4 files changed, 47 insertions(+), 10 deletions(-)

diff --git a/src/quicer.erl b/src/quicer.erl
index 01b4637f..49f1a111 100644
--- a/src/quicer.erl
+++ b/src/quicer.erl
@@ -856,7 +856,7 @@ do_recv(Stream, Count, Buff) ->
 %% @see send/2 send_dgram/2
 -spec async_send_dgram(connection_handle(), binary()) ->
     {ok, non_neg_integer()}
-    | {error, badarg | not_enough_mem | closed}
+    | {error, badarg | not_enough_mem | invalid_parameter | closed}
     | {error, dgram_send_error, atom_reason()}.
 async_send_dgram(Conn, Data) ->
     quicer_nif:send_dgram(Conn, Data, _IsSyncRel = 1).
@@ -870,7 +870,7 @@ async_send_dgram(Conn, Data) ->
 %% @see send/2, async_send_dgram/2
 -spec send_dgram(connection_handle(), binary()) ->
     {ok, BytesSent :: non_neg_integer()}
-    | {error, badarg | not_enough_mem | closed}
+    | {error, badarg | not_enough_mem | invalid_parameter | closed}
     | {error, dgram_send_error, atom_reason()}.
 send_dgram(Conn, Data) ->
     case quicer_nif:send_dgram(Conn, Data, _IsSync = 1) of
diff --git a/src/quicer_nif.erl b/src/quicer_nif.erl
index be95e7b6..d116d0ac 100644
--- a/src/quicer_nif.erl
+++ b/src/quicer_nif.erl
@@ -324,7 +324,7 @@ recv(_Stream, _Len) ->
 
 -spec send_dgram(connection_handle(), iodata(), send_flags()) ->
     {ok, BytesSent :: pos_integer()}
-    | {error, badarg | not_enough_memory | closed}
+    | {error, badarg | not_enough_memory | invalid_parameter | closed}
     | {error, dgram_send_error, atom_reason()}.
 send_dgram(_Conn, _Data, _Flags) ->
     erlang:nif_error(nif_library_not_loaded).
diff --git a/test/prop_stateful_server_conn.erl b/test/prop_stateful_server_conn.erl
index 728c4615..55252fcf 100644
--- a/test/prop_stateful_server_conn.erl
+++ b/test/prop_stateful_server_conn.erl
@@ -160,6 +160,10 @@ postcondition(#{state := S}, {call, quicer, handshake, _Args}, {error, invalid_s
     S =/= accepted
 ->
     true;
+postcondition(#{state := S}, {call, quicer, handshake, _Args}, {error, timeout}) when
+    S =/= accepted
+->
+    true;
 postcondition(_State, {call, quicer, getopt, _Args}, {ok, _}) ->
     true;
 postcondition(_State, {call, quicer, getopt, [_, password]}, {error, badarg}) ->
@@ -335,7 +339,9 @@ postcondition(_State, {call, _Mod, _Fun, _Args} = _Call, _Res) ->
 next_state(State, Res, Call) ->
     step_calls(do_next_state(State, Res, Call)).
 
-do_next_state(#{state := _} = State, {error, closed}, {call, quicer, _, _Args}) ->
+do_next_state(
+    #{state := _} = State, {error, closed}, {call, _M, _F, _A}
+) ->
     State#{state := closed};
 do_next_state(#{state := accepted} = State, {error, _}, {call, quicer, handshake, _Args}) ->
     State;
@@ -353,10 +359,6 @@ do_next_state(
     #{state := _} = State, ok, {call, quicer, controlling_process, [_, Owner]}
 ) ->
     State#{owner := Owner};
-do_next_state(
-    #{state := _} = State, {error, closed}, {call, _M, _F, _A}
-) ->
-    State#{state := closed};
 do_next_state(State, _Res, {call, _Mod, _Fun, _Args}) ->
     State.
 
@@ -405,7 +407,8 @@ default_listen_opts() ->
         {handshake_idle_timeout_ms, 100},
         % QUIC_SERVER_RESUME_AND_ZERORTT
         {server_resumption_level, 2},
-        {peer_bidi_stream_count, 10}
+        {peer_bidi_stream_count, 10},
+        {datagram_receive_enabled, true}
     ].
 
 default_conn_opts() ->
@@ -416,5 +419,6 @@ default_conn_opts() ->
         {idle_timeout_ms, 5000},
         {cacertfile, "./msquic/submodules/openssl/test/certs/rootCA.pem"},
         {certfile, "./msquic/submodules/openssl/test/certs/servercert.pem"},
-        {keyfile, "./msquic/submodules/openssl/test/certs/serverkey.pem"}
+        {keyfile, "./msquic/submodules/openssl/test/certs/serverkey.pem"},
+        {datagram_receive_enabled, true}
     ].
diff --git a/test/quicer_SUITE.erl b/test/quicer_SUITE.erl
index fd3d6293..65a3358b 100644
--- a/test/quicer_SUITE.erl
+++ b/test/quicer_SUITE.erl
@@ -76,6 +76,7 @@
     tc_dgram_client_send/1,
     tc_dgram_client_send_iolist/1,
     tc_dgram_client_send_fail/1,
+    tc_dgram_client_send_exceed_mtu/1,
 
     % , tc_getopt_raw/1
     tc_getopt/1,
@@ -894,6 +895,38 @@ tc_dgram_client_send_fail(_) ->
     ),
     ok.
 
+tc_dgram_client_send_exceed_mtu(Config) ->
+    Port = select_port(),
+    Owner = self(),
+    {SPid, Ref} = spawn_monitor(fun() -> ping_pong_server_dgram(Owner, Config, Port) end),
+    receive
+        listener_ready ->
+            ok
+    after 1000 ->
+        ct:fail("timeout here")
+    end,
+    %% GIVEN: datagram is enabled
+    Opts = default_conn_opts() ++ [{datagram_receive_enabled, 1}],
+    {ok, Conn} = quicer:connect("localhost", Port, Opts, 5000),
+    {ok, Stm} = quicer:start_stream(Conn, []),
+    {ok, 4} = quicer:send(Stm, <<"ping">>),
+    {ok, V2stats} = quicer:getopt(Conn, statistics_v2),
+    MtuMax = proplists:get_value(send_path_mtu, V2stats),
+    %% WHEN: send a datagram that is less than MTU-100
+    Length = MtuMax - 100,
+    %% THEN: send should success
+    {ok, Length} = quicer:send_dgram(Conn, crypto:strong_rand_bytes(Length)),
+    ?assertEqual(
+        {error, dgram_send_error, invalid_parameter},
+        quicer:send_dgram(Conn, crypto:strong_rand_bytes(MtuMax * 2))
+    ),
+    flush_streams_available(Conn),
+    flush_datagram_state_changed(Conn),
+    %% WHEN: send a datagram that is 2xMTU size
+    %% THEN: send should fail
+    SPid ! done,
+    ok = ensure_server_exit_normal(Ref).
+
 tc_dgram_client_send(Config) ->
     Port = select_port(),
     Owner = self(),

From 93cc21c26e408025c33e7121d2517a3f27538e9a Mon Sep 17 00:00:00 2001
From: William Yang <mscame@gmail.com>
Date: Thu, 17 Oct 2024 17:39:33 +0200
Subject: [PATCH 12/12] chore: correct some comments

---
 src/quicer_lib.erl | 7 +++----
 1 file changed, 3 insertions(+), 4 deletions(-)

diff --git a/src/quicer_lib.erl b/src/quicer_lib.erl
index 0eb3ff4e..a66632c9 100644
--- a/src/quicer_lib.erl
+++ b/src/quicer_lib.erl
@@ -111,13 +111,11 @@ handle_dgram_send_states(init, Conn, {Fun, CallbackState}, Timeout) ->
         {quic, dgram_send_state, Conn, #{state := Final}} ->
             Fun(Conn, Final, CallbackState)
     after 5000 ->
+        %% @TODO proper test caught this, may fire a bug report to msquic
         Fun(Conn, timeout, CallbackState)
     end;
 handle_dgram_send_states(sent, Conn, {Fun, CallbackState}, Timeout) ->
     receive
-        %% {quic, dgram_send_state, Conn, #{state := ?QUIC_DATAGRAM_SEND_ACKNOWLEDGED}} ->
-        %%     %% Happy Track
-        %%     Fun(Conn, ?QUIC_DATAGRAM_SEND_ACKNOWLEDGED, CallbackState);
         {quic, dgram_send_state, Conn, #{state := ?QUIC_DATAGRAM_SEND_LOST_SUSPECT}} ->
             %% Lost suspected, call the callback for the return hits.
             %% however, we still need to wait for the final state.
@@ -126,12 +124,13 @@ handle_dgram_send_states(sent, Conn, {Fun, CallbackState}, Timeout) ->
                 {quic, dgram_send_state, Conn, #{state := EState}} ->
                     Fun(Conn, EState, NewCBState)
             after Timeout ->
+                %% @TODO proper test caught this, may fire a bug report to msquic
                 Fun(Conn, timeout, CallbackState)
             end;
         {quic, dgram_send_state, Conn, #{state := Final}} ->
-            %% Unrecoverable Errors.
             Fun(Conn, Final, CallbackState)
     after Timeout ->
+        %% @TODO proper test caught this, may fire a bug report to msquic
         Fun(Conn, timeout, CallbackState)
     end.