session: improve error reporting

Type: improvement

Change-Id: I9dd850a1ce85b0adb5136233f176117e0ee38817
Signed-off-by: Florin Coras <fcoras@cisco.com>
diff --git a/src/vnet/session/application.c b/src/vnet/session/application.c
index 7923d2d..8d7c909 100644
--- a/src/vnet/session/application.c
+++ b/src/vnet/session/application.c
@@ -1019,7 +1019,7 @@
   ASSERT (vlib_thread_is_main_w_barrier ());
 
   if (session_endpoint_is_zero (&a->sep))
-    return VNET_API_ERROR_INVALID_VALUE;
+    return SESSION_E_INVALID_RMT_IP;
 
   client = application_get (a->app_index);
   session_endpoint_update_for_app (&a->sep_ext, client, 1 /* is_connect */ );
@@ -1039,13 +1039,12 @@
       rv = app_worker_connect_session (client_wrk, &a->sep, a->api_context);
       if (rv <= 0)
 	return rv;
+      a->sep_ext.transport_proto = a->sep_ext.original_tp;
     }
   /*
    * Not connecting to a local server, propagate to transport
    */
-  if (app_worker_connect_session (client_wrk, &a->sep, a->api_context))
-    return VNET_API_ERROR_SESSION_CONNECT;
-  return 0;
+  return app_worker_connect_session (client_wrk, &a->sep, a->api_context);
 }
 
 int
diff --git a/src/vnet/session/application.h b/src/vnet/session/application.h
index 90b5a5a..00b60c3 100644
--- a/src/vnet/session/application.h
+++ b/src/vnet/session/application.h
@@ -250,7 +250,7 @@
 int app_worker_accept_notify (app_worker_t * app_wrk, session_t * s);
 int app_worker_init_connected (app_worker_t * app_wrk, session_t * s);
 int app_worker_connect_notify (app_worker_t * app_wrk, session_t * s,
-			       u32 opaque);
+			       session_error_t err, u32 opaque);
 int app_worker_close_notify (app_worker_t * app_wrk, session_t * s);
 int app_worker_transport_closed_notify (app_worker_t * app_wrk,
 					session_t * s);
@@ -299,7 +299,7 @@
 int mq_send_session_bound_cb (u32 app_wrk_index, u32 api_context,
 			      session_handle_t handle, int rv);
 int mq_send_session_connected_cb (u32 app_wrk_index, u32 api_context,
-				  session_t * s, u8 is_fail);
+				  session_t * s, session_error_t err);
 void mq_send_unlisten_reply (app_worker_t * app_wrk, session_handle_t sh,
 			     u32 context, int rv);
 
diff --git a/src/vnet/session/application_interface.h b/src/vnet/session/application_interface.h
index b1ab847..7d5d044 100644
--- a/src/vnet/session/application_interface.h
+++ b/src/vnet/session/application_interface.h
@@ -42,7 +42,7 @@
 
   /** Connection request callback */
   int (*session_connected_callback) (u32 app_wrk_index, u32 opaque,
-				     session_t * s, u8 code);
+				     session_t * s, session_error_t code);
 
   /** Notify app that session is closing */
   void (*session_disconnect_callback) (session_t * s);
@@ -741,6 +741,24 @@
   return app_recv_stream (s, data, len);
 }
 
+/* *INDENT-OFF* */
+static char *session_error_str[] = {
+#define _(sym, str) str,
+    foreach_session_error
+#undef _
+};
+/* *INDENT-ON* */
+
+static inline u8 *
+format_session_error (u8 * s, va_list * args)
+{
+  session_error_t error = va_arg (*args, session_error_t);
+  if (-error >= 0 && -error < SESSION_N_ERRORS)
+    s = format (s, "%s", session_error_str[-error]);
+  else
+    s = format (s, "invalid session err %u", -error);
+  return s;
+}
 #endif /* __included_uri_h__ */
 
 /*
diff --git a/src/vnet/session/application_local.c b/src/vnet/session/application_local.c
index e2c1082..d54d111 100644
--- a/src/vnet/session/application_local.c
+++ b/src/vnet/session/application_local.c
@@ -96,7 +96,7 @@
   segment_manager_t *sm;
   fifo_segment_t *seg;
   u64 segment_handle;
-  int is_fail = 0;
+  int err = 0;
   session_t *cs;
   u32 ss_index;
 
@@ -108,13 +108,12 @@
   seg = segment_manager_get_segment_w_lock (sm, ss->rx_fifo->segment_index);
   segment_handle = segment_manager_segment_handle (sm, seg);
 
-  if (app_worker_add_segment_notify (client_wrk, segment_handle))
+  if ((err = app_worker_add_segment_notify (client_wrk, segment_handle)))
     {
       clib_warning ("failed to notify client %u of new segment",
 		    sct->client_wrk);
       segment_manager_segment_reader_unlock (sm);
       session_close (ss);
-      is_fail = 1;
     }
   else
     {
@@ -149,8 +148,7 @@
       return -1;
     }
 
-  if (app_worker_connect_notify (client_wrk, is_fail ? 0 : cs,
-				 sct->client_opaque))
+  if (app_worker_connect_notify (client_wrk, cs, err, sct->client_opaque))
     {
       session_close (ss);
       return -1;
@@ -371,7 +369,7 @@
   table_index = application_local_session_table (app);
   lh = session_lookup_local_endpoint (table_index, sep);
   if (lh == SESSION_DROP_HANDLE)
-    return VNET_API_ERROR_APP_CONNECT_FILTERED;
+    return SESSION_E_FILTERED;
 
   if (lh == SESSION_INVALID_HANDLE)
     goto global_scope;
@@ -396,10 +394,10 @@
 
 global_scope:
   if (session_endpoint_is_local (sep))
-    return VNET_API_ERROR_SESSION_CONNECT;
+    return SESSION_E_NOROUTE;
 
   if (!application_has_global_scope (app))
-    return VNET_API_ERROR_APP_CONNECT_SCOPE;
+    return SESSION_E_SCOPE;
 
   fib_proto = session_endpoint_fib_proto (sep);
   table_index = session_lookup_get_index_for_fib (fib_proto, sep->fib_index);
diff --git a/src/vnet/session/application_worker.c b/src/vnet/session/application_worker.c
index 142b662..c17f43e 100644
--- a/src/vnet/session/application_worker.c
+++ b/src/vnet/session/application_worker.c
@@ -173,7 +173,7 @@
   /* Allocate segment manager. All sessions derived out of a listen session
    * have fifos allocated by the same segment manager. */
   if (!(sm = app_worker_alloc_segment_manager (app_wrk)))
-    return -1;
+    return SESSION_E_ALLOC;
 
   /* Keep track of the segment manager for the listener or this worker */
   hash_set (app_wrk->listeners_table, listen_session_get_handle (ls),
@@ -182,12 +182,8 @@
   if (transport_connection_is_cless (session_get_transport (ls)))
     {
       if (ls->rx_fifo)
-	{
-	  clib_warning ("sharing of connectionless listeners not supported");
-	  return -1;
-	}
-      if (app_worker_alloc_session_fifos (sm, ls))
-	return -1;
+	return SESSION_E_NOSUPPORT;
+      return app_worker_alloc_session_fifos (sm, ls);
     }
   return 0;
 }
@@ -197,9 +193,10 @@
 			 app_listener_t * app_listener)
 {
   session_t *ls;
+  int rv;
 
   if (clib_bitmap_get (app_listener->workers, app_wrk->wrk_map_index))
-    return VNET_API_ERROR_ADDRESS_IN_USE;
+    return SESSION_E_ALREADY_LISTENING;
 
   app_listener->workers = clib_bitmap_set (app_listener->workers,
 					   app_wrk->wrk_map_index, 1);
@@ -207,15 +204,15 @@
   if (app_listener->session_index != SESSION_INVALID_INDEX)
     {
       ls = session_get (app_listener->session_index, 0);
-      if (app_worker_init_listener (app_wrk, ls))
-	return -1;
+      if ((rv = app_worker_init_listener (app_wrk, ls)))
+	return rv;
     }
 
   if (app_listener->local_index != SESSION_INVALID_INDEX)
     {
       ls = session_get (app_listener->local_index, 0);
-      if (app_worker_init_listener (app_wrk, ls))
-	return -1;
+      if ((rv = app_worker_init_listener (app_wrk, ls)))
+	return rv;
     }
 
   return 0;
@@ -321,8 +318,7 @@
   if (!application_is_builtin_proxy (app))
     {
       sm = app_worker_get_connect_segment_manager (app_wrk);
-      if (app_worker_alloc_session_fifos (sm, s))
-	return -1;
+      return app_worker_alloc_session_fifos (sm, s);
     }
 
   if (app->cb_fns.fifo_tuning_callback)
@@ -332,11 +328,12 @@
 }
 
 int
-app_worker_connect_notify (app_worker_t * app_wrk, session_t * s, u32 opaque)
+app_worker_connect_notify (app_worker_t * app_wrk, session_t * s,
+			   session_error_t err, u32 opaque)
 {
   application_t *app = application_get (app_wrk->app_index);
   return app->cb_fns.session_connected_callback (app_wrk->wrk_index, opaque,
-						 s, s == 0 /* is_fail */ );
+						 s, err);
 }
 
 int
diff --git a/src/vnet/session/segment_manager.c b/src/vnet/session/segment_manager.c
index ad5db65..154c7a6 100644
--- a/src/vnet/session/segment_manager.c
+++ b/src/vnet/session/segment_manager.c
@@ -664,12 +664,12 @@
 	{
 	  clib_warning ("Added a segment, still can't allocate a fifo");
 	  segment_manager_segment_reader_unlock (sm);
-	  return SESSION_ERROR_NEW_SEG_NO_SPACE;
+	  return SESSION_E_SEG_NO_SPACE2;
 	}
       if ((new_fs_index = segment_manager_add_segment (sm, 0)) < 0)
 	{
 	  clib_warning ("Failed to add new segment");
-	  return SESSION_ERROR_SEG_CREATE;
+	  return SESSION_E_SEG_CREATE;
 	}
       fs = segment_manager_get_segment_w_lock (sm, new_fs_index);
       alloc_fail = segment_manager_try_alloc_fifos (fs, thread_index,
@@ -682,7 +682,7 @@
   else
     {
       clib_warning ("Can't add new seg and no space to allocate fifos!");
-      return SESSION_ERROR_NO_SPACE;
+      return SESSION_E_SEG_NO_SPACE;
     }
 }
 
diff --git a/src/vnet/session/session.c b/src/vnet/session/session.c
index 3351400..ac29627 100644
--- a/src/vnet/session/session.c
+++ b/src/vnet/session/session.c
@@ -736,7 +736,8 @@
 }
 
 static inline int
-session_stream_connect_notify_inline (transport_connection_t * tc, u8 is_fail,
+session_stream_connect_notify_inline (transport_connection_t * tc,
+				      session_error_t err,
 				      session_state_t opened_state)
 {
   u32 opaque = 0, new_ti, new_si;
@@ -764,8 +765,8 @@
 
   opaque = tc->s_index;
 
-  if (is_fail)
-    return app_worker_connect_notify (app_wrk, s, opaque);
+  if (err)
+    return app_worker_connect_notify (app_wrk, s, err, opaque);
 
   s = session_alloc_for_connection (tc);
   s->session_state = SESSION_STATE_CONNECTING;
@@ -773,10 +774,10 @@
   new_si = s->session_index;
   new_ti = s->thread_index;
 
-  if (app_worker_init_connected (app_wrk, s))
+  if ((err = app_worker_init_connected (app_wrk, s)))
     {
       session_free (s);
-      app_worker_connect_notify (app_wrk, 0, opaque);
+      app_worker_connect_notify (app_wrk, 0, err, opaque);
       return -1;
     }
 
@@ -784,7 +785,7 @@
   s->session_state = opened_state;
   session_lookup_add_connection (tc, session_handle (s));
 
-  if (app_worker_connect_notify (app_wrk, s, opaque))
+  if (app_worker_connect_notify (app_wrk, s, SESSION_E_NONE, opaque))
     {
       s = session_get (new_si, new_ti);
       session_free_w_fifos (s);
@@ -795,17 +796,17 @@
 }
 
 int
-session_stream_connect_notify (transport_connection_t * tc, u8 is_fail)
+session_stream_connect_notify (transport_connection_t * tc,
+			       session_error_t err)
 {
-  return session_stream_connect_notify_inline (tc, is_fail,
-					       SESSION_STATE_READY);
+  return session_stream_connect_notify_inline (tc, err, SESSION_STATE_READY);
 }
 
 int
-session_ho_stream_connect_notify (transport_connection_t * tc, u8 is_fail)
+session_ho_stream_connect_notify (transport_connection_t * tc,
+				  session_error_t err)
 {
-  return session_stream_connect_notify_inline (tc, is_fail,
-					       SESSION_STATE_OPENED);
+  return session_stream_connect_notify_inline (tc, err, SESSION_STATE_OPENED);
 }
 
 typedef struct _session_switch_pool_args
@@ -1116,7 +1117,7 @@
 
   sh = session_handle (s);
   session_lookup_add_connection (tc, sh);
-  return app_worker_connect_notify (app_wrk, s, opaque);
+  return app_worker_connect_notify (app_wrk, s, SESSION_E_NONE, opaque);
 }
 
 int
diff --git a/src/vnet/session/session.h b/src/vnet/session/session.h
index 681b42d..a9f965d 100644
--- a/src/vnet/session/session.h
+++ b/src/vnet/session/session.h
@@ -30,9 +30,6 @@
 _(FIFO_FULL, "Packets dropped for lack of rx fifo space")               \
 _(EVENT_FIFO_FULL, "Events not sent for lack of event fifo space")      \
 _(API_QUEUE_FULL, "Sessions not created for lack of API queue space")   \
-_(NEW_SEG_NO_SPACE, "Created segment, couldn't allocate a fifo pair")   \
-_(NO_SPACE, "Couldn't allocate a fifo pair")				\
-_(SEG_CREATE, "Couldn't create a new segment")
 
 typedef enum
 {
@@ -40,7 +37,7 @@
   foreach_session_input_error
 #undef _
     SESSION_N_ERROR,
-} session_error_t;
+} session_input_error_t;
 
 typedef struct session_tx_context_
 {
@@ -456,7 +453,8 @@
 				      session_dgram_hdr_t * hdr,
 				      vlib_buffer_t * b, u8 proto,
 				      u8 queue_event);
-int session_stream_connect_notify (transport_connection_t * tc, u8 is_fail);
+int session_stream_connect_notify (transport_connection_t * tc,
+				   session_error_t err);
 int session_dgram_connect_notify (transport_connection_t * tc,
 				  u32 old_thread_index,
 				  session_t ** new_session);
diff --git a/src/vnet/session/session_api.c b/src/vnet/session/session_api.c
index 6dcf06b..43ecf2b 100644
--- a/src/vnet/session/session_api.c
+++ b/src/vnet/session/session_api.c
@@ -100,15 +100,12 @@
 {
   clib_error_t *error;
   if (vl_api_registration_file_index (reg) == VL_API_INVALID_FI)
-    {
-      clib_warning ("can't send memfd fd");
-      return -1;
-    }
+    return SESSION_E_BAPI_NO_FD;
   error = vl_api_send_fd_msg (reg, fds, n_fds);
   if (error)
     {
       clib_error_report (error);
-      return -1;
+      return SESSION_E_BAPI_SEND_FD;
     }
   return 0;
 }
@@ -146,7 +143,7 @@
   app = application_get (app_wrk->app_index);
   app_mq = app_wrk->event_queue;
   if (mq_try_lock_and_alloc_msg (app_mq, msg))
-    return -1;
+    return SESSION_E_MQ_MSG_ALLOC;
 
   evt = svm_msg_q_msg_data (app_mq, msg);
   clib_memset (evt, 0, sizeof (*evt));
@@ -265,7 +262,7 @@
 
 int
 mq_send_session_connected_cb (u32 app_wrk_index, u32 api_context,
-			      session_t * s, u8 is_fail)
+			      session_t * s, session_error_t err)
 {
   svm_msg_q_msg_t _msg, *msg = &_msg;
   session_connected_msg_t *mp;
@@ -284,7 +281,7 @@
     }
 
   if (mq_try_lock_and_alloc_msg (app_mq, msg))
-    return -1;
+    return SESSION_E_MQ_MSG_ALLOC;
 
   evt = svm_msg_q_msg_data (app_mq, msg);
   clib_memset (evt, 0, sizeof (*evt));
@@ -293,7 +290,7 @@
   clib_memset (mp, 0, sizeof (*mp));
   mp->context = api_context;
 
-  if (is_fail)
+  if (err)
     goto done;
 
   if (session_has_transport (s))
@@ -301,7 +298,8 @@
       tc = session_get_transport (s);
       if (!tc)
 	{
-	  is_fail = 1;
+	  clib_warning ("failed to retrieve transport!");
+	  err = SESSION_E_REFUSED;
 	  goto done;
 	}
 
@@ -336,8 +334,7 @@
     }
 
 done:
-  mp->retval = is_fail ?
-    clib_host_to_net_u32 (VNET_API_ERROR_SESSION_CONNECT) : 0;
+  mp->retval = err;
 
   svm_msg_q_add_and_unlock (app_mq, msg);
   return 0;
@@ -365,7 +362,7 @@
     }
 
   if (mq_try_lock_and_alloc_msg (app_mq, msg))
-    return -1;
+    return SESSION_E_MQ_MSG_ALLOC;
 
   evt = svm_msg_q_msg_data (app_mq, msg);
   clib_memset (evt, 0, sizeof (*evt));
diff --git a/src/vnet/session/session_node.c b/src/vnet/session/session_node.c
index 0290fe6..c2a6ec9 100644
--- a/src/vnet/session/session_node.c
+++ b/src/vnet/session/session_node.c
@@ -139,8 +139,7 @@
     {
       clib_warning ("connect returned: %U", format_vnet_api_errno, rv);
       app_wrk = application_get_worker (app, mp->wrk_index);
-      mq_send_session_connected_cb (app_wrk->wrk_index, mp->context, 0,
-				    /* is_fail */ 1);
+      mq_send_session_connected_cb (app_wrk->wrk_index, mp->context, 0, rv);
     }
 
   vec_free (a->sep_ext.hostname);
@@ -169,8 +168,7 @@
     {
       clib_warning ("connect_uri returned: %d", rv);
       app_wrk = application_get_worker (app, 0 /* default wrk only */ );
-      mq_send_session_connected_cb (app_wrk->wrk_index, mp->context, 0,
-				    /* is_fail */ 1);
+      mq_send_session_connected_cb (app_wrk->wrk_index, mp->context, 0, rv);
     }
 }
 
diff --git a/src/vnet/session/session_types.h b/src/vnet/session/session_types.h
index a036013..ca5dcc1 100644
--- a/src/vnet/session/session_types.h
+++ b/src/vnet/session/session_types.h
@@ -433,6 +433,53 @@
 
 STATIC_ASSERT (sizeof (session_dgram_hdr_t) == (SESSION_CONN_ID_LEN + 8),
 	       "session conn id wrong length");
+
+#define foreach_session_error						\
+  _(NONE, "no error")							\
+  _(UNKNOWN, "generic/unknown error")					\
+  _(REFUSED, "refused")							\
+  _(TIMEDOUT, "timedout")						\
+  _(ALLOC, "obj/memory allocation error")				\
+  _(NOROUTE, "no route")						\
+  _(NOINTF, "no resolving interface")					\
+  _(NOIP, "no ip for lcl interface")					\
+  _(NOPORT, "no lcl port")						\
+  _(NOSUPPORT, "not supported")						\
+  _(PORTINUSE, "lcl port in use")					\
+  _(IPINUSE, "ip in use")						\
+  _(ALREADY_LISTENING, "ip port pair already listened on")		\
+  _(INVALID_RMT_IP, "invalid remote ip")				\
+  _(SEG_NO_SPACE, "Couldn't allocate a fifo pair")			\
+  _(SEG_NO_SPACE2, "Created segment, couldn't allocate a fifo pair") 	\
+  _(SEG_CREATE, "Couldn't create a new segment")			\
+  _(FILTERED, "session filtered")					\
+  _(SCOPE, "scope not supported")					\
+  _(BAPI_NO_FD, "bapi doesn't have a socket fd")			\
+  _(BAPI_SEND_FD, "couldn't send fd over bapi socket fd")		\
+  _(BAPI_NO_REG, "app bapi registration not found")			\
+  _(MQ_MSG_ALLOC, "failed to alloc mq msg")				\
+  _(TLS_HANDSHAKE, "failed tls handshake")				\
+
+typedef enum session_error_p_
+{
+#define _(sym, str) SESSION_EP_##sym,
+  foreach_session_error
+#undef _
+  SESSION_N_ERRORS
+} session_error_p_t;
+
+typedef enum session_error_
+{
+#define _(sym, str) SESSION_E_##sym = -SESSION_EP_##sym,
+  foreach_session_error
+#undef _
+} session_error_t;
+
+/* Maintained for compatibility. Will be deprecated */
+#define SESSION_ERROR_SEG_CREATE SESSION_E_SEG_CREATE
+#define SESSION_ERROR_NO_SPACE SESSION_E_SEG_NO_SPACE
+#define SESSION_ERROR_NEW_SEG_NO_SPACE SESSION_E_SEG_NO_SPACE2
+
 #endif /* SRC_VNET_SESSION_SESSION_TYPES_H_ */
 
 /*
diff --git a/src/vnet/session/transport.c b/src/vnet/session/transport.c
index 6904685..29c94f3 100644
--- a/src/vnet/session/transport.c
+++ b/src/vnet/session/transport.c
@@ -480,7 +480,7 @@
   return -1;
 }
 
-static clib_error_t *
+static session_error_t
 transport_get_interface_ip (u32 sw_if_index, u8 is_ip4, ip46_address_t * addr)
 {
   if (is_ip4)
@@ -488,9 +488,7 @@
       ip4_address_t *ip4;
       ip4 = ip_interface_get_first_ip (sw_if_index, 1);
       if (!ip4)
-	return clib_error_return (0, "no routable ip4 address on %U",
-				  format_vnet_sw_if_index_name,
-				  vnet_get_main (), sw_if_index);
+	return SESSION_E_NOIP;
       addr->ip4.as_u32 = ip4->as_u32;
     }
   else
@@ -498,15 +496,13 @@
       ip6_address_t *ip6;
       ip6 = ip_interface_get_first_ip (sw_if_index, 0);
       if (ip6 == 0)
-	return clib_error_return (0, "no routable ip6 addresses on %U",
-				  format_vnet_sw_if_index_name,
-				  vnet_get_main (), sw_if_index);
+	return SESSION_E_NOIP;
       clib_memcpy_fast (&addr->ip6, ip6, sizeof (*ip6));
     }
   return 0;
 }
 
-static clib_error_t *
+static session_error_t
 transport_find_local_ip_for_remote (u32 sw_if_index,
 				    transport_endpoint_t * rmt,
 				    ip46_address_t * lcl_addr)
@@ -526,14 +522,11 @@
 
       /* Couldn't find route to destination. Bail out. */
       if (fei == FIB_NODE_INDEX_INVALID)
-	return clib_error_return (0, "no route to %U", format_ip46_address,
-				  &rmt->ip, (rmt->is_ip4 == 0) + 1);
+	return SESSION_E_NOROUTE;
 
       sw_if_index = fib_entry_get_resolving_interface (fei);
       if (sw_if_index == ENDPOINT_INVALID_INDEX)
-	return clib_error_return (0, "no resolving interface for %U",
-				  format_ip46_address, &rmt->ip,
-				  (rmt->is_ip4 == 0) + 1);
+	return SESSION_E_NOINTF;
     }
 
   clib_memset (lcl_addr, 0, sizeof (*lcl_addr));
@@ -545,7 +538,7 @@
 				ip46_address_t * lcl_addr, u16 * lcl_port)
 {
   transport_endpoint_t *rmt = (transport_endpoint_t *) rmt_cfg;
-  clib_error_t *error;
+  session_error_t error;
   int port;
   u32 tei;
 
@@ -557,10 +550,7 @@
       error = transport_find_local_ip_for_remote (rmt_cfg->peer.sw_if_index,
 						  rmt, lcl_addr);
       if (error)
-	{
-	  clib_error_report (error);
-	  return -1;
-	}
+	return error;
     }
   else
     {
@@ -576,10 +566,7 @@
     {
       port = transport_alloc_local_port (proto, lcl_addr);
       if (port < 1)
-	{
-	  clib_warning ("Failed to allocate src port");
-	  return -1;
-	}
+	return SESSION_E_NOPORT;
       *lcl_port = port;
     }
   else
@@ -588,7 +575,7 @@
       tei = transport_endpoint_lookup (&local_endpoints_table, proto,
 				       lcl_addr, port);
       if (tei != ENDPOINT_INVALID_INDEX)
-	return -1;
+	return SESSION_E_PORTINUSE;
 
       transport_endpoint_mark_used (proto, lcl_addr, port);
       *lcl_port = port;