blob: 9e01ac95bec34ed9cc9b83959f736f8fd9f0c74f [file] [log] [blame]
Keith Burns (alagalah)b327c2b2017-10-09 08:52:59 -07001/*
Florin Coras5e062572019-03-14 19:07:51 -07002 * Copyright (c) 2016-2019 Cisco and/or its affiliates.
Keith Burns (alagalah)b327c2b2017-10-09 08:52:59 -07003 * Licensed under the Apache License, Version 2.0 (the "License");
4 * you may not use this file except in compliance with the License.
5 * You may obtain a copy of the License at:
6 *
7 * http://www.apache.org/licenses/LICENSE-2.0
8 *
9 * Unless required by applicable law or agreed to in writing, software
10 * distributed under the License is distributed on an "AS IS" BASIS,
11 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12 * See the License for the specific language governing permissions and
13 * limitations under the License.
14 */
Florin Coras36847942023-02-02 12:56:16 -080015
16#ifdef HAVE_GNU_SOURCE
17#define _GNU_SOURCE
18#endif
19
Keith Burns (alagalah)b327c2b2017-10-09 08:52:59 -070020#include <unistd.h>
21#include <stdio.h>
22#include <signal.h>
23#include <dlfcn.h>
24#include <pthread.h>
25#include <time.h>
26#include <stdarg.h>
shrinivasan ganapathy1d359632017-10-15 15:46:09 -070027#include <sys/resource.h>
Dave Wallace048b1d62018-01-03 22:24:41 -050028#include <netinet/tcp.h>
Florin Coraseff5f7a2023-02-07 17:36:17 -080029#include <netinet/udp.h>
Keith Burns (alagalah)b327c2b2017-10-09 08:52:59 -070030
Dave Wallace2a865272018-02-07 21:00:42 -050031#include <vcl/ldp_socket_wrapper.h>
32#include <vcl/ldp.h>
Keith Burns (alagalah)b327c2b2017-10-09 08:52:59 -070033#include <sys/time.h>
34
Florin Coras7baeb712019-01-04 17:05:43 -080035#include <vcl/vcl_locked.h>
Dave Wallace048b1d62018-01-03 22:24:41 -050036#include <vppinfra/time.h>
37#include <vppinfra/bitmap.h>
Florin Coras30e273b2018-11-27 00:04:59 -080038#include <vppinfra/lock.h>
39#include <vppinfra/pool.h>
40#include <vppinfra/hash.h>
Keith Burns (alagalah)b327c2b2017-10-09 08:52:59 -070041
42#define HAVE_CONSTRUCTOR_ATTRIBUTE
43#ifdef HAVE_CONSTRUCTOR_ATTRIBUTE
44#define CONSTRUCTOR_ATTRIBUTE \
45 __attribute__ ((constructor))
46#else
47#define CONSTRUCTOR_ATTRIBUTE
48#endif /* HAVE_CONSTRUCTOR_ATTRIBUTE */
49
50#define HAVE_DESTRUCTOR_ATTRIBUTE
51#ifdef HAVE_DESTRUCTOR_ATTRIBUTE
52#define DESTRUCTOR_ATTRIBUTE \
53 __attribute__ ((destructor))
54#else
55#define DESTRUCTOR_ATTRIBUTE
56#endif
57
Florin Corasdfe4cf42018-11-28 22:13:45 -080058#define LDP_MAX_NWORKERS 32
59
Florin Coras36847942023-02-02 12:56:16 -080060#ifdef HAVE_GNU_SOURCE
61#define SOCKADDR_GET_SA(__addr) __addr.__sockaddr__;
62#else
63#define SOCKADDR_GET_SA(__addr) _addr;
64#endif
65
Tianyu Li2073fe42023-02-27 09:14:34 +000066#ifndef UDP_SEGMENT
67#define UDP_SEGMENT 103
68#endif
69
Florin Corasdfe4cf42018-11-28 22:13:45 -080070typedef struct ldp_worker_ctx_
Dave Wallace048b1d62018-01-03 22:24:41 -050071{
Dave Wallace048b1d62018-01-03 22:24:41 -050072 u8 *io_buffer;
73 clib_time_t clib_time;
Florin Corasdfe4cf42018-11-28 22:13:45 -080074
75 /*
76 * Select state
77 */
Dave Wallace048b1d62018-01-03 22:24:41 -050078 clib_bitmap_t *rd_bitmap;
79 clib_bitmap_t *wr_bitmap;
80 clib_bitmap_t *ex_bitmap;
Florin Coras294afe22019-01-07 17:49:17 -080081 clib_bitmap_t *si_rd_bitmap;
82 clib_bitmap_t *si_wr_bitmap;
83 clib_bitmap_t *si_ex_bitmap;
Dave Wallace048b1d62018-01-03 22:24:41 -050084 clib_bitmap_t *libc_rd_bitmap;
85 clib_bitmap_t *libc_wr_bitmap;
86 clib_bitmap_t *libc_ex_bitmap;
Florin Corasdfe4cf42018-11-28 22:13:45 -080087
88 /*
89 * Poll state
90 */
Dave Wallace048b1d62018-01-03 22:24:41 -050091 vcl_poll_t *vcl_poll;
Florin Coras6917b942018-11-13 22:44:54 -080092 struct pollfd *libc_poll;
93 u16 *libc_poll_idxs;
Florin Corasdfe4cf42018-11-28 22:13:45 -080094
95 /*
96 * Epoll state
97 */
Dave Wallace048b1d62018-01-03 22:24:41 -050098 u8 epoll_wait_vcl;
hanlin4266d4d2020-05-19 17:34:17 +080099 u8 mq_epfd_added;
Florin Coras99368312018-08-02 10:45:44 -0700100 int vcl_mq_epfd;
Florin Corasdfe4cf42018-11-28 22:13:45 -0800101
102} ldp_worker_ctx_t;
103
Florin Coras294afe22019-01-07 17:49:17 -0800104/* clib_bitmap_t, fd_mask and vcl_si_set are used interchangeably. Make sure
105 * they are the same size */
106STATIC_ASSERT (sizeof (clib_bitmap_t) == sizeof (fd_mask),
107 "ldp bitmap size mismatch");
108STATIC_ASSERT (sizeof (vcl_si_set) == sizeof (fd_mask),
109 "ldp bitmap size mismatch");
110
Florin Corasdfe4cf42018-11-28 22:13:45 -0800111typedef struct
112{
113 ldp_worker_ctx_t *workers;
114 int init;
115 char app_name[LDP_APP_NAME_MAX];
Florin Coras7baeb712019-01-04 17:05:43 -0800116 u32 vlsh_bit_val;
117 u32 vlsh_bit_mask;
Florin Corasdfe4cf42018-11-28 22:13:45 -0800118 u32 debug;
Florin Corasdfe4cf42018-11-28 22:13:45 -0800119
120 /** vcl needs next epoll_create to go to libc_epoll */
121 u8 vcl_needs_real_epoll;
Florin Corasa5a9efd2021-01-05 17:03:29 -0800122
123 /**
124 * crypto state used only for testing
125 */
126 u8 transparent_tls;
127 u32 ckpair_index;
Dave Wallace2a865272018-02-07 21:00:42 -0500128} ldp_main_t;
Florin Corasdfe4cf42018-11-28 22:13:45 -0800129
Dave Wallace2a865272018-02-07 21:00:42 -0500130#define LDP_DEBUG ldp->debug
Keith Burns (alagalah)b327c2b2017-10-09 08:52:59 -0700131
Florin Coras99368312018-08-02 10:45:44 -0700132#define LDBG(_lvl, _fmt, _args...) \
133 if (ldp->debug > _lvl) \
hanlin9f3f18f2019-12-30 16:25:20 +0800134 { \
135 int errno_saved = errno; \
Florin Coras585c86a2020-10-16 17:57:36 -0700136 fprintf (stderr, "ldp<%d>: " _fmt "\n", getpid(), ##_args); \
hanlin9f3f18f2019-12-30 16:25:20 +0800137 errno = errno_saved; \
138 }
Florin Coras99368312018-08-02 10:45:44 -0700139
Dave Wallace2a865272018-02-07 21:00:42 -0500140static ldp_main_t ldp_main = {
Florin Coras7baeb712019-01-04 17:05:43 -0800141 .vlsh_bit_val = (1 << LDP_SID_BIT_MIN),
142 .vlsh_bit_mask = (1 << LDP_SID_BIT_MIN) - 1,
Dave Wallace2a865272018-02-07 21:00:42 -0500143 .debug = LDP_DEBUG_INIT,
Yu Ping7b74b072019-05-08 00:40:24 +0800144 .transparent_tls = 0,
Florin Corasa5a9efd2021-01-05 17:03:29 -0800145 .ckpair_index = ~0,
Dave Wallace048b1d62018-01-03 22:24:41 -0500146};
Keith Burns (alagalah)b327c2b2017-10-09 08:52:59 -0700147
Dave Wallace2a865272018-02-07 21:00:42 -0500148static ldp_main_t *ldp = &ldp_main;
Keith Burns (alagalah)b327c2b2017-10-09 08:52:59 -0700149
Florin Corasdfe4cf42018-11-28 22:13:45 -0800150static inline ldp_worker_ctx_t *
151ldp_worker_get_current (void)
152{
153 return (ldp->workers + vppcom_worker_index ());
154}
155
Keith Burns (alagalah)b327c2b2017-10-09 08:52:59 -0700156/*
157 * RETURN: 0 on success or -1 on error.
158 * */
Dave Wallace048b1d62018-01-03 22:24:41 -0500159static inline void
Dave Wallace2a865272018-02-07 21:00:42 -0500160ldp_set_app_name (char *app_name)
Keith Burns (alagalah)b327c2b2017-10-09 08:52:59 -0700161{
Florin Coras729b9c92023-02-28 10:58:08 -0800162 snprintf (ldp->app_name, LDP_APP_NAME_MAX, "%s-ldp-%d", app_name, getpid ());
Keith Burns (alagalah)b327c2b2017-10-09 08:52:59 -0700163}
164
Dave Wallace048b1d62018-01-03 22:24:41 -0500165static inline char *
Dave Wallace2a865272018-02-07 21:00:42 -0500166ldp_get_app_name ()
Keith Burns (alagalah)b327c2b2017-10-09 08:52:59 -0700167{
Dave Wallace2a865272018-02-07 21:00:42 -0500168 if (ldp->app_name[0] == '\0')
Florin Coras729b9c92023-02-28 10:58:08 -0800169 ldp_set_app_name (program_invocation_short_name);
Dave Wallace048b1d62018-01-03 22:24:41 -0500170
Dave Wallace2a865272018-02-07 21:00:42 -0500171 return ldp->app_name;
Keith Burns (alagalah)b327c2b2017-10-09 08:52:59 -0700172}
173
Dave Wallace048b1d62018-01-03 22:24:41 -0500174static inline int
Florin Coras7baeb712019-01-04 17:05:43 -0800175ldp_vlsh_to_fd (vls_handle_t vlsh)
Dave Wallace048b1d62018-01-03 22:24:41 -0500176{
Florin Coras7baeb712019-01-04 17:05:43 -0800177 return (vlsh + ldp->vlsh_bit_val);
Dave Wallace048b1d62018-01-03 22:24:41 -0500178}
Keith Burns (alagalah)b327c2b2017-10-09 08:52:59 -0700179
Florin Coras7baeb712019-01-04 17:05:43 -0800180static inline vls_handle_t
181ldp_fd_to_vlsh (int fd)
Dave Wallace048b1d62018-01-03 22:24:41 -0500182{
Florin Coras7baeb712019-01-04 17:05:43 -0800183 if (fd < ldp->vlsh_bit_val)
184 return VLS_INVALID_HANDLE;
Keith Burns (alagalah)b327c2b2017-10-09 08:52:59 -0700185
Florin Coras7baeb712019-01-04 17:05:43 -0800186 return (fd - ldp->vlsh_bit_val);
Dave Wallace048b1d62018-01-03 22:24:41 -0500187}
188
Florin Coras2d9b4272019-03-11 10:14:37 -0700189static void
190ldp_alloc_workers (void)
191{
192 if (ldp->workers)
193 return;
194 pool_alloc (ldp->workers, LDP_MAX_NWORKERS);
195}
196
Florin Coras5f33d0d2021-06-02 21:22:21 -0700197static int
Dave Wallace2a865272018-02-07 21:00:42 -0500198ldp_init (void)
Keith Burns (alagalah)b327c2b2017-10-09 08:52:59 -0700199{
Florin Corasdfe4cf42018-11-28 22:13:45 -0800200 ldp_worker_ctx_t *ldpw;
Florin Coras99368312018-08-02 10:45:44 -0700201 int rv;
Keith Burns (alagalah)b327c2b2017-10-09 08:52:59 -0700202
Florin Coras5f33d0d2021-06-02 21:22:21 -0700203 ASSERT (!ldp->init);
Florin Coras99368312018-08-02 10:45:44 -0700204
205 ldp->init = 1;
206 ldp->vcl_needs_real_epoll = 1;
Florin Coras7baeb712019-01-04 17:05:43 -0800207 rv = vls_app_create (ldp_get_app_name ());
Florin Coras99368312018-08-02 10:45:44 -0700208 if (rv != VPPCOM_OK)
Keith Burns (alagalah)b327c2b2017-10-09 08:52:59 -0700209 {
Florin Coras955bfbb2018-12-04 13:43:45 -0800210 ldp->vcl_needs_real_epoll = 0;
211 if (rv == VPPCOM_EEXIST)
212 return 0;
Florin Coras05ecfcc2018-12-12 18:19:39 -0800213 LDBG (2, "\nERROR: ldp_init: vppcom_app_create()"
214 " failed! rv = %d (%s)\n", rv, vppcom_retval_str (rv));
Florin Coras99368312018-08-02 10:45:44 -0700215 ldp->init = 0;
216 return rv;
217 }
218 ldp->vcl_needs_real_epoll = 0;
Florin Coras2d9b4272019-03-11 10:14:37 -0700219 ldp_alloc_workers ();
Florin Corasdfe4cf42018-11-28 22:13:45 -0800220 ldpw = ldp_worker_get_current ();
Florin Coras99368312018-08-02 10:45:44 -0700221
222 char *env_var_str = getenv (LDP_ENV_DEBUG);
223 if (env_var_str)
224 {
225 u32 tmp;
226 if (sscanf (env_var_str, "%u", &tmp) != 1)
227 clib_warning ("LDP<%d>: WARNING: Invalid LDP debug level specified in"
228 " the env var " LDP_ENV_DEBUG " (%s)!", getpid (),
229 env_var_str);
230 else
Keith Burns (alagalah)b327c2b2017-10-09 08:52:59 -0700231 {
Florin Coras99368312018-08-02 10:45:44 -0700232 ldp->debug = tmp;
Florin Coras05ecfcc2018-12-12 18:19:39 -0800233 LDBG (0, "configured LDP debug level (%u) from env var "
234 LDP_ENV_DEBUG "!", ldp->debug);
Florin Coras99368312018-08-02 10:45:44 -0700235 }
236 }
Dave Wallace048b1d62018-01-03 22:24:41 -0500237
Florin Coras99368312018-08-02 10:45:44 -0700238 env_var_str = getenv (LDP_ENV_APP_NAME);
239 if (env_var_str)
240 {
241 ldp_set_app_name (env_var_str);
Florin Coras05ecfcc2018-12-12 18:19:39 -0800242 LDBG (0, "configured LDP app name (%s) from the env var "
243 LDP_ENV_APP_NAME "!", ldp->app_name);
Florin Coras99368312018-08-02 10:45:44 -0700244 }
Dave Wallace048b1d62018-01-03 22:24:41 -0500245
Florin Coras99368312018-08-02 10:45:44 -0700246 env_var_str = getenv (LDP_ENV_SID_BIT);
247 if (env_var_str)
248 {
249 u32 sb;
250 if (sscanf (env_var_str, "%u", &sb) != 1)
251 {
Florin Coras294afe22019-01-07 17:49:17 -0800252 LDBG (0, "WARNING: Invalid LDP sid bit specified in the env var "
253 LDP_ENV_SID_BIT " (%s)! sid bit value %d (0x%x)", env_var_str,
254 ldp->vlsh_bit_val, ldp->vlsh_bit_val);
Florin Coras99368312018-08-02 10:45:44 -0700255 }
256 else if (sb < LDP_SID_BIT_MIN)
257 {
Florin Coras7baeb712019-01-04 17:05:43 -0800258 ldp->vlsh_bit_val = (1 << LDP_SID_BIT_MIN);
259 ldp->vlsh_bit_mask = ldp->vlsh_bit_val - 1;
Dave Wallace048b1d62018-01-03 22:24:41 -0500260
Florin Coras294afe22019-01-07 17:49:17 -0800261 LDBG (0, "WARNING: LDP sid bit (%u) specified in the env var "
262 LDP_ENV_SID_BIT " (%s) is too small. Using LDP_SID_BIT_MIN"
263 " (%d)! sid bit value %d (0x%x)", sb, env_var_str,
264 LDP_SID_BIT_MIN, ldp->vlsh_bit_val, ldp->vlsh_bit_val);
Florin Coras99368312018-08-02 10:45:44 -0700265 }
266 else if (sb > LDP_SID_BIT_MAX)
267 {
Florin Coras7baeb712019-01-04 17:05:43 -0800268 ldp->vlsh_bit_val = (1 << LDP_SID_BIT_MAX);
269 ldp->vlsh_bit_mask = ldp->vlsh_bit_val - 1;
Dave Wallace048b1d62018-01-03 22:24:41 -0500270
Florin Coras294afe22019-01-07 17:49:17 -0800271 LDBG (0, "WARNING: LDP sid bit (%u) specified in the env var "
272 LDP_ENV_SID_BIT " (%s) is too big. Using LDP_SID_BIT_MAX"
273 " (%d)! sid bit value %d (0x%x)", sb, env_var_str,
274 LDP_SID_BIT_MAX, ldp->vlsh_bit_val, ldp->vlsh_bit_val);
Dave Wallace048b1d62018-01-03 22:24:41 -0500275 }
276 else
277 {
Florin Coras7baeb712019-01-04 17:05:43 -0800278 ldp->vlsh_bit_val = (1 << sb);
279 ldp->vlsh_bit_mask = ldp->vlsh_bit_val - 1;
Florin Coras99368312018-08-02 10:45:44 -0700280
Florin Coras05ecfcc2018-12-12 18:19:39 -0800281 LDBG (0, "configured LDP sid bit (%u) from "
282 LDP_ENV_SID_BIT "! sid bit value %d (0x%x)", sb,
Florin Coras7baeb712019-01-04 17:05:43 -0800283 ldp->vlsh_bit_val, ldp->vlsh_bit_val);
Dave Wallace048b1d62018-01-03 22:24:41 -0500284 }
Florin Coras294afe22019-01-07 17:49:17 -0800285
286 /* Make sure there are enough bits in the fd set for vcl sessions */
287 if (ldp->vlsh_bit_val > FD_SETSIZE / 2)
288 {
wanghanlin1eb8fea2021-10-14 11:10:26 +0800289 /* Only valid for select/pselect, so just WARNING and not exit */
290 LDBG (0,
291 "WARNING: LDP vlsh bit value %d > FD_SETSIZE/2 %d, "
292 "select/pselect not supported now!",
Florin Coras294afe22019-01-07 17:49:17 -0800293 ldp->vlsh_bit_val, FD_SETSIZE / 2);
Florin Coras294afe22019-01-07 17:49:17 -0800294 }
Dave Wallace048b1d62018-01-03 22:24:41 -0500295 }
Yu Ping7b74b072019-05-08 00:40:24 +0800296 env_var_str = getenv (LDP_ENV_TLS_TRANS);
297 if (env_var_str)
298 {
299 ldp->transparent_tls = 1;
300 }
Florin Coras99368312018-08-02 10:45:44 -0700301
Florin Coras4dee8cd2019-01-29 21:28:16 -0800302 /* *INDENT-OFF* */
Damjan Marionb2c31b62020-12-13 21:47:40 +0100303 pool_foreach (ldpw, ldp->workers) {
Florin Coras4dee8cd2019-01-29 21:28:16 -0800304 clib_memset (&ldpw->clib_time, 0, sizeof (ldpw->clib_time));
Damjan Marionb2c31b62020-12-13 21:47:40 +0100305 }
Florin Coras4dee8cd2019-01-29 21:28:16 -0800306 /* *INDENT-ON* */
307
Florin Coras05ecfcc2018-12-12 18:19:39 -0800308 LDBG (0, "LDP initialization: done!");
Florin Coras99368312018-08-02 10:45:44 -0700309
310 return 0;
Dave Wallace048b1d62018-01-03 22:24:41 -0500311}
312
Florin Coras5f33d0d2021-06-02 21:22:21 -0700313#define ldp_init_check() \
314 if (PREDICT_FALSE (!ldp->init)) \
315 { \
316 if ((errno = -ldp_init ())) \
317 return -1; \
318 }
319
Dave Wallace048b1d62018-01-03 22:24:41 -0500320int
321close (int fd)
322{
Florin Coras7baeb712019-01-04 17:05:43 -0800323 vls_handle_t vlsh;
324 int rv, epfd;
Dave Wallace048b1d62018-01-03 22:24:41 -0500325
Florin Coras5f33d0d2021-06-02 21:22:21 -0700326 ldp_init_check ();
Dave Wallace048b1d62018-01-03 22:24:41 -0500327
Florin Coras7baeb712019-01-04 17:05:43 -0800328 vlsh = ldp_fd_to_vlsh (fd);
329 if (vlsh != VLS_INVALID_HANDLE)
Dave Wallace048b1d62018-01-03 22:24:41 -0500330 {
Florin Coras7baeb712019-01-04 17:05:43 -0800331 epfd = vls_attr (vlsh, VPPCOM_ATTR_GET_LIBC_EPFD, 0, 0);
Dave Wallace048b1d62018-01-03 22:24:41 -0500332 if (epfd > 0)
333 {
Florin Coras06defbc2023-02-28 22:32:31 -0800334 ldp_worker_ctx_t *ldpw = ldp_worker_get_current ();
335 u32 size = sizeof (epfd);
336
Florin Coras7baeb712019-01-04 17:05:43 -0800337 LDBG (0, "fd %d: calling libc_close: epfd %u", fd, epfd);
Dave Wallace048b1d62018-01-03 22:24:41 -0500338
Florin Coras06defbc2023-02-28 22:32:31 -0800339 libc_close (epfd);
340 ldpw->mq_epfd_added = 0;
Dave Wallace048b1d62018-01-03 22:24:41 -0500341
Florin Coras06defbc2023-02-28 22:32:31 -0800342 epfd = 0;
343 (void) vls_attr (vlsh, VPPCOM_ATTR_SET_LIBC_EPFD, &epfd, &size);
Dave Wallace048b1d62018-01-03 22:24:41 -0500344 }
345 else if (PREDICT_FALSE (epfd < 0))
346 {
347 errno = -epfd;
348 rv = -1;
349 goto done;
350 }
351
Florin Coras7baeb712019-01-04 17:05:43 -0800352 LDBG (0, "fd %d: calling vls_close: vlsh %u", fd, vlsh);
Dave Wallace048b1d62018-01-03 22:24:41 -0500353
Florin Coras7baeb712019-01-04 17:05:43 -0800354 rv = vls_close (vlsh);
Dave Wallace048b1d62018-01-03 22:24:41 -0500355 if (rv != VPPCOM_OK)
356 {
357 errno = -rv;
358 rv = -1;
359 }
360 }
361 else
362 {
Florin Coras7baeb712019-01-04 17:05:43 -0800363 LDBG (0, "fd %d: calling libc_close", fd);
Dave Wallace048b1d62018-01-03 22:24:41 -0500364 rv = libc_close (fd);
365 }
366
367done:
Dave Wallace048b1d62018-01-03 22:24:41 -0500368 return rv;
369}
370
371ssize_t
372read (int fd, void *buf, size_t nbytes)
373{
Florin Coras7baeb712019-01-04 17:05:43 -0800374 vls_handle_t vlsh;
Dave Wallace048b1d62018-01-03 22:24:41 -0500375 ssize_t size;
Dave Wallace048b1d62018-01-03 22:24:41 -0500376
Florin Coras5f33d0d2021-06-02 21:22:21 -0700377 ldp_init_check ();
Dave Wallace048b1d62018-01-03 22:24:41 -0500378
Florin Coras7baeb712019-01-04 17:05:43 -0800379 vlsh = ldp_fd_to_vlsh (fd);
380 if (vlsh != VLS_INVALID_HANDLE)
Dave Wallace048b1d62018-01-03 22:24:41 -0500381 {
Florin Coras7baeb712019-01-04 17:05:43 -0800382 size = vls_read (vlsh, buf, nbytes);
Dave Wallace048b1d62018-01-03 22:24:41 -0500383 if (size < 0)
384 {
385 errno = -size;
386 size = -1;
387 }
388 }
389 else
390 {
Dave Wallace048b1d62018-01-03 22:24:41 -0500391 size = libc_read (fd, buf, nbytes);
392 }
393
Dave Wallace048b1d62018-01-03 22:24:41 -0500394 return size;
395}
396
397ssize_t
398readv (int fd, const struct iovec * iov, int iovcnt)
399{
Dave Wallace8aaba562018-01-18 17:21:19 -0500400 int rv = 0, i, total = 0;
Florin Coras7baeb712019-01-04 17:05:43 -0800401 vls_handle_t vlsh;
402 ssize_t size = 0;
Dave Wallace048b1d62018-01-03 22:24:41 -0500403
Florin Coras5f33d0d2021-06-02 21:22:21 -0700404 ldp_init_check ();
Dave Wallace048b1d62018-01-03 22:24:41 -0500405
Florin Coras7baeb712019-01-04 17:05:43 -0800406 vlsh = ldp_fd_to_vlsh (fd);
407 if (vlsh != VLS_INVALID_HANDLE)
Dave Wallace048b1d62018-01-03 22:24:41 -0500408 {
Florin Coras067f9542020-02-14 05:33:46 +0000409 for (i = 0; i < iovcnt; ++i)
Dave Wallace048b1d62018-01-03 22:24:41 -0500410 {
Florin Coras067f9542020-02-14 05:33:46 +0000411 rv = vls_read (vlsh, iov[i].iov_base, iov[i].iov_len);
412 if (rv <= 0)
413 break;
414 else
Dave Wallace048b1d62018-01-03 22:24:41 -0500415 {
Florin Coras067f9542020-02-14 05:33:46 +0000416 total += rv;
417 if (rv < iov[i].iov_len)
Dave Wallace048b1d62018-01-03 22:24:41 -0500418 break;
Keith Burns (alagalah)b327c2b2017-10-09 08:52:59 -0700419 }
420 }
Florin Coras067f9542020-02-14 05:33:46 +0000421 if (rv < 0 && total == 0)
Keith Burns (alagalah)b327c2b2017-10-09 08:52:59 -0700422 {
Dave Wallace048b1d62018-01-03 22:24:41 -0500423 errno = -rv;
424 size = -1;
Keith Burns (alagalah)b327c2b2017-10-09 08:52:59 -0700425 }
Dave Wallace048b1d62018-01-03 22:24:41 -0500426 else
427 size = total;
Keith Burns (alagalah)b327c2b2017-10-09 08:52:59 -0700428 }
429 else
430 {
Dave Wallace048b1d62018-01-03 22:24:41 -0500431 size = libc_readv (fd, iov, iovcnt);
Keith Burns (alagalah)b327c2b2017-10-09 08:52:59 -0700432 }
433
Dave Wallace048b1d62018-01-03 22:24:41 -0500434 return size;
435}
Keith Burns (alagalah)b327c2b2017-10-09 08:52:59 -0700436
Dave Wallace048b1d62018-01-03 22:24:41 -0500437ssize_t
438write (int fd, const void *buf, size_t nbytes)
439{
Florin Coras7baeb712019-01-04 17:05:43 -0800440 vls_handle_t vlsh;
Dave Wallace048b1d62018-01-03 22:24:41 -0500441 ssize_t size = 0;
Dave Wallace048b1d62018-01-03 22:24:41 -0500442
Florin Coras5f33d0d2021-06-02 21:22:21 -0700443 ldp_init_check ();
Dave Wallace048b1d62018-01-03 22:24:41 -0500444
Florin Coras7baeb712019-01-04 17:05:43 -0800445 vlsh = ldp_fd_to_vlsh (fd);
446 if (vlsh != VLS_INVALID_HANDLE)
Dave Wallace048b1d62018-01-03 22:24:41 -0500447 {
Florin Coras7baeb712019-01-04 17:05:43 -0800448 size = vls_write_msg (vlsh, (void *) buf, nbytes);
Dave Wallace048b1d62018-01-03 22:24:41 -0500449 if (size < 0)
450 {
451 errno = -size;
452 size = -1;
453 }
Keith Burns (alagalah)b327c2b2017-10-09 08:52:59 -0700454 }
455 else
456 {
Dave Wallace048b1d62018-01-03 22:24:41 -0500457 size = libc_write (fd, buf, nbytes);
Keith Burns (alagalah)b327c2b2017-10-09 08:52:59 -0700458 }
459
Dave Wallace048b1d62018-01-03 22:24:41 -0500460 return size;
461}
Keith Burns (alagalah)b327c2b2017-10-09 08:52:59 -0700462
Dave Wallace048b1d62018-01-03 22:24:41 -0500463ssize_t
464writev (int fd, const struct iovec * iov, int iovcnt)
465{
Dave Wallace048b1d62018-01-03 22:24:41 -0500466 ssize_t size = 0, total = 0;
Florin Coras7baeb712019-01-04 17:05:43 -0800467 vls_handle_t vlsh;
Dave Wallace8aaba562018-01-18 17:21:19 -0500468 int i, rv = 0;
Dave Wallace048b1d62018-01-03 22:24:41 -0500469
Florin Coras5f33d0d2021-06-02 21:22:21 -0700470 ldp_init_check ();
Dave Wallace048b1d62018-01-03 22:24:41 -0500471
Florin Coras7baeb712019-01-04 17:05:43 -0800472 vlsh = ldp_fd_to_vlsh (fd);
473 if (vlsh != VLS_INVALID_HANDLE)
Dave Wallace048b1d62018-01-03 22:24:41 -0500474 {
Florin Coraseda1b8c2020-03-23 16:00:35 +0000475 for (i = 0; i < iovcnt; ++i)
Dave Wallace048b1d62018-01-03 22:24:41 -0500476 {
Florin Coraseda1b8c2020-03-23 16:00:35 +0000477 rv = vls_write_msg (vlsh, iov[i].iov_base, iov[i].iov_len);
478 if (rv < 0)
479 break;
480 else
Dave Wallace048b1d62018-01-03 22:24:41 -0500481 {
Florin Coraseda1b8c2020-03-23 16:00:35 +0000482 total += rv;
483 if (rv < iov[i].iov_len)
Dave Wallace048b1d62018-01-03 22:24:41 -0500484 break;
Dave Wallace048b1d62018-01-03 22:24:41 -0500485 }
486 }
Dave Wallace048b1d62018-01-03 22:24:41 -0500487
Florin Coraseda1b8c2020-03-23 16:00:35 +0000488 if (rv < 0 && total == 0)
Dave Wallace048b1d62018-01-03 22:24:41 -0500489 {
490 errno = -rv;
491 size = -1;
492 }
493 else
494 size = total;
Keith Burns (alagalah)b327c2b2017-10-09 08:52:59 -0700495 }
496 else
497 {
Dave Wallace048b1d62018-01-03 22:24:41 -0500498 size = libc_writev (fd, iov, iovcnt);
Keith Burns (alagalah)b327c2b2017-10-09 08:52:59 -0700499 }
500
Dave Wallace048b1d62018-01-03 22:24:41 -0500501 return size;
Keith Burns (alagalah)b327c2b2017-10-09 08:52:59 -0700502}
503
Florin Coras0ab36f52020-05-26 19:45:45 +0000504static int
505fcntl_internal (int fd, int cmd, va_list ap)
Keith Burns (alagalah)b327c2b2017-10-09 08:52:59 -0700506{
Florin Coras7baeb712019-01-04 17:05:43 -0800507 vls_handle_t vlsh;
Keith Burns (alagalah)b327c2b2017-10-09 08:52:59 -0700508 int rv = 0;
Florin Coras7baeb712019-01-04 17:05:43 -0800509
510 vlsh = ldp_fd_to_vlsh (fd);
511 LDBG (0, "fd %u vlsh %d, cmd %u", fd, vlsh, cmd);
512 if (vlsh != VLS_INVALID_HANDLE)
Keith Burns (alagalah)b327c2b2017-10-09 08:52:59 -0700513 {
Dave Wallace048b1d62018-01-03 22:24:41 -0500514 int flags = va_arg (ap, int);
515 u32 size;
Keith Burns (alagalah)b327c2b2017-10-09 08:52:59 -0700516
Dave Wallace048b1d62018-01-03 22:24:41 -0500517 size = sizeof (flags);
518 rv = -EOPNOTSUPP;
519 switch (cmd)
Keith Burns (alagalah)b327c2b2017-10-09 08:52:59 -0700520 {
Dave Wallace048b1d62018-01-03 22:24:41 -0500521 case F_SETFL:
Florin Coras7baeb712019-01-04 17:05:43 -0800522 rv = vls_attr (vlsh, VPPCOM_ATTR_SET_FLAGS, &flags, &size);
Dave Wallace048b1d62018-01-03 22:24:41 -0500523 break;
524
525 case F_GETFL:
Florin Coras7baeb712019-01-04 17:05:43 -0800526 rv = vls_attr (vlsh, VPPCOM_ATTR_GET_FLAGS, &flags, &size);
Dave Wallace048b1d62018-01-03 22:24:41 -0500527 if (rv == VPPCOM_OK)
Florin Coras7baeb712019-01-04 17:05:43 -0800528 rv = flags;
Keith Burns (alagalah)b327c2b2017-10-09 08:52:59 -0700529 break;
Florin Coras173bae32018-11-16 18:56:28 -0800530 case F_SETFD:
531 /* TODO handle this */
532 LDBG (0, "F_SETFD ignored flags %u", flags);
533 rv = 0;
534 break;
Keith Burns (alagalah)b327c2b2017-10-09 08:52:59 -0700535 default:
Dave Wallace048b1d62018-01-03 22:24:41 -0500536 rv = -EOPNOTSUPP;
Keith Burns (alagalah)b327c2b2017-10-09 08:52:59 -0700537 break;
538 }
Dave Wallace048b1d62018-01-03 22:24:41 -0500539 if (rv < 0)
540 {
541 errno = -rv;
542 rv = -1;
543 }
544 }
545 else
546 {
Carl Smithe16707b2019-11-13 14:37:39 +1300547#ifdef HAVE_FCNTL64
548 rv = libc_vfcntl64 (fd, cmd, ap);
549#else
Dave Wallace048b1d62018-01-03 22:24:41 -0500550 rv = libc_vfcntl (fd, cmd, ap);
Carl Smithe16707b2019-11-13 14:37:39 +1300551#endif
Keith Burns (alagalah)b327c2b2017-10-09 08:52:59 -0700552 }
553
Florin Coras0ab36f52020-05-26 19:45:45 +0000554 return rv;
555}
556
557int
558fcntl (int fd, int cmd, ...)
559{
560 va_list ap;
561 int rv;
562
Florin Coras5f33d0d2021-06-02 21:22:21 -0700563 ldp_init_check ();
Florin Coras0ab36f52020-05-26 19:45:45 +0000564
565 va_start (ap, cmd);
566 rv = fcntl_internal (fd, cmd, ap);
Dave Wallace048b1d62018-01-03 22:24:41 -0500567 va_end (ap);
568
Dave Wallace048b1d62018-01-03 22:24:41 -0500569 return rv;
570}
571
572int
Florin Corasd7586d52020-04-29 02:19:51 +0000573fcntl64 (int fd, int cmd, ...)
574{
575 va_list ap;
576 int rv;
577
Florin Coras5f33d0d2021-06-02 21:22:21 -0700578 ldp_init_check ();
Florin Coras0ab36f52020-05-26 19:45:45 +0000579
Florin Corasd7586d52020-04-29 02:19:51 +0000580 va_start (ap, cmd);
Florin Coras0ab36f52020-05-26 19:45:45 +0000581 rv = fcntl_internal (fd, cmd, ap);
Florin Corasd7586d52020-04-29 02:19:51 +0000582 va_end (ap);
583 return rv;
584}
585
586int
Dave Wallace048b1d62018-01-03 22:24:41 -0500587ioctl (int fd, unsigned long int cmd, ...)
588{
Florin Coras7baeb712019-01-04 17:05:43 -0800589 vls_handle_t vlsh;
Dave Wallace048b1d62018-01-03 22:24:41 -0500590 va_list ap;
Florin Coras7baeb712019-01-04 17:05:43 -0800591 int rv;
Dave Wallace048b1d62018-01-03 22:24:41 -0500592
Florin Coras5f33d0d2021-06-02 21:22:21 -0700593 ldp_init_check ();
Dave Wallace048b1d62018-01-03 22:24:41 -0500594
595 va_start (ap, cmd);
Dave Wallace048b1d62018-01-03 22:24:41 -0500596
Florin Coras7baeb712019-01-04 17:05:43 -0800597 vlsh = ldp_fd_to_vlsh (fd);
598 if (vlsh != VLS_INVALID_HANDLE)
599 {
Dave Wallace048b1d62018-01-03 22:24:41 -0500600 switch (cmd)
601 {
602 case FIONREAD:
Florin Coras7baeb712019-01-04 17:05:43 -0800603 rv = vls_attr (vlsh, VPPCOM_ATTR_GET_NREAD, 0, 0);
Dave Wallace048b1d62018-01-03 22:24:41 -0500604 break;
605
606 case FIONBIO:
607 {
608 u32 flags = va_arg (ap, int) ? O_NONBLOCK : 0;
609 u32 size = sizeof (flags);
610
611 /* TBD: When VPPCOM_ATTR_[GS]ET_FLAGS supports flags other than
612 * non-blocking, the flags should be read here and merged
613 * with O_NONBLOCK.
614 */
Florin Coras7baeb712019-01-04 17:05:43 -0800615 rv = vls_attr (vlsh, VPPCOM_ATTR_SET_FLAGS, &flags, &size);
Dave Wallace048b1d62018-01-03 22:24:41 -0500616 }
617 break;
618
619 default:
620 rv = -EOPNOTSUPP;
621 break;
622 }
623 if (rv < 0)
624 {
625 errno = -rv;
626 rv = -1;
627 }
628 }
629 else
630 {
Dave Wallace048b1d62018-01-03 22:24:41 -0500631 rv = libc_vioctl (fd, cmd, ap);
632 }
633
Dave Wallace048b1d62018-01-03 22:24:41 -0500634 va_end (ap);
635 return rv;
636}
637
Florin Coras294afe22019-01-07 17:49:17 -0800638always_inline void
639ldp_select_init_maps (fd_set * __restrict original,
640 clib_bitmap_t ** resultb, clib_bitmap_t ** libcb,
641 clib_bitmap_t ** vclb, int nfds, u32 minbits,
642 u32 n_bytes, uword * si_bits, uword * libc_bits)
643{
644 uword si_bits_set, libc_bits_set;
645 vls_handle_t vlsh;
646 int fd;
647
648 clib_bitmap_validate (*vclb, minbits);
649 clib_bitmap_validate (*libcb, minbits);
650 clib_bitmap_validate (*resultb, minbits);
651 clib_memcpy_fast (*resultb, original, n_bytes);
652 memset (original, 0, n_bytes);
653
654 /* *INDENT-OFF* */
Damjan Marionf0ca1e82020-12-13 23:26:56 +0100655 clib_bitmap_foreach (fd, *resultb) {
Florin Coras294afe22019-01-07 17:49:17 -0800656 if (fd > nfds)
657 break;
658 vlsh = ldp_fd_to_vlsh (fd);
659 if (vlsh == VLS_INVALID_HANDLE)
660 clib_bitmap_set_no_check (*libcb, fd, 1);
661 else
Florin Corascbce80a2020-04-20 01:32:38 +0000662 *vclb = clib_bitmap_set (*vclb, vlsh_to_session_index (vlsh), 1);
Damjan Marionf0ca1e82020-12-13 23:26:56 +0100663 }
Florin Coras294afe22019-01-07 17:49:17 -0800664 /* *INDENT-ON* */
665
666 si_bits_set = clib_bitmap_last_set (*vclb) + 1;
667 *si_bits = (si_bits_set > *si_bits) ? si_bits_set : *si_bits;
Florin Corascbce80a2020-04-20 01:32:38 +0000668 clib_bitmap_validate (*resultb, *si_bits);
Florin Coras294afe22019-01-07 17:49:17 -0800669
670 libc_bits_set = clib_bitmap_last_set (*libcb) + 1;
671 *libc_bits = (libc_bits_set > *libc_bits) ? libc_bits_set : *libc_bits;
672}
673
674always_inline int
675ldp_select_vcl_map_to_libc (clib_bitmap_t * vclb, fd_set * __restrict libcb)
676{
677 vls_handle_t vlsh;
678 uword si;
679 int fd;
680
681 if (!libcb)
682 return 0;
683
684 /* *INDENT-OFF* */
Damjan Marionf0ca1e82020-12-13 23:26:56 +0100685 clib_bitmap_foreach (si, vclb) {
Florin Coras294afe22019-01-07 17:49:17 -0800686 vlsh = vls_session_index_to_vlsh (si);
Florin Coras54140622020-02-04 19:04:34 +0000687 ASSERT (vlsh != VLS_INVALID_HANDLE);
Florin Coras294afe22019-01-07 17:49:17 -0800688 fd = ldp_vlsh_to_fd (vlsh);
689 if (PREDICT_FALSE (fd < 0))
690 {
691 errno = EBADFD;
692 return -1;
693 }
694 FD_SET (fd, libcb);
Damjan Marionf0ca1e82020-12-13 23:26:56 +0100695 }
Florin Coras294afe22019-01-07 17:49:17 -0800696 /* *INDENT-ON* */
697
698 return 0;
699}
700
701always_inline void
702ldp_select_libc_map_merge (clib_bitmap_t * result, fd_set * __restrict libcb)
703{
704 uword fd;
705
Florin Coras78b5fa62019-02-21 20:04:15 -0800706 if (!libcb)
707 return;
708
Florin Coras294afe22019-01-07 17:49:17 -0800709 /* *INDENT-OFF* */
Damjan Marionf0ca1e82020-12-13 23:26:56 +0100710 clib_bitmap_foreach (fd, result)
Florin Coras294afe22019-01-07 17:49:17 -0800711 FD_SET ((int)fd, libcb);
Florin Coras294afe22019-01-07 17:49:17 -0800712 /* *INDENT-ON* */
713}
714
Dave Wallace048b1d62018-01-03 22:24:41 -0500715int
Dave Wallace2a865272018-02-07 21:00:42 -0500716ldp_pselect (int nfds, fd_set * __restrict readfds,
717 fd_set * __restrict writefds,
718 fd_set * __restrict exceptfds,
719 const struct timespec *__restrict timeout,
720 const __sigset_t * __restrict sigmask)
Dave Wallace048b1d62018-01-03 22:24:41 -0500721{
Florin Coras294afe22019-01-07 17:49:17 -0800722 u32 minbits = clib_max (nfds, BITS (uword)), n_bytes;
Florin Corasdfe4cf42018-11-28 22:13:45 -0800723 ldp_worker_ctx_t *ldpw = ldp_worker_get_current ();
Florin Coras294afe22019-01-07 17:49:17 -0800724 struct timespec libc_tspec = { 0 };
725 f64 time_out, vcl_timeout = 0;
726 uword si_bits, libc_bits;
727 int rv, bits_set = 0;
Dave Wallace048b1d62018-01-03 22:24:41 -0500728
729 if (nfds < 0)
730 {
731 errno = EINVAL;
732 return -1;
733 }
734
Florin Coras4dee8cd2019-01-29 21:28:16 -0800735 if (PREDICT_FALSE (ldpw->clib_time.init_cpu_time == 0))
736 clib_time_init (&ldpw->clib_time);
737
Dave Wallace3ee1fe12018-02-23 01:09:11 -0500738 if (timeout)
739 {
740 time_out = (timeout->tv_sec == 0 && timeout->tv_nsec == 0) ?
Florin Coras7baeb712019-01-04 17:05:43 -0800741 (f64) 0 : (f64) timeout->tv_sec + (f64) timeout->tv_nsec / (f64) 1e9;
Dave Wallace3ee1fe12018-02-23 01:09:11 -0500742
liuyacanf71796e2021-08-02 10:01:39 +0800743 time_out += clib_time_now (&ldpw->clib_time);
744
Dave Wallace3ee1fe12018-02-23 01:09:11 -0500745 /* select as fine grained sleep */
746 if (!nfds)
747 {
Florin Corasdfe4cf42018-11-28 22:13:45 -0800748 while (clib_time_now (&ldpw->clib_time) < time_out)
Dave Wallace3ee1fe12018-02-23 01:09:11 -0500749 ;
750 return 0;
751 }
752 }
753 else if (!nfds)
754 {
755 errno = EINVAL;
756 return -1;
757 }
758 else
759 time_out = -1;
760
Florin Coras7baeb712019-01-04 17:05:43 -0800761 if (nfds <= ldp->vlsh_bit_val)
Dave Wallace048b1d62018-01-03 22:24:41 -0500762 {
Dave Wallace048b1d62018-01-03 22:24:41 -0500763 rv = libc_pselect (nfds, readfds, writefds, exceptfds,
764 timeout, sigmask);
765 goto done;
766 }
767
Florin Coras294afe22019-01-07 17:49:17 -0800768 si_bits = libc_bits = 0;
769 n_bytes = nfds / 8 + ((nfds % 8) ? 1 : 0);
Florin Coras7baeb712019-01-04 17:05:43 -0800770
Dave Wallace048b1d62018-01-03 22:24:41 -0500771 if (readfds)
Florin Coras294afe22019-01-07 17:49:17 -0800772 ldp_select_init_maps (readfds, &ldpw->rd_bitmap, &ldpw->libc_rd_bitmap,
773 &ldpw->si_rd_bitmap, nfds, minbits, n_bytes,
774 &si_bits, &libc_bits);
Dave Wallace048b1d62018-01-03 22:24:41 -0500775 if (writefds)
Florin Coras294afe22019-01-07 17:49:17 -0800776 ldp_select_init_maps (writefds, &ldpw->wr_bitmap,
777 &ldpw->libc_wr_bitmap, &ldpw->si_wr_bitmap, nfds,
778 minbits, n_bytes, &si_bits, &libc_bits);
Dave Wallace048b1d62018-01-03 22:24:41 -0500779 if (exceptfds)
Florin Coras294afe22019-01-07 17:49:17 -0800780 ldp_select_init_maps (exceptfds, &ldpw->ex_bitmap,
781 &ldpw->libc_ex_bitmap, &ldpw->si_ex_bitmap, nfds,
782 minbits, n_bytes, &si_bits, &libc_bits);
Dave Wallace048b1d62018-01-03 22:24:41 -0500783
Florin Coras294afe22019-01-07 17:49:17 -0800784 if (PREDICT_FALSE (!si_bits && !libc_bits))
Dave Wallace048b1d62018-01-03 22:24:41 -0500785 {
786 errno = EINVAL;
787 rv = -1;
788 goto done;
789 }
790
Florin Coras78b5fa62019-02-21 20:04:15 -0800791 if (!si_bits)
792 libc_tspec = timeout ? *timeout : libc_tspec;
Florin Coras294afe22019-01-07 17:49:17 -0800793
Dave Wallace048b1d62018-01-03 22:24:41 -0500794 do
795 {
Florin Coras294afe22019-01-07 17:49:17 -0800796 if (si_bits)
Dave Wallace048b1d62018-01-03 22:24:41 -0500797 {
Dave Wallace048b1d62018-01-03 22:24:41 -0500798 if (readfds)
Florin Coras294afe22019-01-07 17:49:17 -0800799 clib_memcpy_fast (ldpw->rd_bitmap, ldpw->si_rd_bitmap,
Florin Corascbce80a2020-04-20 01:32:38 +0000800 vec_len (ldpw->si_rd_bitmap) *
Dave Barach178cf492018-11-13 16:34:13 -0500801 sizeof (clib_bitmap_t));
Dave Wallace048b1d62018-01-03 22:24:41 -0500802 if (writefds)
Florin Coras294afe22019-01-07 17:49:17 -0800803 clib_memcpy_fast (ldpw->wr_bitmap, ldpw->si_wr_bitmap,
Florin Corascbce80a2020-04-20 01:32:38 +0000804 vec_len (ldpw->si_wr_bitmap) *
Dave Barach178cf492018-11-13 16:34:13 -0500805 sizeof (clib_bitmap_t));
Dave Wallace048b1d62018-01-03 22:24:41 -0500806 if (exceptfds)
Florin Coras294afe22019-01-07 17:49:17 -0800807 clib_memcpy_fast (ldpw->ex_bitmap, ldpw->si_ex_bitmap,
Florin Corascbce80a2020-04-20 01:32:38 +0000808 vec_len (ldpw->si_ex_bitmap) *
Dave Barach178cf492018-11-13 16:34:13 -0500809 sizeof (clib_bitmap_t));
Florin Coras294afe22019-01-07 17:49:17 -0800810
Florin Coras0ef8ef22019-01-18 08:37:13 -0800811 rv = vls_select (si_bits, readfds ? ldpw->rd_bitmap : NULL,
812 writefds ? ldpw->wr_bitmap : NULL,
813 exceptfds ? ldpw->ex_bitmap : NULL, vcl_timeout);
Florin Coras294afe22019-01-07 17:49:17 -0800814 if (rv < 0)
815 {
816 errno = -rv;
817 rv = -1;
Florin Coras5e6222a2020-04-24 17:09:25 +0000818 goto done;
Florin Coras294afe22019-01-07 17:49:17 -0800819 }
820 else if (rv > 0)
821 {
822 if (ldp_select_vcl_map_to_libc (ldpw->rd_bitmap, readfds))
823 {
824 rv = -1;
825 goto done;
826 }
827
828 if (ldp_select_vcl_map_to_libc (ldpw->wr_bitmap, writefds))
829 {
830 rv = -1;
831 goto done;
832 }
833
834 if (ldp_select_vcl_map_to_libc (ldpw->ex_bitmap, exceptfds))
835 {
836 rv = -1;
837 goto done;
838 }
839 bits_set = rv;
840 }
841 }
842 if (libc_bits)
843 {
844 if (readfds)
845 clib_memcpy_fast (ldpw->rd_bitmap, ldpw->libc_rd_bitmap,
846 vec_len (ldpw->libc_rd_bitmap) *
847 sizeof (clib_bitmap_t));
848 if (writefds)
849 clib_memcpy_fast (ldpw->wr_bitmap, ldpw->libc_wr_bitmap,
850 vec_len (ldpw->libc_wr_bitmap) *
851 sizeof (clib_bitmap_t));
852 if (exceptfds)
853 clib_memcpy_fast (ldpw->ex_bitmap, ldpw->libc_ex_bitmap,
854 vec_len (ldpw->libc_ex_bitmap) *
855 sizeof (clib_bitmap_t));
856
Dave Wallace048b1d62018-01-03 22:24:41 -0500857 rv = libc_pselect (libc_bits,
Florin Coras294afe22019-01-07 17:49:17 -0800858 readfds ? (fd_set *) ldpw->rd_bitmap : NULL,
859 writefds ? (fd_set *) ldpw->wr_bitmap : NULL,
860 exceptfds ? (fd_set *) ldpw->ex_bitmap : NULL,
861 &libc_tspec, sigmask);
862 if (rv > 0)
863 {
864 ldp_select_libc_map_merge (ldpw->rd_bitmap, readfds);
865 ldp_select_libc_map_merge (ldpw->wr_bitmap, writefds);
866 ldp_select_libc_map_merge (ldpw->ex_bitmap, exceptfds);
867 bits_set += rv;
868 }
869 }
870
871 if (bits_set)
872 {
873 rv = bits_set;
874 goto done;
Dave Wallace048b1d62018-01-03 22:24:41 -0500875 }
876 }
Florin Corasdfe4cf42018-11-28 22:13:45 -0800877 while ((time_out == -1) || (clib_time_now (&ldpw->clib_time) < time_out));
Dave Wallace048b1d62018-01-03 22:24:41 -0500878 rv = 0;
879
880done:
881 /* TBD: set timeout to amount of time left */
Florin Corasdfe4cf42018-11-28 22:13:45 -0800882 clib_bitmap_zero (ldpw->rd_bitmap);
Florin Coras294afe22019-01-07 17:49:17 -0800883 clib_bitmap_zero (ldpw->si_rd_bitmap);
Florin Corasdfe4cf42018-11-28 22:13:45 -0800884 clib_bitmap_zero (ldpw->libc_rd_bitmap);
885 clib_bitmap_zero (ldpw->wr_bitmap);
Florin Coras294afe22019-01-07 17:49:17 -0800886 clib_bitmap_zero (ldpw->si_wr_bitmap);
Florin Corasdfe4cf42018-11-28 22:13:45 -0800887 clib_bitmap_zero (ldpw->libc_wr_bitmap);
888 clib_bitmap_zero (ldpw->ex_bitmap);
Florin Coras294afe22019-01-07 17:49:17 -0800889 clib_bitmap_zero (ldpw->si_ex_bitmap);
Florin Corasdfe4cf42018-11-28 22:13:45 -0800890 clib_bitmap_zero (ldpw->libc_ex_bitmap);
Dave Wallace048b1d62018-01-03 22:24:41 -0500891
Dave Wallace048b1d62018-01-03 22:24:41 -0500892 return rv;
893}
894
895int
896select (int nfds, fd_set * __restrict readfds,
897 fd_set * __restrict writefds,
898 fd_set * __restrict exceptfds, struct timeval *__restrict timeout)
899{
900 struct timespec tspec;
901
902 if (timeout)
903 {
904 tspec.tv_sec = timeout->tv_sec;
905 tspec.tv_nsec = timeout->tv_usec * 1000;
906 }
Dave Wallace2a865272018-02-07 21:00:42 -0500907 return ldp_pselect (nfds, readfds, writefds, exceptfds,
908 timeout ? &tspec : NULL, NULL);
Dave Wallace048b1d62018-01-03 22:24:41 -0500909}
910
911#ifdef __USE_XOPEN2K
912int
913pselect (int nfds, fd_set * __restrict readfds,
914 fd_set * __restrict writefds,
915 fd_set * __restrict exceptfds,
916 const struct timespec *__restrict timeout,
917 const __sigset_t * __restrict sigmask)
918{
Dave Wallace2a865272018-02-07 21:00:42 -0500919 return ldp_pselect (nfds, readfds, writefds, exceptfds, timeout, 0);
Dave Wallace048b1d62018-01-03 22:24:41 -0500920}
921#endif
922
Yu Ping7b74b072019-05-08 00:40:24 +0800923/* If transparent TLS mode is turned on, then ldp will load key and cert.
924 */
925static int
Florin Corasa5a9efd2021-01-05 17:03:29 -0800926load_cert_key_pair (void)
Yu Ping7b74b072019-05-08 00:40:24 +0800927{
Florin Corasa5a9efd2021-01-05 17:03:29 -0800928 char *cert_str = getenv (LDP_ENV_TLS_CERT);
929 char *key_str = getenv (LDP_ENV_TLS_KEY);
930 char cert_buf[4096], key_buf[4096];
931 int cert_size, key_size;
932 vppcom_cert_key_pair_t crypto;
933 int ckp_index;
Yu Ping7b74b072019-05-08 00:40:24 +0800934 FILE *fp;
935
Florin Corasa5a9efd2021-01-05 17:03:29 -0800936 if (!cert_str || !key_str)
Yu Ping7b74b072019-05-08 00:40:24 +0800937 {
938 LDBG (0, "ERROR: failed to read LDP environment %s\n",
939 LDP_ENV_TLS_CERT);
940 return -1;
941 }
Florin Corasa5a9efd2021-01-05 17:03:29 -0800942
943 fp = fopen (cert_str, "r");
944 if (fp == NULL)
945 {
946 LDBG (0, "ERROR: failed to open cert file %s \n", cert_str);
947 return -1;
948 }
949 cert_size = fread (cert_buf, sizeof (char), sizeof (cert_buf), fp);
950 fclose (fp);
951
952 fp = fopen (key_str, "r");
953 if (fp == NULL)
954 {
955 LDBG (0, "ERROR: failed to open key file %s \n", key_str);
956 return -1;
957 }
958 key_size = fread (key_buf, sizeof (char), sizeof (key_buf), fp);
959 fclose (fp);
960
961 crypto.cert = cert_buf;
962 crypto.key = key_buf;
963 crypto.cert_len = cert_size;
964 crypto.key_len = key_size;
965 ckp_index = vppcom_add_cert_key_pair (&crypto);
966 if (ckp_index < 0)
967 {
968 LDBG (0, "ERROR: failed to add cert key pair\n");
969 return -1;
970 }
971
972 ldp->ckpair_index = ckp_index;
973
Yu Ping7b74b072019-05-08 00:40:24 +0800974 return 0;
975}
976
977static int
Florin Corasa5a9efd2021-01-05 17:03:29 -0800978assign_cert_key_pair (vls_handle_t vlsh)
Yu Ping7b74b072019-05-08 00:40:24 +0800979{
Florin Corasa5a9efd2021-01-05 17:03:29 -0800980 uint32_t ckp_len;
Yu Ping7b74b072019-05-08 00:40:24 +0800981
Florin Corasa5a9efd2021-01-05 17:03:29 -0800982 if (ldp->ckpair_index == ~0 && load_cert_key_pair () < 0)
983 return -1;
984
985 ckp_len = sizeof (ldp->ckpair_index);
liuyacan276675e2021-08-30 13:24:53 +0800986 return vls_attr (vlsh, VPPCOM_ATTR_SET_CKPAIR, &ldp->ckpair_index, &ckp_len);
Yu Ping7b74b072019-05-08 00:40:24 +0800987}
988
Dave Wallace048b1d62018-01-03 22:24:41 -0500989int
990socket (int domain, int type, int protocol)
991{
Florin Coras7baeb712019-01-04 17:05:43 -0800992 int rv, sock_type = type & ~(SOCK_CLOEXEC | SOCK_NONBLOCK);
Dave Wallace048b1d62018-01-03 22:24:41 -0500993 u8 is_nonblocking = type & SOCK_NONBLOCK ? 1 : 0;
Florin Coras7baeb712019-01-04 17:05:43 -0800994 vls_handle_t vlsh;
Dave Wallace048b1d62018-01-03 22:24:41 -0500995
Florin Coras5f33d0d2021-06-02 21:22:21 -0700996 ldp_init_check ();
Dave Wallace048b1d62018-01-03 22:24:41 -0500997
998 if (((domain == AF_INET) || (domain == AF_INET6)) &&
999 ((sock_type == SOCK_STREAM) || (sock_type == SOCK_DGRAM)))
1000 {
Yu Ping7b74b072019-05-08 00:40:24 +08001001 u8 proto;
1002 if (ldp->transparent_tls)
1003 {
1004 proto = VPPCOM_PROTO_TLS;
1005 }
1006 else
1007 proto = ((sock_type == SOCK_DGRAM) ?
1008 VPPCOM_PROTO_UDP : VPPCOM_PROTO_TCP);
Dave Wallace048b1d62018-01-03 22:24:41 -05001009
Florin Coras7baeb712019-01-04 17:05:43 -08001010 LDBG (0, "calling vls_create: proto %u (%s), is_nonblocking %u",
1011 proto, vppcom_proto_str (proto), is_nonblocking);
Dave Wallace048b1d62018-01-03 22:24:41 -05001012
Florin Coras7baeb712019-01-04 17:05:43 -08001013 vlsh = vls_create (proto, is_nonblocking);
1014 if (vlsh < 0)
Dave Wallace048b1d62018-01-03 22:24:41 -05001015 {
Florin Coras7baeb712019-01-04 17:05:43 -08001016 errno = -vlsh;
Dave Wallace048b1d62018-01-03 22:24:41 -05001017 rv = -1;
1018 }
1019 else
1020 {
Yu Ping7b74b072019-05-08 00:40:24 +08001021 if (ldp->transparent_tls)
1022 {
Florin Corasa5a9efd2021-01-05 17:03:29 -08001023 if (assign_cert_key_pair (vlsh) < 0)
1024 return -1;
Yu Ping7b74b072019-05-08 00:40:24 +08001025 }
Florin Coras7baeb712019-01-04 17:05:43 -08001026 rv = ldp_vlsh_to_fd (vlsh);
Dave Wallace048b1d62018-01-03 22:24:41 -05001027 }
1028 }
1029 else
1030 {
Florin Coras7baeb712019-01-04 17:05:43 -08001031 LDBG (0, "calling libc_socket");
Dave Wallace048b1d62018-01-03 22:24:41 -05001032 rv = libc_socket (domain, type, protocol);
1033 }
1034
Dave Wallace048b1d62018-01-03 22:24:41 -05001035 return rv;
Keith Burns (alagalah)b327c2b2017-10-09 08:52:59 -07001036}
1037
1038/*
1039 * Create two new sockets, of type TYPE in domain DOMAIN and using
1040 * protocol PROTOCOL, which are connected to each other, and put file
1041 * descriptors for them in FDS[0] and FDS[1]. If PROTOCOL is zero,
1042 * one will be chosen automatically.
1043 * Returns 0 on success, -1 for errors.
1044 * */
1045int
Dave Wallace048b1d62018-01-03 22:24:41 -05001046socketpair (int domain, int type, int protocol, int fds[2])
Keith Burns (alagalah)b327c2b2017-10-09 08:52:59 -07001047{
Florin Coras7baeb712019-01-04 17:05:43 -08001048 int rv, sock_type = type & ~(SOCK_CLOEXEC | SOCK_NONBLOCK);
Dave Wallace048b1d62018-01-03 22:24:41 -05001049
Florin Coras5f33d0d2021-06-02 21:22:21 -07001050 ldp_init_check ();
Dave Wallace048b1d62018-01-03 22:24:41 -05001051
1052 if (((domain == AF_INET) || (domain == AF_INET6)) &&
1053 ((sock_type == SOCK_STREAM) || (sock_type == SOCK_DGRAM)))
Keith Burns (alagalah)b327c2b2017-10-09 08:52:59 -07001054 {
Florin Coras7baeb712019-01-04 17:05:43 -08001055 LDBG (0, "LDP-TBD");
Dave Wallace048b1d62018-01-03 22:24:41 -05001056 errno = ENOSYS;
1057 rv = -1;
1058 }
1059 else
1060 {
Florin Coras7baeb712019-01-04 17:05:43 -08001061 LDBG (1, "calling libc_socketpair");
Florin Coras173bae32018-11-16 18:56:28 -08001062 rv = libc_socketpair (domain, type, protocol, fds);
Keith Burns (alagalah)b327c2b2017-10-09 08:52:59 -07001063 }
1064
Dave Wallace048b1d62018-01-03 22:24:41 -05001065 return rv;
Keith Burns (alagalah)b327c2b2017-10-09 08:52:59 -07001066}
1067
1068int
Florin Coras36847942023-02-02 12:56:16 -08001069bind (int fd, __CONST_SOCKADDR_ARG _addr, socklen_t len)
Keith Burns (alagalah)b327c2b2017-10-09 08:52:59 -07001070{
Florin Coras36847942023-02-02 12:56:16 -08001071 const struct sockaddr *addr = SOCKADDR_GET_SA (_addr);
Florin Coras7baeb712019-01-04 17:05:43 -08001072 vls_handle_t vlsh;
Keith Burns (alagalah)b327c2b2017-10-09 08:52:59 -07001073 int rv;
Keith Burns (alagalah)b327c2b2017-10-09 08:52:59 -07001074
Florin Coras5f33d0d2021-06-02 21:22:21 -07001075 ldp_init_check ();
Dave Wallace048b1d62018-01-03 22:24:41 -05001076
Florin Coras7baeb712019-01-04 17:05:43 -08001077 vlsh = ldp_fd_to_vlsh (fd);
1078 if (vlsh != VLS_INVALID_HANDLE)
Keith Burns (alagalah)b327c2b2017-10-09 08:52:59 -07001079 {
Dave Wallace048b1d62018-01-03 22:24:41 -05001080 vppcom_endpt_t ep;
1081
Dave Wallace048b1d62018-01-03 22:24:41 -05001082 switch (addr->sa_family)
Keith Burns (alagalah)b327c2b2017-10-09 08:52:59 -07001083 {
Dave Wallace048b1d62018-01-03 22:24:41 -05001084 case AF_INET:
1085 if (len != sizeof (struct sockaddr_in))
Keith Burns (alagalah)b327c2b2017-10-09 08:52:59 -07001086 {
Florin Coras7baeb712019-01-04 17:05:43 -08001087 LDBG (0, "ERROR: fd %d: vlsh %u: Invalid AF_INET addr len %u!",
1088 fd, vlsh, len);
Dave Wallace048b1d62018-01-03 22:24:41 -05001089 errno = EINVAL;
1090 rv = -1;
1091 goto done;
Keith Burns (alagalah)b327c2b2017-10-09 08:52:59 -07001092 }
Dave Wallace048b1d62018-01-03 22:24:41 -05001093 ep.is_ip4 = VPPCOM_IS_IP4;
1094 ep.ip = (u8 *) & ((const struct sockaddr_in *) addr)->sin_addr;
1095 ep.port = (u16) ((const struct sockaddr_in *) addr)->sin_port;
1096 break;
1097
1098 case AF_INET6:
1099 if (len != sizeof (struct sockaddr_in6))
1100 {
Florin Coras7baeb712019-01-04 17:05:43 -08001101 LDBG (0, "ERROR: fd %d: vlsh %u: Invalid AF_INET6 addr len %u!",
1102 fd, vlsh, len);
Dave Wallace048b1d62018-01-03 22:24:41 -05001103 errno = EINVAL;
1104 rv = -1;
1105 goto done;
1106 }
1107 ep.is_ip4 = VPPCOM_IS_IP6;
1108 ep.ip = (u8 *) & ((const struct sockaddr_in6 *) addr)->sin6_addr;
1109 ep.port = (u16) ((const struct sockaddr_in6 *) addr)->sin6_port;
Keith Burns (alagalah)b327c2b2017-10-09 08:52:59 -07001110 break;
1111
1112 default:
Florin Coras7baeb712019-01-04 17:05:43 -08001113 LDBG (0, "ERROR: fd %d: vlsh %u: Unsupported address family %u!",
1114 fd, vlsh, addr->sa_family);
Dave Wallace048b1d62018-01-03 22:24:41 -05001115 errno = EAFNOSUPPORT;
1116 rv = -1;
1117 goto done;
Keith Burns (alagalah)b327c2b2017-10-09 08:52:59 -07001118 }
Florin Coras7baeb712019-01-04 17:05:43 -08001119 LDBG (0, "fd %d: calling vls_bind: vlsh %u, addr %p, len %u", fd, vlsh,
1120 addr, len);
Dave Wallace048b1d62018-01-03 22:24:41 -05001121
Florin Coras7baeb712019-01-04 17:05:43 -08001122 rv = vls_bind (vlsh, &ep);
Dave Wallace048b1d62018-01-03 22:24:41 -05001123 if (rv != VPPCOM_OK)
Keith Burns (alagalah)b327c2b2017-10-09 08:52:59 -07001124 {
1125 errno = -rv;
Dave Wallace048b1d62018-01-03 22:24:41 -05001126 rv = -1;
Keith Burns (alagalah)b327c2b2017-10-09 08:52:59 -07001127 }
Keith Burns (alagalah)b327c2b2017-10-09 08:52:59 -07001128 }
Dave Wallace048b1d62018-01-03 22:24:41 -05001129 else
Keith Burns (alagalah)b327c2b2017-10-09 08:52:59 -07001130 {
Florin Coras7baeb712019-01-04 17:05:43 -08001131 LDBG (0, "fd %d: calling libc_bind: addr %p, len %u", fd, addr, len);
Dave Wallace048b1d62018-01-03 22:24:41 -05001132 rv = libc_bind (fd, addr, len);
Keith Burns (alagalah)b327c2b2017-10-09 08:52:59 -07001133 }
1134
Dave Wallace048b1d62018-01-03 22:24:41 -05001135done:
Florin Coras7baeb712019-01-04 17:05:43 -08001136 LDBG (1, "fd %d: returning %d", fd, rv);
Florin Coras05ecfcc2018-12-12 18:19:39 -08001137
Keith Burns (alagalah)b327c2b2017-10-09 08:52:59 -07001138 return rv;
1139}
1140
1141static inline int
Florin Coras36847942023-02-02 12:56:16 -08001142ldp_copy_ep_to_sockaddr (struct sockaddr *addr, socklen_t *__restrict len,
1143 vppcom_endpt_t *ep)
Keith Burns (alagalah)b327c2b2017-10-09 08:52:59 -07001144{
Florin Coras36847942023-02-02 12:56:16 -08001145 int rv = 0, sa_len, copy_len;
Keith Burns (alagalah)b327c2b2017-10-09 08:52:59 -07001146
Florin Coras5f33d0d2021-06-02 21:22:21 -07001147 ldp_init_check ();
Dave Wallace048b1d62018-01-03 22:24:41 -05001148
1149 if (addr && len && ep)
Keith Burns (alagalah)b327c2b2017-10-09 08:52:59 -07001150 {
Dave Wallace048b1d62018-01-03 22:24:41 -05001151 addr->sa_family = (ep->is_ip4 == VPPCOM_IS_IP4) ? AF_INET : AF_INET6;
1152 switch (addr->sa_family)
1153 {
1154 case AF_INET:
1155 ((struct sockaddr_in *) addr)->sin_port = ep->port;
1156 if (*len > sizeof (struct sockaddr_in))
1157 *len = sizeof (struct sockaddr_in);
1158 sa_len = sizeof (struct sockaddr_in) - sizeof (struct in_addr);
1159 copy_len = *len - sa_len;
1160 if (copy_len > 0)
1161 memcpy (&((struct sockaddr_in *) addr)->sin_addr, ep->ip,
1162 copy_len);
1163 break;
Keith Burns (alagalah)b327c2b2017-10-09 08:52:59 -07001164
Dave Wallace048b1d62018-01-03 22:24:41 -05001165 case AF_INET6:
1166 ((struct sockaddr_in6 *) addr)->sin6_port = ep->port;
1167 if (*len > sizeof (struct sockaddr_in6))
1168 *len = sizeof (struct sockaddr_in6);
1169 sa_len = sizeof (struct sockaddr_in6) - sizeof (struct in6_addr);
1170 copy_len = *len - sa_len;
1171 if (copy_len > 0)
1172 memcpy (((struct sockaddr_in6 *) addr)->sin6_addr.
1173 __in6_u.__u6_addr8, ep->ip, copy_len);
1174 break;
Keith Burns (alagalah)b327c2b2017-10-09 08:52:59 -07001175
Dave Wallace048b1d62018-01-03 22:24:41 -05001176 default:
1177 /* Not possible */
1178 rv = -EAFNOSUPPORT;
1179 break;
1180 }
Keith Burns (alagalah)b327c2b2017-10-09 08:52:59 -07001181 }
Dave Wallacee695cb42017-11-02 22:04:42 -04001182 return rv;
Keith Burns (alagalah)b327c2b2017-10-09 08:52:59 -07001183}
1184
Keith Burns (alagalah)b327c2b2017-10-09 08:52:59 -07001185int
Florin Coras36847942023-02-02 12:56:16 -08001186getsockname (int fd, __SOCKADDR_ARG _addr, socklen_t *__restrict len)
Keith Burns (alagalah)b327c2b2017-10-09 08:52:59 -07001187{
Florin Coras36847942023-02-02 12:56:16 -08001188 struct sockaddr *addr = SOCKADDR_GET_SA (_addr);
Florin Coras7baeb712019-01-04 17:05:43 -08001189 vls_handle_t vlsh;
Keith Burns (alagalah)b327c2b2017-10-09 08:52:59 -07001190 int rv;
Keith Burns (alagalah)b327c2b2017-10-09 08:52:59 -07001191
Florin Coras5f33d0d2021-06-02 21:22:21 -07001192 ldp_init_check ();
Keith Burns (alagalah)b327c2b2017-10-09 08:52:59 -07001193
Florin Coras7baeb712019-01-04 17:05:43 -08001194 vlsh = ldp_fd_to_vlsh (fd);
1195 if (vlsh != VLS_INVALID_HANDLE)
Keith Burns (alagalah)b327c2b2017-10-09 08:52:59 -07001196 {
Dave Wallace048b1d62018-01-03 22:24:41 -05001197 vppcom_endpt_t ep;
1198 u8 addr_buf[sizeof (struct in6_addr)];
1199 u32 size = sizeof (ep);
1200
1201 ep.ip = addr_buf;
Dave Wallace048b1d62018-01-03 22:24:41 -05001202
Florin Coras7baeb712019-01-04 17:05:43 -08001203 rv = vls_attr (vlsh, VPPCOM_ATTR_GET_LCL_ADDR, &ep, &size);
Dave Wallace048b1d62018-01-03 22:24:41 -05001204 if (rv != VPPCOM_OK)
Keith Burns (alagalah)b327c2b2017-10-09 08:52:59 -07001205 {
1206 errno = -rv;
Dave Wallace048b1d62018-01-03 22:24:41 -05001207 rv = -1;
Keith Burns (alagalah)b327c2b2017-10-09 08:52:59 -07001208 }
Dave Wallace048b1d62018-01-03 22:24:41 -05001209 else
1210 {
Dave Wallace2a865272018-02-07 21:00:42 -05001211 rv = ldp_copy_ep_to_sockaddr (addr, len, &ep);
Dave Wallace048b1d62018-01-03 22:24:41 -05001212 if (rv != VPPCOM_OK)
1213 {
1214 errno = -rv;
1215 rv = -1;
1216 }
1217 }
Keith Burns (alagalah)b327c2b2017-10-09 08:52:59 -07001218 }
1219 else
1220 {
Florin Coras36847942023-02-02 12:56:16 -08001221 rv = libc_getsockname (fd, _addr, len);
Keith Burns (alagalah)b327c2b2017-10-09 08:52:59 -07001222 }
1223
Dave Wallace048b1d62018-01-03 22:24:41 -05001224 return rv;
Keith Burns (alagalah)b327c2b2017-10-09 08:52:59 -07001225}
1226
Keith Burns (alagalah)b327c2b2017-10-09 08:52:59 -07001227int
Florin Coras36847942023-02-02 12:56:16 -08001228connect (int fd, __CONST_SOCKADDR_ARG _addr, socklen_t len)
Keith Burns (alagalah)b327c2b2017-10-09 08:52:59 -07001229{
Florin Coras36847942023-02-02 12:56:16 -08001230 const struct sockaddr *addr = SOCKADDR_GET_SA (_addr);
Florin Coras7baeb712019-01-04 17:05:43 -08001231 vls_handle_t vlsh;
Dave Wallace048b1d62018-01-03 22:24:41 -05001232 int rv;
shrinivasan ganapathy1d359632017-10-15 15:46:09 -07001233
Florin Coras5f33d0d2021-06-02 21:22:21 -07001234 ldp_init_check ();
shrinivasan ganapathy1d359632017-10-15 15:46:09 -07001235
Dave Wallace048b1d62018-01-03 22:24:41 -05001236 if (!addr)
Keith Burns (alagalah)b327c2b2017-10-09 08:52:59 -07001237 {
Florin Coras7baeb712019-01-04 17:05:43 -08001238 LDBG (0, "ERROR: fd %d: NULL addr, len %u", fd, len);
Dave Wallace048b1d62018-01-03 22:24:41 -05001239 errno = EINVAL;
1240 rv = -1;
1241 goto done;
Keith Burns (alagalah)b327c2b2017-10-09 08:52:59 -07001242 }
1243
Florin Coras7baeb712019-01-04 17:05:43 -08001244 vlsh = ldp_fd_to_vlsh (fd);
1245 if (vlsh != VLS_INVALID_HANDLE)
shrinivasan ganapathy1d359632017-10-15 15:46:09 -07001246 {
Dave Wallace048b1d62018-01-03 22:24:41 -05001247 vppcom_endpt_t ep;
1248
Dave Wallace048b1d62018-01-03 22:24:41 -05001249 switch (addr->sa_family)
shrinivasan ganapathy1d359632017-10-15 15:46:09 -07001250 {
Dave Wallace048b1d62018-01-03 22:24:41 -05001251 case AF_INET:
1252 if (len != sizeof (struct sockaddr_in))
1253 {
Florin Coras7baeb712019-01-04 17:05:43 -08001254 LDBG (0, "fd %d: ERROR vlsh %u: Invalid AF_INET addr len %u!",
1255 fd, vlsh, len);
Dave Wallace048b1d62018-01-03 22:24:41 -05001256 errno = EINVAL;
1257 rv = -1;
1258 goto done;
1259 }
1260 ep.is_ip4 = VPPCOM_IS_IP4;
1261 ep.ip = (u8 *) & ((const struct sockaddr_in *) addr)->sin_addr;
1262 ep.port = (u16) ((const struct sockaddr_in *) addr)->sin_port;
1263 break;
1264
1265 case AF_INET6:
1266 if (len != sizeof (struct sockaddr_in6))
1267 {
Florin Coras7baeb712019-01-04 17:05:43 -08001268 LDBG (0, "fd %d: ERROR vlsh %u: Invalid AF_INET6 addr len %u!",
1269 fd, vlsh, len);
Dave Wallace048b1d62018-01-03 22:24:41 -05001270 errno = EINVAL;
1271 rv = -1;
1272 goto done;
1273 }
1274 ep.is_ip4 = VPPCOM_IS_IP6;
1275 ep.ip = (u8 *) & ((const struct sockaddr_in6 *) addr)->sin6_addr;
1276 ep.port = (u16) ((const struct sockaddr_in6 *) addr)->sin6_port;
1277 break;
1278
1279 default:
Florin Coras7baeb712019-01-04 17:05:43 -08001280 LDBG (0, "fd %d: ERROR vlsh %u: Unsupported address family %u!",
1281 fd, vlsh, addr->sa_family);
Dave Wallace048b1d62018-01-03 22:24:41 -05001282 errno = EAFNOSUPPORT;
1283 rv = -1;
1284 goto done;
1285 }
Florin Coras7baeb712019-01-04 17:05:43 -08001286 LDBG (0, "fd %d: calling vls_connect(): vlsh %u addr %p len %u", fd,
1287 vlsh, addr, len);
Dave Wallace048b1d62018-01-03 22:24:41 -05001288
Florin Coras7baeb712019-01-04 17:05:43 -08001289 rv = vls_connect (vlsh, &ep);
Dave Wallace048b1d62018-01-03 22:24:41 -05001290 if (rv != VPPCOM_OK)
1291 {
1292 errno = -rv;
1293 rv = -1;
1294 }
1295 }
1296 else
1297 {
Florin Coras7baeb712019-01-04 17:05:43 -08001298 LDBG (0, "fd %d: calling libc_connect(): addr %p, len %u",
1299 fd, addr, len);
Dave Wallace048b1d62018-01-03 22:24:41 -05001300
1301 rv = libc_connect (fd, addr, len);
1302 }
1303
1304done:
Florin Coras7baeb712019-01-04 17:05:43 -08001305 LDBG (1, "fd %d: returning %d (0x%x)", fd, rv, rv);
Dave Wallace048b1d62018-01-03 22:24:41 -05001306 return rv;
1307}
1308
1309int
Florin Coras36847942023-02-02 12:56:16 -08001310getpeername (int fd, __SOCKADDR_ARG _addr, socklen_t *__restrict len)
Dave Wallace048b1d62018-01-03 22:24:41 -05001311{
Florin Coras36847942023-02-02 12:56:16 -08001312 struct sockaddr *addr = SOCKADDR_GET_SA (_addr);
Florin Coras7baeb712019-01-04 17:05:43 -08001313 vls_handle_t vlsh;
Dave Wallace048b1d62018-01-03 22:24:41 -05001314 int rv;
Dave Wallace048b1d62018-01-03 22:24:41 -05001315
Florin Coras5f33d0d2021-06-02 21:22:21 -07001316 ldp_init_check ();
Dave Wallace048b1d62018-01-03 22:24:41 -05001317
Florin Coras7baeb712019-01-04 17:05:43 -08001318 vlsh = ldp_fd_to_vlsh (fd);
1319 if (vlsh != VLS_INVALID_HANDLE)
Dave Wallace048b1d62018-01-03 22:24:41 -05001320 {
1321 vppcom_endpt_t ep;
1322 u8 addr_buf[sizeof (struct in6_addr)];
1323 u32 size = sizeof (ep);
1324
1325 ep.ip = addr_buf;
Florin Coras7baeb712019-01-04 17:05:43 -08001326 rv = vls_attr (vlsh, VPPCOM_ATTR_GET_PEER_ADDR, &ep, &size);
Dave Wallace048b1d62018-01-03 22:24:41 -05001327 if (rv != VPPCOM_OK)
1328 {
1329 errno = -rv;
1330 rv = -1;
shrinivasan ganapathy1d359632017-10-15 15:46:09 -07001331 }
1332 else
1333 {
Dave Wallace2a865272018-02-07 21:00:42 -05001334 rv = ldp_copy_ep_to_sockaddr (addr, len, &ep);
Dave Wallace048b1d62018-01-03 22:24:41 -05001335 if (rv != VPPCOM_OK)
1336 {
1337 errno = -rv;
1338 rv = -1;
1339 }
shrinivasan ganapathy1d359632017-10-15 15:46:09 -07001340 }
1341 }
Dave Wallace048b1d62018-01-03 22:24:41 -05001342 else
shrinivasan ganapathy1d359632017-10-15 15:46:09 -07001343 {
Dave Wallace048b1d62018-01-03 22:24:41 -05001344 rv = libc_getpeername (fd, addr, len);
shrinivasan ganapathy1d359632017-10-15 15:46:09 -07001345 }
1346
Dave Wallace048b1d62018-01-03 22:24:41 -05001347 return rv;
1348}
1349
1350ssize_t
1351send (int fd, const void *buf, size_t n, int flags)
1352{
Florin Coras7baeb712019-01-04 17:05:43 -08001353 vls_handle_t vlsh = ldp_fd_to_vlsh (fd);
Dave Wallace048b1d62018-01-03 22:24:41 -05001354 ssize_t size;
Dave Wallace048b1d62018-01-03 22:24:41 -05001355
Florin Coras5f33d0d2021-06-02 21:22:21 -07001356 ldp_init_check ();
Dave Wallace048b1d62018-01-03 22:24:41 -05001357
Florin Coras7baeb712019-01-04 17:05:43 -08001358 if (vlsh != VLS_INVALID_HANDLE)
Dave Wallace048b1d62018-01-03 22:24:41 -05001359 {
Florin Coras7baeb712019-01-04 17:05:43 -08001360 size = vls_sendto (vlsh, (void *) buf, n, flags, NULL);
qchangaa8f63c2018-05-30 11:44:18 -07001361 if (size < VPPCOM_OK)
Dave Wallace048b1d62018-01-03 22:24:41 -05001362 {
1363 errno = -size;
1364 size = -1;
shrinivasan ganapathy1d359632017-10-15 15:46:09 -07001365 }
1366 }
Dave Wallace048b1d62018-01-03 22:24:41 -05001367 else
1368 {
Dave Wallace048b1d62018-01-03 22:24:41 -05001369 size = libc_send (fd, buf, n, flags);
1370 }
1371
Dave Wallace048b1d62018-01-03 22:24:41 -05001372 return size;
1373}
1374
1375ssize_t
1376sendfile (int out_fd, int in_fd, off_t * offset, size_t len)
1377{
Florin Corasdfe4cf42018-11-28 22:13:45 -08001378 ldp_worker_ctx_t *ldpw = ldp_worker_get_current ();
Florin Coras7baeb712019-01-04 17:05:43 -08001379 vls_handle_t vlsh;
Dave Wallace048b1d62018-01-03 22:24:41 -05001380 ssize_t size = 0;
Dave Wallace048b1d62018-01-03 22:24:41 -05001381
Florin Coras5f33d0d2021-06-02 21:22:21 -07001382 ldp_init_check ();
Dave Wallace048b1d62018-01-03 22:24:41 -05001383
Florin Coras7baeb712019-01-04 17:05:43 -08001384 vlsh = ldp_fd_to_vlsh (out_fd);
1385 if (vlsh != VLS_INVALID_HANDLE)
Dave Wallace048b1d62018-01-03 22:24:41 -05001386 {
1387 int rv;
1388 ssize_t results = 0;
1389 size_t n_bytes_left = len;
1390 size_t bytes_to_read;
1391 int nbytes;
Dave Wallace048b1d62018-01-03 22:24:41 -05001392 u8 eagain = 0;
1393 u32 flags, flags_len = sizeof (flags);
1394
Florin Coras7baeb712019-01-04 17:05:43 -08001395 rv = vls_attr (vlsh, VPPCOM_ATTR_GET_FLAGS, &flags, &flags_len);
Dave Wallace048b1d62018-01-03 22:24:41 -05001396 if (PREDICT_FALSE (rv != VPPCOM_OK))
1397 {
Florin Coras7baeb712019-01-04 17:05:43 -08001398 LDBG (0, "ERROR: out fd %d: vls_attr: vlsh %u, returned %d (%s)!",
1399 out_fd, vlsh, rv, vppcom_retval_str (rv));
Dave Wallace048b1d62018-01-03 22:24:41 -05001400
Florin Corasdfe4cf42018-11-28 22:13:45 -08001401 vec_reset_length (ldpw->io_buffer);
Dave Wallace048b1d62018-01-03 22:24:41 -05001402 errno = -rv;
1403 size = -1;
1404 goto done;
1405 }
1406
1407 if (offset)
1408 {
1409 off_t off = lseek (in_fd, *offset, SEEK_SET);
1410 if (PREDICT_FALSE (off == -1))
1411 {
Dave Wallace048b1d62018-01-03 22:24:41 -05001412 size = -1;
1413 goto done;
1414 }
1415
1416 ASSERT (off == *offset);
1417 }
1418
1419 do
1420 {
Florin Coras7baeb712019-01-04 17:05:43 -08001421 size = vls_attr (vlsh, VPPCOM_ATTR_GET_NWRITE, 0, 0);
Dave Wallace048b1d62018-01-03 22:24:41 -05001422 if (size < 0)
1423 {
Florin Coraseb801d02020-09-16 17:44:58 -07001424 LDBG (0, "ERROR: fd %d: vls_attr: vlsh %u returned %ld (%s)!",
Florin Coras7baeb712019-01-04 17:05:43 -08001425 out_fd, vlsh, size, vppcom_retval_str (size));
Florin Corasdfe4cf42018-11-28 22:13:45 -08001426 vec_reset_length (ldpw->io_buffer);
Dave Wallace048b1d62018-01-03 22:24:41 -05001427 errno = -size;
1428 size = -1;
1429 goto done;
1430 }
1431
1432 bytes_to_read = size;
Dave Wallace048b1d62018-01-03 22:24:41 -05001433 if (bytes_to_read == 0)
1434 {
1435 if (flags & O_NONBLOCK)
1436 {
1437 if (!results)
Florin Coras7baeb712019-01-04 17:05:43 -08001438 eagain = 1;
Dave Wallace048b1d62018-01-03 22:24:41 -05001439 goto update_offset;
1440 }
1441 else
1442 continue;
1443 }
1444 bytes_to_read = clib_min (n_bytes_left, bytes_to_read);
Florin Corasdfe4cf42018-11-28 22:13:45 -08001445 vec_validate (ldpw->io_buffer, bytes_to_read);
1446 nbytes = libc_read (in_fd, ldpw->io_buffer, bytes_to_read);
Dave Wallace048b1d62018-01-03 22:24:41 -05001447 if (nbytes < 0)
1448 {
Dave Wallace048b1d62018-01-03 22:24:41 -05001449 if (results == 0)
1450 {
Florin Corasdfe4cf42018-11-28 22:13:45 -08001451 vec_reset_length (ldpw->io_buffer);
Dave Wallace048b1d62018-01-03 22:24:41 -05001452 size = -1;
1453 goto done;
1454 }
1455 goto update_offset;
1456 }
Dave Wallace048b1d62018-01-03 22:24:41 -05001457
Florin Coras7baeb712019-01-04 17:05:43 -08001458 size = vls_write (vlsh, ldpw->io_buffer, nbytes);
Dave Wallace048b1d62018-01-03 22:24:41 -05001459 if (size < 0)
1460 {
1461 if (size == VPPCOM_EAGAIN)
1462 {
1463 if (flags & O_NONBLOCK)
1464 {
1465 if (!results)
Florin Coras7baeb712019-01-04 17:05:43 -08001466 eagain = 1;
Dave Wallace048b1d62018-01-03 22:24:41 -05001467 goto update_offset;
1468 }
1469 else
1470 continue;
1471 }
Dave Wallace048b1d62018-01-03 22:24:41 -05001472 if (results == 0)
1473 {
Florin Corasdfe4cf42018-11-28 22:13:45 -08001474 vec_reset_length (ldpw->io_buffer);
Dave Wallace048b1d62018-01-03 22:24:41 -05001475 errno = -size;
1476 size = -1;
1477 goto done;
1478 }
1479 goto update_offset;
1480 }
1481
1482 results += nbytes;
1483 ASSERT (n_bytes_left >= nbytes);
1484 n_bytes_left = n_bytes_left - nbytes;
1485 }
1486 while (n_bytes_left > 0);
1487
1488 update_offset:
Florin Corasdfe4cf42018-11-28 22:13:45 -08001489 vec_reset_length (ldpw->io_buffer);
Dave Wallace048b1d62018-01-03 22:24:41 -05001490 if (offset)
1491 {
1492 off_t off = lseek (in_fd, *offset, SEEK_SET);
1493 if (PREDICT_FALSE (off == -1))
1494 {
Dave Wallace048b1d62018-01-03 22:24:41 -05001495 size = -1;
1496 goto done;
1497 }
1498
1499 ASSERT (off == *offset);
1500 *offset += results + 1;
1501 }
1502 if (eagain)
1503 {
1504 errno = EAGAIN;
1505 size = -1;
1506 }
1507 else
1508 size = results;
1509 }
1510 else
1511 {
Dave Wallace048b1d62018-01-03 22:24:41 -05001512 size = libc_sendfile (out_fd, in_fd, offset, len);
1513 }
1514
1515done:
Dave Wallace048b1d62018-01-03 22:24:41 -05001516 return size;
1517}
1518
1519ssize_t
1520sendfile64 (int out_fd, int in_fd, off_t * offset, size_t len)
1521{
1522 return sendfile (out_fd, in_fd, offset, len);
1523}
1524
1525ssize_t
1526recv (int fd, void *buf, size_t n, int flags)
1527{
Florin Coras7baeb712019-01-04 17:05:43 -08001528 vls_handle_t vlsh;
Dave Wallace048b1d62018-01-03 22:24:41 -05001529 ssize_t size;
Dave Wallace048b1d62018-01-03 22:24:41 -05001530
Florin Coras5f33d0d2021-06-02 21:22:21 -07001531 ldp_init_check ();
Dave Wallace048b1d62018-01-03 22:24:41 -05001532
Florin Coras7baeb712019-01-04 17:05:43 -08001533 vlsh = ldp_fd_to_vlsh (fd);
1534 if (vlsh != VLS_INVALID_HANDLE)
Dave Wallace048b1d62018-01-03 22:24:41 -05001535 {
Florin Coras7baeb712019-01-04 17:05:43 -08001536 size = vls_recvfrom (vlsh, buf, n, flags, NULL);
Dave Wallace048b1d62018-01-03 22:24:41 -05001537 if (size < 0)
Florin Coras2a6642e2020-03-24 15:24:29 +00001538 {
1539 errno = -size;
1540 size = -1;
1541 }
Dave Wallace048b1d62018-01-03 22:24:41 -05001542 }
1543 else
1544 {
Dave Wallace048b1d62018-01-03 22:24:41 -05001545 size = libc_recv (fd, buf, n, flags);
1546 }
1547
Dave Wallace048b1d62018-01-03 22:24:41 -05001548 return size;
1549}
1550
Sivaprasad Tummalafdcbd382021-07-31 21:38:19 +05301551ssize_t
1552__recv_chk (int fd, void *buf, size_t n, size_t buflen, int flags)
1553{
1554 if (n > buflen)
1555 return -1;
1556
1557 return recv (fd, buf, n, flags);
1558}
1559
Dou Chao243a0432022-11-29 19:41:34 +08001560static inline int
1561ldp_vls_sendo (vls_handle_t vlsh, const void *buf, size_t n,
Florin Coraseff5f7a2023-02-07 17:36:17 -08001562 vppcom_endpt_tlv_t *app_tlvs, int flags,
Florin Coras36847942023-02-02 12:56:16 -08001563 __CONST_SOCKADDR_ARG _addr, socklen_t addr_len)
Florin Corasce17f462020-05-22 20:36:29 +00001564{
Florin Coras36847942023-02-02 12:56:16 -08001565 const struct sockaddr *addr = SOCKADDR_GET_SA (_addr);
Florin Corasce17f462020-05-22 20:36:29 +00001566 vppcom_endpt_t *ep = 0;
1567 vppcom_endpt_t _ep;
1568
Florin Coraseff5f7a2023-02-07 17:36:17 -08001569 _ep.app_tlvs = app_tlvs;
Dou Chao243a0432022-11-29 19:41:34 +08001570
Florin Corasce17f462020-05-22 20:36:29 +00001571 if (addr)
1572 {
1573 ep = &_ep;
1574 switch (addr->sa_family)
1575 {
1576 case AF_INET:
1577 ep->is_ip4 = VPPCOM_IS_IP4;
1578 ep->ip =
1579 (uint8_t *) & ((const struct sockaddr_in *) addr)->sin_addr;
1580 ep->port = (uint16_t) ((const struct sockaddr_in *) addr)->sin_port;
1581 break;
1582
1583 case AF_INET6:
1584 ep->is_ip4 = VPPCOM_IS_IP6;
1585 ep->ip =
1586 (uint8_t *) & ((const struct sockaddr_in6 *) addr)->sin6_addr;
1587 ep->port =
1588 (uint16_t) ((const struct sockaddr_in6 *) addr)->sin6_port;
1589 break;
1590
1591 default:
1592 return EAFNOSUPPORT;
1593 }
1594 }
1595
1596 return vls_sendto (vlsh, (void *) buf, n, flags, ep);
1597}
1598
1599static int
Florin Coras36847942023-02-02 12:56:16 -08001600ldp_vls_recvfrom (vls_handle_t vlsh, void *__restrict buf, size_t n, int flags,
1601 __SOCKADDR_ARG _addr, socklen_t *__restrict addr_len)
Florin Corasce17f462020-05-22 20:36:29 +00001602{
1603 u8 src_addr[sizeof (struct sockaddr_in6)];
Florin Coras36847942023-02-02 12:56:16 -08001604 struct sockaddr *addr = SOCKADDR_GET_SA (_addr);
Florin Corasce17f462020-05-22 20:36:29 +00001605 vppcom_endpt_t ep;
1606 ssize_t size;
1607 int rv;
1608
1609 if (addr)
1610 {
1611 ep.ip = src_addr;
1612 size = vls_recvfrom (vlsh, buf, n, flags, &ep);
1613
1614 if (size > 0)
1615 {
1616 rv = ldp_copy_ep_to_sockaddr (addr, addr_len, &ep);
1617 if (rv < 0)
1618 size = rv;
1619 }
1620 }
1621 else
1622 size = vls_recvfrom (vlsh, buf, n, flags, NULL);
1623
1624 return size;
1625}
1626
Dave Wallace048b1d62018-01-03 22:24:41 -05001627ssize_t
1628sendto (int fd, const void *buf, size_t n, int flags,
Florin Coras36847942023-02-02 12:56:16 -08001629 __CONST_SOCKADDR_ARG _addr, socklen_t addr_len)
Dave Wallace048b1d62018-01-03 22:24:41 -05001630{
Florin Coras36847942023-02-02 12:56:16 -08001631 const struct sockaddr *addr = SOCKADDR_GET_SA (_addr);
Florin Coras7baeb712019-01-04 17:05:43 -08001632 vls_handle_t vlsh;
Dave Wallace048b1d62018-01-03 22:24:41 -05001633 ssize_t size;
Dave Wallace048b1d62018-01-03 22:24:41 -05001634
Florin Coras5f33d0d2021-06-02 21:22:21 -07001635 ldp_init_check ();
Dave Wallace048b1d62018-01-03 22:24:41 -05001636
Florin Coras7baeb712019-01-04 17:05:43 -08001637 vlsh = ldp_fd_to_vlsh (fd);
wanghanlin97c6e0d2021-07-06 15:01:48 +08001638 if (vlsh != VLS_INVALID_HANDLE)
Dave Wallace048b1d62018-01-03 22:24:41 -05001639 {
Dou Chao243a0432022-11-29 19:41:34 +08001640 size = ldp_vls_sendo (vlsh, buf, n, NULL, flags, addr, addr_len);
Dave Wallace048b1d62018-01-03 22:24:41 -05001641 if (size < 0)
1642 {
1643 errno = -size;
1644 size = -1;
1645 }
1646 }
1647 else
1648 {
Dave Wallace048b1d62018-01-03 22:24:41 -05001649 size = libc_sendto (fd, buf, n, flags, addr, addr_len);
1650 }
1651
Dave Wallace048b1d62018-01-03 22:24:41 -05001652 return size;
1653}
1654
1655ssize_t
1656recvfrom (int fd, void *__restrict buf, size_t n, int flags,
1657 __SOCKADDR_ARG addr, socklen_t * __restrict addr_len)
1658{
Florin Corasce17f462020-05-22 20:36:29 +00001659 vls_handle_t vlsh;
1660 ssize_t size;
Dave Wallace048b1d62018-01-03 22:24:41 -05001661
Florin Coras5f33d0d2021-06-02 21:22:21 -07001662 ldp_init_check ();
Dave Wallace048b1d62018-01-03 22:24:41 -05001663
Florin Corasce17f462020-05-22 20:36:29 +00001664 vlsh = ldp_fd_to_vlsh (fd);
1665 if (vlsh != VLS_INVALID_HANDLE)
Dave Wallace048b1d62018-01-03 22:24:41 -05001666 {
Florin Corasce17f462020-05-22 20:36:29 +00001667 size = ldp_vls_recvfrom (vlsh, buf, n, flags, addr, addr_len);
Dave Wallace048b1d62018-01-03 22:24:41 -05001668 if (size < 0)
1669 {
1670 errno = -size;
1671 size = -1;
1672 }
1673 }
1674 else
1675 {
Dave Wallace048b1d62018-01-03 22:24:41 -05001676 size = libc_recvfrom (fd, buf, n, flags, addr, addr_len);
1677 }
1678
Dave Wallace048b1d62018-01-03 22:24:41 -05001679 return size;
1680}
1681
Florin Coraseff5f7a2023-02-07 17:36:17 -08001682static int
1683ldp_parse_cmsg (vls_handle_t vlsh, const struct msghdr *msg,
1684 vppcom_endpt_tlv_t **app_tlvs)
1685{
1686 uint8_t *ad, *at = (uint8_t *) *app_tlvs;
1687 vppcom_endpt_tlv_t *adh;
1688 struct in_pktinfo *pi;
1689 struct cmsghdr *cmsg;
1690
1691 cmsg = CMSG_FIRSTHDR (msg);
1692
1693 while (cmsg != NULL)
1694 {
1695 switch (cmsg->cmsg_level)
1696 {
1697 case SOL_UDP:
1698 switch (cmsg->cmsg_type)
1699 {
1700 case UDP_SEGMENT:
1701 vec_add2 (at, adh, sizeof (*adh));
1702 adh->data_type = VCL_UDP_SEGMENT;
1703 adh->data_len = sizeof (uint16_t);
1704 vec_add2 (at, ad, sizeof (uint16_t));
1705 *(uint16_t *) ad = *(uint16_t *) CMSG_DATA (cmsg);
1706 break;
1707 default:
1708 LDBG (1, "SOL_UDP cmsg_type %u not supported", cmsg->cmsg_type);
1709 break;
1710 }
1711 break;
1712 case SOL_IP:
1713 switch (cmsg->cmsg_type)
1714 {
1715 case IP_PKTINFO:
1716 vec_add2 (at, adh, sizeof (*adh));
1717 adh->data_type = VCL_IP_PKTINFO;
1718 adh->data_len = sizeof (struct in_addr);
1719 vec_add2 (at, ad, sizeof (struct in_addr));
1720 pi = (void *) CMSG_DATA (cmsg);
1721 clib_memcpy_fast (ad, &pi->ipi_spec_dst,
1722 sizeof (struct in_addr));
1723 break;
1724 default:
1725 LDBG (1, "SOL_IP cmsg_type %u not supported", cmsg->cmsg_type);
1726 break;
1727 }
1728 break;
1729 default:
1730 LDBG (1, "cmsg_level %u not supported", cmsg->cmsg_level);
1731 break;
1732 }
1733 cmsg = CMSG_NXTHDR ((struct msghdr *) msg, cmsg);
1734 }
1735 *app_tlvs = (vppcom_endpt_tlv_t *) at;
1736 return 0;
1737}
1738
1739static int
1740ldp_make_cmsg (vls_handle_t vlsh, struct msghdr *msg)
1741{
1742 u32 optval, optlen = sizeof (optval);
1743 struct cmsghdr *cmsg;
1744
1745 cmsg = CMSG_FIRSTHDR (msg);
1746
1747 if (!vls_attr (vlsh, VPPCOM_ATTR_GET_IP_PKTINFO, (void *) &optval, &optlen))
1748 return 0;
1749
1750 if (optval)
1751 {
1752 vppcom_endpt_t ep;
1753 u8 addr_buf[sizeof (struct in_addr)];
1754 u32 size = sizeof (ep);
1755
1756 ep.ip = addr_buf;
1757
1758 if (!vls_attr (vlsh, VPPCOM_ATTR_GET_LCL_ADDR, &ep, &size))
1759 {
1760 struct in_pktinfo pi = {};
1761
1762 clib_memcpy (&pi.ipi_addr, ep.ip, sizeof (struct in_addr));
1763 cmsg->cmsg_level = SOL_IP;
1764 cmsg->cmsg_type = IP_PKTINFO;
1765 cmsg->cmsg_len = CMSG_LEN (sizeof (pi));
1766 clib_memcpy (CMSG_DATA (cmsg), &pi, sizeof (pi));
1767 }
1768 }
1769
1770 return 0;
1771}
1772
Dave Wallace048b1d62018-01-03 22:24:41 -05001773ssize_t
Florin Corasce17f462020-05-22 20:36:29 +00001774sendmsg (int fd, const struct msghdr * msg, int flags)
Dave Wallace048b1d62018-01-03 22:24:41 -05001775{
Florin Coras7baeb712019-01-04 17:05:43 -08001776 vls_handle_t vlsh;
Dave Wallace048b1d62018-01-03 22:24:41 -05001777 ssize_t size;
Dave Wallace048b1d62018-01-03 22:24:41 -05001778
Florin Coras5f33d0d2021-06-02 21:22:21 -07001779 ldp_init_check ();
Dave Wallace048b1d62018-01-03 22:24:41 -05001780
Florin Coras7baeb712019-01-04 17:05:43 -08001781 vlsh = ldp_fd_to_vlsh (fd);
1782 if (vlsh != VLS_INVALID_HANDLE)
Dave Wallace048b1d62018-01-03 22:24:41 -05001783 {
Florin Coraseff5f7a2023-02-07 17:36:17 -08001784 vppcom_endpt_tlv_t *app_tlvs = 0;
Florin Corasce17f462020-05-22 20:36:29 +00001785 struct iovec *iov = msg->msg_iov;
1786 ssize_t total = 0;
Florin Corascba1c222021-11-29 08:12:27 -08001787 int i, rv = 0;
Dou Chao243a0432022-11-29 19:41:34 +08001788
Florin Coraseff5f7a2023-02-07 17:36:17 -08001789 ldp_parse_cmsg (vlsh, msg, &app_tlvs);
Florin Corasce17f462020-05-22 20:36:29 +00001790
1791 for (i = 0; i < msg->msg_iovlen; ++i)
1792 {
Florin Coraseff5f7a2023-02-07 17:36:17 -08001793 rv = ldp_vls_sendo (vlsh, iov[i].iov_base, iov[i].iov_len, app_tlvs,
1794 flags, msg->msg_name, msg->msg_namelen);
Florin Corasce17f462020-05-22 20:36:29 +00001795 if (rv < 0)
1796 break;
1797 else
1798 {
1799 total += rv;
1800 if (rv < iov[i].iov_len)
1801 break;
1802 }
1803 }
1804
Florin Coraseff5f7a2023-02-07 17:36:17 -08001805 vec_free (app_tlvs);
1806
Florin Corasce17f462020-05-22 20:36:29 +00001807 if (rv < 0 && total == 0)
1808 {
1809 errno = -rv;
1810 size = -1;
1811 }
1812 else
1813 size = total;
Dave Wallace048b1d62018-01-03 22:24:41 -05001814 }
1815 else
1816 {
Florin Corasce17f462020-05-22 20:36:29 +00001817 size = libc_sendmsg (fd, msg, flags);
Dave Wallace048b1d62018-01-03 22:24:41 -05001818 }
1819
Dave Wallace048b1d62018-01-03 22:24:41 -05001820 return size;
1821}
1822
Florin Coras36847942023-02-02 12:56:16 -08001823#ifdef _GNU_SOURCE
Dave Wallace048b1d62018-01-03 22:24:41 -05001824int
1825sendmmsg (int fd, struct mmsghdr *vmessages, unsigned int vlen, int flags)
1826{
1827 ssize_t size;
1828 const char *func_str;
Florin Coras7baeb712019-01-04 17:05:43 -08001829 u32 sh = ldp_fd_to_vlsh (fd);
Dave Wallace048b1d62018-01-03 22:24:41 -05001830
Florin Coras5f33d0d2021-06-02 21:22:21 -07001831 ldp_init_check ();
Dave Wallace048b1d62018-01-03 22:24:41 -05001832
wanghanlin97c6e0d2021-07-06 15:01:48 +08001833 if (sh != VLS_INVALID_HANDLE)
Dave Wallace048b1d62018-01-03 22:24:41 -05001834 {
1835 clib_warning ("LDP<%d>: LDP-TBD", getpid ());
1836 errno = ENOSYS;
1837 size = -1;
1838 }
1839 else
1840 {
1841 func_str = "libc_sendmmsg";
1842
Dave Wallace2a865272018-02-07 21:00:42 -05001843 if (LDP_DEBUG > 2)
Dave Wallace048b1d62018-01-03 22:24:41 -05001844 clib_warning ("LDP<%d>: fd %d (0x%x): calling %s(): "
1845 "vmessages %p, vlen %u, flags 0x%x",
1846 getpid (), fd, fd, func_str, vmessages, vlen, flags);
1847
1848 size = libc_sendmmsg (fd, vmessages, vlen, flags);
1849 }
1850
Dave Wallace2a865272018-02-07 21:00:42 -05001851 if (LDP_DEBUG > 2)
Dave Wallace048b1d62018-01-03 22:24:41 -05001852 {
1853 if (size < 0)
1854 {
1855 int errno_val = errno;
1856 perror (func_str);
1857 clib_warning ("LDP<%d>: ERROR: fd %d (0x%x): %s() failed! "
1858 "rv %d, errno = %d", getpid (), fd, fd,
1859 func_str, size, errno_val);
1860 errno = errno_val;
1861 }
1862 else
1863 clib_warning ("LDP<%d>: fd %d (0x%x): returning %d (0x%x)",
1864 getpid (), fd, fd, size, size);
1865 }
1866 return size;
1867}
shrinivasan ganapathy1d359632017-10-15 15:46:09 -07001868#endif
1869
Dave Wallace048b1d62018-01-03 22:24:41 -05001870ssize_t
Florin Corasce17f462020-05-22 20:36:29 +00001871recvmsg (int fd, struct msghdr * msg, int flags)
Dave Wallace048b1d62018-01-03 22:24:41 -05001872{
Florin Coras7baeb712019-01-04 17:05:43 -08001873 vls_handle_t vlsh;
Dave Wallace048b1d62018-01-03 22:24:41 -05001874 ssize_t size;
Dave Wallace048b1d62018-01-03 22:24:41 -05001875
Florin Coras5f33d0d2021-06-02 21:22:21 -07001876 ldp_init_check ();
Dave Wallace048b1d62018-01-03 22:24:41 -05001877
Florin Coras7baeb712019-01-04 17:05:43 -08001878 vlsh = ldp_fd_to_vlsh (fd);
1879 if (vlsh != VLS_INVALID_HANDLE)
shrinivasan ganapathy1d359632017-10-15 15:46:09 -07001880 {
Florin Corasce17f462020-05-22 20:36:29 +00001881 struct iovec *iov = msg->msg_iov;
1882 ssize_t max_deq, total = 0;
1883 int i, rv;
1884
1885 max_deq = vls_attr (vlsh, VPPCOM_ATTR_GET_NREAD, 0, 0);
1886 if (!max_deq)
1887 return 0;
1888
1889 for (i = 0; i < msg->msg_iovlen; i++)
1890 {
1891 rv = ldp_vls_recvfrom (vlsh, iov[i].iov_base, iov[i].iov_len, flags,
1892 (i == 0 ? msg->msg_name : NULL),
1893 (i == 0 ? &msg->msg_namelen : NULL));
1894 if (rv <= 0)
1895 break;
1896 else
1897 {
1898 total += rv;
1899 if (rv < iov[i].iov_len)
1900 break;
1901 }
1902 if (total >= max_deq)
1903 break;
1904 }
1905
1906 if (rv < 0 && total == 0)
1907 {
1908 errno = -rv;
1909 size = -1;
1910 }
1911 else
Florin Coraseff5f7a2023-02-07 17:36:17 -08001912 {
1913 if (msg->msg_controllen)
1914 ldp_make_cmsg (vlsh, msg);
1915 size = total;
1916 }
Dave Wallace048b1d62018-01-03 22:24:41 -05001917 }
1918 else
1919 {
Florin Corasce17f462020-05-22 20:36:29 +00001920 size = libc_recvmsg (fd, msg, flags);
Dave Wallace048b1d62018-01-03 22:24:41 -05001921 }
1922
Dave Wallace048b1d62018-01-03 22:24:41 -05001923 return size;
1924}
1925
Florin Coras36847942023-02-02 12:56:16 -08001926#ifdef _GNU_SOURCE
Dave Wallace048b1d62018-01-03 22:24:41 -05001927int
1928recvmmsg (int fd, struct mmsghdr *vmessages,
1929 unsigned int vlen, int flags, struct timespec *tmo)
1930{
Florin Corasf1a232f2023-02-02 22:56:03 -08001931 ldp_worker_ctx_t *ldpw = ldp_worker_get_current ();
1932 u32 sh;
Dave Wallace048b1d62018-01-03 22:24:41 -05001933
Florin Coras5f33d0d2021-06-02 21:22:21 -07001934 ldp_init_check ();
Dave Wallace048b1d62018-01-03 22:24:41 -05001935
Florin Corasf1a232f2023-02-02 22:56:03 -08001936 sh = ldp_fd_to_vlsh (fd);
1937
wanghanlin97c6e0d2021-07-06 15:01:48 +08001938 if (sh != VLS_INVALID_HANDLE)
Dave Wallace048b1d62018-01-03 22:24:41 -05001939 {
Florin Corasf1a232f2023-02-02 22:56:03 -08001940 struct mmsghdr *mh;
1941 ssize_t rv = 0;
1942 u32 nvecs = 0;
1943 f64 time_out;
1944
1945 if (PREDICT_FALSE (ldpw->clib_time.init_cpu_time == 0))
1946 clib_time_init (&ldpw->clib_time);
1947 if (tmo)
1948 {
1949 time_out = (f64) tmo->tv_sec + (f64) tmo->tv_nsec / (f64) 1e9;
1950 time_out += clib_time_now (&ldpw->clib_time);
1951 }
1952 else
1953 {
1954 time_out = (f64) ~0;
1955 }
1956
1957 while (nvecs < vlen)
1958 {
1959 mh = &vmessages[nvecs];
1960 rv = recvmsg (fd, &mh->msg_hdr, flags);
1961 if (rv > 0)
1962 {
1963 mh->msg_len = rv;
1964 nvecs += 1;
1965 continue;
1966 }
1967
1968 if (!time_out || clib_time_now (&ldpw->clib_time) >= time_out)
1969 break;
1970
1971 usleep (1);
1972 }
1973
1974 return nvecs > 0 ? nvecs : rv;
Dave Wallace048b1d62018-01-03 22:24:41 -05001975 }
1976 else
1977 {
Florin Corasf1a232f2023-02-02 22:56:03 -08001978 return libc_recvmmsg (fd, vmessages, vlen, flags, tmo);
Dave Wallace048b1d62018-01-03 22:24:41 -05001979 }
Dave Wallace048b1d62018-01-03 22:24:41 -05001980}
1981#endif
1982
1983int
1984getsockopt (int fd, int level, int optname,
1985 void *__restrict optval, socklen_t * __restrict optlen)
1986{
Florin Coras7baeb712019-01-04 17:05:43 -08001987 vls_handle_t vlsh;
Dave Wallace048b1d62018-01-03 22:24:41 -05001988 int rv;
Dave Wallace048b1d62018-01-03 22:24:41 -05001989
Florin Coras5f33d0d2021-06-02 21:22:21 -07001990 ldp_init_check ();
Dave Wallace048b1d62018-01-03 22:24:41 -05001991
Florin Coras7baeb712019-01-04 17:05:43 -08001992 vlsh = ldp_fd_to_vlsh (fd);
1993 if (vlsh != VLS_INVALID_HANDLE)
Dave Wallace048b1d62018-01-03 22:24:41 -05001994 {
1995 rv = -EOPNOTSUPP;
1996
1997 switch (level)
1998 {
1999 case SOL_TCP:
2000 switch (optname)
2001 {
2002 case TCP_NODELAY:
Florin Coras7baeb712019-01-04 17:05:43 -08002003 rv = vls_attr (vlsh, VPPCOM_ATTR_GET_TCP_NODELAY,
2004 optval, optlen);
Dave Wallace048b1d62018-01-03 22:24:41 -05002005 break;
2006 case TCP_MAXSEG:
Florin Coras7baeb712019-01-04 17:05:43 -08002007 rv = vls_attr (vlsh, VPPCOM_ATTR_GET_TCP_USER_MSS,
2008 optval, optlen);
Dave Wallace048b1d62018-01-03 22:24:41 -05002009 break;
2010 case TCP_KEEPIDLE:
Florin Coras7baeb712019-01-04 17:05:43 -08002011 rv = vls_attr (vlsh, VPPCOM_ATTR_GET_TCP_KEEPIDLE,
2012 optval, optlen);
Dave Wallace048b1d62018-01-03 22:24:41 -05002013 break;
2014 case TCP_KEEPINTVL:
Florin Coras7baeb712019-01-04 17:05:43 -08002015 rv = vls_attr (vlsh, VPPCOM_ATTR_GET_TCP_KEEPINTVL,
2016 optval, optlen);
Dave Wallace048b1d62018-01-03 22:24:41 -05002017 break;
2018 case TCP_INFO:
2019 if (optval && optlen && (*optlen == sizeof (struct tcp_info)))
2020 {
Florin Coras7baeb712019-01-04 17:05:43 -08002021 LDBG (1, "fd %d: vlsh %u SOL_TCP, TCP_INFO, optval %p, "
2022 "optlen %d: #LDP-NOP#", fd, vlsh, optval, *optlen);
Dave Wallace048b1d62018-01-03 22:24:41 -05002023 memset (optval, 0, *optlen);
2024 rv = VPPCOM_OK;
2025 }
2026 else
2027 rv = -EFAULT;
2028 break;
Florin Coras0ed24e92019-01-21 09:03:10 -08002029 case TCP_CONGESTION:
Florin Coras0ed24e92019-01-21 09:03:10 -08002030 *optlen = strlen ("cubic");
Dave Barach02500902020-04-04 18:34:41 -04002031 strncpy (optval, "cubic", *optlen + 1);
Florin Coras0ed24e92019-01-21 09:03:10 -08002032 rv = 0;
2033 break;
Dave Wallace048b1d62018-01-03 22:24:41 -05002034 default:
Florin Coras7baeb712019-01-04 17:05:43 -08002035 LDBG (0, "ERROR: fd %d: getsockopt SOL_TCP: sid %u, "
2036 "optname %d unsupported!", fd, vlsh, optname);
Dave Wallace048b1d62018-01-03 22:24:41 -05002037 break;
2038 }
2039 break;
2040 case SOL_IPV6:
2041 switch (optname)
2042 {
2043 case IPV6_V6ONLY:
Florin Coras7baeb712019-01-04 17:05:43 -08002044 rv = vls_attr (vlsh, VPPCOM_ATTR_GET_V6ONLY, optval, optlen);
Dave Wallace048b1d62018-01-03 22:24:41 -05002045 break;
2046 default:
Florin Coras7baeb712019-01-04 17:05:43 -08002047 LDBG (0, "ERROR: fd %d: getsockopt SOL_IPV6: vlsh %u "
2048 "optname %d unsupported!", fd, vlsh, optname);
Dave Wallace048b1d62018-01-03 22:24:41 -05002049 break;
2050 }
2051 break;
2052 case SOL_SOCKET:
2053 switch (optname)
2054 {
2055 case SO_ACCEPTCONN:
Florin Coras7baeb712019-01-04 17:05:43 -08002056 rv = vls_attr (vlsh, VPPCOM_ATTR_GET_LISTEN, optval, optlen);
Dave Wallace048b1d62018-01-03 22:24:41 -05002057 break;
2058 case SO_KEEPALIVE:
Florin Coras7baeb712019-01-04 17:05:43 -08002059 rv = vls_attr (vlsh, VPPCOM_ATTR_GET_KEEPALIVE, optval, optlen);
Dave Wallace048b1d62018-01-03 22:24:41 -05002060 break;
2061 case SO_PROTOCOL:
Florin Coras7baeb712019-01-04 17:05:43 -08002062 rv = vls_attr (vlsh, VPPCOM_ATTR_GET_PROTOCOL, optval, optlen);
Dave Wallace048b1d62018-01-03 22:24:41 -05002063 *(int *) optval = *(int *) optval ? SOCK_DGRAM : SOCK_STREAM;
2064 break;
2065 case SO_SNDBUF:
Florin Coras7baeb712019-01-04 17:05:43 -08002066 rv = vls_attr (vlsh, VPPCOM_ATTR_GET_TX_FIFO_LEN,
2067 optval, optlen);
Dave Wallace048b1d62018-01-03 22:24:41 -05002068 break;
2069 case SO_RCVBUF:
Florin Coras7baeb712019-01-04 17:05:43 -08002070 rv = vls_attr (vlsh, VPPCOM_ATTR_GET_RX_FIFO_LEN,
2071 optval, optlen);
Dave Wallace048b1d62018-01-03 22:24:41 -05002072 break;
2073 case SO_REUSEADDR:
Florin Coras7baeb712019-01-04 17:05:43 -08002074 rv = vls_attr (vlsh, VPPCOM_ATTR_GET_REUSEADDR, optval, optlen);
Dave Wallace048b1d62018-01-03 22:24:41 -05002075 break;
wanghanlin0674f852021-02-22 10:38:36 +08002076 case SO_REUSEPORT:
2077 rv = vls_attr (vlsh, VPPCOM_ATTR_GET_REUSEPORT, optval, optlen);
2078 break;
Dave Wallace048b1d62018-01-03 22:24:41 -05002079 case SO_BROADCAST:
Florin Coras7baeb712019-01-04 17:05:43 -08002080 rv = vls_attr (vlsh, VPPCOM_ATTR_GET_BROADCAST, optval, optlen);
Dave Wallace048b1d62018-01-03 22:24:41 -05002081 break;
wanghanlin0674f852021-02-22 10:38:36 +08002082 case SO_DOMAIN:
2083 rv = vls_attr (vlsh, VPPCOM_ATTR_GET_DOMAIN, optval, optlen);
2084 break;
Dave Wallace048b1d62018-01-03 22:24:41 -05002085 case SO_ERROR:
Florin Coras7baeb712019-01-04 17:05:43 -08002086 rv = vls_attr (vlsh, VPPCOM_ATTR_GET_ERROR, optval, optlen);
Dave Wallace048b1d62018-01-03 22:24:41 -05002087 break;
Florin Coras8eb8d502021-06-16 14:46:57 -07002088 case SO_BINDTODEVICE:
2089 rv = 0;
2090 break;
Dave Wallace048b1d62018-01-03 22:24:41 -05002091 default:
Florin Coras7baeb712019-01-04 17:05:43 -08002092 LDBG (0, "ERROR: fd %d: getsockopt SOL_SOCKET: vlsh %u "
2093 "optname %d unsupported!", fd, vlsh, optname);
Dave Wallace048b1d62018-01-03 22:24:41 -05002094 break;
2095 }
2096 break;
2097 default:
2098 break;
shrinivasan ganapathy1d359632017-10-15 15:46:09 -07002099 }
2100
Dave Wallace048b1d62018-01-03 22:24:41 -05002101 if (rv != VPPCOM_OK)
shrinivasan ganapathy1d359632017-10-15 15:46:09 -07002102 {
Dave Wallace048b1d62018-01-03 22:24:41 -05002103 errno = -rv;
2104 rv = -1;
2105 }
2106 }
2107 else
2108 {
Dave Wallace048b1d62018-01-03 22:24:41 -05002109 rv = libc_getsockopt (fd, level, optname, optval, optlen);
2110 }
2111
Dave Wallace048b1d62018-01-03 22:24:41 -05002112 return rv;
2113}
2114
2115int
2116setsockopt (int fd, int level, int optname,
2117 const void *optval, socklen_t optlen)
2118{
Florin Coras7baeb712019-01-04 17:05:43 -08002119 vls_handle_t vlsh;
Dave Wallace048b1d62018-01-03 22:24:41 -05002120 int rv;
Dave Wallace048b1d62018-01-03 22:24:41 -05002121
Florin Coras5f33d0d2021-06-02 21:22:21 -07002122 ldp_init_check ();
Dave Wallace048b1d62018-01-03 22:24:41 -05002123
Florin Coras7baeb712019-01-04 17:05:43 -08002124 vlsh = ldp_fd_to_vlsh (fd);
2125 if (vlsh != VLS_INVALID_HANDLE)
Dave Wallace048b1d62018-01-03 22:24:41 -05002126 {
2127 rv = -EOPNOTSUPP;
2128
2129 switch (level)
2130 {
2131 case SOL_TCP:
2132 switch (optname)
2133 {
2134 case TCP_NODELAY:
Florin Coras7baeb712019-01-04 17:05:43 -08002135 rv = vls_attr (vlsh, VPPCOM_ATTR_SET_TCP_NODELAY,
2136 (void *) optval, &optlen);
Dave Wallace048b1d62018-01-03 22:24:41 -05002137 break;
2138 case TCP_MAXSEG:
Florin Coras7baeb712019-01-04 17:05:43 -08002139 rv = vls_attr (vlsh, VPPCOM_ATTR_SET_TCP_USER_MSS,
2140 (void *) optval, &optlen);
Dave Wallace048b1d62018-01-03 22:24:41 -05002141 break;
2142 case TCP_KEEPIDLE:
Florin Coras7baeb712019-01-04 17:05:43 -08002143 rv = vls_attr (vlsh, VPPCOM_ATTR_SET_TCP_KEEPIDLE,
2144 (void *) optval, &optlen);
Dave Wallace048b1d62018-01-03 22:24:41 -05002145 break;
2146 case TCP_KEEPINTVL:
Florin Coras7baeb712019-01-04 17:05:43 -08002147 rv = vls_attr (vlsh, VPPCOM_ATTR_SET_TCP_KEEPINTVL,
2148 (void *) optval, &optlen);
Dave Wallace048b1d62018-01-03 22:24:41 -05002149 break;
Florin Coras0ed24e92019-01-21 09:03:10 -08002150 case TCP_CONGESTION:
Florin Coras8509aa22019-04-04 12:55:30 -07002151 case TCP_CORK:
Florin Coras0ed24e92019-01-21 09:03:10 -08002152 /* Ignore */
2153 rv = 0;
2154 break;
Dave Wallace048b1d62018-01-03 22:24:41 -05002155 default:
Florin Coras7baeb712019-01-04 17:05:43 -08002156 LDBG (0, "ERROR: fd %d: setsockopt() SOL_TCP: vlsh %u"
2157 "optname %d unsupported!", fd, vlsh, optname);
Dave Wallace048b1d62018-01-03 22:24:41 -05002158 break;
2159 }
2160 break;
2161 case SOL_IPV6:
2162 switch (optname)
2163 {
2164 case IPV6_V6ONLY:
Florin Coras7baeb712019-01-04 17:05:43 -08002165 rv = vls_attr (vlsh, VPPCOM_ATTR_SET_V6ONLY,
2166 (void *) optval, &optlen);
Dave Wallace048b1d62018-01-03 22:24:41 -05002167 break;
2168 default:
Florin Coras7baeb712019-01-04 17:05:43 -08002169 LDBG (0, "ERROR: fd %d: setsockopt SOL_IPV6: vlsh %u"
2170 "optname %d unsupported!", fd, vlsh, optname);
Dave Wallace048b1d62018-01-03 22:24:41 -05002171 break;
2172 }
2173 break;
2174 case SOL_SOCKET:
2175 switch (optname)
2176 {
2177 case SO_KEEPALIVE:
Florin Coras7baeb712019-01-04 17:05:43 -08002178 rv = vls_attr (vlsh, VPPCOM_ATTR_SET_KEEPALIVE,
2179 (void *) optval, &optlen);
Dave Wallace048b1d62018-01-03 22:24:41 -05002180 break;
2181 case SO_REUSEADDR:
Florin Coras7baeb712019-01-04 17:05:43 -08002182 rv = vls_attr (vlsh, VPPCOM_ATTR_SET_REUSEADDR,
2183 (void *) optval, &optlen);
Dave Wallace048b1d62018-01-03 22:24:41 -05002184 break;
wanghanlin0674f852021-02-22 10:38:36 +08002185 case SO_REUSEPORT:
2186 rv = vls_attr (vlsh, VPPCOM_ATTR_SET_REUSEPORT, (void *) optval,
2187 &optlen);
2188 break;
Dave Wallace048b1d62018-01-03 22:24:41 -05002189 case SO_BROADCAST:
Florin Coras7baeb712019-01-04 17:05:43 -08002190 rv = vls_attr (vlsh, VPPCOM_ATTR_SET_BROADCAST,
2191 (void *) optval, &optlen);
Dave Wallace048b1d62018-01-03 22:24:41 -05002192 break;
Florin Coras2f647902021-06-02 08:23:50 -07002193 case SO_LINGER:
2194 rv = 0;
2195 break;
Dave Wallace048b1d62018-01-03 22:24:41 -05002196 default:
Florin Coras7baeb712019-01-04 17:05:43 -08002197 LDBG (0, "ERROR: fd %d: setsockopt SOL_SOCKET: vlsh %u "
2198 "optname %d unsupported!", fd, vlsh, optname);
Dave Wallace048b1d62018-01-03 22:24:41 -05002199 break;
2200 }
2201 break;
Florin Coraseff5f7a2023-02-07 17:36:17 -08002202 case SOL_IP:
2203 switch (optname)
2204 {
2205 case IP_PKTINFO:
2206 rv = vls_attr (vlsh, VPPCOM_ATTR_SET_IP_PKTINFO, (void *) optval,
2207 &optlen);
2208 break;
2209 default:
2210 LDBG (0,
2211 "ERROR: fd %d: setsockopt SOL_IP: vlsh %u optname %d"
2212 "unsupported!",
2213 fd, vlsh, optname);
2214 break;
2215 }
2216 break;
Dave Wallace048b1d62018-01-03 22:24:41 -05002217 default:
2218 break;
2219 }
2220
2221 if (rv != VPPCOM_OK)
2222 {
2223 errno = -rv;
2224 rv = -1;
2225 }
2226 }
2227 else
2228 {
Dave Wallace048b1d62018-01-03 22:24:41 -05002229 rv = libc_setsockopt (fd, level, optname, optval, optlen);
2230 }
2231
Dave Wallace048b1d62018-01-03 22:24:41 -05002232 return rv;
2233}
2234
2235int
2236listen (int fd, int n)
2237{
Florin Coras7baeb712019-01-04 17:05:43 -08002238 vls_handle_t vlsh;
Dave Wallace048b1d62018-01-03 22:24:41 -05002239 int rv;
Dave Wallace048b1d62018-01-03 22:24:41 -05002240
Florin Coras5f33d0d2021-06-02 21:22:21 -07002241 ldp_init_check ();
Dave Wallace048b1d62018-01-03 22:24:41 -05002242
Florin Coras7baeb712019-01-04 17:05:43 -08002243 vlsh = ldp_fd_to_vlsh (fd);
2244 if (vlsh != VLS_INVALID_HANDLE)
Dave Wallace048b1d62018-01-03 22:24:41 -05002245 {
Florin Coras7baeb712019-01-04 17:05:43 -08002246 LDBG (0, "fd %d: calling vls_listen: vlsh %u, n %d", fd, vlsh, n);
Dave Wallace048b1d62018-01-03 22:24:41 -05002247
Florin Coras7baeb712019-01-04 17:05:43 -08002248 rv = vls_listen (vlsh, n);
Dave Wallace048b1d62018-01-03 22:24:41 -05002249 if (rv != VPPCOM_OK)
2250 {
2251 errno = -rv;
2252 rv = -1;
2253 }
2254 }
2255 else
2256 {
Florin Coras7baeb712019-01-04 17:05:43 -08002257 LDBG (0, "fd %d: calling libc_listen(): n %d", fd, n);
Dave Wallace048b1d62018-01-03 22:24:41 -05002258 rv = libc_listen (fd, n);
2259 }
2260
Florin Coras7baeb712019-01-04 17:05:43 -08002261 LDBG (1, "fd %d: returning %d", fd, rv);
Dave Wallace048b1d62018-01-03 22:24:41 -05002262 return rv;
2263}
2264
2265static inline int
Florin Coras36847942023-02-02 12:56:16 -08002266ldp_accept4 (int listen_fd, __SOCKADDR_ARG _addr,
2267 socklen_t *__restrict addr_len, int flags)
Dave Wallace048b1d62018-01-03 22:24:41 -05002268{
Florin Coras36847942023-02-02 12:56:16 -08002269 struct sockaddr *addr = SOCKADDR_GET_SA (_addr);
Florin Coras7baeb712019-01-04 17:05:43 -08002270 vls_handle_t listen_vlsh, accept_vlsh;
Dave Wallace048b1d62018-01-03 22:24:41 -05002271 int rv;
Dave Wallace048b1d62018-01-03 22:24:41 -05002272
Florin Coras5f33d0d2021-06-02 21:22:21 -07002273 ldp_init_check ();
Dave Wallace048b1d62018-01-03 22:24:41 -05002274
Florin Coras7baeb712019-01-04 17:05:43 -08002275 listen_vlsh = ldp_fd_to_vlsh (listen_fd);
2276 if (listen_vlsh != VLS_INVALID_HANDLE)
Dave Wallace048b1d62018-01-03 22:24:41 -05002277 {
2278 vppcom_endpt_t ep;
2279 u8 src_addr[sizeof (struct sockaddr_in6)];
Dave Wallace8aaba562018-01-18 17:21:19 -05002280 memset (&ep, 0, sizeof (ep));
Dave Wallace048b1d62018-01-03 22:24:41 -05002281 ep.ip = src_addr;
2282
Florin Coras7baeb712019-01-04 17:05:43 -08002283 LDBG (0, "listen fd %d: calling vppcom_session_accept: listen sid %u,"
Florin Coraseb801d02020-09-16 17:44:58 -07002284 " ep %p, flags 0x%x", listen_fd, listen_vlsh, &ep, flags);
Dave Wallace048b1d62018-01-03 22:24:41 -05002285
Florin Coras7baeb712019-01-04 17:05:43 -08002286 accept_vlsh = vls_accept (listen_vlsh, &ep, flags);
2287 if (accept_vlsh < 0)
Dave Wallace048b1d62018-01-03 22:24:41 -05002288 {
Florin Coras7baeb712019-01-04 17:05:43 -08002289 errno = -accept_vlsh;
Dave Wallace048b1d62018-01-03 22:24:41 -05002290 rv = -1;
shrinivasan ganapathy1d359632017-10-15 15:46:09 -07002291 }
2292 else
2293 {
Dave Wallace2a865272018-02-07 21:00:42 -05002294 rv = ldp_copy_ep_to_sockaddr (addr, addr_len, &ep);
Dave Wallace048b1d62018-01-03 22:24:41 -05002295 if (rv != VPPCOM_OK)
shrinivasan ganapathy1d359632017-10-15 15:46:09 -07002296 {
Florin Coras7baeb712019-01-04 17:05:43 -08002297 (void) vls_close (accept_vlsh);
Dave Wallace048b1d62018-01-03 22:24:41 -05002298 errno = -rv;
2299 rv = -1;
shrinivasan ganapathy1d359632017-10-15 15:46:09 -07002300 }
Dave Wallace048b1d62018-01-03 22:24:41 -05002301 else
2302 {
Florin Coras7baeb712019-01-04 17:05:43 -08002303 rv = ldp_vlsh_to_fd (accept_vlsh);
Dave Wallace048b1d62018-01-03 22:24:41 -05002304 }
shrinivasan ganapathy1d359632017-10-15 15:46:09 -07002305 }
2306 }
Dave Wallace048b1d62018-01-03 22:24:41 -05002307 else
shrinivasan ganapathy1d359632017-10-15 15:46:09 -07002308 {
Florin Coras7baeb712019-01-04 17:05:43 -08002309 LDBG (0, "listen fd %d: calling libc_accept4(): addr %p, addr_len %p,"
2310 " flags 0x%x", listen_fd, addr, addr_len, flags);
shrinivasan ganapathy1d359632017-10-15 15:46:09 -07002311
Dave Wallace048b1d62018-01-03 22:24:41 -05002312 rv = libc_accept4 (listen_fd, addr, addr_len, flags);
shrinivasan ganapathy1d359632017-10-15 15:46:09 -07002313 }
2314
Florin Coras7baeb712019-01-04 17:05:43 -08002315 LDBG (1, "listen fd %d: accept returning %d", listen_fd, rv);
Florin Coras05ecfcc2018-12-12 18:19:39 -08002316
shrinivasan ganapathy1d359632017-10-15 15:46:09 -07002317 return rv;
Keith Burns (alagalah)b327c2b2017-10-09 08:52:59 -07002318}
2319
Dave Wallace048b1d62018-01-03 22:24:41 -05002320int
2321accept4 (int fd, __SOCKADDR_ARG addr, socklen_t * __restrict addr_len,
2322 int flags)
2323{
Dave Wallace2a865272018-02-07 21:00:42 -05002324 return ldp_accept4 (fd, addr, addr_len, flags);
Dave Wallace048b1d62018-01-03 22:24:41 -05002325}
shrinivasan ganapathy1d359632017-10-15 15:46:09 -07002326
Keith Burns (alagalah)b327c2b2017-10-09 08:52:59 -07002327int
Dave Wallace048b1d62018-01-03 22:24:41 -05002328accept (int fd, __SOCKADDR_ARG addr, socklen_t * __restrict addr_len)
Keith Burns (alagalah)b327c2b2017-10-09 08:52:59 -07002329{
Dave Wallace2a865272018-02-07 21:00:42 -05002330 return ldp_accept4 (fd, addr, addr_len, 0);
Dave Wallace048b1d62018-01-03 22:24:41 -05002331}
Keith Burns (alagalah)b327c2b2017-10-09 08:52:59 -07002332
Dave Wallace048b1d62018-01-03 22:24:41 -05002333int
2334shutdown (int fd, int how)
2335{
Florin Coras7baeb712019-01-04 17:05:43 -08002336 vls_handle_t vlsh;
liuyacan55c952e2021-06-13 14:54:55 +08002337 int rv = 0;
shrinivasan ganapathy1d359632017-10-15 15:46:09 -07002338
Florin Coras5f33d0d2021-06-02 21:22:21 -07002339 ldp_init_check ();
Dave Wallace048b1d62018-01-03 22:24:41 -05002340
Florin Coras7baeb712019-01-04 17:05:43 -08002341 vlsh = ldp_fd_to_vlsh (fd);
2342 if (vlsh != VLS_INVALID_HANDLE)
Dave Wallace048b1d62018-01-03 22:24:41 -05002343 {
Florin Coras7baeb712019-01-04 17:05:43 -08002344 LDBG (0, "called shutdown: fd %u vlsh %u how %d", fd, vlsh, how);
liuyacan55c952e2021-06-13 14:54:55 +08002345 rv = vls_shutdown (vlsh, how);
Dave Wallace048b1d62018-01-03 22:24:41 -05002346 }
2347 else
2348 {
Florin Coras7baeb712019-01-04 17:05:43 -08002349 LDBG (0, "fd %d: calling libc_shutdown: how %d", fd, how);
Dave Wallace048b1d62018-01-03 22:24:41 -05002350 rv = libc_shutdown (fd, how);
2351 }
2352
Dave Wallace048b1d62018-01-03 22:24:41 -05002353 return rv;
2354}
2355
2356int
2357epoll_create1 (int flags)
2358{
Florin Corasdfe4cf42018-11-28 22:13:45 -08002359 ldp_worker_ctx_t *ldpw = ldp_worker_get_current ();
Florin Coras7baeb712019-01-04 17:05:43 -08002360 vls_handle_t vlsh;
Dave Wallace048b1d62018-01-03 22:24:41 -05002361 int rv;
2362
Florin Coras5f33d0d2021-06-02 21:22:21 -07002363 ldp_init_check ();
Dave Wallace048b1d62018-01-03 22:24:41 -05002364
hanlina3a48962020-07-13 11:09:15 +08002365 if (ldp->vcl_needs_real_epoll || vls_use_real_epoll ())
Florin Coras99368312018-08-02 10:45:44 -07002366 {
Florin Coras2d9b4272019-03-11 10:14:37 -07002367 /* Make sure workers have been allocated */
2368 if (!ldp->workers)
2369 {
2370 ldp_alloc_workers ();
2371 ldpw = ldp_worker_get_current ();
2372 }
Florin Coras99368312018-08-02 10:45:44 -07002373 rv = libc_epoll_create1 (flags);
2374 ldp->vcl_needs_real_epoll = 0;
Florin Corasdfe4cf42018-11-28 22:13:45 -08002375 ldpw->vcl_mq_epfd = rv;
Florin Coras05ecfcc2018-12-12 18:19:39 -08002376 LDBG (0, "created vcl epfd %u", rv);
Florin Coras99368312018-08-02 10:45:44 -07002377 return rv;
2378 }
Dave Wallace048b1d62018-01-03 22:24:41 -05002379
Florin Coras7baeb712019-01-04 17:05:43 -08002380 vlsh = vls_epoll_create ();
2381 if (PREDICT_FALSE (vlsh == VLS_INVALID_HANDLE))
shrinivasan ganapathy1d359632017-10-15 15:46:09 -07002382 {
Florin Coras7baeb712019-01-04 17:05:43 -08002383 errno = -vlsh;
Dave Wallace048b1d62018-01-03 22:24:41 -05002384 rv = -1;
shrinivasan ganapathy1d359632017-10-15 15:46:09 -07002385 }
Dave Wallace048b1d62018-01-03 22:24:41 -05002386 else
Keith Burns (alagalah)b327c2b2017-10-09 08:52:59 -07002387 {
Florin Coras7baeb712019-01-04 17:05:43 -08002388 rv = ldp_vlsh_to_fd (vlsh);
Keith Burns (alagalah)b327c2b2017-10-09 08:52:59 -07002389 }
Florin Coras7baeb712019-01-04 17:05:43 -08002390 LDBG (0, "epoll_create epfd %u vlsh %u", rv, vlsh);
Dave Wallace048b1d62018-01-03 22:24:41 -05002391 return rv;
Keith Burns (alagalah)b327c2b2017-10-09 08:52:59 -07002392}
2393
2394int
Dave Wallace048b1d62018-01-03 22:24:41 -05002395epoll_create (int size)
Keith Burns (alagalah)b327c2b2017-10-09 08:52:59 -07002396{
Dave Wallace048b1d62018-01-03 22:24:41 -05002397 return epoll_create1 (0);
2398}
Keith Burns (alagalah)b327c2b2017-10-09 08:52:59 -07002399
Dave Wallace048b1d62018-01-03 22:24:41 -05002400int
2401epoll_ctl (int epfd, int op, int fd, struct epoll_event *event)
2402{
Florin Coras7baeb712019-01-04 17:05:43 -08002403 vls_handle_t vep_vlsh, vlsh;
Florin Coras99368312018-08-02 10:45:44 -07002404 int rv;
Dave Wallace048b1d62018-01-03 22:24:41 -05002405
Florin Coras5f33d0d2021-06-02 21:22:21 -07002406 ldp_init_check ();
Dave Wallace048b1d62018-01-03 22:24:41 -05002407
Florin Coras7baeb712019-01-04 17:05:43 -08002408 vep_vlsh = ldp_fd_to_vlsh (epfd);
2409 if (PREDICT_FALSE (vep_vlsh == VLS_INVALID_HANDLE))
Dave Wallace048b1d62018-01-03 22:24:41 -05002410 {
Dave Wallace3ee1fe12018-02-23 01:09:11 -05002411 /* The LDP epoll_create1 always creates VCL epfd's.
2412 * The app should never have a kernel base epoll fd unless it
2413 * was acquired outside of the LD_PRELOAD process context.
2414 * In any case, if we get one, punt it to libc_epoll_ctl.
2415 */
Florin Coras7baeb712019-01-04 17:05:43 -08002416 LDBG (1, "epfd %d: calling libc_epoll_ctl: op %d, fd %d"
2417 " event %p", epfd, op, fd, event);
Dave Wallace048b1d62018-01-03 22:24:41 -05002418
2419 rv = libc_epoll_ctl (epfd, op, fd, event);
Florin Coras99368312018-08-02 10:45:44 -07002420 goto done;
2421 }
2422
Florin Coras7baeb712019-01-04 17:05:43 -08002423 vlsh = ldp_fd_to_vlsh (fd);
Florin Coras99368312018-08-02 10:45:44 -07002424
Florin Coras7baeb712019-01-04 17:05:43 -08002425 LDBG (0, "epfd %d ep_vlsh %d, fd %u vlsh %d, op %u", epfd, vep_vlsh, fd,
2426 vlsh, op);
Florin Coras99368312018-08-02 10:45:44 -07002427
Florin Coras7baeb712019-01-04 17:05:43 -08002428 if (vlsh != VLS_INVALID_HANDLE)
Florin Coras99368312018-08-02 10:45:44 -07002429 {
Florin Coras7baeb712019-01-04 17:05:43 -08002430 LDBG (1, "epfd %d: calling vls_epoll_ctl: ep_vlsh %d op %d, vlsh %u,"
Florin Coraseb801d02020-09-16 17:44:58 -07002431 " event %p", epfd, vep_vlsh, op, vlsh, event);
Florin Coras99368312018-08-02 10:45:44 -07002432
Florin Coras7baeb712019-01-04 17:05:43 -08002433 rv = vls_epoll_ctl (vep_vlsh, op, vlsh, event);
Florin Coras99368312018-08-02 10:45:44 -07002434 if (rv != VPPCOM_OK)
2435 {
2436 errno = -rv;
2437 rv = -1;
2438 }
2439 }
2440 else
2441 {
2442 int libc_epfd;
2443 u32 size = sizeof (epfd);
2444
Florin Coras7baeb712019-01-04 17:05:43 -08002445 libc_epfd = vls_attr (vep_vlsh, VPPCOM_ATTR_GET_LIBC_EPFD, 0, 0);
Florin Coras99368312018-08-02 10:45:44 -07002446 if (!libc_epfd)
2447 {
Florin Coras7baeb712019-01-04 17:05:43 -08002448 LDBG (1, "epfd %d, vep_vlsh %d calling libc_epoll_create1: "
2449 "EPOLL_CLOEXEC", epfd, vep_vlsh);
Florin Coras99368312018-08-02 10:45:44 -07002450
2451 libc_epfd = libc_epoll_create1 (EPOLL_CLOEXEC);
2452 if (libc_epfd < 0)
2453 {
2454 rv = libc_epfd;
2455 goto done;
2456 }
2457
Florin Coras7baeb712019-01-04 17:05:43 -08002458 rv = vls_attr (vep_vlsh, VPPCOM_ATTR_SET_LIBC_EPFD, &libc_epfd,
2459 &size);
Florin Coras99368312018-08-02 10:45:44 -07002460 if (rv < 0)
2461 {
2462 errno = -rv;
2463 rv = -1;
2464 goto done;
2465 }
2466 }
2467 else if (PREDICT_FALSE (libc_epfd < 0))
2468 {
2469 errno = -epfd;
2470 rv = -1;
2471 goto done;
2472 }
2473
Florin Coras7baeb712019-01-04 17:05:43 -08002474 LDBG (1, "epfd %d: calling libc_epoll_ctl: libc_epfd %d, op %d, fd %d,"
2475 " event %p", epfd, libc_epfd, op, fd, event);
Florin Coras99368312018-08-02 10:45:44 -07002476
2477 rv = libc_epoll_ctl (libc_epfd, op, fd, event);
Dave Wallace048b1d62018-01-03 22:24:41 -05002478 }
2479
2480done:
Keith Burns (alagalah)b327c2b2017-10-09 08:52:59 -07002481 return rv;
2482}
Dave Wallace048b1d62018-01-03 22:24:41 -05002483
2484static inline int
Florin Coras99368312018-08-02 10:45:44 -07002485ldp_epoll_pwait (int epfd, struct epoll_event *events, int maxevents,
2486 int timeout, const sigset_t * sigmask)
Dave Wallace048b1d62018-01-03 22:24:41 -05002487{
Liangxing Wang7c7231f2023-02-16 09:31:01 +00002488 ldp_worker_ctx_t *ldpw;
Florin Coras72f77822019-01-22 19:05:52 -08002489 double time_to_wait = (double) 0, max_time;
Florin Coras99368312018-08-02 10:45:44 -07002490 int libc_epfd, rv = 0;
Florin Coras7baeb712019-01-04 17:05:43 -08002491 vls_handle_t ep_vlsh;
Dave Wallace048b1d62018-01-03 22:24:41 -05002492
Florin Coras5f33d0d2021-06-02 21:22:21 -07002493 ldp_init_check ();
Dave Wallace048b1d62018-01-03 22:24:41 -05002494
2495 if (PREDICT_FALSE (!events || (timeout < -1)))
2496 {
2497 errno = EFAULT;
2498 return -1;
2499 }
2500
Liangxing Wang7c7231f2023-02-16 09:31:01 +00002501 if (PREDICT_FALSE (vppcom_worker_index () == ~0))
2502 vls_register_vcl_worker ();
2503
2504 ldpw = ldp_worker_get_current ();
Florin Corasdfe4cf42018-11-28 22:13:45 -08002505 if (epfd == ldpw->vcl_mq_epfd)
Florin Coras99368312018-08-02 10:45:44 -07002506 return libc_epoll_pwait (epfd, events, maxevents, timeout, sigmask);
2507
Florin Coras7baeb712019-01-04 17:05:43 -08002508 ep_vlsh = ldp_fd_to_vlsh (epfd);
2509 if (PREDICT_FALSE (ep_vlsh == VLS_INVALID_HANDLE))
Dave Wallace048b1d62018-01-03 22:24:41 -05002510 {
Florin Coras7baeb712019-01-04 17:05:43 -08002511 LDBG (0, "epfd %d: bad ep_vlsh %d!", epfd, ep_vlsh);
Dave Wallace048b1d62018-01-03 22:24:41 -05002512 errno = EBADFD;
2513 return -1;
2514 }
2515
Florin Coras4dee8cd2019-01-29 21:28:16 -08002516 if (PREDICT_FALSE (ldpw->clib_time.init_cpu_time == 0))
2517 clib_time_init (&ldpw->clib_time);
Florin Corasb0f662f2018-12-27 14:51:46 -08002518 time_to_wait = ((timeout >= 0) ? (double) timeout / 1000 : 0);
Florin Coras72f77822019-01-22 19:05:52 -08002519 max_time = clib_time_now (&ldpw->clib_time) + time_to_wait;
Dave Wallace048b1d62018-01-03 22:24:41 -05002520
Florin Coras7baeb712019-01-04 17:05:43 -08002521 libc_epfd = vls_attr (ep_vlsh, VPPCOM_ATTR_GET_LIBC_EPFD, 0, 0);
Dave Wallace048b1d62018-01-03 22:24:41 -05002522 if (PREDICT_FALSE (libc_epfd < 0))
2523 {
2524 errno = -libc_epfd;
2525 rv = -1;
2526 goto done;
2527 }
2528
Florin Coras7baeb712019-01-04 17:05:43 -08002529 LDBG (2, "epfd %d: vep_idx %d, libc_epfd %d, events %p, maxevents %d, "
2530 "timeout %d, sigmask %p: time_to_wait %.02f", epfd, ep_vlsh,
Florin Coras72f77822019-01-22 19:05:52 -08002531 libc_epfd, events, maxevents, timeout, sigmask, time_to_wait);
Dave Wallace048b1d62018-01-03 22:24:41 -05002532 do
2533 {
Florin Corasdfe4cf42018-11-28 22:13:45 -08002534 if (!ldpw->epoll_wait_vcl)
Dave Wallace048b1d62018-01-03 22:24:41 -05002535 {
Florin Coras7baeb712019-01-04 17:05:43 -08002536 rv = vls_epoll_wait (ep_vlsh, events, maxevents, 0);
Dave Wallace048b1d62018-01-03 22:24:41 -05002537 if (rv > 0)
2538 {
Florin Corasdfe4cf42018-11-28 22:13:45 -08002539 ldpw->epoll_wait_vcl = 1;
Dave Wallace048b1d62018-01-03 22:24:41 -05002540 goto done;
2541 }
2542 else if (rv < 0)
2543 {
2544 errno = -rv;
2545 rv = -1;
2546 goto done;
2547 }
2548 }
2549 else
Florin Corasdfe4cf42018-11-28 22:13:45 -08002550 ldpw->epoll_wait_vcl = 0;
Dave Wallace048b1d62018-01-03 22:24:41 -05002551
2552 if (libc_epfd > 0)
2553 {
Florin Corasb0f662f2018-12-27 14:51:46 -08002554 rv = libc_epoll_pwait (libc_epfd, events, maxevents, 0, sigmask);
Dave Wallace048b1d62018-01-03 22:24:41 -05002555 if (rv != 0)
2556 goto done;
2557 }
Dave Wallace048b1d62018-01-03 22:24:41 -05002558 }
Florin Coras72f77822019-01-22 19:05:52 -08002559 while ((timeout == -1) || (clib_time_now (&ldpw->clib_time) < max_time));
Dave Wallace048b1d62018-01-03 22:24:41 -05002560
2561done:
Dave Wallace048b1d62018-01-03 22:24:41 -05002562 return rv;
2563}
2564
hanlin4266d4d2020-05-19 17:34:17 +08002565static inline int
2566ldp_epoll_pwait_eventfd (int epfd, struct epoll_event *events,
2567 int maxevents, int timeout, const sigset_t * sigmask)
2568{
hanlina3a48962020-07-13 11:09:15 +08002569 ldp_worker_ctx_t *ldpw;
Florin Coras7a1e95a2023-03-01 00:45:31 -08002570 int libc_epfd, rv = 0, num_ev, vcl_wups = 0;
hanlin4266d4d2020-05-19 17:34:17 +08002571 vls_handle_t ep_vlsh;
2572
Florin Coras5f33d0d2021-06-02 21:22:21 -07002573 ldp_init_check ();
hanlin4266d4d2020-05-19 17:34:17 +08002574
2575 if (PREDICT_FALSE (!events || (timeout < -1)))
2576 {
2577 errno = EFAULT;
2578 return -1;
2579 }
2580
Florin Corasff40d8f2020-08-11 22:05:28 -07002581 /* Make sure the vcl worker is valid. Could be that epoll fd was created on
2582 * one thread but it is now used on another */
2583 if (PREDICT_FALSE (vppcom_worker_index () == ~0))
2584 vls_register_vcl_worker ();
hanlina3a48962020-07-13 11:09:15 +08002585
2586 ldpw = ldp_worker_get_current ();
hanlin4266d4d2020-05-19 17:34:17 +08002587 if (epfd == ldpw->vcl_mq_epfd)
2588 return libc_epoll_pwait (epfd, events, maxevents, timeout, sigmask);
2589
2590 ep_vlsh = ldp_fd_to_vlsh (epfd);
2591 if (PREDICT_FALSE (ep_vlsh == VLS_INVALID_HANDLE))
2592 {
2593 LDBG (0, "epfd %d: bad ep_vlsh %d!", epfd, ep_vlsh);
2594 errno = EBADFD;
2595 return -1;
2596 }
2597
2598 libc_epfd = vls_attr (ep_vlsh, VPPCOM_ATTR_GET_LIBC_EPFD, 0, 0);
2599 if (PREDICT_FALSE (!libc_epfd))
2600 {
2601 u32 size = sizeof (epfd);
2602
2603 LDBG (1, "epfd %d, vep_vlsh %d calling libc_epoll_create1: "
2604 "EPOLL_CLOEXEC", epfd, ep_vlsh);
2605 libc_epfd = libc_epoll_create1 (EPOLL_CLOEXEC);
2606 if (libc_epfd < 0)
2607 {
2608 rv = libc_epfd;
2609 goto done;
2610 }
2611
2612 rv = vls_attr (ep_vlsh, VPPCOM_ATTR_SET_LIBC_EPFD, &libc_epfd, &size);
2613 if (rv < 0)
2614 {
2615 errno = -rv;
2616 rv = -1;
2617 goto done;
2618 }
2619 }
2620 if (PREDICT_FALSE (libc_epfd <= 0))
2621 {
2622 errno = -libc_epfd;
2623 rv = -1;
2624 goto done;
2625 }
2626
2627 if (PREDICT_FALSE (!ldpw->mq_epfd_added))
2628 {
2629 struct epoll_event e = { 0 };
2630 e.events = EPOLLIN;
2631 e.data.fd = ldpw->vcl_mq_epfd;
2632 if (libc_epoll_ctl (libc_epfd, EPOLL_CTL_ADD, ldpw->vcl_mq_epfd, &e) <
2633 0)
2634 {
2635 LDBG (0, "epfd %d, add libc mq epoll fd %d to libc epoll fd %d",
2636 epfd, ldpw->vcl_mq_epfd, libc_epfd);
2637 rv = -1;
2638 goto done;
2639 }
2640 ldpw->mq_epfd_added = 1;
2641 }
2642
wanghanlin8919fec2021-03-18 20:00:41 +08002643 /* Request to only drain unhandled to prevent libc_epoll_wait starved */
2644 rv = vls_epoll_wait (ep_vlsh, events, maxevents, -2);
hanlin4266d4d2020-05-19 17:34:17 +08002645 if (rv > 0)
2646 goto done;
hanlina3a48962020-07-13 11:09:15 +08002647 else if (PREDICT_FALSE (rv < 0))
hanlin4266d4d2020-05-19 17:34:17 +08002648 {
2649 errno = -rv;
2650 rv = -1;
2651 goto done;
2652 }
2653
Florin Coras7a1e95a2023-03-01 00:45:31 -08002654epoll_again:
hanlin4266d4d2020-05-19 17:34:17 +08002655 rv = libc_epoll_pwait (libc_epfd, events, maxevents, timeout, sigmask);
2656 if (rv <= 0)
2657 goto done;
2658 for (int i = 0; i < rv; i++)
2659 {
2660 if (events[i].data.fd == ldpw->vcl_mq_epfd)
2661 {
2662 /* We should remove mq epoll fd from events. */
2663 rv--;
2664 if (i != rv)
2665 {
2666 events[i].events = events[rv].events;
2667 events[i].data.u64 = events[rv].data.u64;
2668 }
2669 num_ev = vls_epoll_wait (ep_vlsh, &events[rv], maxevents - rv, 0);
2670 if (PREDICT_TRUE (num_ev > 0))
2671 rv += num_ev;
Florin Coras7a1e95a2023-03-01 00:45:31 -08002672 /* Woken up by vcl but no events generated. Accept it once */
2673 if (rv == 0 && vcl_wups++ < 1)
2674 goto epoll_again;
hanlin4266d4d2020-05-19 17:34:17 +08002675 break;
2676 }
2677 }
2678
2679done:
2680 return rv;
2681}
2682
Dave Wallace048b1d62018-01-03 22:24:41 -05002683int
2684epoll_pwait (int epfd, struct epoll_event *events,
2685 int maxevents, int timeout, const sigset_t * sigmask)
2686{
hanlin4266d4d2020-05-19 17:34:17 +08002687 if (vls_use_eventfd ())
2688 return ldp_epoll_pwait_eventfd (epfd, events, maxevents, timeout,
2689 sigmask);
2690 else
2691 return ldp_epoll_pwait (epfd, events, maxevents, timeout, sigmask);
Dave Wallace048b1d62018-01-03 22:24:41 -05002692}
2693
2694int
2695epoll_wait (int epfd, struct epoll_event *events, int maxevents, int timeout)
2696{
hanlin4266d4d2020-05-19 17:34:17 +08002697 if (vls_use_eventfd ())
2698 return ldp_epoll_pwait_eventfd (epfd, events, maxevents, timeout, NULL);
2699 else
2700 return ldp_epoll_pwait (epfd, events, maxevents, timeout, NULL);
Dave Wallace048b1d62018-01-03 22:24:41 -05002701}
2702
2703int
2704poll (struct pollfd *fds, nfds_t nfds, int timeout)
2705{
Florin Corasdfe4cf42018-11-28 22:13:45 -08002706 ldp_worker_ctx_t *ldpw = ldp_worker_get_current ();
Florin Coras6917b942018-11-13 22:44:54 -08002707 int rv, i, n_revents = 0;
Florin Coras7baeb712019-01-04 17:05:43 -08002708 vls_handle_t vlsh;
Dave Wallace048b1d62018-01-03 22:24:41 -05002709 vcl_poll_t *vp;
Florin Coras4dee8cd2019-01-29 21:28:16 -08002710 double max_time;
Dave Wallace048b1d62018-01-03 22:24:41 -05002711
Florin Coraseb801d02020-09-16 17:44:58 -07002712 LDBG (3, "fds %p, nfds %ld, timeout %d", fds, nfds, timeout);
Dave Wallace048b1d62018-01-03 22:24:41 -05002713
Florin Coras4dee8cd2019-01-29 21:28:16 -08002714 if (PREDICT_FALSE (ldpw->clib_time.init_cpu_time == 0))
2715 clib_time_init (&ldpw->clib_time);
2716
2717 max_time = (timeout >= 0) ? (f64) timeout / 1000 : 0;
2718 max_time += clib_time_now (&ldpw->clib_time);
Dave Wallace048b1d62018-01-03 22:24:41 -05002719
Dave Wallace048b1d62018-01-03 22:24:41 -05002720 for (i = 0; i < nfds; i++)
2721 {
Florin Coras6917b942018-11-13 22:44:54 -08002722 if (fds[i].fd < 0)
2723 continue;
Dave Wallace048b1d62018-01-03 22:24:41 -05002724
Florin Coras7baeb712019-01-04 17:05:43 -08002725 vlsh = ldp_fd_to_vlsh (fds[i].fd);
2726 if (vlsh != VLS_INVALID_HANDLE)
Florin Coras6917b942018-11-13 22:44:54 -08002727 {
2728 fds[i].fd = -fds[i].fd;
Florin Corasdfe4cf42018-11-28 22:13:45 -08002729 vec_add2 (ldpw->vcl_poll, vp, 1);
Florin Coras6917b942018-11-13 22:44:54 -08002730 vp->fds_ndx = i;
Florin Coras7baeb712019-01-04 17:05:43 -08002731 vp->sh = vlsh_to_sh (vlsh);
Florin Coras6917b942018-11-13 22:44:54 -08002732 vp->events = fds[i].events;
Dave Wallace048b1d62018-01-03 22:24:41 -05002733#ifdef __USE_XOPEN2K
Florin Coras6917b942018-11-13 22:44:54 -08002734 if (fds[i].events & POLLRDNORM)
2735 vp->events |= POLLIN;
2736 if (fds[i].events & POLLWRNORM)
2737 vp->events |= POLLOUT;
Dave Wallace048b1d62018-01-03 22:24:41 -05002738#endif
Florin Coras6917b942018-11-13 22:44:54 -08002739 vp->revents = fds[i].revents;
2740 }
2741 else
2742 {
Florin Corasdfe4cf42018-11-28 22:13:45 -08002743 vec_add1 (ldpw->libc_poll, fds[i]);
2744 vec_add1 (ldpw->libc_poll_idxs, i);
Dave Wallace048b1d62018-01-03 22:24:41 -05002745 }
2746 }
2747
Dave Wallace048b1d62018-01-03 22:24:41 -05002748 do
2749 {
Florin Corasdfe4cf42018-11-28 22:13:45 -08002750 if (vec_len (ldpw->vcl_poll))
Dave Wallace048b1d62018-01-03 22:24:41 -05002751 {
Florin Corasdfe4cf42018-11-28 22:13:45 -08002752 rv = vppcom_poll (ldpw->vcl_poll, vec_len (ldpw->vcl_poll), 0);
Dave Wallace048b1d62018-01-03 22:24:41 -05002753 if (rv < 0)
2754 {
2755 errno = -rv;
2756 rv = -1;
2757 goto done;
2758 }
2759 else
2760 n_revents += rv;
2761 }
2762
Florin Corasdfe4cf42018-11-28 22:13:45 -08002763 if (vec_len (ldpw->libc_poll))
Dave Wallace048b1d62018-01-03 22:24:41 -05002764 {
Florin Corasdfe4cf42018-11-28 22:13:45 -08002765 rv = libc_poll (ldpw->libc_poll, vec_len (ldpw->libc_poll), 0);
Dave Wallace048b1d62018-01-03 22:24:41 -05002766 if (rv < 0)
2767 goto done;
2768 else
2769 n_revents += rv;
2770 }
2771
2772 if (n_revents)
2773 {
2774 rv = n_revents;
2775 goto done;
2776 }
2777 }
Florin Coras4dee8cd2019-01-29 21:28:16 -08002778 while ((timeout < 0) || (clib_time_now (&ldpw->clib_time) < max_time));
Dave Wallace048b1d62018-01-03 22:24:41 -05002779 rv = 0;
2780
2781done:
Florin Corasdfe4cf42018-11-28 22:13:45 -08002782 vec_foreach (vp, ldpw->vcl_poll)
Dave Wallace048b1d62018-01-03 22:24:41 -05002783 {
2784 fds[vp->fds_ndx].fd = -fds[vp->fds_ndx].fd;
Florin Coras6917b942018-11-13 22:44:54 -08002785 fds[vp->fds_ndx].revents = vp->revents;
Dave Wallace048b1d62018-01-03 22:24:41 -05002786#ifdef __USE_XOPEN2K
2787 if ((fds[vp->fds_ndx].revents & POLLIN) &&
2788 (fds[vp->fds_ndx].events & POLLRDNORM))
2789 fds[vp->fds_ndx].revents |= POLLRDNORM;
2790 if ((fds[vp->fds_ndx].revents & POLLOUT) &&
2791 (fds[vp->fds_ndx].events & POLLWRNORM))
2792 fds[vp->fds_ndx].revents |= POLLWRNORM;
2793#endif
2794 }
Florin Corasdfe4cf42018-11-28 22:13:45 -08002795 vec_reset_length (ldpw->vcl_poll);
Dave Wallace048b1d62018-01-03 22:24:41 -05002796
Florin Corasdfe4cf42018-11-28 22:13:45 -08002797 for (i = 0; i < vec_len (ldpw->libc_poll); i++)
Florin Coras6917b942018-11-13 22:44:54 -08002798 {
Florin Corasdfe4cf42018-11-28 22:13:45 -08002799 fds[ldpw->libc_poll_idxs[i]].revents = ldpw->libc_poll[i].revents;
Florin Coras6917b942018-11-13 22:44:54 -08002800 }
Florin Corasdfe4cf42018-11-28 22:13:45 -08002801 vec_reset_length (ldpw->libc_poll_idxs);
2802 vec_reset_length (ldpw->libc_poll);
Florin Coras6917b942018-11-13 22:44:54 -08002803
Dave Wallace048b1d62018-01-03 22:24:41 -05002804 return rv;
2805}
2806
Florin Coras36847942023-02-02 12:56:16 -08002807#ifdef _GNU_SOURCE
Dave Wallace048b1d62018-01-03 22:24:41 -05002808int
2809ppoll (struct pollfd *fds, nfds_t nfds,
2810 const struct timespec *timeout, const sigset_t * sigmask)
2811{
Florin Coras5f33d0d2021-06-02 21:22:21 -07002812 ldp_init_check ();
Dave Wallace048b1d62018-01-03 22:24:41 -05002813
2814 clib_warning ("LDP<%d>: LDP-TBD", getpid ());
2815 errno = ENOSYS;
2816
2817
2818 return -1;
2819}
Keith Burns (alagalah)b327c2b2017-10-09 08:52:59 -07002820#endif
2821
Dave Wallace2a865272018-02-07 21:00:42 -05002822void CONSTRUCTOR_ATTRIBUTE ldp_constructor (void);
Keith Burns (alagalah)b327c2b2017-10-09 08:52:59 -07002823
Dave Wallace2a865272018-02-07 21:00:42 -05002824void DESTRUCTOR_ATTRIBUTE ldp_destructor (void);
Keith Burns (alagalah)b327c2b2017-10-09 08:52:59 -07002825
Dave Wallace048b1d62018-01-03 22:24:41 -05002826/*
2827 * This function is called when the library is loaded
2828 */
Keith Burns (alagalah)b327c2b2017-10-09 08:52:59 -07002829void
Dave Wallace2a865272018-02-07 21:00:42 -05002830ldp_constructor (void)
Keith Burns (alagalah)b327c2b2017-10-09 08:52:59 -07002831{
Keith Burns (alagalah)b327c2b2017-10-09 08:52:59 -07002832 swrap_constructor ();
Dave Wallace2a865272018-02-07 21:00:42 -05002833 if (ldp_init () != 0)
Florin Corasd89411e2019-03-19 19:44:51 -07002834 {
2835 fprintf (stderr, "\nLDP<%d>: ERROR: ldp_constructor: failed!\n",
2836 getpid ());
2837 _exit (1);
2838 }
Dave Wallace69d01192018-02-22 16:22:09 -05002839 else if (LDP_DEBUG > 0)
Dave Wallace2a865272018-02-07 21:00:42 -05002840 clib_warning ("LDP<%d>: LDP constructor: done!\n", getpid ());
Keith Burns (alagalah)b327c2b2017-10-09 08:52:59 -07002841}
2842
2843/*
2844 * This function is called when the library is unloaded
2845 */
2846void
Dave Wallace2a865272018-02-07 21:00:42 -05002847ldp_destructor (void)
Keith Burns (alagalah)b327c2b2017-10-09 08:52:59 -07002848{
Florin Coras0ef8ef22019-01-18 08:37:13 -08002849 /*
2850 swrap_destructor ();
2851 if (ldp->init)
2852 ldp->init = 0;
2853 */
Dave Wallace048b1d62018-01-03 22:24:41 -05002854
2855 /* Don't use clib_warning() here because that calls writev()
Dave Wallace2a865272018-02-07 21:00:42 -05002856 * which will call ldp_init().
Dave Wallace048b1d62018-01-03 22:24:41 -05002857 */
Dave Wallace69d01192018-02-22 16:22:09 -05002858 if (LDP_DEBUG > 0)
Florin Coras0ef8ef22019-01-18 08:37:13 -08002859 fprintf (stderr, "%s:%d: LDP<%d>: LDP destructor: done!\n",
2860 __func__, __LINE__, getpid ());
Keith Burns (alagalah)b327c2b2017-10-09 08:52:59 -07002861}
2862
2863
2864/*
2865 * fd.io coding-style-patch-verification: ON
2866 *
2867 * Local Variables:
2868 * eval: (c-set-style "gnu")
2869 * End:
2870 */