Florin Coras | 8c1be05 | 2022-10-17 11:52:34 -0700 | [diff] [blame] | 1 | /* SPDX-License-Identifier: Apache-2.0 |
| 2 | * Copyright(c) 2022 Cisco Systems, Inc. |
| 3 | */ |
| 4 | |
| 5 | #include <vnet/udp/udp.h> |
| 6 | #include <vnet/ip/ip4_inlines.h> |
| 7 | #include <vnet/ip/ip6_inlines.h> |
| 8 | |
| 9 | #define udp_node_index(node_id, is_ip4) \ |
| 10 | ((is_ip4) ? udp4_##node_id##_node.index : udp6_##node_id##_node.index) |
| 11 | |
| 12 | typedef enum udp_output_next_ |
| 13 | { |
| 14 | UDP_OUTPUT_NEXT_DROP, |
| 15 | UDP_OUTPUT_NEXT_IP_LOOKUP, |
| 16 | UDP_OUTPUT_N_NEXT |
| 17 | } udp_output_next_t; |
| 18 | |
| 19 | #define foreach_udp4_output_next \ |
| 20 | _ (DROP, "error-drop") \ |
| 21 | _ (IP_LOOKUP, "ip4-lookup") |
| 22 | |
| 23 | #define foreach_udp6_output_next \ |
| 24 | _ (DROP, "error-drop") \ |
| 25 | _ (IP_LOOKUP, "ip6-lookup") |
| 26 | |
| 27 | static vlib_error_desc_t udp_output_error_counters[] = { |
| 28 | #define udp_error(f, n, s, d) { #n, d, VL_COUNTER_SEVERITY_##s }, |
| 29 | #include <vnet/udp/udp_error.def> |
| 30 | #undef udp_error |
| 31 | }; |
| 32 | |
| 33 | typedef struct udp_tx_trace_ |
| 34 | { |
| 35 | udp_header_t udp_header; |
| 36 | udp_connection_t udp_connection; |
| 37 | } udp_tx_trace_t; |
| 38 | |
| 39 | static u8 * |
| 40 | format_udp_tx_trace (u8 *s, va_list *args) |
| 41 | { |
| 42 | CLIB_UNUSED (vlib_main_t * vm) = va_arg (*args, vlib_main_t *); |
| 43 | CLIB_UNUSED (vlib_node_t * node) = va_arg (*args, vlib_node_t *); |
| 44 | udp_tx_trace_t *t = va_arg (*args, udp_tx_trace_t *); |
| 45 | udp_connection_t *uc = &t->udp_connection; |
| 46 | u32 indent = format_get_indent (s); |
| 47 | |
Steven Luong | 16581f5 | 2023-03-06 20:28:51 -0800 | [diff] [blame] | 48 | s = format (s, "%U\n%U%U", format_udp_connection, uc, 1, format_white_space, |
Florin Coras | 8c1be05 | 2022-10-17 11:52:34 -0700 | [diff] [blame] | 49 | indent, format_udp_header, &t->udp_header, 128); |
| 50 | |
| 51 | return s; |
| 52 | } |
| 53 | |
Steven Luong | a361a39 | 2023-03-13 11:07:40 -0700 | [diff] [blame] | 54 | always_inline udp_connection_t * |
| 55 | udp_output_get_connection (vlib_buffer_t *b, u32 thread_index) |
| 56 | { |
| 57 | if (PREDICT_FALSE (vnet_buffer (b)->tcp.flags & UDP_CONN_F_LISTEN)) |
| 58 | return udp_listener_get (vnet_buffer (b)->tcp.connection_index); |
| 59 | |
| 60 | return udp_connection_get (vnet_buffer (b)->tcp.connection_index, |
| 61 | thread_index); |
| 62 | } |
| 63 | |
Florin Coras | 8c1be05 | 2022-10-17 11:52:34 -0700 | [diff] [blame] | 64 | static void |
| 65 | udp46_output_trace_frame (vlib_main_t *vm, vlib_node_runtime_t *node, |
| 66 | u32 *to_next, u32 n_bufs) |
| 67 | { |
| 68 | udp_connection_t *uc; |
| 69 | udp_tx_trace_t *t; |
| 70 | vlib_buffer_t *b; |
| 71 | udp_header_t *uh; |
| 72 | int i; |
| 73 | |
| 74 | for (i = 0; i < n_bufs; i++) |
| 75 | { |
| 76 | b = vlib_get_buffer (vm, to_next[i]); |
| 77 | if (!(b->flags & VLIB_BUFFER_IS_TRACED)) |
| 78 | continue; |
| 79 | uh = vlib_buffer_get_current (b); |
Steven Luong | a361a39 | 2023-03-13 11:07:40 -0700 | [diff] [blame] | 80 | uc = udp_output_get_connection (b, vm->thread_index); |
Florin Coras | 8c1be05 | 2022-10-17 11:52:34 -0700 | [diff] [blame] | 81 | t = vlib_add_trace (vm, node, b, sizeof (*t)); |
| 82 | clib_memcpy_fast (&t->udp_header, uh, sizeof (t->udp_header)); |
| 83 | clib_memcpy_fast (&t->udp_connection, uc, sizeof (t->udp_connection)); |
| 84 | } |
| 85 | } |
| 86 | |
| 87 | always_inline void |
Florin Coras | 8c1be05 | 2022-10-17 11:52:34 -0700 | [diff] [blame] | 88 | udp_output_handle_packet (udp_connection_t *uc0, vlib_buffer_t *b0, |
| 89 | vlib_node_runtime_t *error_node, u16 *next0, |
| 90 | u8 is_ip4) |
| 91 | { |
| 92 | /* If next_index is not drop use it */ |
| 93 | if (uc0->next_node_index) |
| 94 | { |
| 95 | *next0 = uc0->next_node_index; |
| 96 | vnet_buffer (b0)->tcp.next_node_opaque = uc0->next_node_opaque; |
| 97 | } |
| 98 | else |
| 99 | { |
| 100 | *next0 = UDP_OUTPUT_NEXT_IP_LOOKUP; |
| 101 | } |
| 102 | |
| 103 | vnet_buffer (b0)->sw_if_index[VLIB_TX] = uc0->c_fib_index; |
| 104 | vnet_buffer (b0)->sw_if_index[VLIB_RX] = uc0->sw_if_index; |
| 105 | } |
| 106 | |
| 107 | always_inline uword |
| 108 | udp46_output_inline (vlib_main_t *vm, vlib_node_runtime_t *node, |
| 109 | vlib_frame_t *frame, int is_ip4) |
| 110 | { |
| 111 | u32 n_left_from, *from, thread_index = vm->thread_index; |
| 112 | vlib_buffer_t *bufs[VLIB_FRAME_SIZE], **b; |
| 113 | u16 nexts[VLIB_FRAME_SIZE], *next; |
| 114 | |
| 115 | from = vlib_frame_vector_args (frame); |
| 116 | n_left_from = frame->n_vectors; |
| 117 | |
| 118 | if (PREDICT_FALSE (node->flags & VLIB_NODE_FLAG_TRACE)) |
| 119 | udp46_output_trace_frame (vm, node, from, n_left_from); |
| 120 | |
| 121 | vlib_get_buffers (vm, from, bufs, n_left_from); |
| 122 | b = bufs; |
| 123 | next = nexts; |
| 124 | |
| 125 | while (n_left_from >= 4) |
| 126 | { |
| 127 | udp_connection_t *uc0, *uc1; |
| 128 | |
| 129 | vlib_prefetch_buffer_header (b[2], STORE); |
| 130 | CLIB_PREFETCH (b[2]->data, 2 * CLIB_CACHE_LINE_BYTES, STORE); |
| 131 | |
| 132 | vlib_prefetch_buffer_header (b[3], STORE); |
| 133 | CLIB_PREFETCH (b[3]->data, 2 * CLIB_CACHE_LINE_BYTES, STORE); |
| 134 | |
Florin Coras | 15952b2 | 2022-12-19 10:55:18 -0800 | [diff] [blame] | 135 | uc0 = udp_output_get_connection (b[0], thread_index); |
| 136 | uc1 = udp_output_get_connection (b[1], thread_index); |
Florin Coras | 8c1be05 | 2022-10-17 11:52:34 -0700 | [diff] [blame] | 137 | |
| 138 | if (PREDICT_TRUE (!uc0 + !uc1 == 0)) |
| 139 | { |
Florin Coras | 8c1be05 | 2022-10-17 11:52:34 -0700 | [diff] [blame] | 140 | udp_output_handle_packet (uc0, b[0], node, &next[0], is_ip4); |
| 141 | udp_output_handle_packet (uc1, b[1], node, &next[1], is_ip4); |
| 142 | } |
| 143 | else |
| 144 | { |
| 145 | if (uc0 != 0) |
| 146 | { |
Florin Coras | 8c1be05 | 2022-10-17 11:52:34 -0700 | [diff] [blame] | 147 | udp_output_handle_packet (uc0, b[0], node, &next[0], is_ip4); |
| 148 | } |
| 149 | else |
| 150 | { |
| 151 | b[0]->error = node->errors[UDP_ERROR_INVALID_CONNECTION]; |
| 152 | next[0] = UDP_OUTPUT_NEXT_DROP; |
| 153 | } |
| 154 | if (uc1 != 0) |
| 155 | { |
Florin Coras | 8c1be05 | 2022-10-17 11:52:34 -0700 | [diff] [blame] | 156 | udp_output_handle_packet (uc1, b[1], node, &next[1], is_ip4); |
| 157 | } |
| 158 | else |
| 159 | { |
| 160 | b[1]->error = node->errors[UDP_ERROR_INVALID_CONNECTION]; |
| 161 | next[1] = UDP_OUTPUT_NEXT_DROP; |
| 162 | } |
| 163 | } |
| 164 | |
| 165 | b += 2; |
| 166 | next += 2; |
| 167 | n_left_from -= 2; |
| 168 | } |
| 169 | while (n_left_from > 0) |
| 170 | { |
| 171 | udp_connection_t *uc0; |
| 172 | |
| 173 | if (n_left_from > 1) |
| 174 | { |
| 175 | vlib_prefetch_buffer_header (b[1], STORE); |
| 176 | CLIB_PREFETCH (b[1]->data, 2 * CLIB_CACHE_LINE_BYTES, STORE); |
| 177 | } |
| 178 | |
Florin Coras | 15952b2 | 2022-12-19 10:55:18 -0800 | [diff] [blame] | 179 | uc0 = udp_output_get_connection (b[0], thread_index); |
Florin Coras | 8c1be05 | 2022-10-17 11:52:34 -0700 | [diff] [blame] | 180 | |
| 181 | if (PREDICT_TRUE (uc0 != 0)) |
| 182 | { |
Florin Coras | 8c1be05 | 2022-10-17 11:52:34 -0700 | [diff] [blame] | 183 | udp_output_handle_packet (uc0, b[0], node, &next[0], is_ip4); |
| 184 | } |
| 185 | else |
| 186 | { |
| 187 | b[0]->error = node->errors[UDP_ERROR_INVALID_CONNECTION]; |
| 188 | next[0] = UDP_OUTPUT_NEXT_DROP; |
| 189 | } |
| 190 | |
| 191 | b += 1; |
| 192 | next += 1; |
| 193 | n_left_from -= 1; |
| 194 | } |
| 195 | |
| 196 | vlib_buffer_enqueue_to_next (vm, node, from, nexts, frame->n_vectors); |
| 197 | vlib_node_increment_counter (vm, udp_node_index (output, is_ip4), |
| 198 | UDP_ERROR_PKTS_SENT, frame->n_vectors); |
| 199 | return frame->n_vectors; |
| 200 | } |
| 201 | |
| 202 | VLIB_NODE_FN (udp4_output_node) |
| 203 | (vlib_main_t *vm, vlib_node_runtime_t *node, vlib_frame_t *from_frame) |
| 204 | { |
| 205 | return udp46_output_inline (vm, node, from_frame, 1 /* is_ip4 */); |
| 206 | } |
| 207 | |
| 208 | VLIB_NODE_FN (udp6_output_node) |
| 209 | (vlib_main_t *vm, vlib_node_runtime_t *node, vlib_frame_t *from_frame) |
| 210 | { |
| 211 | return udp46_output_inline (vm, node, from_frame, 0 /* is_ip4 */); |
| 212 | } |
| 213 | |
| 214 | VLIB_REGISTER_NODE (udp4_output_node) = |
| 215 | { |
| 216 | .name = "udp4-output", |
| 217 | .vector_size = sizeof (u32), |
| 218 | .n_errors = UDP_N_ERROR, |
| 219 | .protocol_hint = VLIB_NODE_PROTO_HINT_UDP, |
| 220 | .error_counters = udp_output_error_counters, |
| 221 | .n_next_nodes = UDP_OUTPUT_N_NEXT, |
| 222 | .next_nodes = { |
| 223 | #define _(s, n) [UDP_OUTPUT_NEXT_##s] = n, |
| 224 | foreach_udp4_output_next |
| 225 | #undef _ |
| 226 | }, |
| 227 | .format_buffer = format_udp_header, |
| 228 | .format_trace = format_udp_tx_trace, |
| 229 | }; |
| 230 | |
| 231 | VLIB_REGISTER_NODE (udp6_output_node) = |
| 232 | { |
| 233 | .name = "udp6-output", |
| 234 | .vector_size = sizeof (u32), |
| 235 | .n_errors = UDP_N_ERROR, |
| 236 | .protocol_hint = VLIB_NODE_PROTO_HINT_UDP, |
| 237 | .error_counters = udp_output_error_counters, |
| 238 | .n_next_nodes = UDP_OUTPUT_N_NEXT, |
| 239 | .next_nodes = { |
| 240 | #define _(s, n) [UDP_OUTPUT_NEXT_##s] = n, |
| 241 | foreach_udp6_output_next |
| 242 | #undef _ |
| 243 | }, |
| 244 | .format_buffer = format_udp_header, |
| 245 | .format_trace = format_udp_tx_trace, |
| 246 | }; |
| 247 | |
| 248 | /* |
| 249 | * fd.io coding-style-patch-verification: ON |
| 250 | * |
| 251 | * Local Variables: |
| 252 | * eval: (c-set-style "gnu") |
| 253 | * End: |
| 254 | */ |