| /* SPDX-License-Identifier: Apache-2.0 |
| * Copyright(c) 2022 Cisco Systems, Inc. |
| */ |
| |
| #include <vnet/udp/udp.h> |
| #include <vnet/ip/ip4_inlines.h> |
| #include <vnet/ip/ip6_inlines.h> |
| |
| #define udp_node_index(node_id, is_ip4) \ |
| ((is_ip4) ? udp4_##node_id##_node.index : udp6_##node_id##_node.index) |
| |
| typedef enum udp_output_next_ |
| { |
| UDP_OUTPUT_NEXT_DROP, |
| UDP_OUTPUT_NEXT_IP_LOOKUP, |
| UDP_OUTPUT_N_NEXT |
| } udp_output_next_t; |
| |
| #define foreach_udp4_output_next \ |
| _ (DROP, "error-drop") \ |
| _ (IP_LOOKUP, "ip4-lookup") |
| |
| #define foreach_udp6_output_next \ |
| _ (DROP, "error-drop") \ |
| _ (IP_LOOKUP, "ip6-lookup") |
| |
| static vlib_error_desc_t udp_output_error_counters[] = { |
| #define udp_error(f, n, s, d) { #n, d, VL_COUNTER_SEVERITY_##s }, |
| #include <vnet/udp/udp_error.def> |
| #undef udp_error |
| }; |
| |
| typedef struct udp_tx_trace_ |
| { |
| udp_header_t udp_header; |
| udp_connection_t udp_connection; |
| } udp_tx_trace_t; |
| |
| static u8 * |
| format_udp_tx_trace (u8 *s, va_list *args) |
| { |
| CLIB_UNUSED (vlib_main_t * vm) = va_arg (*args, vlib_main_t *); |
| CLIB_UNUSED (vlib_node_t * node) = va_arg (*args, vlib_node_t *); |
| udp_tx_trace_t *t = va_arg (*args, udp_tx_trace_t *); |
| udp_connection_t *uc = &t->udp_connection; |
| u32 indent = format_get_indent (s); |
| |
| s = format (s, "%U\n%U%U", format_udp_connection, uc, format_white_space, |
| indent, format_udp_header, &t->udp_header, 128); |
| |
| return s; |
| } |
| |
| static void |
| udp46_output_trace_frame (vlib_main_t *vm, vlib_node_runtime_t *node, |
| u32 *to_next, u32 n_bufs) |
| { |
| udp_connection_t *uc; |
| udp_tx_trace_t *t; |
| vlib_buffer_t *b; |
| udp_header_t *uh; |
| int i; |
| |
| for (i = 0; i < n_bufs; i++) |
| { |
| b = vlib_get_buffer (vm, to_next[i]); |
| if (!(b->flags & VLIB_BUFFER_IS_TRACED)) |
| continue; |
| uh = vlib_buffer_get_current (b); |
| uc = udp_connection_get (vnet_buffer (b)->tcp.connection_index, |
| vm->thread_index); |
| t = vlib_add_trace (vm, node, b, sizeof (*t)); |
| clib_memcpy_fast (&t->udp_header, uh, sizeof (t->udp_header)); |
| clib_memcpy_fast (&t->udp_connection, uc, sizeof (t->udp_connection)); |
| } |
| } |
| |
| always_inline void |
| udp_output_push_ip (vlib_main_t *vm, vlib_buffer_t *b, udp_connection_t *uc, |
| u8 is_ip4) |
| { |
| if (uc->c_is_ip4) |
| vlib_buffer_push_ip4_custom (vm, b, &uc->c_lcl_ip4, &uc->c_rmt_ip4, |
| IP_PROTOCOL_UDP, 1 /* csum offload */, |
| 0 /* is_df */, uc->c_dscp); |
| else |
| vlib_buffer_push_ip6 (vm, b, &uc->c_lcl_ip6, &uc->c_rmt_ip6, |
| IP_PROTOCOL_UDP); |
| vnet_buffer (b)->sw_if_index[VLIB_RX] = uc->sw_if_index; |
| vnet_buffer (b)->sw_if_index[VLIB_TX] = uc->c_fib_index; |
| } |
| |
| always_inline void |
| udp_output_handle_packet (udp_connection_t *uc0, vlib_buffer_t *b0, |
| vlib_node_runtime_t *error_node, u16 *next0, |
| u8 is_ip4) |
| { |
| /* If next_index is not drop use it */ |
| if (uc0->next_node_index) |
| { |
| *next0 = uc0->next_node_index; |
| vnet_buffer (b0)->tcp.next_node_opaque = uc0->next_node_opaque; |
| } |
| else |
| { |
| *next0 = UDP_OUTPUT_NEXT_IP_LOOKUP; |
| } |
| |
| vnet_buffer (b0)->sw_if_index[VLIB_TX] = uc0->c_fib_index; |
| vnet_buffer (b0)->sw_if_index[VLIB_RX] = uc0->sw_if_index; |
| } |
| |
| always_inline uword |
| udp46_output_inline (vlib_main_t *vm, vlib_node_runtime_t *node, |
| vlib_frame_t *frame, int is_ip4) |
| { |
| u32 n_left_from, *from, thread_index = vm->thread_index; |
| vlib_buffer_t *bufs[VLIB_FRAME_SIZE], **b; |
| u16 nexts[VLIB_FRAME_SIZE], *next; |
| |
| from = vlib_frame_vector_args (frame); |
| n_left_from = frame->n_vectors; |
| |
| if (PREDICT_FALSE (node->flags & VLIB_NODE_FLAG_TRACE)) |
| udp46_output_trace_frame (vm, node, from, n_left_from); |
| |
| vlib_get_buffers (vm, from, bufs, n_left_from); |
| b = bufs; |
| next = nexts; |
| |
| while (n_left_from >= 4) |
| { |
| udp_connection_t *uc0, *uc1; |
| |
| vlib_prefetch_buffer_header (b[2], STORE); |
| CLIB_PREFETCH (b[2]->data, 2 * CLIB_CACHE_LINE_BYTES, STORE); |
| |
| vlib_prefetch_buffer_header (b[3], STORE); |
| CLIB_PREFETCH (b[3]->data, 2 * CLIB_CACHE_LINE_BYTES, STORE); |
| |
| uc0 = udp_connection_get (vnet_buffer (b[0])->tcp.connection_index, |
| thread_index); |
| uc1 = udp_connection_get (vnet_buffer (b[1])->tcp.connection_index, |
| thread_index); |
| |
| if (PREDICT_TRUE (!uc0 + !uc1 == 0)) |
| { |
| udp_output_push_ip (vm, b[0], uc0, is_ip4); |
| udp_output_push_ip (vm, b[1], uc1, is_ip4); |
| |
| udp_output_handle_packet (uc0, b[0], node, &next[0], is_ip4); |
| udp_output_handle_packet (uc1, b[1], node, &next[1], is_ip4); |
| } |
| else |
| { |
| if (uc0 != 0) |
| { |
| udp_output_push_ip (vm, b[0], uc0, is_ip4); |
| udp_output_handle_packet (uc0, b[0], node, &next[0], is_ip4); |
| } |
| else |
| { |
| b[0]->error = node->errors[UDP_ERROR_INVALID_CONNECTION]; |
| next[0] = UDP_OUTPUT_NEXT_DROP; |
| } |
| if (uc1 != 0) |
| { |
| udp_output_push_ip (vm, b[1], uc1, is_ip4); |
| udp_output_handle_packet (uc1, b[1], node, &next[1], is_ip4); |
| } |
| else |
| { |
| b[1]->error = node->errors[UDP_ERROR_INVALID_CONNECTION]; |
| next[1] = UDP_OUTPUT_NEXT_DROP; |
| } |
| } |
| |
| b += 2; |
| next += 2; |
| n_left_from -= 2; |
| } |
| while (n_left_from > 0) |
| { |
| udp_connection_t *uc0; |
| |
| if (n_left_from > 1) |
| { |
| vlib_prefetch_buffer_header (b[1], STORE); |
| CLIB_PREFETCH (b[1]->data, 2 * CLIB_CACHE_LINE_BYTES, STORE); |
| } |
| |
| uc0 = udp_connection_get (vnet_buffer (b[0])->tcp.connection_index, |
| thread_index); |
| |
| if (PREDICT_TRUE (uc0 != 0)) |
| { |
| udp_output_push_ip (vm, b[0], uc0, is_ip4); |
| udp_output_handle_packet (uc0, b[0], node, &next[0], is_ip4); |
| } |
| else |
| { |
| b[0]->error = node->errors[UDP_ERROR_INVALID_CONNECTION]; |
| next[0] = UDP_OUTPUT_NEXT_DROP; |
| } |
| |
| b += 1; |
| next += 1; |
| n_left_from -= 1; |
| } |
| |
| vlib_buffer_enqueue_to_next (vm, node, from, nexts, frame->n_vectors); |
| vlib_node_increment_counter (vm, udp_node_index (output, is_ip4), |
| UDP_ERROR_PKTS_SENT, frame->n_vectors); |
| return frame->n_vectors; |
| } |
| |
| VLIB_NODE_FN (udp4_output_node) |
| (vlib_main_t *vm, vlib_node_runtime_t *node, vlib_frame_t *from_frame) |
| { |
| return udp46_output_inline (vm, node, from_frame, 1 /* is_ip4 */); |
| } |
| |
| VLIB_NODE_FN (udp6_output_node) |
| (vlib_main_t *vm, vlib_node_runtime_t *node, vlib_frame_t *from_frame) |
| { |
| return udp46_output_inline (vm, node, from_frame, 0 /* is_ip4 */); |
| } |
| |
| VLIB_REGISTER_NODE (udp4_output_node) = |
| { |
| .name = "udp4-output", |
| .vector_size = sizeof (u32), |
| .n_errors = UDP_N_ERROR, |
| .protocol_hint = VLIB_NODE_PROTO_HINT_UDP, |
| .error_counters = udp_output_error_counters, |
| .n_next_nodes = UDP_OUTPUT_N_NEXT, |
| .next_nodes = { |
| #define _(s, n) [UDP_OUTPUT_NEXT_##s] = n, |
| foreach_udp4_output_next |
| #undef _ |
| }, |
| .format_buffer = format_udp_header, |
| .format_trace = format_udp_tx_trace, |
| }; |
| |
| VLIB_REGISTER_NODE (udp6_output_node) = |
| { |
| .name = "udp6-output", |
| .vector_size = sizeof (u32), |
| .n_errors = UDP_N_ERROR, |
| .protocol_hint = VLIB_NODE_PROTO_HINT_UDP, |
| .error_counters = udp_output_error_counters, |
| .n_next_nodes = UDP_OUTPUT_N_NEXT, |
| .next_nodes = { |
| #define _(s, n) [UDP_OUTPUT_NEXT_##s] = n, |
| foreach_udp6_output_next |
| #undef _ |
| }, |
| .format_buffer = format_udp_header, |
| .format_trace = format_udp_tx_trace, |
| }; |
| |
| /* |
| * fd.io coding-style-patch-verification: ON |
| * |
| * Local Variables: |
| * eval: (c-set-style "gnu") |
| * End: |
| */ |