Ed Warnicke | cb9cada | 2015-12-08 15:45:58 -0700 | [diff] [blame] | 1 | /* |
| 2 | * replication.c : packet replication |
| 3 | * |
| 4 | * Copyright (c) 2013 Cisco and/or its affiliates. |
| 5 | * Licensed under the Apache License, Version 2.0 (the "License"); |
| 6 | * you may not use this file except in compliance with the License. |
| 7 | * You may obtain a copy of the License at: |
| 8 | * |
| 9 | * http://www.apache.org/licenses/LICENSE-2.0 |
| 10 | * |
| 11 | * Unless required by applicable law or agreed to in writing, software |
| 12 | * distributed under the License is distributed on an "AS IS" BASIS, |
| 13 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. |
| 14 | * See the License for the specific language governing permissions and |
| 15 | * limitations under the License. |
| 16 | */ |
| 17 | |
| 18 | #include <vlib/vlib.h> |
| 19 | #include <vnet/vnet.h> |
| 20 | #include <vppinfra/error.h> |
| 21 | #include <vnet/ip/ip4_packet.h> |
| 22 | #include <vnet/replication.h> |
| 23 | |
| 24 | |
| 25 | replication_main_t replication_main; |
| 26 | |
| 27 | |
| 28 | replication_context_t * |
| 29 | replication_prep (vlib_main_t * vm, |
Dave Barach | ba868bb | 2016-08-08 09:51:21 -0400 | [diff] [blame] | 30 | vlib_buffer_t * b0, u32 recycle_node_index, u32 l2_packet) |
Ed Warnicke | cb9cada | 2015-12-08 15:45:58 -0700 | [diff] [blame] | 31 | { |
Dave Barach | ba868bb | 2016-08-08 09:51:21 -0400 | [diff] [blame] | 32 | replication_main_t *rm = &replication_main; |
| 33 | replication_context_t *ctx; |
Damjan Marion | 586afd7 | 2017-04-05 19:18:20 +0200 | [diff] [blame] | 34 | uword thread_index = vm->thread_index; |
Dave Barach | ba868bb | 2016-08-08 09:51:21 -0400 | [diff] [blame] | 35 | ip4_header_t *ip; |
Ed Warnicke | cb9cada | 2015-12-08 15:45:58 -0700 | [diff] [blame] | 36 | u32 ctx_id; |
| 37 | |
Dave Barach | ba868bb | 2016-08-08 09:51:21 -0400 | [diff] [blame] | 38 | /* Allocate a context, reserve context 0 */ |
Damjan Marion | 586afd7 | 2017-04-05 19:18:20 +0200 | [diff] [blame] | 39 | if (PREDICT_FALSE (rm->contexts[thread_index] == 0)) |
| 40 | pool_get_aligned (rm->contexts[thread_index], ctx, CLIB_CACHE_LINE_BYTES); |
Dave Barach | ba868bb | 2016-08-08 09:51:21 -0400 | [diff] [blame] | 41 | |
Damjan Marion | 586afd7 | 2017-04-05 19:18:20 +0200 | [diff] [blame] | 42 | pool_get_aligned (rm->contexts[thread_index], ctx, CLIB_CACHE_LINE_BYTES); |
| 43 | ctx_id = ctx - rm->contexts[thread_index]; |
Ed Warnicke | cb9cada | 2015-12-08 15:45:58 -0700 | [diff] [blame] | 44 | |
Dave Barach | ba868bb | 2016-08-08 09:51:21 -0400 | [diff] [blame] | 45 | /* Save state from vlib buffer */ |
Damjan Marion | 072401e | 2017-07-13 18:53:27 +0200 | [diff] [blame^] | 46 | ctx->saved_free_list_index = vlib_buffer_get_free_list_index (b0); |
Ed Warnicke | cb9cada | 2015-12-08 15:45:58 -0700 | [diff] [blame] | 47 | ctx->current_data = b0->current_data; |
| 48 | |
Dave Barach | ba868bb | 2016-08-08 09:51:21 -0400 | [diff] [blame] | 49 | /* Set up vlib buffer hooks */ |
Dave Barach | b5adaea | 2016-06-17 14:09:56 -0400 | [diff] [blame] | 50 | b0->recycle_count = ctx_id; |
Damjan Marion | 072401e | 2017-07-13 18:53:27 +0200 | [diff] [blame^] | 51 | vlib_buffer_set_free_list_index (b0, rm->recycle_list_index); |
Dave Barach | b5adaea | 2016-06-17 14:09:56 -0400 | [diff] [blame] | 52 | b0->flags |= VLIB_BUFFER_RECYCLE; |
Ed Warnicke | cb9cada | 2015-12-08 15:45:58 -0700 | [diff] [blame] | 53 | |
Dave Barach | ba868bb | 2016-08-08 09:51:21 -0400 | [diff] [blame] | 54 | /* Save feature state */ |
Ed Warnicke | cb9cada | 2015-12-08 15:45:58 -0700 | [diff] [blame] | 55 | ctx->recycle_node_index = recycle_node_index; |
| 56 | |
Dave Barach | ba868bb | 2016-08-08 09:51:21 -0400 | [diff] [blame] | 57 | /* Save vnet state */ |
| 58 | clib_memcpy (ctx->vnet_buffer, vnet_buffer (b0), |
| 59 | sizeof (vnet_buffer_opaque_t)); |
Ed Warnicke | cb9cada | 2015-12-08 15:45:58 -0700 | [diff] [blame] | 60 | |
Dave Barach | ba868bb | 2016-08-08 09:51:21 -0400 | [diff] [blame] | 61 | /* Save packet contents */ |
Ed Warnicke | cb9cada | 2015-12-08 15:45:58 -0700 | [diff] [blame] | 62 | ctx->l2_packet = l2_packet; |
Dave Barach | ba868bb | 2016-08-08 09:51:21 -0400 | [diff] [blame] | 63 | ip = (ip4_header_t *) vlib_buffer_get_current (b0); |
| 64 | if (l2_packet) |
| 65 | { |
| 66 | /* Save ethernet header */ |
| 67 | ctx->l2_header[0] = ((u64 *) ip)[0]; |
| 68 | ctx->l2_header[1] = ((u64 *) ip)[1]; |
| 69 | ctx->l2_header[2] = ((u64 *) ip)[2]; |
| 70 | /* set ip to the true ip header */ |
| 71 | ip = (ip4_header_t *) (((u8 *) ip) + vnet_buffer (b0)->l2.l2_len); |
| 72 | } |
Ed Warnicke | cb9cada | 2015-12-08 15:45:58 -0700 | [diff] [blame] | 73 | |
Dave Barach | ba868bb | 2016-08-08 09:51:21 -0400 | [diff] [blame] | 74 | /* |
| 75 | * Copy L3 fields. |
| 76 | * We need to save TOS for ip4 and ip6 packets. |
| 77 | * Fortunately the TOS field is |
| 78 | * in the first two bytes of both the ip4 and ip6 headers. |
| 79 | */ |
| 80 | ctx->ip_tos = *((u16 *) (ip)); |
Ed Warnicke | cb9cada | 2015-12-08 15:45:58 -0700 | [diff] [blame] | 81 | |
Dave Barach | ba868bb | 2016-08-08 09:51:21 -0400 | [diff] [blame] | 82 | /* |
| 83 | * Save the ip4 checksum as well. We just blindly save the corresponding two |
| 84 | * bytes even for ip6 packets. |
| 85 | */ |
Ed Warnicke | cb9cada | 2015-12-08 15:45:58 -0700 | [diff] [blame] | 86 | ctx->ip4_checksum = ip->checksum; |
| 87 | |
| 88 | return ctx; |
| 89 | } |
| 90 | |
| 91 | |
| 92 | replication_context_t * |
Dave Barach | ba868bb | 2016-08-08 09:51:21 -0400 | [diff] [blame] | 93 | replication_recycle (vlib_main_t * vm, vlib_buffer_t * b0, u32 is_last) |
Ed Warnicke | cb9cada | 2015-12-08 15:45:58 -0700 | [diff] [blame] | 94 | { |
Dave Barach | ba868bb | 2016-08-08 09:51:21 -0400 | [diff] [blame] | 95 | replication_main_t *rm = &replication_main; |
| 96 | replication_context_t *ctx; |
Damjan Marion | 586afd7 | 2017-04-05 19:18:20 +0200 | [diff] [blame] | 97 | uword thread_index = vm->thread_index; |
Dave Barach | ba868bb | 2016-08-08 09:51:21 -0400 | [diff] [blame] | 98 | ip4_header_t *ip; |
Ed Warnicke | cb9cada | 2015-12-08 15:45:58 -0700 | [diff] [blame] | 99 | |
Dave Barach | ba868bb | 2016-08-08 09:51:21 -0400 | [diff] [blame] | 100 | /* Get access to the replication context */ |
Damjan Marion | 586afd7 | 2017-04-05 19:18:20 +0200 | [diff] [blame] | 101 | ctx = pool_elt_at_index (rm->contexts[thread_index], b0->recycle_count); |
Ed Warnicke | cb9cada | 2015-12-08 15:45:58 -0700 | [diff] [blame] | 102 | |
Dave Barach | ba868bb | 2016-08-08 09:51:21 -0400 | [diff] [blame] | 103 | /* Restore vnet buffer state */ |
| 104 | clib_memcpy (vnet_buffer (b0), ctx->vnet_buffer, |
| 105 | sizeof (vnet_buffer_opaque_t)); |
Ed Warnicke | cb9cada | 2015-12-08 15:45:58 -0700 | [diff] [blame] | 106 | |
Dave Barach | ba868bb | 2016-08-08 09:51:21 -0400 | [diff] [blame] | 107 | /* Restore the packet start (current_data) and length */ |
| 108 | vlib_buffer_advance (b0, ctx->current_data - b0->current_data); |
Ed Warnicke | cb9cada | 2015-12-08 15:45:58 -0700 | [diff] [blame] | 109 | |
Dave Barach | ba868bb | 2016-08-08 09:51:21 -0400 | [diff] [blame] | 110 | /* Restore packet contents */ |
| 111 | ip = (ip4_header_t *) vlib_buffer_get_current (b0); |
| 112 | if (ctx->l2_packet) |
| 113 | { |
| 114 | /* Restore ethernet header */ |
| 115 | ((u64 *) ip)[0] = ctx->l2_header[0]; |
| 116 | ((u64 *) ip)[1] = ctx->l2_header[1]; |
| 117 | ((u64 *) ip)[2] = ctx->l2_header[2]; |
| 118 | /* set ip to the true ip header */ |
| 119 | ip = (ip4_header_t *) (((u8 *) ip) + vnet_buffer (b0)->l2.l2_len); |
| 120 | } |
Ed Warnicke | cb9cada | 2015-12-08 15:45:58 -0700 | [diff] [blame] | 121 | |
| 122 | // Restore L3 fields |
Dave Barach | ba868bb | 2016-08-08 09:51:21 -0400 | [diff] [blame] | 123 | *((u16 *) (ip)) = ctx->ip_tos; |
Ed Warnicke | cb9cada | 2015-12-08 15:45:58 -0700 | [diff] [blame] | 124 | ip->checksum = ctx->ip4_checksum; |
| 125 | |
Dave Barach | ba868bb | 2016-08-08 09:51:21 -0400 | [diff] [blame] | 126 | if (is_last) |
| 127 | { |
| 128 | /* |
| 129 | * This is the last replication in the list. |
| 130 | * Restore original buffer free functionality. |
| 131 | */ |
Damjan Marion | 072401e | 2017-07-13 18:53:27 +0200 | [diff] [blame^] | 132 | vlib_buffer_set_free_list_index (b0, ctx->saved_free_list_index); |
Dave Barach | ba868bb | 2016-08-08 09:51:21 -0400 | [diff] [blame] | 133 | b0->flags &= ~VLIB_BUFFER_RECYCLE; |
Ed Warnicke | cb9cada | 2015-12-08 15:45:58 -0700 | [diff] [blame] | 134 | |
Dave Barach | ba868bb | 2016-08-08 09:51:21 -0400 | [diff] [blame] | 135 | /* Free context back to its pool */ |
Damjan Marion | 586afd7 | 2017-04-05 19:18:20 +0200 | [diff] [blame] | 136 | pool_put (rm->contexts[thread_index], ctx); |
Dave Barach | ba868bb | 2016-08-08 09:51:21 -0400 | [diff] [blame] | 137 | } |
Ed Warnicke | cb9cada | 2015-12-08 15:45:58 -0700 | [diff] [blame] | 138 | |
| 139 | return ctx; |
| 140 | } |
| 141 | |
| 142 | |
| 143 | |
| 144 | /* |
| 145 | * fish pkts back from the recycle queue/freelist |
| 146 | * un-flatten the context chains |
| 147 | */ |
Dave Barach | ba868bb | 2016-08-08 09:51:21 -0400 | [diff] [blame] | 148 | static void |
| 149 | replication_recycle_callback (vlib_main_t * vm, vlib_buffer_free_list_t * fl) |
Ed Warnicke | cb9cada | 2015-12-08 15:45:58 -0700 | [diff] [blame] | 150 | { |
Dave Barach | ba868bb | 2016-08-08 09:51:21 -0400 | [diff] [blame] | 151 | vlib_frame_t *f = 0; |
Ed Warnicke | cb9cada | 2015-12-08 15:45:58 -0700 | [diff] [blame] | 152 | u32 n_left_from; |
| 153 | u32 n_left_to_next = 0; |
| 154 | u32 n_this_frame = 0; |
Dave Barach | ba868bb | 2016-08-08 09:51:21 -0400 | [diff] [blame] | 155 | u32 *from; |
| 156 | u32 *to_next = 0; |
Ed Warnicke | cb9cada | 2015-12-08 15:45:58 -0700 | [diff] [blame] | 157 | u32 bi0, pi0; |
| 158 | vlib_buffer_t *b0; |
Ed Warnicke | cb9cada | 2015-12-08 15:45:58 -0700 | [diff] [blame] | 159 | int i; |
Dave Barach | ba868bb | 2016-08-08 09:51:21 -0400 | [diff] [blame] | 160 | replication_main_t *rm = &replication_main; |
| 161 | replication_context_t *ctx; |
| 162 | u32 feature_node_index = 0; |
Damjan Marion | 586afd7 | 2017-04-05 19:18:20 +0200 | [diff] [blame] | 163 | uword thread_index = vm->thread_index; |
Ed Warnicke | cb9cada | 2015-12-08 15:45:58 -0700 | [diff] [blame] | 164 | |
Dave Barach | ba868bb | 2016-08-08 09:51:21 -0400 | [diff] [blame] | 165 | /* |
| 166 | * All buffers in the list are destined to the same recycle node. |
| 167 | * Pull the recycle node index from the first buffer. |
| 168 | * Note: this could be sped up if the node index were stuffed into |
| 169 | * the freelist itself. |
| 170 | */ |
Damjan Marion | bd69a5f | 2017-02-05 23:44:42 +0100 | [diff] [blame] | 171 | if (vec_len (fl->buffers) > 0) |
Dave Barach | ba868bb | 2016-08-08 09:51:21 -0400 | [diff] [blame] | 172 | { |
Damjan Marion | bd69a5f | 2017-02-05 23:44:42 +0100 | [diff] [blame] | 173 | bi0 = fl->buffers[0]; |
Dave Barach | ba868bb | 2016-08-08 09:51:21 -0400 | [diff] [blame] | 174 | b0 = vlib_get_buffer (vm, bi0); |
Damjan Marion | 586afd7 | 2017-04-05 19:18:20 +0200 | [diff] [blame] | 175 | ctx = pool_elt_at_index (rm->contexts[thread_index], b0->recycle_count); |
Dave Barach | ba868bb | 2016-08-08 09:51:21 -0400 | [diff] [blame] | 176 | feature_node_index = ctx->recycle_node_index; |
| 177 | } |
Ed Warnicke | cb9cada | 2015-12-08 15:45:58 -0700 | [diff] [blame] | 178 | |
Damjan Marion | bd69a5f | 2017-02-05 23:44:42 +0100 | [diff] [blame] | 179 | /* buffers */ |
Dave Barach | ba868bb | 2016-08-08 09:51:21 -0400 | [diff] [blame] | 180 | for (i = 0; i < 2; i++) |
Ed Warnicke | cb9cada | 2015-12-08 15:45:58 -0700 | [diff] [blame] | 181 | { |
| 182 | if (i == 0) |
Dave Barach | ba868bb | 2016-08-08 09:51:21 -0400 | [diff] [blame] | 183 | { |
Damjan Marion | bd69a5f | 2017-02-05 23:44:42 +0100 | [diff] [blame] | 184 | from = fl->buffers; |
Dave Barach | ba868bb | 2016-08-08 09:51:21 -0400 | [diff] [blame] | 185 | n_left_from = vec_len (from); |
| 186 | } |
| 187 | |
Ed Warnicke | cb9cada | 2015-12-08 15:45:58 -0700 | [diff] [blame] | 188 | while (n_left_from > 0) |
Dave Barach | ba868bb | 2016-08-08 09:51:21 -0400 | [diff] [blame] | 189 | { |
| 190 | if (PREDICT_FALSE (n_left_to_next == 0)) |
| 191 | { |
| 192 | if (f) |
| 193 | { |
| 194 | f->n_vectors = n_this_frame; |
| 195 | vlib_put_frame_to_node (vm, feature_node_index, f); |
| 196 | } |
| 197 | |
| 198 | f = vlib_get_frame_to_node (vm, feature_node_index); |
| 199 | to_next = vlib_frame_vector_args (f); |
| 200 | n_left_to_next = VLIB_FRAME_SIZE; |
| 201 | n_this_frame = 0; |
| 202 | } |
| 203 | |
| 204 | bi0 = from[0]; |
| 205 | if (PREDICT_TRUE (n_left_from > 1)) |
| 206 | { |
| 207 | pi0 = from[1]; |
| 208 | vlib_prefetch_buffer_with_index (vm, pi0, LOAD); |
| 209 | } |
John Lo | 17f17cb | 2016-05-31 16:25:13 -0400 | [diff] [blame] | 210 | |
| 211 | b0 = vlib_get_buffer (vm, bi0); |
Ed Warnicke | cb9cada | 2015-12-08 15:45:58 -0700 | [diff] [blame] | 212 | |
Dave Barach | ba868bb | 2016-08-08 09:51:21 -0400 | [diff] [blame] | 213 | /* Mark that this buffer was just recycled */ |
| 214 | b0->flags |= VLIB_BUFFER_IS_RECYCLED; |
Ed Warnicke | cb9cada | 2015-12-08 15:45:58 -0700 | [diff] [blame] | 215 | |
Neale Ranns | b80c536 | 2016-10-08 13:03:40 +0100 | [diff] [blame] | 216 | #if (CLIB_DEBUG > 0) |
Damjan Marion | 04a7f05 | 2017-07-10 15:06:17 +0200 | [diff] [blame] | 217 | if (vm->buffer_main->callbacks_registered == 0) |
Damjan Marion | c3a814b | 2017-02-28 19:22:22 +0100 | [diff] [blame] | 218 | vlib_buffer_set_known_state (vm, bi0, |
| 219 | VLIB_BUFFER_KNOWN_ALLOCATED); |
Neale Ranns | b80c536 | 2016-10-08 13:03:40 +0100 | [diff] [blame] | 220 | #endif |
| 221 | |
Dave Barach | ba868bb | 2016-08-08 09:51:21 -0400 | [diff] [blame] | 222 | /* If buffer is traced, mark frame as traced */ |
| 223 | if (PREDICT_FALSE (b0->flags & VLIB_BUFFER_IS_TRACED)) |
| 224 | f->flags |= VLIB_FRAME_TRACE; |
Ed Warnicke | cb9cada | 2015-12-08 15:45:58 -0700 | [diff] [blame] | 225 | |
Dave Barach | ba868bb | 2016-08-08 09:51:21 -0400 | [diff] [blame] | 226 | to_next[0] = bi0; |
Ed Warnicke | cb9cada | 2015-12-08 15:45:58 -0700 | [diff] [blame] | 227 | |
Dave Barach | ba868bb | 2016-08-08 09:51:21 -0400 | [diff] [blame] | 228 | from++; |
| 229 | to_next++; |
| 230 | n_this_frame++; |
| 231 | n_left_to_next--; |
| 232 | n_left_from--; |
| 233 | } |
Ed Warnicke | cb9cada | 2015-12-08 15:45:58 -0700 | [diff] [blame] | 234 | } |
Dave Barach | ba868bb | 2016-08-08 09:51:21 -0400 | [diff] [blame] | 235 | |
Damjan Marion | bd69a5f | 2017-02-05 23:44:42 +0100 | [diff] [blame] | 236 | vec_reset_length (fl->buffers); |
Ed Warnicke | cb9cada | 2015-12-08 15:45:58 -0700 | [diff] [blame] | 237 | |
| 238 | if (f) |
| 239 | { |
Dave Barach | ba868bb | 2016-08-08 09:51:21 -0400 | [diff] [blame] | 240 | ASSERT (n_this_frame); |
Ed Warnicke | cb9cada | 2015-12-08 15:45:58 -0700 | [diff] [blame] | 241 | f->n_vectors = n_this_frame; |
| 242 | vlib_put_frame_to_node (vm, feature_node_index, f); |
| 243 | } |
| 244 | } |
| 245 | |
Dave Barach | ba868bb | 2016-08-08 09:51:21 -0400 | [diff] [blame] | 246 | clib_error_t * |
| 247 | replication_init (vlib_main_t * vm) |
Ed Warnicke | cb9cada | 2015-12-08 15:45:58 -0700 | [diff] [blame] | 248 | { |
Dave Barach | ba868bb | 2016-08-08 09:51:21 -0400 | [diff] [blame] | 249 | replication_main_t *rm = &replication_main; |
| 250 | vlib_buffer_main_t *bm = vm->buffer_main; |
| 251 | vlib_buffer_free_list_t *fl; |
| 252 | __attribute__ ((unused)) replication_context_t *ctx; |
| 253 | vlib_thread_main_t *tm = vlib_get_thread_main (); |
Ed Warnicke | cb9cada | 2015-12-08 15:45:58 -0700 | [diff] [blame] | 254 | |
Dave Barach | ba868bb | 2016-08-08 09:51:21 -0400 | [diff] [blame] | 255 | rm->vlib_main = vm; |
| 256 | rm->vnet_main = vnet_get_main (); |
| 257 | rm->recycle_list_index = |
| 258 | vlib_buffer_create_free_list (vm, 1024 /* fictional */ , |
| 259 | "replication-recycle"); |
| 260 | |
| 261 | fl = pool_elt_at_index (bm->buffer_free_list_pool, rm->recycle_list_index); |
Ed Warnicke | cb9cada | 2015-12-08 15:45:58 -0700 | [diff] [blame] | 262 | |
| 263 | fl->buffers_added_to_freelist_function = replication_recycle_callback; |
| 264 | |
Dave Barach | ba868bb | 2016-08-08 09:51:21 -0400 | [diff] [blame] | 265 | /* Verify the replication context is the expected size */ |
| 266 | ASSERT (sizeof (replication_context_t) == 128); /* 2 cache lines */ |
Ed Warnicke | cb9cada | 2015-12-08 15:45:58 -0700 | [diff] [blame] | 267 | |
| 268 | vec_validate (rm->contexts, tm->n_vlib_mains - 1); |
| 269 | return 0; |
| 270 | } |
| 271 | |
| 272 | VLIB_INIT_FUNCTION (replication_init); |
Dave Barach | ba868bb | 2016-08-08 09:51:21 -0400 | [diff] [blame] | 273 | |
| 274 | /* |
| 275 | * fd.io coding-style-patch-verification: ON |
| 276 | * |
| 277 | * Local Variables: |
| 278 | * eval: (c-set-style "gnu") |
| 279 | * End: |
| 280 | */ |