Vijayabhaskar Katamreddy | acbde66 | 2018-01-23 13:39:40 -0800 | [diff] [blame] | 1 | /* |
| 2 | * Copyright (c) 2015 Cisco and/or its affiliates. |
| 3 | * Licensed under the Apache License, Version 2.0 (the "License"); |
| 4 | * you may not use this file except in compliance with the License. |
| 5 | * You may obtain a copy of the License at: |
| 6 | * |
| 7 | * http://www.apache.org/licenses/LICENSE-2.0 |
| 8 | * |
| 9 | * Unless required by applicable law or agreed to in writing, software |
| 10 | * distributed under the License is distributed on an "AS IS" BASIS, |
| 11 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. |
| 12 | * See the License for the specific language governing permissions and |
| 13 | * limitations under the License. |
| 14 | */ |
| 15 | /* |
| 16 | * ip/ip4_forward.h: IP v4 forwarding |
| 17 | * |
| 18 | * Copyright (c) 2008 Eliot Dresselhaus |
| 19 | * |
| 20 | * Permission is hereby granted, free of charge, to any person obtaining |
| 21 | * a copy of this software and associated documentation files (the |
| 22 | * "Software"), to deal in the Software without restriction, including |
| 23 | * without limitation the rights to use, copy, modify, merge, publish, |
| 24 | * distribute, sublicense, and/or sell copies of the Software, and to |
| 25 | * permit persons to whom the Software is furnished to do so, subject to |
| 26 | * the following conditions: |
| 27 | * |
| 28 | * The above copyright notice and this permission notice shall be |
| 29 | * included in all copies or substantial portions of the Software. |
| 30 | * |
| 31 | * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, |
| 32 | * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF |
| 33 | * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND |
| 34 | * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE |
| 35 | * LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION |
| 36 | * OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION |
| 37 | * WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. |
| 38 | */ |
| 39 | |
| 40 | #ifndef __included_ip4_forward_h__ |
| 41 | #define __included_ip4_forward_h__ |
| 42 | |
| 43 | #include <vnet/fib/ip4_fib.h> |
| 44 | #include <vnet/dpo/load_balance_map.h> |
| 45 | |
| 46 | /** |
| 47 | * @file |
| 48 | * @brief IPv4 Forwarding. |
| 49 | * |
| 50 | * This file contains the source code for IPv4 forwarding. |
| 51 | */ |
| 52 | |
| 53 | always_inline uword |
| 54 | ip4_lookup_inline (vlib_main_t * vm, |
| 55 | vlib_node_runtime_t * node, |
| 56 | vlib_frame_t * frame, |
| 57 | int lookup_for_responses_to_locally_received_packets) |
| 58 | { |
| 59 | ip4_main_t *im = &ip4_main; |
| 60 | vlib_combined_counter_main_t *cm = &load_balance_main.lbm_to_counters; |
| 61 | u32 n_left_from, n_left_to_next, *from, *to_next; |
| 62 | ip_lookup_next_t next; |
Damjan Marion | 067cd62 | 2018-07-11 12:47:43 +0200 | [diff] [blame^] | 63 | u32 thread_index = vm->thread_index; |
Vijayabhaskar Katamreddy | acbde66 | 2018-01-23 13:39:40 -0800 | [diff] [blame] | 64 | |
| 65 | from = vlib_frame_vector_args (frame); |
| 66 | n_left_from = frame->n_vectors; |
| 67 | next = node->cached_next_index; |
| 68 | |
| 69 | while (n_left_from > 0) |
| 70 | { |
| 71 | vlib_get_next_frame (vm, node, next, to_next, n_left_to_next); |
| 72 | |
| 73 | while (n_left_from >= 8 && n_left_to_next >= 4) |
| 74 | { |
| 75 | vlib_buffer_t *p0, *p1, *p2, *p3; |
| 76 | ip4_header_t *ip0, *ip1, *ip2, *ip3; |
| 77 | ip_lookup_next_t next0, next1, next2, next3; |
| 78 | const load_balance_t *lb0, *lb1, *lb2, *lb3; |
| 79 | ip4_fib_mtrie_t *mtrie0, *mtrie1, *mtrie2, *mtrie3; |
| 80 | ip4_fib_mtrie_leaf_t leaf0, leaf1, leaf2, leaf3; |
| 81 | ip4_address_t *dst_addr0, *dst_addr1, *dst_addr2, *dst_addr3; |
Florin Coras | 3a0325f | 2018-06-01 12:22:23 -0700 | [diff] [blame] | 82 | u32 pi0, pi1, pi2, pi3, lb_index0, lb_index1, lb_index2, lb_index3; |
Vijayabhaskar Katamreddy | acbde66 | 2018-01-23 13:39:40 -0800 | [diff] [blame] | 83 | flow_hash_config_t flow_hash_config0, flow_hash_config1; |
| 84 | flow_hash_config_t flow_hash_config2, flow_hash_config3; |
| 85 | u32 hash_c0, hash_c1, hash_c2, hash_c3; |
| 86 | const dpo_id_t *dpo0, *dpo1, *dpo2, *dpo3; |
| 87 | |
| 88 | /* Prefetch next iteration. */ |
| 89 | { |
| 90 | vlib_buffer_t *p4, *p5, *p6, *p7; |
| 91 | |
| 92 | p4 = vlib_get_buffer (vm, from[4]); |
| 93 | p5 = vlib_get_buffer (vm, from[5]); |
| 94 | p6 = vlib_get_buffer (vm, from[6]); |
| 95 | p7 = vlib_get_buffer (vm, from[7]); |
| 96 | |
| 97 | vlib_prefetch_buffer_header (p4, LOAD); |
| 98 | vlib_prefetch_buffer_header (p5, LOAD); |
| 99 | vlib_prefetch_buffer_header (p6, LOAD); |
| 100 | vlib_prefetch_buffer_header (p7, LOAD); |
| 101 | |
| 102 | CLIB_PREFETCH (p4->data, sizeof (ip0[0]), LOAD); |
| 103 | CLIB_PREFETCH (p5->data, sizeof (ip0[0]), LOAD); |
| 104 | CLIB_PREFETCH (p6->data, sizeof (ip0[0]), LOAD); |
| 105 | CLIB_PREFETCH (p7->data, sizeof (ip0[0]), LOAD); |
| 106 | } |
| 107 | |
| 108 | pi0 = to_next[0] = from[0]; |
| 109 | pi1 = to_next[1] = from[1]; |
| 110 | pi2 = to_next[2] = from[2]; |
| 111 | pi3 = to_next[3] = from[3]; |
| 112 | |
| 113 | from += 4; |
| 114 | to_next += 4; |
| 115 | n_left_to_next -= 4; |
| 116 | n_left_from -= 4; |
| 117 | |
| 118 | p0 = vlib_get_buffer (vm, pi0); |
| 119 | p1 = vlib_get_buffer (vm, pi1); |
| 120 | p2 = vlib_get_buffer (vm, pi2); |
| 121 | p3 = vlib_get_buffer (vm, pi3); |
| 122 | |
| 123 | ip0 = vlib_buffer_get_current (p0); |
| 124 | ip1 = vlib_buffer_get_current (p1); |
| 125 | ip2 = vlib_buffer_get_current (p2); |
| 126 | ip3 = vlib_buffer_get_current (p3); |
| 127 | |
| 128 | dst_addr0 = &ip0->dst_address; |
| 129 | dst_addr1 = &ip1->dst_address; |
| 130 | dst_addr2 = &ip2->dst_address; |
| 131 | dst_addr3 = &ip3->dst_address; |
| 132 | |
Florin Coras | 3a0325f | 2018-06-01 12:22:23 -0700 | [diff] [blame] | 133 | ip_lookup_set_buffer_fib_index (im->fib_index_by_sw_if_index, p0); |
| 134 | ip_lookup_set_buffer_fib_index (im->fib_index_by_sw_if_index, p1); |
| 135 | ip_lookup_set_buffer_fib_index (im->fib_index_by_sw_if_index, p2); |
| 136 | ip_lookup_set_buffer_fib_index (im->fib_index_by_sw_if_index, p3); |
Vijayabhaskar Katamreddy | acbde66 | 2018-01-23 13:39:40 -0800 | [diff] [blame] | 137 | |
| 138 | if (!lookup_for_responses_to_locally_received_packets) |
| 139 | { |
Florin Coras | 3a0325f | 2018-06-01 12:22:23 -0700 | [diff] [blame] | 140 | mtrie0 = &ip4_fib_get (vnet_buffer (p0)->ip.fib_index)->mtrie; |
| 141 | mtrie1 = &ip4_fib_get (vnet_buffer (p1)->ip.fib_index)->mtrie; |
| 142 | mtrie2 = &ip4_fib_get (vnet_buffer (p2)->ip.fib_index)->mtrie; |
| 143 | mtrie3 = &ip4_fib_get (vnet_buffer (p3)->ip.fib_index)->mtrie; |
Vijayabhaskar Katamreddy | acbde66 | 2018-01-23 13:39:40 -0800 | [diff] [blame] | 144 | |
| 145 | leaf0 = ip4_fib_mtrie_lookup_step_one (mtrie0, dst_addr0); |
| 146 | leaf1 = ip4_fib_mtrie_lookup_step_one (mtrie1, dst_addr1); |
| 147 | leaf2 = ip4_fib_mtrie_lookup_step_one (mtrie2, dst_addr2); |
| 148 | leaf3 = ip4_fib_mtrie_lookup_step_one (mtrie3, dst_addr3); |
| 149 | } |
| 150 | |
| 151 | if (!lookup_for_responses_to_locally_received_packets) |
| 152 | { |
| 153 | leaf0 = ip4_fib_mtrie_lookup_step (mtrie0, leaf0, dst_addr0, 2); |
| 154 | leaf1 = ip4_fib_mtrie_lookup_step (mtrie1, leaf1, dst_addr1, 2); |
| 155 | leaf2 = ip4_fib_mtrie_lookup_step (mtrie2, leaf2, dst_addr2, 2); |
| 156 | leaf3 = ip4_fib_mtrie_lookup_step (mtrie3, leaf3, dst_addr3, 2); |
| 157 | } |
| 158 | |
| 159 | if (!lookup_for_responses_to_locally_received_packets) |
| 160 | { |
| 161 | leaf0 = ip4_fib_mtrie_lookup_step (mtrie0, leaf0, dst_addr0, 3); |
| 162 | leaf1 = ip4_fib_mtrie_lookup_step (mtrie1, leaf1, dst_addr1, 3); |
| 163 | leaf2 = ip4_fib_mtrie_lookup_step (mtrie2, leaf2, dst_addr2, 3); |
| 164 | leaf3 = ip4_fib_mtrie_lookup_step (mtrie3, leaf3, dst_addr3, 3); |
| 165 | } |
| 166 | |
| 167 | if (lookup_for_responses_to_locally_received_packets) |
| 168 | { |
| 169 | lb_index0 = vnet_buffer (p0)->ip.adj_index[VLIB_RX]; |
| 170 | lb_index1 = vnet_buffer (p1)->ip.adj_index[VLIB_RX]; |
| 171 | lb_index2 = vnet_buffer (p2)->ip.adj_index[VLIB_RX]; |
| 172 | lb_index3 = vnet_buffer (p3)->ip.adj_index[VLIB_RX]; |
| 173 | } |
| 174 | else |
| 175 | { |
| 176 | lb_index0 = ip4_fib_mtrie_leaf_get_adj_index (leaf0); |
| 177 | lb_index1 = ip4_fib_mtrie_leaf_get_adj_index (leaf1); |
| 178 | lb_index2 = ip4_fib_mtrie_leaf_get_adj_index (leaf2); |
| 179 | lb_index3 = ip4_fib_mtrie_leaf_get_adj_index (leaf3); |
| 180 | } |
| 181 | |
| 182 | ASSERT (lb_index0 && lb_index1 && lb_index2 && lb_index3); |
| 183 | lb0 = load_balance_get (lb_index0); |
| 184 | lb1 = load_balance_get (lb_index1); |
| 185 | lb2 = load_balance_get (lb_index2); |
| 186 | lb3 = load_balance_get (lb_index3); |
| 187 | |
| 188 | ASSERT (lb0->lb_n_buckets > 0); |
| 189 | ASSERT (is_pow2 (lb0->lb_n_buckets)); |
| 190 | ASSERT (lb1->lb_n_buckets > 0); |
| 191 | ASSERT (is_pow2 (lb1->lb_n_buckets)); |
| 192 | ASSERT (lb2->lb_n_buckets > 0); |
| 193 | ASSERT (is_pow2 (lb2->lb_n_buckets)); |
| 194 | ASSERT (lb3->lb_n_buckets > 0); |
| 195 | ASSERT (is_pow2 (lb3->lb_n_buckets)); |
| 196 | |
| 197 | /* Use flow hash to compute multipath adjacency. */ |
| 198 | hash_c0 = vnet_buffer (p0)->ip.flow_hash = 0; |
| 199 | hash_c1 = vnet_buffer (p1)->ip.flow_hash = 0; |
| 200 | hash_c2 = vnet_buffer (p2)->ip.flow_hash = 0; |
| 201 | hash_c3 = vnet_buffer (p3)->ip.flow_hash = 0; |
| 202 | if (PREDICT_FALSE (lb0->lb_n_buckets > 1)) |
| 203 | { |
| 204 | flow_hash_config0 = lb0->lb_hash_config; |
| 205 | hash_c0 = vnet_buffer (p0)->ip.flow_hash = |
| 206 | ip4_compute_flow_hash (ip0, flow_hash_config0); |
| 207 | dpo0 = |
| 208 | load_balance_get_fwd_bucket (lb0, |
| 209 | (hash_c0 & |
| 210 | (lb0->lb_n_buckets_minus_1))); |
| 211 | } |
| 212 | else |
| 213 | { |
| 214 | dpo0 = load_balance_get_bucket_i (lb0, 0); |
| 215 | } |
| 216 | if (PREDICT_FALSE (lb1->lb_n_buckets > 1)) |
| 217 | { |
| 218 | flow_hash_config1 = lb1->lb_hash_config; |
| 219 | hash_c1 = vnet_buffer (p1)->ip.flow_hash = |
| 220 | ip4_compute_flow_hash (ip1, flow_hash_config1); |
| 221 | dpo1 = |
| 222 | load_balance_get_fwd_bucket (lb1, |
| 223 | (hash_c1 & |
| 224 | (lb1->lb_n_buckets_minus_1))); |
| 225 | } |
| 226 | else |
| 227 | { |
| 228 | dpo1 = load_balance_get_bucket_i (lb1, 0); |
| 229 | } |
| 230 | if (PREDICT_FALSE (lb2->lb_n_buckets > 1)) |
| 231 | { |
| 232 | flow_hash_config2 = lb2->lb_hash_config; |
| 233 | hash_c2 = vnet_buffer (p2)->ip.flow_hash = |
| 234 | ip4_compute_flow_hash (ip2, flow_hash_config2); |
| 235 | dpo2 = |
| 236 | load_balance_get_fwd_bucket (lb2, |
| 237 | (hash_c2 & |
| 238 | (lb2->lb_n_buckets_minus_1))); |
| 239 | } |
| 240 | else |
| 241 | { |
| 242 | dpo2 = load_balance_get_bucket_i (lb2, 0); |
| 243 | } |
| 244 | if (PREDICT_FALSE (lb3->lb_n_buckets > 1)) |
| 245 | { |
| 246 | flow_hash_config3 = lb3->lb_hash_config; |
| 247 | hash_c3 = vnet_buffer (p3)->ip.flow_hash = |
| 248 | ip4_compute_flow_hash (ip3, flow_hash_config3); |
| 249 | dpo3 = |
| 250 | load_balance_get_fwd_bucket (lb3, |
| 251 | (hash_c3 & |
| 252 | (lb3->lb_n_buckets_minus_1))); |
| 253 | } |
| 254 | else |
| 255 | { |
| 256 | dpo3 = load_balance_get_bucket_i (lb3, 0); |
| 257 | } |
| 258 | |
| 259 | next0 = dpo0->dpoi_next_node; |
| 260 | vnet_buffer (p0)->ip.adj_index[VLIB_TX] = dpo0->dpoi_index; |
| 261 | next1 = dpo1->dpoi_next_node; |
| 262 | vnet_buffer (p1)->ip.adj_index[VLIB_TX] = dpo1->dpoi_index; |
| 263 | next2 = dpo2->dpoi_next_node; |
| 264 | vnet_buffer (p2)->ip.adj_index[VLIB_TX] = dpo2->dpoi_index; |
| 265 | next3 = dpo3->dpoi_next_node; |
| 266 | vnet_buffer (p3)->ip.adj_index[VLIB_TX] = dpo3->dpoi_index; |
| 267 | |
| 268 | vlib_increment_combined_counter |
| 269 | (cm, thread_index, lb_index0, 1, |
| 270 | vlib_buffer_length_in_chain (vm, p0)); |
| 271 | vlib_increment_combined_counter |
| 272 | (cm, thread_index, lb_index1, 1, |
| 273 | vlib_buffer_length_in_chain (vm, p1)); |
| 274 | vlib_increment_combined_counter |
| 275 | (cm, thread_index, lb_index2, 1, |
| 276 | vlib_buffer_length_in_chain (vm, p2)); |
| 277 | vlib_increment_combined_counter |
| 278 | (cm, thread_index, lb_index3, 1, |
| 279 | vlib_buffer_length_in_chain (vm, p3)); |
| 280 | |
| 281 | vlib_validate_buffer_enqueue_x4 (vm, node, next, |
| 282 | to_next, n_left_to_next, |
| 283 | pi0, pi1, pi2, pi3, |
| 284 | next0, next1, next2, next3); |
| 285 | } |
| 286 | |
| 287 | while (n_left_from > 0 && n_left_to_next > 0) |
| 288 | { |
| 289 | vlib_buffer_t *p0; |
| 290 | ip4_header_t *ip0; |
| 291 | ip_lookup_next_t next0; |
| 292 | const load_balance_t *lb0; |
| 293 | ip4_fib_mtrie_t *mtrie0; |
| 294 | ip4_fib_mtrie_leaf_t leaf0; |
| 295 | ip4_address_t *dst_addr0; |
Florin Coras | 3a0325f | 2018-06-01 12:22:23 -0700 | [diff] [blame] | 296 | u32 pi0, lbi0; |
Vijayabhaskar Katamreddy | acbde66 | 2018-01-23 13:39:40 -0800 | [diff] [blame] | 297 | flow_hash_config_t flow_hash_config0; |
| 298 | const dpo_id_t *dpo0; |
| 299 | u32 hash_c0; |
| 300 | |
| 301 | pi0 = from[0]; |
| 302 | to_next[0] = pi0; |
| 303 | |
| 304 | p0 = vlib_get_buffer (vm, pi0); |
Vijayabhaskar Katamreddy | acbde66 | 2018-01-23 13:39:40 -0800 | [diff] [blame] | 305 | ip0 = vlib_buffer_get_current (p0); |
Vijayabhaskar Katamreddy | acbde66 | 2018-01-23 13:39:40 -0800 | [diff] [blame] | 306 | dst_addr0 = &ip0->dst_address; |
Florin Coras | 3a0325f | 2018-06-01 12:22:23 -0700 | [diff] [blame] | 307 | ip_lookup_set_buffer_fib_index (im->fib_index_by_sw_if_index, p0); |
Vijayabhaskar Katamreddy | acbde66 | 2018-01-23 13:39:40 -0800 | [diff] [blame] | 308 | |
| 309 | if (!lookup_for_responses_to_locally_received_packets) |
| 310 | { |
Florin Coras | 3a0325f | 2018-06-01 12:22:23 -0700 | [diff] [blame] | 311 | mtrie0 = &ip4_fib_get (vnet_buffer (p0)->ip.fib_index)->mtrie; |
Vijayabhaskar Katamreddy | acbde66 | 2018-01-23 13:39:40 -0800 | [diff] [blame] | 312 | leaf0 = ip4_fib_mtrie_lookup_step_one (mtrie0, dst_addr0); |
| 313 | } |
| 314 | |
| 315 | if (!lookup_for_responses_to_locally_received_packets) |
| 316 | leaf0 = ip4_fib_mtrie_lookup_step (mtrie0, leaf0, dst_addr0, 2); |
| 317 | |
| 318 | if (!lookup_for_responses_to_locally_received_packets) |
| 319 | leaf0 = ip4_fib_mtrie_lookup_step (mtrie0, leaf0, dst_addr0, 3); |
| 320 | |
| 321 | if (lookup_for_responses_to_locally_received_packets) |
| 322 | lbi0 = vnet_buffer (p0)->ip.adj_index[VLIB_RX]; |
| 323 | else |
| 324 | { |
| 325 | /* Handle default route. */ |
| 326 | lbi0 = ip4_fib_mtrie_leaf_get_adj_index (leaf0); |
| 327 | } |
| 328 | |
| 329 | ASSERT (lbi0); |
| 330 | lb0 = load_balance_get (lbi0); |
| 331 | |
| 332 | ASSERT (lb0->lb_n_buckets > 0); |
| 333 | ASSERT (is_pow2 (lb0->lb_n_buckets)); |
| 334 | |
| 335 | /* Use flow hash to compute multipath adjacency. */ |
| 336 | hash_c0 = vnet_buffer (p0)->ip.flow_hash = 0; |
| 337 | if (PREDICT_FALSE (lb0->lb_n_buckets > 1)) |
| 338 | { |
| 339 | flow_hash_config0 = lb0->lb_hash_config; |
| 340 | |
| 341 | hash_c0 = vnet_buffer (p0)->ip.flow_hash = |
| 342 | ip4_compute_flow_hash (ip0, flow_hash_config0); |
| 343 | dpo0 = |
| 344 | load_balance_get_fwd_bucket (lb0, |
| 345 | (hash_c0 & |
| 346 | (lb0->lb_n_buckets_minus_1))); |
| 347 | } |
| 348 | else |
| 349 | { |
| 350 | dpo0 = load_balance_get_bucket_i (lb0, 0); |
| 351 | } |
| 352 | |
| 353 | next0 = dpo0->dpoi_next_node; |
| 354 | vnet_buffer (p0)->ip.adj_index[VLIB_TX] = dpo0->dpoi_index; |
| 355 | |
| 356 | vlib_increment_combined_counter (cm, thread_index, lbi0, 1, |
| 357 | vlib_buffer_length_in_chain (vm, |
| 358 | p0)); |
| 359 | |
| 360 | from += 1; |
| 361 | to_next += 1; |
| 362 | n_left_to_next -= 1; |
| 363 | n_left_from -= 1; |
| 364 | |
| 365 | if (PREDICT_FALSE (next0 != next)) |
| 366 | { |
| 367 | n_left_to_next += 1; |
| 368 | vlib_put_next_frame (vm, node, next, n_left_to_next); |
| 369 | next = next0; |
| 370 | vlib_get_next_frame (vm, node, next, to_next, n_left_to_next); |
| 371 | to_next[0] = pi0; |
| 372 | to_next += 1; |
| 373 | n_left_to_next -= 1; |
| 374 | } |
| 375 | } |
| 376 | |
| 377 | vlib_put_next_frame (vm, node, next, n_left_to_next); |
| 378 | } |
| 379 | |
| 380 | if (node->flags & VLIB_NODE_FLAG_TRACE) |
| 381 | ip4_forward_next_trace (vm, node, frame, VLIB_TX); |
| 382 | |
| 383 | return frame->n_vectors; |
| 384 | } |
| 385 | |
| 386 | #endif /* __included_ip4_forward_h__ */ |
| 387 | |
| 388 | /* |
| 389 | * fd.io coding-style-patch-verification: ON |
| 390 | * |
| 391 | * Local Variables: |
| 392 | * eval: (c-set-style "gnu") |
| 393 | * End: |
| 394 | */ |