Ed Warnicke | cb9cada | 2015-12-08 15:45:58 -0700 | [diff] [blame] | 1 | /* |
| 2 | * esp_decrypt.c : IPSec ESP decrypt node |
| 3 | * |
| 4 | * Copyright (c) 2015 Cisco and/or its affiliates. |
| 5 | * Licensed under the Apache License, Version 2.0 (the "License"); |
| 6 | * you may not use this file except in compliance with the License. |
| 7 | * You may obtain a copy of the License at: |
| 8 | * |
| 9 | * http://www.apache.org/licenses/LICENSE-2.0 |
| 10 | * |
| 11 | * Unless required by applicable law or agreed to in writing, software |
| 12 | * distributed under the License is distributed on an "AS IS" BASIS, |
| 13 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. |
| 14 | * See the License for the specific language governing permissions and |
| 15 | * limitations under the License. |
| 16 | */ |
| 17 | |
| 18 | #include <vnet/vnet.h> |
| 19 | #include <vnet/api_errno.h> |
| 20 | #include <vnet/ip/ip.h> |
| 21 | |
| 22 | #include <vnet/ipsec/ipsec.h> |
| 23 | #include <vnet/ipsec/esp.h> |
Neale Ranns | 918c161 | 2019-02-21 23:34:59 -0800 | [diff] [blame] | 24 | #include <vnet/ipsec/ipsec_io.h> |
Ed Warnicke | cb9cada | 2015-12-08 15:45:58 -0700 | [diff] [blame] | 25 | |
Ed Warnicke | cb9cada | 2015-12-08 15:45:58 -0700 | [diff] [blame] | 26 | #define foreach_esp_decrypt_next \ |
| 27 | _(DROP, "error-drop") \ |
Kingwel Xie | 561d1ca | 2019-03-05 22:56:17 -0500 | [diff] [blame] | 28 | _(IP4_INPUT, "ip4-input-no-checksum") \ |
Matus Fabian | 694265d | 2016-08-10 01:55:36 -0700 | [diff] [blame] | 29 | _(IP6_INPUT, "ip6-input") \ |
| 30 | _(IPSEC_GRE_INPUT, "ipsec-gre-input") |
Ed Warnicke | cb9cada | 2015-12-08 15:45:58 -0700 | [diff] [blame] | 31 | |
| 32 | #define _(v, s) ESP_DECRYPT_NEXT_##v, |
Keith Burns (alagalah) | 166a9d4 | 2016-08-06 11:00:56 -0700 | [diff] [blame] | 33 | typedef enum |
| 34 | { |
Ed Warnicke | cb9cada | 2015-12-08 15:45:58 -0700 | [diff] [blame] | 35 | foreach_esp_decrypt_next |
| 36 | #undef _ |
Keith Burns (alagalah) | 166a9d4 | 2016-08-06 11:00:56 -0700 | [diff] [blame] | 37 | ESP_DECRYPT_N_NEXT, |
Ed Warnicke | cb9cada | 2015-12-08 15:45:58 -0700 | [diff] [blame] | 38 | } esp_decrypt_next_t; |
| 39 | |
| 40 | |
Damjan Marion | b4fff3a | 2019-03-25 15:54:40 +0100 | [diff] [blame] | 41 | #define foreach_esp_decrypt_error \ |
| 42 | _(RX_PKTS, "ESP pkts received") \ |
| 43 | _(DECRYPTION_FAILED, "ESP decryption failed") \ |
| 44 | _(INTEG_ERROR, "Integrity check failed") \ |
| 45 | _(CRYPTO_ENGINE_ERROR, "crypto engine error (packet dropped)") \ |
| 46 | _(REPLAY, "SA replayed packet") \ |
| 47 | _(CHAINED_BUFFER, "chained buffers (packet dropped)") \ |
| 48 | _(OVERSIZED_HEADER, "buffer with oversized header (dropped)") \ |
| 49 | _(NO_TAIL_SPACE, "no enough buffer tail space (dropped)") |
Ed Warnicke | cb9cada | 2015-12-08 15:45:58 -0700 | [diff] [blame] | 50 | |
| 51 | |
Keith Burns (alagalah) | 166a9d4 | 2016-08-06 11:00:56 -0700 | [diff] [blame] | 52 | typedef enum |
| 53 | { |
Ed Warnicke | cb9cada | 2015-12-08 15:45:58 -0700 | [diff] [blame] | 54 | #define _(sym,str) ESP_DECRYPT_ERROR_##sym, |
| 55 | foreach_esp_decrypt_error |
| 56 | #undef _ |
Keith Burns (alagalah) | 166a9d4 | 2016-08-06 11:00:56 -0700 | [diff] [blame] | 57 | ESP_DECRYPT_N_ERROR, |
Ed Warnicke | cb9cada | 2015-12-08 15:45:58 -0700 | [diff] [blame] | 58 | } esp_decrypt_error_t; |
| 59 | |
Keith Burns (alagalah) | 166a9d4 | 2016-08-06 11:00:56 -0700 | [diff] [blame] | 60 | static char *esp_decrypt_error_strings[] = { |
Ed Warnicke | cb9cada | 2015-12-08 15:45:58 -0700 | [diff] [blame] | 61 | #define _(sym,string) string, |
| 62 | foreach_esp_decrypt_error |
| 63 | #undef _ |
| 64 | }; |
| 65 | |
Keith Burns (alagalah) | 166a9d4 | 2016-08-06 11:00:56 -0700 | [diff] [blame] | 66 | typedef struct |
| 67 | { |
Damjan Marion | b4fff3a | 2019-03-25 15:54:40 +0100 | [diff] [blame] | 68 | u32 seq; |
Ed Warnicke | cb9cada | 2015-12-08 15:45:58 -0700 | [diff] [blame] | 69 | ipsec_crypto_alg_t crypto_alg; |
| 70 | ipsec_integ_alg_t integ_alg; |
| 71 | } esp_decrypt_trace_t; |
| 72 | |
| 73 | /* packet trace format function */ |
Keith Burns (alagalah) | 166a9d4 | 2016-08-06 11:00:56 -0700 | [diff] [blame] | 74 | static u8 * |
| 75 | format_esp_decrypt_trace (u8 * s, va_list * args) |
Ed Warnicke | cb9cada | 2015-12-08 15:45:58 -0700 | [diff] [blame] | 76 | { |
| 77 | CLIB_UNUSED (vlib_main_t * vm) = va_arg (*args, vlib_main_t *); |
| 78 | CLIB_UNUSED (vlib_node_t * node) = va_arg (*args, vlib_node_t *); |
Keith Burns (alagalah) | 166a9d4 | 2016-08-06 11:00:56 -0700 | [diff] [blame] | 79 | esp_decrypt_trace_t *t = va_arg (*args, esp_decrypt_trace_t *); |
Ed Warnicke | cb9cada | 2015-12-08 15:45:58 -0700 | [diff] [blame] | 80 | |
Damjan Marion | b4fff3a | 2019-03-25 15:54:40 +0100 | [diff] [blame] | 81 | s = format (s, "esp: crypto %U integrity %U seq %u", |
Keith Burns (alagalah) | 166a9d4 | 2016-08-06 11:00:56 -0700 | [diff] [blame] | 82 | format_ipsec_crypto_alg, t->crypto_alg, |
Damjan Marion | b4fff3a | 2019-03-25 15:54:40 +0100 | [diff] [blame] | 83 | format_ipsec_integ_alg, t->integ_alg, t->seq); |
Ed Warnicke | cb9cada | 2015-12-08 15:45:58 -0700 | [diff] [blame] | 84 | return s; |
| 85 | } |
| 86 | |
Damjan Marion | b4fff3a | 2019-03-25 15:54:40 +0100 | [diff] [blame] | 87 | typedef struct |
Ed Warnicke | cb9cada | 2015-12-08 15:45:58 -0700 | [diff] [blame] | 88 | { |
Damjan Marion | b4fff3a | 2019-03-25 15:54:40 +0100 | [diff] [blame] | 89 | union |
| 90 | { |
| 91 | struct |
| 92 | { |
| 93 | u8 icv_sz; |
| 94 | u8 iv_sz; |
| 95 | ipsec_sa_flags_t flags:8; |
| 96 | u32 sa_index; |
| 97 | }; |
| 98 | u64 sa_data; |
| 99 | }; |
Ed Warnicke | cb9cada | 2015-12-08 15:45:58 -0700 | [diff] [blame] | 100 | |
Damjan Marion | b4fff3a | 2019-03-25 15:54:40 +0100 | [diff] [blame] | 101 | i16 current_data; |
| 102 | i16 current_length; |
| 103 | u16 hdr_sz; |
| 104 | } esp_decrypt_packet_data_t; |
Ed Warnicke | cb9cada | 2015-12-08 15:45:58 -0700 | [diff] [blame] | 105 | |
Damjan Marion | b4fff3a | 2019-03-25 15:54:40 +0100 | [diff] [blame] | 106 | STATIC_ASSERT_SIZEOF (esp_decrypt_packet_data_t, 2 * sizeof (u64)); |
Ed Warnicke | cb9cada | 2015-12-08 15:45:58 -0700 | [diff] [blame] | 107 | |
Damjan Marion | b4fff3a | 2019-03-25 15:54:40 +0100 | [diff] [blame] | 108 | #define ESP_ENCRYPT_PD_F_FD_TRANSPORT (1 << 2) |
Ed Warnicke | cb9cada | 2015-12-08 15:45:58 -0700 | [diff] [blame] | 109 | |
Klement Sekera | be5a5dd | 2018-10-09 16:05:48 +0200 | [diff] [blame] | 110 | always_inline uword |
| 111 | esp_decrypt_inline (vlib_main_t * vm, |
| 112 | vlib_node_runtime_t * node, vlib_frame_t * from_frame, |
| 113 | int is_ip6) |
Ed Warnicke | cb9cada | 2015-12-08 15:45:58 -0700 | [diff] [blame] | 114 | { |
Ed Warnicke | cb9cada | 2015-12-08 15:45:58 -0700 | [diff] [blame] | 115 | ipsec_main_t *im = &ipsec_main; |
Damjan Marion | b4fff3a | 2019-03-25 15:54:40 +0100 | [diff] [blame] | 116 | u32 thread_index = vm->thread_index; |
| 117 | u16 buffer_data_size = vlib_buffer_get_default_data_size (vm); |
| 118 | u16 len; |
| 119 | ipsec_per_thread_data_t *ptd = vec_elt_at_index (im->ptd, thread_index); |
Damjan Marion | c98275f | 2019-03-06 14:05:01 +0100 | [diff] [blame] | 120 | u32 *from = vlib_frame_vector_args (from_frame); |
Damjan Marion | b4fff3a | 2019-03-25 15:54:40 +0100 | [diff] [blame] | 121 | u32 n, n_left = from_frame->n_vectors; |
| 122 | vlib_buffer_t *bufs[VLIB_FRAME_SIZE], **b = bufs; |
Damjan Marion | c98275f | 2019-03-06 14:05:01 +0100 | [diff] [blame] | 123 | u16 nexts[VLIB_FRAME_SIZE], *next = nexts; |
Damjan Marion | b4fff3a | 2019-03-25 15:54:40 +0100 | [diff] [blame] | 124 | esp_decrypt_packet_data_t pkt_data[VLIB_FRAME_SIZE], *pd = pkt_data; |
| 125 | esp_decrypt_packet_data_t cpd = { }; |
| 126 | u32 current_sa_index = ~0, current_sa_bytes = 0, current_sa_pkts = 0; |
| 127 | const u8 esp_sz = sizeof (esp_header_t); |
| 128 | ipsec_sa_t *sa0 = 0; |
Ed Warnicke | cb9cada | 2015-12-08 15:45:58 -0700 | [diff] [blame] | 129 | |
Damjan Marion | b4fff3a | 2019-03-25 15:54:40 +0100 | [diff] [blame] | 130 | vlib_get_buffers (vm, from, b, n_left); |
| 131 | vec_reset_length (ptd->crypto_ops); |
| 132 | vec_reset_length (ptd->integ_ops); |
| 133 | clib_memset_u16 (nexts, -1, n_left); |
| 134 | |
| 135 | while (n_left > 0) |
Keith Burns (alagalah) | 166a9d4 | 2016-08-06 11:00:56 -0700 | [diff] [blame] | 136 | { |
Damjan Marion | b4fff3a | 2019-03-25 15:54:40 +0100 | [diff] [blame] | 137 | u8 *payload; |
Ed Warnicke | cb9cada | 2015-12-08 15:45:58 -0700 | [diff] [blame] | 138 | |
Damjan Marion | b4fff3a | 2019-03-25 15:54:40 +0100 | [diff] [blame] | 139 | if (n_left > 2) |
Keith Burns (alagalah) | 166a9d4 | 2016-08-06 11:00:56 -0700 | [diff] [blame] | 140 | { |
Damjan Marion | b4fff3a | 2019-03-25 15:54:40 +0100 | [diff] [blame] | 141 | u8 *p; |
| 142 | vlib_prefetch_buffer_header (b[2], LOAD); |
| 143 | p = vlib_buffer_get_current (b[1]); |
| 144 | CLIB_PREFETCH (p, CLIB_CACHE_LINE_BYTES, LOAD); |
| 145 | p -= CLIB_CACHE_LINE_BYTES; |
| 146 | CLIB_PREFETCH (p, CLIB_CACHE_LINE_BYTES, LOAD); |
| 147 | } |
| 148 | |
| 149 | if (vlib_buffer_chain_linearize (vm, b[0]) != 1) |
| 150 | { |
| 151 | b[0]->error = node->errors[ESP_DECRYPT_ERROR_CHAINED_BUFFER]; |
Damjan Marion | 1f4e1cb | 2019-03-28 19:19:31 +0100 | [diff] [blame] | 152 | next[0] = ESP_DECRYPT_NEXT_DROP; |
Damjan Marion | b4fff3a | 2019-03-25 15:54:40 +0100 | [diff] [blame] | 153 | goto next; |
Keith Burns (alagalah) | 166a9d4 | 2016-08-06 11:00:56 -0700 | [diff] [blame] | 154 | } |
Damjan Marion | c98275f | 2019-03-06 14:05:01 +0100 | [diff] [blame] | 155 | |
Damjan Marion | b4fff3a | 2019-03-25 15:54:40 +0100 | [diff] [blame] | 156 | if (vnet_buffer (b[0])->ipsec.sad_index != current_sa_index) |
Damjan Marion | c98275f | 2019-03-06 14:05:01 +0100 | [diff] [blame] | 157 | { |
Damjan Marion | b4fff3a | 2019-03-25 15:54:40 +0100 | [diff] [blame] | 158 | current_sa_index = vnet_buffer (b[0])->ipsec.sad_index; |
| 159 | sa0 = pool_elt_at_index (im->sad, current_sa_index); |
| 160 | cpd.icv_sz = sa0->integ_trunc_size; |
| 161 | cpd.iv_sz = sa0->crypto_iv_size; |
| 162 | cpd.flags = sa0->flags; |
| 163 | cpd.sa_index = current_sa_index; |
Damjan Marion | c98275f | 2019-03-06 14:05:01 +0100 | [diff] [blame] | 164 | |
Damjan Marion | b4fff3a | 2019-03-25 15:54:40 +0100 | [diff] [blame] | 165 | vlib_increment_combined_counter (&ipsec_sa_counters, thread_index, |
| 166 | current_sa_index, current_sa_pkts, |
| 167 | current_sa_bytes); |
Damjan Marion | c98275f | 2019-03-06 14:05:01 +0100 | [diff] [blame] | 168 | |
Damjan Marion | b4fff3a | 2019-03-25 15:54:40 +0100 | [diff] [blame] | 169 | current_sa_bytes = current_sa_pkts = 0; |
| 170 | } |
| 171 | |
| 172 | /* store packet data for next round for easier prefetch */ |
| 173 | pd->sa_data = cpd.sa_data; |
| 174 | pd->current_data = b[0]->current_data; |
| 175 | pd->current_length = b[0]->current_length; |
| 176 | pd->hdr_sz = pd->current_data - vnet_buffer (b[0])->l3_hdr_offset; |
| 177 | payload = b[0]->data + pd->current_data; |
| 178 | |
| 179 | /* we need 4 extra bytes for HMAC calculation when ESN are used */ |
| 180 | if ((sa0->flags & IPSEC_SA_FLAG_USE_ESN) && pd->icv_sz && |
| 181 | (pd->current_data + pd->current_length + 4 > buffer_data_size)) |
| 182 | { |
| 183 | b[0]->error = node->errors[ESP_DECRYPT_ERROR_NO_TAIL_SPACE]; |
| 184 | next[0] = ESP_DECRYPT_NEXT_DROP; |
| 185 | goto next; |
| 186 | } |
| 187 | |
| 188 | /* anti-reply check */ |
| 189 | if (ipsec_sa_anti_replay_check (sa0, &((esp_header_t *) payload)->seq)) |
| 190 | { |
| 191 | b[0]->error = node->errors[ESP_DECRYPT_ERROR_REPLAY]; |
| 192 | next[0] = ESP_DECRYPT_NEXT_DROP; |
| 193 | goto next; |
| 194 | } |
| 195 | |
| 196 | len = pd->current_length - cpd.icv_sz; |
| 197 | current_sa_pkts += 1; |
| 198 | current_sa_bytes += pd->current_length; |
| 199 | |
| 200 | if (PREDICT_TRUE (cpd.icv_sz > 0)) |
| 201 | { |
| 202 | vnet_crypto_op_t *op; |
| 203 | vec_add2_aligned (ptd->integ_ops, op, 1, CLIB_CACHE_LINE_BYTES); |
| 204 | |
| 205 | vnet_crypto_op_init (op, sa0->integ_op_type); |
| 206 | op->key = sa0->integ_key.data; |
| 207 | op->key_len = sa0->integ_key.len; |
| 208 | op->src = payload; |
| 209 | op->hmac_trunc_len = cpd.icv_sz; |
| 210 | op->flags = VNET_CRYPTO_OP_FLAG_HMAC_CHECK; |
| 211 | op->user_data = b - bufs; |
| 212 | op->dst = payload + len; |
| 213 | op->len = len; |
| 214 | if (PREDICT_TRUE (sa0->flags & IPSEC_SA_FLAG_USE_ESN)) |
Damjan Marion | c98275f | 2019-03-06 14:05:01 +0100 | [diff] [blame] | 215 | { |
Damjan Marion | b4fff3a | 2019-03-25 15:54:40 +0100 | [diff] [blame] | 216 | /* shift ICV for 4 bytes to insert ESN */ |
| 217 | u8 tmp[ESP_MAX_ICV_SIZE], sz = sizeof (sa0->seq_hi); |
| 218 | clib_memcpy_fast (tmp, payload + len, ESP_MAX_ICV_SIZE); |
| 219 | clib_memcpy_fast (payload + len, &sa0->seq_hi, sz); |
| 220 | clib_memcpy_fast (payload + len + sz, tmp, ESP_MAX_ICV_SIZE); |
| 221 | op->len += sz; |
| 222 | op->dst += sz; |
Damjan Marion | c98275f | 2019-03-06 14:05:01 +0100 | [diff] [blame] | 223 | } |
| 224 | } |
| 225 | |
Damjan Marion | b4fff3a | 2019-03-25 15:54:40 +0100 | [diff] [blame] | 226 | payload += esp_sz; |
| 227 | len -= esp_sz; |
Damjan Marion | c98275f | 2019-03-06 14:05:01 +0100 | [diff] [blame] | 228 | |
Damjan Marion | b4fff3a | 2019-03-25 15:54:40 +0100 | [diff] [blame] | 229 | if (sa0->crypto_enc_op_type != VNET_CRYPTO_OP_NONE) |
Damjan Marion | c98275f | 2019-03-06 14:05:01 +0100 | [diff] [blame] | 230 | { |
Damjan Marion | b4fff3a | 2019-03-25 15:54:40 +0100 | [diff] [blame] | 231 | vnet_crypto_op_t *op; |
| 232 | vec_add2_aligned (ptd->crypto_ops, op, 1, CLIB_CACHE_LINE_BYTES); |
| 233 | vnet_crypto_op_init (op, sa0->crypto_dec_op_type); |
| 234 | op->key = sa0->crypto_key.data; |
| 235 | op->iv = payload; |
| 236 | op->src = op->dst = payload += cpd.iv_sz; |
| 237 | op->len = len; |
| 238 | op->user_data = b - bufs; |
Damjan Marion | c98275f | 2019-03-06 14:05:01 +0100 | [diff] [blame] | 239 | } |
| 240 | |
| 241 | /* next */ |
Damjan Marion | b4fff3a | 2019-03-25 15:54:40 +0100 | [diff] [blame] | 242 | next: |
| 243 | n_left -= 1; |
Damjan Marion | c98275f | 2019-03-06 14:05:01 +0100 | [diff] [blame] | 244 | next += 1; |
Damjan Marion | b4fff3a | 2019-03-25 15:54:40 +0100 | [diff] [blame] | 245 | pd += 1; |
| 246 | b += 1; |
Ed Warnicke | cb9cada | 2015-12-08 15:45:58 -0700 | [diff] [blame] | 247 | } |
Damjan Marion | c98275f | 2019-03-06 14:05:01 +0100 | [diff] [blame] | 248 | |
Damjan Marion | b4fff3a | 2019-03-25 15:54:40 +0100 | [diff] [blame] | 249 | vlib_increment_combined_counter (&ipsec_sa_counters, thread_index, |
| 250 | current_sa_index, current_sa_pkts, |
| 251 | current_sa_bytes); |
Klement Sekera | be5a5dd | 2018-10-09 16:05:48 +0200 | [diff] [blame] | 252 | |
Damjan Marion | b4fff3a | 2019-03-25 15:54:40 +0100 | [diff] [blame] | 253 | if ((n = vec_len (ptd->integ_ops))) |
| 254 | { |
| 255 | vnet_crypto_op_t *op = ptd->integ_ops; |
| 256 | n -= vnet_crypto_process_ops (vm, op, n); |
| 257 | while (n) |
| 258 | { |
| 259 | ASSERT (op - ptd->integ_ops < vec_len (ptd->integ_ops)); |
| 260 | if (op->status != VNET_CRYPTO_OP_STATUS_COMPLETED) |
| 261 | { |
| 262 | u32 err, bi = op->user_data; |
| 263 | if (op->status == VNET_CRYPTO_OP_STATUS_FAIL_BAD_HMAC) |
| 264 | err = ESP_DECRYPT_ERROR_INTEG_ERROR; |
| 265 | else |
| 266 | err = ESP_DECRYPT_ERROR_CRYPTO_ENGINE_ERROR; |
| 267 | bufs[bi]->error = node->errors[err]; |
| 268 | nexts[bi] = ESP_DECRYPT_NEXT_DROP; |
| 269 | n--; |
| 270 | } |
| 271 | op++; |
| 272 | } |
| 273 | } |
| 274 | |
| 275 | if ((n = vec_len (ptd->crypto_ops))) |
| 276 | { |
| 277 | vnet_crypto_op_t *op = ptd->crypto_ops; |
| 278 | n -= vnet_crypto_process_ops (vm, op, n); |
| 279 | while (n) |
| 280 | { |
| 281 | ASSERT (op - ptd->crypto_ops < vec_len (ptd->crypto_ops)); |
| 282 | if (op->status != VNET_CRYPTO_OP_STATUS_COMPLETED) |
| 283 | { |
| 284 | u32 bi = op->user_data; |
| 285 | u32 err = ESP_DECRYPT_ERROR_CRYPTO_ENGINE_ERROR; |
| 286 | bufs[bi]->error = node->errors[err]; |
| 287 | nexts[bi] = ESP_DECRYPT_NEXT_DROP; |
| 288 | n--; |
| 289 | } |
| 290 | op++; |
| 291 | } |
| 292 | } |
| 293 | |
| 294 | /* Post decryption ronud - adjust packet data start and length and next |
| 295 | node */ |
| 296 | |
| 297 | n_left = from_frame->n_vectors; |
| 298 | next = nexts; |
| 299 | pd = pkt_data; |
| 300 | b = bufs; |
| 301 | |
| 302 | while (n_left) |
| 303 | { |
| 304 | const u8 tun_flags = IPSEC_SA_FLAG_IS_TUNNEL | |
| 305 | IPSEC_SA_FLAG_IS_TUNNEL_V6; |
| 306 | |
| 307 | if (n_left >= 2) |
| 308 | { |
| 309 | void *data = b[1]->data + pd[1].current_data; |
| 310 | |
| 311 | /* buffer metadata */ |
| 312 | vlib_prefetch_buffer_header (b[1], LOAD); |
| 313 | |
| 314 | /* esp_footer_t */ |
| 315 | CLIB_PREFETCH (data + pd[1].current_length - pd[1].icv_sz - 2, |
| 316 | CLIB_CACHE_LINE_BYTES, LOAD); |
| 317 | |
| 318 | /* packet headers */ |
| 319 | CLIB_PREFETCH (data - CLIB_CACHE_LINE_BYTES, |
| 320 | CLIB_CACHE_LINE_BYTES * 2, LOAD); |
| 321 | } |
| 322 | |
| 323 | if (next[0] < ESP_DECRYPT_N_NEXT) |
| 324 | goto trace; |
| 325 | |
| 326 | sa0 = vec_elt_at_index (im->sad, pd->sa_index); |
| 327 | u8 *payload = b[0]->data + pd->current_data; |
| 328 | |
| 329 | ipsec_sa_anti_replay_advance (sa0, &((esp_header_t *) payload)->seq); |
| 330 | |
| 331 | esp_footer_t *f = (esp_footer_t *) (b[0]->data + pd->current_data + |
| 332 | pd->current_length - sizeof (*f) - |
| 333 | pd->icv_sz); |
| 334 | u16 adv = pd->iv_sz + esp_sz; |
| 335 | u16 tail = sizeof (esp_footer_t) + f->pad_length + pd->icv_sz; |
| 336 | |
| 337 | if ((pd->flags & tun_flags) == 0) /* transport mode */ |
| 338 | { |
| 339 | u8 udp_sz = (is_ip6 == 0 && pd->flags & IPSEC_SA_FLAG_UDP_ENCAP) ? |
| 340 | sizeof (udp_header_t) : 0; |
| 341 | u16 ip_hdr_sz = pd->hdr_sz - udp_sz; |
| 342 | u8 *old_ip = b[0]->data + pd->current_data - ip_hdr_sz - udp_sz; |
| 343 | u8 *ip = old_ip + adv + udp_sz; |
| 344 | |
| 345 | if (is_ip6 && ip_hdr_sz > 64) |
| 346 | memmove (ip, old_ip, ip_hdr_sz); |
| 347 | else |
| 348 | clib_memcpy_le64 (ip, old_ip, ip_hdr_sz); |
| 349 | |
| 350 | b[0]->current_data = pd->current_data + adv - ip_hdr_sz; |
| 351 | b[0]->current_length = pd->current_length + ip_hdr_sz - tail - adv; |
| 352 | |
| 353 | if (is_ip6) |
| 354 | { |
| 355 | ip6_header_t *ip6 = (ip6_header_t *) ip; |
| 356 | u16 len = clib_net_to_host_u16 (ip6->payload_length); |
| 357 | len -= adv + tail; |
| 358 | ip6->payload_length = clib_host_to_net_u16 (len); |
| 359 | ip6->protocol = f->next_header; |
| 360 | next[0] = ESP_DECRYPT_NEXT_IP6_INPUT; |
| 361 | } |
| 362 | else |
| 363 | { |
| 364 | ip4_header_t *ip4 = (ip4_header_t *) ip; |
| 365 | ip_csum_t sum = ip4->checksum; |
| 366 | u16 len = clib_net_to_host_u16 (ip4->length); |
| 367 | len = clib_host_to_net_u16 (len - adv - tail - udp_sz); |
| 368 | sum = ip_csum_update (sum, ip4->protocol, f->next_header, |
| 369 | ip4_header_t, protocol); |
| 370 | sum = ip_csum_update (sum, ip4->length, len, |
| 371 | ip4_header_t, length); |
| 372 | ip4->checksum = ip_csum_fold (sum); |
| 373 | ip4->protocol = f->next_header; |
| 374 | ip4->length = len; |
| 375 | next[0] = ESP_DECRYPT_NEXT_IP4_INPUT; |
| 376 | } |
| 377 | } |
| 378 | else |
| 379 | { |
| 380 | if (PREDICT_TRUE (f->next_header == IP_PROTOCOL_IP_IN_IP)) |
| 381 | { |
| 382 | next[0] = ESP_DECRYPT_NEXT_IP4_INPUT; |
| 383 | b[0]->current_data = pd->current_data + adv; |
| 384 | b[0]->current_length = pd->current_length + adv - tail; |
| 385 | } |
| 386 | else if (f->next_header == IP_PROTOCOL_IPV6) |
| 387 | { |
| 388 | next[0] = ESP_DECRYPT_NEXT_IP6_INPUT; |
| 389 | b[0]->current_data = pd->current_data + adv; |
| 390 | b[0]->current_length = pd->current_length + adv - tail; |
| 391 | } |
| 392 | else |
| 393 | { |
| 394 | next[0] = ESP_DECRYPT_NEXT_DROP; |
| 395 | b[0]->error = node->errors[ESP_DECRYPT_ERROR_DECRYPTION_FAILED]; |
| 396 | } |
| 397 | } |
| 398 | |
| 399 | if (vnet_buffer (b[0])->ipsec.flags & IPSEC_FLAG_IPSEC_GRE_TUNNEL) |
| 400 | next[0] = ESP_DECRYPT_NEXT_IPSEC_GRE_INPUT; |
| 401 | |
| 402 | trace: |
| 403 | if (PREDICT_FALSE (b[0]->flags & VLIB_BUFFER_IS_TRACED)) |
| 404 | { |
| 405 | esp_decrypt_trace_t *tr; |
| 406 | u8 *payload = b[0]->data + pd->current_data; |
| 407 | tr = vlib_add_trace (vm, node, b[0], sizeof (*tr)); |
| 408 | sa0 = pool_elt_at_index (im->sad, |
| 409 | vnet_buffer (b[0])->ipsec.sad_index); |
| 410 | tr->crypto_alg = sa0->crypto_alg; |
| 411 | tr->integ_alg = sa0->integ_alg; |
| 412 | tr->seq = clib_host_to_net_u32 (((esp_header_t *) payload)->seq); |
| 413 | } |
| 414 | |
| 415 | /* next */ |
| 416 | n_left -= 1; |
| 417 | next += 1; |
| 418 | pd += 1; |
| 419 | b += 1; |
| 420 | } |
| 421 | |
| 422 | n_left = from_frame->n_vectors; |
| 423 | vlib_node_increment_counter (vm, node->node_index, |
| 424 | ESP_DECRYPT_ERROR_RX_PKTS, n_left); |
| 425 | |
| 426 | vlib_buffer_enqueue_to_next (vm, node, from, nexts, n_left); |
| 427 | |
| 428 | b = bufs; |
| 429 | return n_left; |
Ed Warnicke | cb9cada | 2015-12-08 15:45:58 -0700 | [diff] [blame] | 430 | } |
| 431 | |
Klement Sekera | b8f3544 | 2018-10-29 13:38:19 +0100 | [diff] [blame] | 432 | VLIB_NODE_FN (esp4_decrypt_node) (vlib_main_t * vm, |
| 433 | vlib_node_runtime_t * node, |
| 434 | vlib_frame_t * from_frame) |
Klement Sekera | be5a5dd | 2018-10-09 16:05:48 +0200 | [diff] [blame] | 435 | { |
| 436 | return esp_decrypt_inline (vm, node, from_frame, 0 /* is_ip6 */ ); |
| 437 | } |
Ed Warnicke | cb9cada | 2015-12-08 15:45:58 -0700 | [diff] [blame] | 438 | |
Keith Burns (alagalah) | 166a9d4 | 2016-08-06 11:00:56 -0700 | [diff] [blame] | 439 | /* *INDENT-OFF* */ |
Klement Sekera | be5a5dd | 2018-10-09 16:05:48 +0200 | [diff] [blame] | 440 | VLIB_REGISTER_NODE (esp4_decrypt_node) = { |
Klement Sekera | be5a5dd | 2018-10-09 16:05:48 +0200 | [diff] [blame] | 441 | .name = "esp4-decrypt", |
Ed Warnicke | cb9cada | 2015-12-08 15:45:58 -0700 | [diff] [blame] | 442 | .vector_size = sizeof (u32), |
| 443 | .format_trace = format_esp_decrypt_trace, |
| 444 | .type = VLIB_NODE_TYPE_INTERNAL, |
| 445 | |
| 446 | .n_errors = ARRAY_LEN(esp_decrypt_error_strings), |
| 447 | .error_strings = esp_decrypt_error_strings, |
| 448 | |
| 449 | .n_next_nodes = ESP_DECRYPT_N_NEXT, |
| 450 | .next_nodes = { |
| 451 | #define _(s,n) [ESP_DECRYPT_NEXT_##s] = n, |
| 452 | foreach_esp_decrypt_next |
| 453 | #undef _ |
| 454 | }, |
| 455 | }; |
Keith Burns (alagalah) | 166a9d4 | 2016-08-06 11:00:56 -0700 | [diff] [blame] | 456 | /* *INDENT-ON* */ |
Ed Warnicke | cb9cada | 2015-12-08 15:45:58 -0700 | [diff] [blame] | 457 | |
Klement Sekera | b8f3544 | 2018-10-29 13:38:19 +0100 | [diff] [blame] | 458 | VLIB_NODE_FN (esp6_decrypt_node) (vlib_main_t * vm, |
| 459 | vlib_node_runtime_t * node, |
| 460 | vlib_frame_t * from_frame) |
Klement Sekera | be5a5dd | 2018-10-09 16:05:48 +0200 | [diff] [blame] | 461 | { |
| 462 | return esp_decrypt_inline (vm, node, from_frame, 1 /* is_ip6 */ ); |
| 463 | } |
| 464 | |
| 465 | /* *INDENT-OFF* */ |
| 466 | VLIB_REGISTER_NODE (esp6_decrypt_node) = { |
Klement Sekera | be5a5dd | 2018-10-09 16:05:48 +0200 | [diff] [blame] | 467 | .name = "esp6-decrypt", |
| 468 | .vector_size = sizeof (u32), |
| 469 | .format_trace = format_esp_decrypt_trace, |
| 470 | .type = VLIB_NODE_TYPE_INTERNAL, |
| 471 | |
| 472 | .n_errors = ARRAY_LEN(esp_decrypt_error_strings), |
| 473 | .error_strings = esp_decrypt_error_strings, |
| 474 | |
| 475 | .n_next_nodes = ESP_DECRYPT_N_NEXT, |
| 476 | .next_nodes = { |
| 477 | #define _(s,n) [ESP_DECRYPT_NEXT_##s] = n, |
| 478 | foreach_esp_decrypt_next |
| 479 | #undef _ |
| 480 | }, |
| 481 | }; |
| 482 | /* *INDENT-ON* */ |
| 483 | |
Keith Burns (alagalah) | 166a9d4 | 2016-08-06 11:00:56 -0700 | [diff] [blame] | 484 | /* |
| 485 | * fd.io coding-style-patch-verification: ON |
| 486 | * |
| 487 | * Local Variables: |
| 488 | * eval: (c-set-style "gnu") |
| 489 | * End: |
| 490 | */ |