Ed Warnicke | cb9cada | 2015-12-08 15:45:58 -0700 | [diff] [blame] | 1 | /* |
| 2 | * esp_encrypt.c : IPSec ESP encrypt node |
| 3 | * |
| 4 | * Copyright (c) 2015 Cisco and/or its affiliates. |
| 5 | * Licensed under the Apache License, Version 2.0 (the "License"); |
| 6 | * you may not use this file except in compliance with the License. |
| 7 | * You may obtain a copy of the License at: |
| 8 | * |
| 9 | * http://www.apache.org/licenses/LICENSE-2.0 |
| 10 | * |
| 11 | * Unless required by applicable law or agreed to in writing, software |
| 12 | * distributed under the License is distributed on an "AS IS" BASIS, |
| 13 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. |
| 14 | * See the License for the specific language governing permissions and |
| 15 | * limitations under the License. |
| 16 | */ |
| 17 | |
| 18 | #include <vnet/vnet.h> |
| 19 | #include <vnet/api_errno.h> |
| 20 | #include <vnet/ip/ip.h> |
| 21 | |
Damjan Marion | 91f17dc | 2019-03-18 18:59:25 +0100 | [diff] [blame] | 22 | #include <vnet/crypto/crypto.h> |
| 23 | |
Ed Warnicke | cb9cada | 2015-12-08 15:45:58 -0700 | [diff] [blame] | 24 | #include <vnet/ipsec/ipsec.h> |
Neale Ranns | 2828721 | 2019-12-16 00:53:11 +0000 | [diff] [blame] | 25 | #include <vnet/ipsec/ipsec_tun.h> |
Ed Warnicke | cb9cada | 2015-12-08 15:45:58 -0700 | [diff] [blame] | 26 | #include <vnet/ipsec/esp.h> |
Neale Ranns | 041add7 | 2020-01-02 04:06:10 +0000 | [diff] [blame] | 27 | #include <vnet/tunnel/tunnel_dp.h> |
Ed Warnicke | cb9cada | 2015-12-08 15:45:58 -0700 | [diff] [blame] | 28 | |
Ed Warnicke | cb9cada | 2015-12-08 15:45:58 -0700 | [diff] [blame] | 29 | #define foreach_esp_encrypt_next \ |
Neale Ranns | b1fd80f | 2020-05-12 13:33:56 +0000 | [diff] [blame] | 30 | _(DROP4, "ip4-drop") \ |
| 31 | _(DROP6, "ip6-drop") \ |
Neale Ranns | b1fd80f | 2020-05-12 13:33:56 +0000 | [diff] [blame] | 32 | _(HANDOFF4, "handoff4") \ |
| 33 | _(HANDOFF6, "handoff6") \ |
Ed Warnicke | cb9cada | 2015-12-08 15:45:58 -0700 | [diff] [blame] | 34 | _(INTERFACE_OUTPUT, "interface-output") |
| 35 | |
| 36 | #define _(v, s) ESP_ENCRYPT_NEXT_##v, |
Keith Burns (alagalah) | 166a9d4 | 2016-08-06 11:00:56 -0700 | [diff] [blame] | 37 | typedef enum |
| 38 | { |
Ed Warnicke | cb9cada | 2015-12-08 15:45:58 -0700 | [diff] [blame] | 39 | foreach_esp_encrypt_next |
| 40 | #undef _ |
Keith Burns (alagalah) | 166a9d4 | 2016-08-06 11:00:56 -0700 | [diff] [blame] | 41 | ESP_ENCRYPT_N_NEXT, |
Ed Warnicke | cb9cada | 2015-12-08 15:45:58 -0700 | [diff] [blame] | 42 | } esp_encrypt_next_t; |
| 43 | |
Damjan Marion | c59b9a2 | 2019-03-19 15:38:40 +0100 | [diff] [blame] | 44 | #define foreach_esp_encrypt_error \ |
| 45 | _(RX_PKTS, "ESP pkts received") \ |
Fan Zhang | f539578 | 2020-04-29 14:00:03 +0100 | [diff] [blame] | 46 | _(POST_RX_PKTS, "ESP-post pkts received") \ |
Damjan Marion | c59b9a2 | 2019-03-19 15:38:40 +0100 | [diff] [blame] | 47 | _(SEQ_CYCLED, "sequence number cycled (packet dropped)") \ |
| 48 | _(CRYPTO_ENGINE_ERROR, "crypto engine error (packet dropped)") \ |
Fan Zhang | f539578 | 2020-04-29 14:00:03 +0100 | [diff] [blame] | 49 | _(CRYPTO_QUEUE_FULL, "crypto queue full (packet dropped)") \ |
Filip Tehlar | efcad1a | 2020-02-04 09:36:04 +0000 | [diff] [blame] | 50 | _(NO_BUFFERS, "no buffers (packet dropped)") \ |
Ed Warnicke | cb9cada | 2015-12-08 15:45:58 -0700 | [diff] [blame] | 51 | |
Keith Burns (alagalah) | 166a9d4 | 2016-08-06 11:00:56 -0700 | [diff] [blame] | 52 | typedef enum |
| 53 | { |
Ed Warnicke | cb9cada | 2015-12-08 15:45:58 -0700 | [diff] [blame] | 54 | #define _(sym,str) ESP_ENCRYPT_ERROR_##sym, |
| 55 | foreach_esp_encrypt_error |
| 56 | #undef _ |
Keith Burns (alagalah) | 166a9d4 | 2016-08-06 11:00:56 -0700 | [diff] [blame] | 57 | ESP_ENCRYPT_N_ERROR, |
Ed Warnicke | cb9cada | 2015-12-08 15:45:58 -0700 | [diff] [blame] | 58 | } esp_encrypt_error_t; |
| 59 | |
Keith Burns (alagalah) | 166a9d4 | 2016-08-06 11:00:56 -0700 | [diff] [blame] | 60 | static char *esp_encrypt_error_strings[] = { |
Ed Warnicke | cb9cada | 2015-12-08 15:45:58 -0700 | [diff] [blame] | 61 | #define _(sym,string) string, |
| 62 | foreach_esp_encrypt_error |
| 63 | #undef _ |
| 64 | }; |
| 65 | |
Keith Burns (alagalah) | 166a9d4 | 2016-08-06 11:00:56 -0700 | [diff] [blame] | 66 | typedef struct |
| 67 | { |
Neale Ranns | 8d7c502 | 2019-02-06 01:41:05 -0800 | [diff] [blame] | 68 | u32 sa_index; |
Ed Warnicke | cb9cada | 2015-12-08 15:45:58 -0700 | [diff] [blame] | 69 | u32 spi; |
| 70 | u32 seq; |
Neale Ranns | 6afaae1 | 2019-07-17 15:07:14 +0000 | [diff] [blame] | 71 | u32 sa_seq_hi; |
Klement Sekera | 4b089f2 | 2018-04-17 18:04:57 +0200 | [diff] [blame] | 72 | u8 udp_encap; |
Ed Warnicke | cb9cada | 2015-12-08 15:45:58 -0700 | [diff] [blame] | 73 | ipsec_crypto_alg_t crypto_alg; |
| 74 | ipsec_integ_alg_t integ_alg; |
| 75 | } esp_encrypt_trace_t; |
| 76 | |
Fan Zhang | f539578 | 2020-04-29 14:00:03 +0100 | [diff] [blame] | 77 | typedef struct |
| 78 | { |
| 79 | u32 next_index; |
| 80 | } esp_encrypt_post_trace_t; |
| 81 | |
Ed Warnicke | cb9cada | 2015-12-08 15:45:58 -0700 | [diff] [blame] | 82 | /* packet trace format function */ |
Keith Burns (alagalah) | 166a9d4 | 2016-08-06 11:00:56 -0700 | [diff] [blame] | 83 | static u8 * |
| 84 | format_esp_encrypt_trace (u8 * s, va_list * args) |
Ed Warnicke | cb9cada | 2015-12-08 15:45:58 -0700 | [diff] [blame] | 85 | { |
| 86 | CLIB_UNUSED (vlib_main_t * vm) = va_arg (*args, vlib_main_t *); |
| 87 | CLIB_UNUSED (vlib_node_t * node) = va_arg (*args, vlib_node_t *); |
Keith Burns (alagalah) | 166a9d4 | 2016-08-06 11:00:56 -0700 | [diff] [blame] | 88 | esp_encrypt_trace_t *t = va_arg (*args, esp_encrypt_trace_t *); |
Ed Warnicke | cb9cada | 2015-12-08 15:45:58 -0700 | [diff] [blame] | 89 | |
Guillaume Solignac | 8f818cc | 2019-05-15 12:02:33 +0200 | [diff] [blame] | 90 | s = |
| 91 | format (s, |
Neale Ranns | 6afaae1 | 2019-07-17 15:07:14 +0000 | [diff] [blame] | 92 | "esp: sa-index %d spi %u (0x%08x) seq %u sa-seq-hi %u crypto %U integrity %U%s", |
| 93 | t->sa_index, t->spi, t->spi, t->seq, t->sa_seq_hi, |
| 94 | format_ipsec_crypto_alg, |
Guillaume Solignac | 8f818cc | 2019-05-15 12:02:33 +0200 | [diff] [blame] | 95 | t->crypto_alg, format_ipsec_integ_alg, t->integ_alg, |
| 96 | t->udp_encap ? " udp-encap-enabled" : ""); |
Ed Warnicke | cb9cada | 2015-12-08 15:45:58 -0700 | [diff] [blame] | 97 | return s; |
| 98 | } |
| 99 | |
Fan Zhang | f539578 | 2020-04-29 14:00:03 +0100 | [diff] [blame] | 100 | static u8 * |
| 101 | format_esp_post_encrypt_trace (u8 * s, va_list * args) |
| 102 | { |
| 103 | CLIB_UNUSED (vlib_main_t * vm) = va_arg (*args, vlib_main_t *); |
| 104 | CLIB_UNUSED (vlib_node_t * node) = va_arg (*args, vlib_node_t *); |
| 105 | esp_encrypt_post_trace_t *t = va_arg (*args, esp_encrypt_post_trace_t *); |
| 106 | |
| 107 | s = format (s, "esp-post: next node index %u", t->next_index); |
| 108 | return s; |
| 109 | } |
| 110 | |
Damjan Marion | c59b9a2 | 2019-03-19 15:38:40 +0100 | [diff] [blame] | 111 | /* pad packet in input buffer */ |
| 112 | static_always_inline u8 * |
Filip Tehlar | e4e8c6b | 2020-02-13 07:49:30 +0000 | [diff] [blame] | 113 | esp_add_footer_and_icv (vlib_main_t * vm, vlib_buffer_t ** last, |
Christian Hopps | fb7e7ed | 2019-11-03 07:02:15 -0500 | [diff] [blame] | 114 | u8 esp_align, u8 icv_sz, |
Filip Tehlar | 8cdb1a0 | 2019-11-18 22:21:37 +0000 | [diff] [blame] | 115 | u16 * next, vlib_node_runtime_t * node, |
Filip Tehlar | efcad1a | 2020-02-04 09:36:04 +0000 | [diff] [blame] | 116 | u16 buffer_data_size, uword total_len) |
Ed Warnicke | cb9cada | 2015-12-08 15:45:58 -0700 | [diff] [blame] | 117 | { |
Damjan Marion | c59b9a2 | 2019-03-19 15:38:40 +0100 | [diff] [blame] | 118 | static const u8 pad_data[ESP_MAX_BLOCK_SIZE] = { |
| 119 | 0x01, 0x02, 0x03, 0x04, 0x05, 0x06, 0x07, 0x08, |
Milan Lenco | 7885c74 | 2020-08-20 13:23:09 +0200 | [diff] [blame] | 120 | 0x09, 0x0a, 0x0b, 0x0c, 0x0d, 0x0e, 0x0f, 0x00, |
Damjan Marion | c59b9a2 | 2019-03-19 15:38:40 +0100 | [diff] [blame] | 121 | }; |
Ed Warnicke | cb9cada | 2015-12-08 15:45:58 -0700 | [diff] [blame] | 122 | |
Filip Tehlar | efcad1a | 2020-02-04 09:36:04 +0000 | [diff] [blame] | 123 | u16 min_length = total_len + sizeof (esp_footer_t); |
Christian Hopps | fb7e7ed | 2019-11-03 07:02:15 -0500 | [diff] [blame] | 124 | u16 new_length = round_pow2 (min_length, esp_align); |
Damjan Marion | c59b9a2 | 2019-03-19 15:38:40 +0100 | [diff] [blame] | 125 | u8 pad_bytes = new_length - min_length; |
Filip Tehlar | e4e8c6b | 2020-02-13 07:49:30 +0000 | [diff] [blame] | 126 | esp_footer_t *f = (esp_footer_t *) (vlib_buffer_get_current (last[0]) + |
| 127 | last[0]->current_length + pad_bytes); |
Filip Tehlar | efcad1a | 2020-02-04 09:36:04 +0000 | [diff] [blame] | 128 | u16 tail_sz = sizeof (esp_footer_t) + pad_bytes + icv_sz; |
Ed Warnicke | cb9cada | 2015-12-08 15:45:58 -0700 | [diff] [blame] | 129 | |
Filip Tehlar | e4e8c6b | 2020-02-13 07:49:30 +0000 | [diff] [blame] | 130 | if (last[0]->current_length + tail_sz > buffer_data_size) |
Filip Tehlar | 8cdb1a0 | 2019-11-18 22:21:37 +0000 | [diff] [blame] | 131 | { |
Filip Tehlar | e4e8c6b | 2020-02-13 07:49:30 +0000 | [diff] [blame] | 132 | u32 tmp_bi = 0; |
| 133 | if (vlib_buffer_alloc (vm, &tmp_bi, 1) != 1) |
| 134 | return 0; |
Filip Tehlar | c2c1bfd | 2020-02-13 07:49:30 +0000 | [diff] [blame] | 135 | |
Filip Tehlar | e4e8c6b | 2020-02-13 07:49:30 +0000 | [diff] [blame] | 136 | vlib_buffer_t *tmp = vlib_get_buffer (vm, tmp_bi); |
| 137 | last[0]->next_buffer = tmp_bi; |
| 138 | last[0]->flags |= VLIB_BUFFER_NEXT_PRESENT; |
| 139 | f = (esp_footer_t *) (vlib_buffer_get_current (tmp) + pad_bytes); |
| 140 | tmp->current_length += tail_sz; |
| 141 | last[0] = tmp; |
| 142 | } |
| 143 | else |
| 144 | last[0]->current_length += tail_sz; |
| 145 | |
| 146 | f->pad_length = pad_bytes; |
Damjan Marion | c59b9a2 | 2019-03-19 15:38:40 +0100 | [diff] [blame] | 147 | if (pad_bytes) |
Benoît Ganne | 4505f01 | 2019-12-07 09:14:27 -0700 | [diff] [blame] | 148 | { |
| 149 | ASSERT (pad_bytes <= ESP_MAX_BLOCK_SIZE); |
| 150 | pad_bytes = clib_min (ESP_MAX_BLOCK_SIZE, pad_bytes); |
| 151 | clib_memcpy_fast ((u8 *) f - pad_bytes, pad_data, pad_bytes); |
| 152 | } |
Damjan Marion | c59b9a2 | 2019-03-19 15:38:40 +0100 | [diff] [blame] | 153 | |
Damjan Marion | c59b9a2 | 2019-03-19 15:38:40 +0100 | [diff] [blame] | 154 | return &f->next_header; |
| 155 | } |
| 156 | |
| 157 | static_always_inline void |
| 158 | esp_update_ip4_hdr (ip4_header_t * ip4, u16 len, int is_transport, int is_udp) |
| 159 | { |
Neale Ranns | 1b582b8 | 2019-04-18 19:49:13 -0700 | [diff] [blame] | 160 | ip_csum_t sum; |
| 161 | u16 old_len; |
| 162 | |
| 163 | len = clib_net_to_host_u16 (len); |
| 164 | old_len = ip4->length; |
Damjan Marion | c59b9a2 | 2019-03-19 15:38:40 +0100 | [diff] [blame] | 165 | |
| 166 | if (is_transport) |
| 167 | { |
| 168 | u8 prot = is_udp ? IP_PROTOCOL_UDP : IP_PROTOCOL_IPSEC_ESP; |
Damjan Marion | c59b9a2 | 2019-03-19 15:38:40 +0100 | [diff] [blame] | 169 | |
Neale Ranns | 1b582b8 | 2019-04-18 19:49:13 -0700 | [diff] [blame] | 170 | sum = ip_csum_update (ip4->checksum, ip4->protocol, |
| 171 | prot, ip4_header_t, protocol); |
| 172 | ip4->protocol = prot; |
| 173 | |
| 174 | sum = ip_csum_update (sum, old_len, len, ip4_header_t, length); |
| 175 | } |
| 176 | else |
| 177 | sum = ip_csum_update (ip4->checksum, old_len, len, ip4_header_t, length); |
| 178 | |
| 179 | ip4->length = len; |
Damjan Marion | c59b9a2 | 2019-03-19 15:38:40 +0100 | [diff] [blame] | 180 | ip4->checksum = ip_csum_fold (sum); |
| 181 | } |
| 182 | |
| 183 | static_always_inline void |
| 184 | esp_fill_udp_hdr (ipsec_sa_t * sa, udp_header_t * udp, u16 len) |
| 185 | { |
| 186 | clib_memcpy_fast (udp, &sa->udp_hdr, sizeof (udp_header_t)); |
| 187 | udp->length = clib_net_to_host_u16 (len); |
| 188 | } |
| 189 | |
| 190 | static_always_inline u8 |
| 191 | ext_hdr_is_pre_esp (u8 nexthdr) |
| 192 | { |
| 193 | #ifdef CLIB_HAVE_VEC128 |
| 194 | static const u8x16 ext_hdr_types = { |
| 195 | IP_PROTOCOL_IP6_HOP_BY_HOP_OPTIONS, |
| 196 | IP_PROTOCOL_IPV6_ROUTE, |
| 197 | IP_PROTOCOL_IPV6_FRAGMENTATION, |
| 198 | }; |
| 199 | |
| 200 | return !u8x16_is_all_zero (ext_hdr_types == u8x16_splat (nexthdr)); |
| 201 | #else |
| 202 | return ((nexthdr ^ IP_PROTOCOL_IP6_HOP_BY_HOP_OPTIONS) | |
| 203 | (nexthdr ^ IP_PROTOCOL_IPV6_ROUTE) | |
| 204 | (nexthdr ^ IP_PROTOCOL_IPV6_FRAGMENTATION) != 0); |
| 205 | #endif |
| 206 | } |
| 207 | |
| 208 | static_always_inline u8 |
Neale Ranns | 0295040 | 2019-12-20 00:54:57 +0000 | [diff] [blame] | 209 | esp_get_ip6_hdr_len (ip6_header_t * ip6, ip6_ext_header_t ** ext_hdr) |
Damjan Marion | c59b9a2 | 2019-03-19 15:38:40 +0100 | [diff] [blame] | 210 | { |
| 211 | /* this code assumes that HbH, route and frag headers will be before |
| 212 | others, if that is not the case, they will end up encrypted */ |
Damjan Marion | c59b9a2 | 2019-03-19 15:38:40 +0100 | [diff] [blame] | 213 | u8 len = sizeof (ip6_header_t); |
| 214 | ip6_ext_header_t *p; |
| 215 | |
| 216 | /* if next packet doesn't have ext header */ |
| 217 | if (ext_hdr_is_pre_esp (ip6->protocol) == 0) |
Neale Ranns | 0295040 | 2019-12-20 00:54:57 +0000 | [diff] [blame] | 218 | { |
| 219 | *ext_hdr = NULL; |
| 220 | return len; |
| 221 | } |
Damjan Marion | c59b9a2 | 2019-03-19 15:38:40 +0100 | [diff] [blame] | 222 | |
| 223 | p = (void *) (ip6 + 1); |
| 224 | len += ip6_ext_header_len (p); |
| 225 | |
| 226 | while (ext_hdr_is_pre_esp (p->next_hdr)) |
| 227 | { |
| 228 | len += ip6_ext_header_len (p); |
| 229 | p = ip6_ext_next_header (p); |
| 230 | } |
| 231 | |
Neale Ranns | 0295040 | 2019-12-20 00:54:57 +0000 | [diff] [blame] | 232 | *ext_hdr = p; |
Damjan Marion | c59b9a2 | 2019-03-19 15:38:40 +0100 | [diff] [blame] | 233 | return len; |
| 234 | } |
| 235 | |
Damjan Marion | c59b9a2 | 2019-03-19 15:38:40 +0100 | [diff] [blame] | 236 | static_always_inline void |
Filip Tehlar | efcad1a | 2020-02-04 09:36:04 +0000 | [diff] [blame] | 237 | esp_process_chained_ops (vlib_main_t * vm, vlib_node_runtime_t * node, |
| 238 | vnet_crypto_op_t * ops, vlib_buffer_t * b[], |
Neale Ranns | b1fd80f | 2020-05-12 13:33:56 +0000 | [diff] [blame] | 239 | u16 * nexts, vnet_crypto_op_chunk_t * chunks, |
| 240 | u16 drop_next) |
Filip Tehlar | efcad1a | 2020-02-04 09:36:04 +0000 | [diff] [blame] | 241 | { |
| 242 | u32 n_fail, n_ops = vec_len (ops); |
| 243 | vnet_crypto_op_t *op = ops; |
| 244 | |
| 245 | if (n_ops == 0) |
| 246 | return; |
| 247 | |
| 248 | n_fail = n_ops - vnet_crypto_process_chained_ops (vm, op, chunks, n_ops); |
| 249 | |
| 250 | while (n_fail) |
| 251 | { |
| 252 | ASSERT (op - ops < n_ops); |
| 253 | |
| 254 | if (op->status != VNET_CRYPTO_OP_STATUS_COMPLETED) |
| 255 | { |
| 256 | u32 bi = op->user_data; |
| 257 | b[bi]->error = node->errors[ESP_ENCRYPT_ERROR_CRYPTO_ENGINE_ERROR]; |
Neale Ranns | b1fd80f | 2020-05-12 13:33:56 +0000 | [diff] [blame] | 258 | nexts[bi] = drop_next; |
Filip Tehlar | efcad1a | 2020-02-04 09:36:04 +0000 | [diff] [blame] | 259 | n_fail--; |
| 260 | } |
| 261 | op++; |
| 262 | } |
| 263 | } |
| 264 | |
| 265 | static_always_inline void |
Damjan Marion | c59b9a2 | 2019-03-19 15:38:40 +0100 | [diff] [blame] | 266 | esp_process_ops (vlib_main_t * vm, vlib_node_runtime_t * node, |
Neale Ranns | b1fd80f | 2020-05-12 13:33:56 +0000 | [diff] [blame] | 267 | vnet_crypto_op_t * ops, vlib_buffer_t * b[], u16 * nexts, |
| 268 | u16 drop_next) |
Damjan Marion | c59b9a2 | 2019-03-19 15:38:40 +0100 | [diff] [blame] | 269 | { |
| 270 | u32 n_fail, n_ops = vec_len (ops); |
| 271 | vnet_crypto_op_t *op = ops; |
| 272 | |
| 273 | if (n_ops == 0) |
Ed Warnicke | cb9cada | 2015-12-08 15:45:58 -0700 | [diff] [blame] | 274 | return; |
| 275 | |
Damjan Marion | c59b9a2 | 2019-03-19 15:38:40 +0100 | [diff] [blame] | 276 | n_fail = n_ops - vnet_crypto_process_ops (vm, op, n_ops); |
Ed Warnicke | cb9cada | 2015-12-08 15:45:58 -0700 | [diff] [blame] | 277 | |
Damjan Marion | c59b9a2 | 2019-03-19 15:38:40 +0100 | [diff] [blame] | 278 | while (n_fail) |
| 279 | { |
| 280 | ASSERT (op - ops < n_ops); |
| 281 | |
| 282 | if (op->status != VNET_CRYPTO_OP_STATUS_COMPLETED) |
| 283 | { |
| 284 | u32 bi = op->user_data; |
| 285 | b[bi]->error = node->errors[ESP_ENCRYPT_ERROR_CRYPTO_ENGINE_ERROR]; |
Neale Ranns | b1fd80f | 2020-05-12 13:33:56 +0000 | [diff] [blame] | 286 | nexts[bi] = drop_next; |
Damjan Marion | c59b9a2 | 2019-03-19 15:38:40 +0100 | [diff] [blame] | 287 | n_fail--; |
| 288 | } |
| 289 | op++; |
| 290 | } |
Ed Warnicke | cb9cada | 2015-12-08 15:45:58 -0700 | [diff] [blame] | 291 | } |
| 292 | |
Damjan Marion | d97918e | 2019-04-25 18:28:31 +0200 | [diff] [blame] | 293 | typedef struct |
| 294 | { |
| 295 | u32 salt; |
| 296 | u64 iv; |
| 297 | } __clib_packed esp_gcm_nonce_t; |
| 298 | |
| 299 | STATIC_ASSERT_SIZEOF (esp_gcm_nonce_t, 12); |
| 300 | |
Fan Zhang | f539578 | 2020-04-29 14:00:03 +0100 | [diff] [blame] | 301 | static_always_inline u32 |
| 302 | esp_encrypt_chain_crypto (vlib_main_t * vm, ipsec_per_thread_data_t * ptd, |
| 303 | ipsec_sa_t * sa0, vlib_buffer_t * b, |
| 304 | vlib_buffer_t * lb, u8 icv_sz, u8 * start, |
| 305 | u32 start_len, u16 * n_ch) |
| 306 | { |
| 307 | vnet_crypto_op_chunk_t *ch; |
| 308 | vlib_buffer_t *cb = b; |
| 309 | u32 n_chunks = 1; |
| 310 | u32 total_len; |
| 311 | vec_add2 (ptd->chunks, ch, 1); |
| 312 | total_len = ch->len = start_len; |
| 313 | ch->src = ch->dst = start; |
| 314 | cb = vlib_get_buffer (vm, cb->next_buffer); |
| 315 | |
| 316 | while (1) |
| 317 | { |
| 318 | vec_add2 (ptd->chunks, ch, 1); |
| 319 | n_chunks += 1; |
| 320 | if (lb == cb) |
| 321 | total_len += ch->len = cb->current_length - icv_sz; |
| 322 | else |
| 323 | total_len += ch->len = cb->current_length; |
| 324 | ch->src = ch->dst = vlib_buffer_get_current (cb); |
| 325 | |
| 326 | if (!(cb->flags & VLIB_BUFFER_NEXT_PRESENT)) |
| 327 | break; |
| 328 | |
| 329 | cb = vlib_get_buffer (vm, cb->next_buffer); |
| 330 | } |
| 331 | |
| 332 | if (n_ch) |
| 333 | *n_ch = n_chunks; |
| 334 | |
| 335 | return total_len; |
| 336 | } |
| 337 | |
| 338 | static_always_inline u32 |
| 339 | esp_encrypt_chain_integ (vlib_main_t * vm, ipsec_per_thread_data_t * ptd, |
| 340 | ipsec_sa_t * sa0, vlib_buffer_t * b, |
| 341 | vlib_buffer_t * lb, u8 icv_sz, u8 * start, |
| 342 | u32 start_len, u8 * digest, u16 * n_ch) |
| 343 | { |
| 344 | vnet_crypto_op_chunk_t *ch; |
| 345 | vlib_buffer_t *cb = b; |
| 346 | u32 n_chunks = 1; |
| 347 | u32 total_len; |
| 348 | vec_add2 (ptd->chunks, ch, 1); |
| 349 | total_len = ch->len = start_len; |
| 350 | ch->src = start; |
| 351 | cb = vlib_get_buffer (vm, cb->next_buffer); |
| 352 | |
| 353 | while (1) |
| 354 | { |
| 355 | vec_add2 (ptd->chunks, ch, 1); |
| 356 | n_chunks += 1; |
| 357 | if (lb == cb) |
| 358 | { |
| 359 | total_len += ch->len = cb->current_length - icv_sz; |
| 360 | if (ipsec_sa_is_set_USE_ESN (sa0)) |
| 361 | { |
| 362 | u32 seq_hi = clib_net_to_host_u32 (sa0->seq_hi); |
| 363 | clib_memcpy_fast (digest, &seq_hi, sizeof (seq_hi)); |
| 364 | ch->len += sizeof (seq_hi); |
| 365 | total_len += sizeof (seq_hi); |
| 366 | } |
| 367 | } |
| 368 | else |
| 369 | total_len += ch->len = cb->current_length; |
| 370 | ch->src = vlib_buffer_get_current (cb); |
| 371 | |
| 372 | if (!(cb->flags & VLIB_BUFFER_NEXT_PRESENT)) |
| 373 | break; |
| 374 | |
| 375 | cb = vlib_get_buffer (vm, cb->next_buffer); |
| 376 | } |
| 377 | |
| 378 | if (n_ch) |
| 379 | *n_ch = n_chunks; |
| 380 | |
| 381 | return total_len; |
| 382 | } |
| 383 | |
| 384 | always_inline void |
| 385 | esp_prepare_sync_op (vlib_main_t * vm, ipsec_per_thread_data_t * ptd, |
| 386 | vnet_crypto_op_t ** crypto_ops, |
| 387 | vnet_crypto_op_t ** integ_ops, ipsec_sa_t * sa0, |
| 388 | u8 * payload, u16 payload_len, u8 iv_sz, u8 icv_sz, |
| 389 | vlib_buffer_t ** bufs, vlib_buffer_t ** b, |
| 390 | vlib_buffer_t * lb, u32 hdr_len, esp_header_t * esp, |
| 391 | esp_gcm_nonce_t * nonce) |
| 392 | { |
| 393 | if (sa0->crypto_enc_op_id) |
| 394 | { |
| 395 | vnet_crypto_op_t *op; |
| 396 | vec_add2_aligned (crypto_ops[0], op, 1, CLIB_CACHE_LINE_BYTES); |
| 397 | vnet_crypto_op_init (op, sa0->crypto_enc_op_id); |
| 398 | |
| 399 | op->src = op->dst = payload; |
| 400 | op->key_index = sa0->crypto_key_index; |
| 401 | op->len = payload_len - icv_sz; |
| 402 | op->user_data = b - bufs; |
| 403 | |
| 404 | if (ipsec_sa_is_set_IS_AEAD (sa0)) |
| 405 | { |
| 406 | /* |
| 407 | * construct the AAD in a scratch space in front |
| 408 | * of the IP header. |
| 409 | */ |
| 410 | op->aad = payload - hdr_len - sizeof (esp_aead_t); |
| 411 | op->aad_len = esp_aad_fill (op->aad, esp, sa0); |
| 412 | |
| 413 | op->tag = payload + op->len; |
| 414 | op->tag_len = 16; |
| 415 | |
| 416 | u64 *iv = (u64 *) (payload - iv_sz); |
| 417 | nonce->salt = sa0->salt; |
| 418 | nonce->iv = *iv = clib_host_to_net_u64 (sa0->gcm_iv_counter++); |
| 419 | op->iv = (u8 *) nonce; |
| 420 | } |
| 421 | else |
| 422 | { |
| 423 | op->iv = payload - iv_sz; |
| 424 | op->flags = VNET_CRYPTO_OP_FLAG_INIT_IV; |
| 425 | } |
| 426 | |
| 427 | if (lb != b[0]) |
| 428 | { |
| 429 | /* is chained */ |
| 430 | op->flags |= VNET_CRYPTO_OP_FLAG_CHAINED_BUFFERS; |
| 431 | op->chunk_index = vec_len (ptd->chunks); |
| 432 | op->tag = vlib_buffer_get_tail (lb) - icv_sz; |
| 433 | esp_encrypt_chain_crypto (vm, ptd, sa0, b[0], lb, icv_sz, payload, |
| 434 | payload_len, &op->n_chunks); |
| 435 | } |
| 436 | } |
| 437 | |
| 438 | if (sa0->integ_op_id) |
| 439 | { |
| 440 | vnet_crypto_op_t *op; |
| 441 | vec_add2_aligned (integ_ops[0], op, 1, CLIB_CACHE_LINE_BYTES); |
| 442 | vnet_crypto_op_init (op, sa0->integ_op_id); |
| 443 | op->src = payload - iv_sz - sizeof (esp_header_t); |
| 444 | op->digest = payload + payload_len - icv_sz; |
| 445 | op->key_index = sa0->integ_key_index; |
| 446 | op->digest_len = icv_sz; |
| 447 | op->len = payload_len - icv_sz + iv_sz + sizeof (esp_header_t); |
| 448 | op->user_data = b - bufs; |
| 449 | |
| 450 | if (lb != b[0]) |
| 451 | { |
| 452 | /* is chained */ |
| 453 | op->flags |= VNET_CRYPTO_OP_FLAG_CHAINED_BUFFERS; |
| 454 | op->chunk_index = vec_len (ptd->chunks); |
| 455 | op->digest = vlib_buffer_get_tail (lb) - icv_sz; |
| 456 | |
| 457 | esp_encrypt_chain_integ (vm, ptd, sa0, b[0], lb, icv_sz, |
| 458 | payload - iv_sz - sizeof (esp_header_t), |
| 459 | payload_len + iv_sz + |
| 460 | sizeof (esp_header_t), op->digest, |
| 461 | &op->n_chunks); |
| 462 | } |
| 463 | else if (ipsec_sa_is_set_USE_ESN (sa0)) |
| 464 | { |
| 465 | u32 seq_hi = clib_net_to_host_u32 (sa0->seq_hi); |
| 466 | clib_memcpy_fast (op->digest, &seq_hi, sizeof (seq_hi)); |
| 467 | op->len += sizeof (seq_hi); |
| 468 | } |
| 469 | } |
| 470 | } |
| 471 | |
| 472 | static_always_inline int |
| 473 | esp_prepare_async_frame (vlib_main_t * vm, ipsec_per_thread_data_t * ptd, |
| 474 | vnet_crypto_async_frame_t ** async_frame, |
| 475 | ipsec_sa_t * sa, vlib_buffer_t * b, |
| 476 | esp_header_t * esp, u8 * payload, u32 payload_len, |
Fan Zhang | 18f0e31 | 2020-10-19 13:08:34 +0100 | [diff] [blame] | 477 | u8 iv_sz, u8 icv_sz, u32 bi, u16 next, u32 hdr_len, |
Fan Zhang | f539578 | 2020-04-29 14:00:03 +0100 | [diff] [blame] | 478 | u16 async_next, vlib_buffer_t * lb) |
| 479 | { |
| 480 | esp_post_data_t *post = esp_post_data (b); |
| 481 | u8 *tag, *iv, *aad = 0; |
| 482 | u8 flag = 0; |
| 483 | u32 key_index; |
| 484 | i16 crypto_start_offset, integ_start_offset = 0; |
| 485 | u16 crypto_total_len, integ_total_len; |
| 486 | |
Fan Zhang | 18f0e31 | 2020-10-19 13:08:34 +0100 | [diff] [blame] | 487 | post->next_index = next; |
Fan Zhang | f539578 | 2020-04-29 14:00:03 +0100 | [diff] [blame] | 488 | |
| 489 | /* crypto */ |
| 490 | crypto_start_offset = payload - b->data; |
| 491 | crypto_total_len = integ_total_len = payload_len - icv_sz; |
| 492 | tag = payload + crypto_total_len; |
| 493 | |
| 494 | /* aead */ |
| 495 | if (ipsec_sa_is_set_IS_AEAD (sa)) |
| 496 | { |
| 497 | esp_gcm_nonce_t *nonce; |
| 498 | u64 *pkt_iv = (u64 *) (payload - iv_sz); |
| 499 | |
| 500 | aad = payload - hdr_len - sizeof (esp_aead_t); |
| 501 | esp_aad_fill (aad, esp, sa); |
| 502 | nonce = (esp_gcm_nonce_t *) (aad - sizeof (*nonce)); |
| 503 | nonce->salt = sa->salt; |
| 504 | nonce->iv = *pkt_iv = clib_host_to_net_u64 (sa->gcm_iv_counter++); |
| 505 | iv = (u8 *) nonce; |
| 506 | key_index = sa->crypto_key_index; |
| 507 | |
| 508 | if (lb != b) |
| 509 | { |
| 510 | /* chain */ |
| 511 | flag |= VNET_CRYPTO_OP_FLAG_CHAINED_BUFFERS; |
| 512 | tag = vlib_buffer_get_tail (lb) - icv_sz; |
| 513 | crypto_total_len = esp_encrypt_chain_crypto (vm, ptd, sa, b, lb, |
| 514 | icv_sz, payload, |
| 515 | payload_len, 0); |
| 516 | } |
| 517 | goto out; |
| 518 | } |
| 519 | |
| 520 | /* cipher then hash */ |
| 521 | iv = payload - iv_sz; |
| 522 | integ_start_offset = crypto_start_offset - iv_sz - sizeof (esp_header_t); |
| 523 | integ_total_len += iv_sz + sizeof (esp_header_t); |
| 524 | flag |= VNET_CRYPTO_OP_FLAG_INIT_IV; |
| 525 | key_index = sa->linked_key_index; |
| 526 | |
| 527 | if (b != lb) |
| 528 | { |
| 529 | flag |= VNET_CRYPTO_OP_FLAG_CHAINED_BUFFERS; |
| 530 | crypto_total_len = esp_encrypt_chain_crypto (vm, ptd, sa, b, lb, |
| 531 | icv_sz, payload, |
| 532 | payload_len, 0); |
| 533 | tag = vlib_buffer_get_tail (lb) - icv_sz; |
| 534 | integ_total_len = esp_encrypt_chain_integ (vm, ptd, sa, b, lb, icv_sz, |
| 535 | payload - iv_sz - |
| 536 | sizeof (esp_header_t), |
| 537 | payload_len + iv_sz + |
| 538 | sizeof (esp_header_t), |
| 539 | tag, 0); |
| 540 | } |
| 541 | else if (ipsec_sa_is_set_USE_ESN (sa) && !ipsec_sa_is_set_IS_AEAD (sa)) |
| 542 | { |
| 543 | u32 seq_hi = clib_net_to_host_u32 (sa->seq_hi); |
| 544 | clib_memcpy_fast (tag, &seq_hi, sizeof (seq_hi)); |
| 545 | integ_total_len += sizeof (seq_hi); |
| 546 | } |
| 547 | |
| 548 | out: |
| 549 | return vnet_crypto_async_add_to_frame (vm, async_frame, key_index, |
| 550 | crypto_total_len, |
| 551 | integ_total_len - crypto_total_len, |
| 552 | crypto_start_offset, |
| 553 | integ_start_offset, bi, async_next, |
| 554 | iv, tag, aad, flag); |
| 555 | } |
| 556 | |
Klement Sekera | be5a5dd | 2018-10-09 16:05:48 +0200 | [diff] [blame] | 557 | always_inline uword |
Damjan Marion | c59b9a2 | 2019-03-19 15:38:40 +0100 | [diff] [blame] | 558 | esp_encrypt_inline (vlib_main_t * vm, vlib_node_runtime_t * node, |
Fan Zhang | f539578 | 2020-04-29 14:00:03 +0100 | [diff] [blame] | 559 | vlib_frame_t * frame, int is_ip6, int is_tun, |
| 560 | u16 async_next) |
Ed Warnicke | cb9cada | 2015-12-08 15:45:58 -0700 | [diff] [blame] | 561 | { |
Ed Warnicke | cb9cada | 2015-12-08 15:45:58 -0700 | [diff] [blame] | 562 | ipsec_main_t *im = &ipsec_main; |
Damjan Marion | c59b9a2 | 2019-03-19 15:38:40 +0100 | [diff] [blame] | 563 | ipsec_per_thread_data_t *ptd = vec_elt_at_index (im->ptd, vm->thread_index); |
| 564 | u32 *from = vlib_frame_vector_args (frame); |
| 565 | u32 n_left = frame->n_vectors; |
| 566 | vlib_buffer_t *bufs[VLIB_FRAME_SIZE], **b = bufs; |
Damjan Marion | c98275f | 2019-03-06 14:05:01 +0100 | [diff] [blame] | 567 | u16 nexts[VLIB_FRAME_SIZE], *next = nexts; |
Damjan Marion | d97918e | 2019-04-25 18:28:31 +0200 | [diff] [blame] | 568 | esp_gcm_nonce_t nonces[VLIB_FRAME_SIZE], *nonce = nonces; |
Damjan Marion | c59b9a2 | 2019-03-19 15:38:40 +0100 | [diff] [blame] | 569 | u32 thread_index = vm->thread_index; |
| 570 | u16 buffer_data_size = vlib_buffer_get_default_data_size (vm); |
| 571 | u32 current_sa_index = ~0, current_sa_packets = 0; |
| 572 | u32 current_sa_bytes = 0, spi = 0; |
Christian Hopps | fb7e7ed | 2019-11-03 07:02:15 -0500 | [diff] [blame] | 573 | u8 esp_align = 4, iv_sz = 0, icv_sz = 0; |
Damjan Marion | c59b9a2 | 2019-03-19 15:38:40 +0100 | [diff] [blame] | 574 | ipsec_sa_t *sa0 = 0; |
Filip Tehlar | efcad1a | 2020-02-04 09:36:04 +0000 | [diff] [blame] | 575 | vlib_buffer_t *lb; |
| 576 | vnet_crypto_op_t **crypto_ops = &ptd->crypto_ops; |
| 577 | vnet_crypto_op_t **integ_ops = &ptd->integ_ops; |
Fan Zhang | f539578 | 2020-04-29 14:00:03 +0100 | [diff] [blame] | 578 | vnet_crypto_async_frame_t *async_frame = 0; |
| 579 | int is_async = im->async_mode; |
| 580 | vnet_crypto_async_op_id_t last_async_op = ~0; |
Neale Ranns | b1fd80f | 2020-05-12 13:33:56 +0000 | [diff] [blame] | 581 | u16 drop_next = (is_ip6 ? ESP_ENCRYPT_NEXT_DROP6 : ESP_ENCRYPT_NEXT_DROP4); |
Fan Zhang | 18f0e31 | 2020-10-19 13:08:34 +0100 | [diff] [blame] | 582 | u16 n_async_drop = 0; |
Ed Warnicke | cb9cada | 2015-12-08 15:45:58 -0700 | [diff] [blame] | 583 | |
Damjan Marion | c59b9a2 | 2019-03-19 15:38:40 +0100 | [diff] [blame] | 584 | vlib_get_buffers (vm, from, b, n_left); |
Fan Zhang | f539578 | 2020-04-29 14:00:03 +0100 | [diff] [blame] | 585 | if (!is_async) |
| 586 | { |
| 587 | vec_reset_length (ptd->crypto_ops); |
| 588 | vec_reset_length (ptd->integ_ops); |
| 589 | vec_reset_length (ptd->chained_crypto_ops); |
| 590 | vec_reset_length (ptd->chained_integ_ops); |
| 591 | } |
Filip Tehlar | efcad1a | 2020-02-04 09:36:04 +0000 | [diff] [blame] | 592 | vec_reset_length (ptd->chunks); |
Damjan Marion | c59b9a2 | 2019-03-19 15:38:40 +0100 | [diff] [blame] | 593 | |
| 594 | while (n_left > 0) |
Keith Burns (alagalah) | 166a9d4 | 2016-08-06 11:00:56 -0700 | [diff] [blame] | 595 | { |
Zhiyong Yang | 1cff643 | 2019-04-30 05:33:53 -0400 | [diff] [blame] | 596 | u32 sa_index0; |
Damjan Marion | c59b9a2 | 2019-03-19 15:38:40 +0100 | [diff] [blame] | 597 | dpo_id_t *dpo; |
| 598 | esp_header_t *esp; |
| 599 | u8 *payload, *next_hdr_ptr; |
Filip Tehlar | efcad1a | 2020-02-04 09:36:04 +0000 | [diff] [blame] | 600 | u16 payload_len, payload_len_total, n_bufs; |
Neale Ranns | 4ec36c5 | 2020-03-31 09:21:29 -0400 | [diff] [blame] | 601 | u32 hdr_len; |
Ed Warnicke | cb9cada | 2015-12-08 15:45:58 -0700 | [diff] [blame] | 602 | |
Damjan Marion | c59b9a2 | 2019-03-19 15:38:40 +0100 | [diff] [blame] | 603 | if (n_left > 2) |
Keith Burns (alagalah) | 166a9d4 | 2016-08-06 11:00:56 -0700 | [diff] [blame] | 604 | { |
Damjan Marion | c59b9a2 | 2019-03-19 15:38:40 +0100 | [diff] [blame] | 605 | u8 *p; |
| 606 | vlib_prefetch_buffer_header (b[2], LOAD); |
| 607 | p = vlib_buffer_get_current (b[1]); |
| 608 | CLIB_PREFETCH (p, CLIB_CACHE_LINE_BYTES, LOAD); |
| 609 | p -= CLIB_CACHE_LINE_BYTES; |
| 610 | CLIB_PREFETCH (p, CLIB_CACHE_LINE_BYTES, LOAD); |
Neale Ranns | 123b5eb | 2020-10-16 14:03:55 +0000 | [diff] [blame] | 611 | /* speculate that the trailer goes in the first buffer */ |
| 612 | CLIB_PREFETCH (vlib_buffer_get_tail (b[1]), |
| 613 | CLIB_CACHE_LINE_BYTES, LOAD); |
Keith Burns (alagalah) | 166a9d4 | 2016-08-06 11:00:56 -0700 | [diff] [blame] | 614 | } |
Ed Warnicke | cb9cada | 2015-12-08 15:45:58 -0700 | [diff] [blame] | 615 | |
Neale Ranns | 25edf14 | 2019-03-22 08:12:48 +0000 | [diff] [blame] | 616 | if (is_tun) |
| 617 | { |
| 618 | /* we are on a ipsec tunnel's feature arc */ |
Neale Ranns | 2828721 | 2019-12-16 00:53:11 +0000 | [diff] [blame] | 619 | vnet_buffer (b[0])->ipsec.sad_index = |
| 620 | sa_index0 = ipsec_tun_protect_get_sa_out |
| 621 | (vnet_buffer (b[0])->ip.adj_index[VLIB_TX]); |
Neale Ranns | 25edf14 | 2019-03-22 08:12:48 +0000 | [diff] [blame] | 622 | } |
| 623 | else |
| 624 | sa_index0 = vnet_buffer (b[0])->ipsec.sad_index; |
| 625 | |
| 626 | if (sa_index0 != current_sa_index) |
Damjan Marion | c59b9a2 | 2019-03-19 15:38:40 +0100 | [diff] [blame] | 627 | { |
Damjan Marion | 21f265f | 2019-06-05 15:45:50 +0200 | [diff] [blame] | 628 | if (current_sa_packets) |
| 629 | vlib_increment_combined_counter (&ipsec_sa_counters, thread_index, |
| 630 | current_sa_index, |
| 631 | current_sa_packets, |
| 632 | current_sa_bytes); |
| 633 | current_sa_packets = current_sa_bytes = 0; |
| 634 | |
Damjan Marion | c59b9a2 | 2019-03-19 15:38:40 +0100 | [diff] [blame] | 635 | sa0 = pool_elt_at_index (im->sad, sa_index0); |
Neale Ranns | 123b5eb | 2020-10-16 14:03:55 +0000 | [diff] [blame] | 636 | |
| 637 | /* fetch the second cacheline ASAP */ |
| 638 | CLIB_PREFETCH (sa0->cacheline1, CLIB_CACHE_LINE_BYTES, LOAD); |
| 639 | |
Damjan Marion | c59b9a2 | 2019-03-19 15:38:40 +0100 | [diff] [blame] | 640 | current_sa_index = sa_index0; |
Damjan Marion | c59b9a2 | 2019-03-19 15:38:40 +0100 | [diff] [blame] | 641 | spi = clib_net_to_host_u32 (sa0->spi); |
Christian Hopps | fb7e7ed | 2019-11-03 07:02:15 -0500 | [diff] [blame] | 642 | esp_align = sa0->esp_block_align; |
Damjan Marion | 7c22ff7 | 2019-04-04 12:25:44 +0200 | [diff] [blame] | 643 | icv_sz = sa0->integ_icv_size; |
Damjan Marion | c59b9a2 | 2019-03-19 15:38:40 +0100 | [diff] [blame] | 644 | iv_sz = sa0->crypto_iv_size; |
Fan Zhang | f539578 | 2020-04-29 14:00:03 +0100 | [diff] [blame] | 645 | |
| 646 | /* submit frame when op_id is different then the old one */ |
| 647 | if (is_async && sa0->crypto_async_enc_op_id != last_async_op) |
| 648 | { |
| 649 | if (async_frame && async_frame->n_elts) |
| 650 | { |
Fan Zhang | 18f0e31 | 2020-10-19 13:08:34 +0100 | [diff] [blame] | 651 | if (vnet_crypto_async_submit_open_frame (vm, async_frame)) |
| 652 | esp_async_recycle_failed_submit (async_frame, b, from, |
| 653 | nexts, &n_async_drop, |
| 654 | drop_next, |
| 655 | ESP_ENCRYPT_ERROR_CRYPTO_ENGINE_ERROR); |
Fan Zhang | f539578 | 2020-04-29 14:00:03 +0100 | [diff] [blame] | 656 | } |
| 657 | async_frame = |
| 658 | vnet_crypto_async_get_frame (vm, sa0->crypto_async_enc_op_id); |
| 659 | last_async_op = sa0->crypto_async_enc_op_id; |
| 660 | } |
Damjan Marion | c59b9a2 | 2019-03-19 15:38:40 +0100 | [diff] [blame] | 661 | } |
| 662 | |
Neale Ranns | f62a8c0 | 2019-04-02 08:13:33 +0000 | [diff] [blame] | 663 | if (PREDICT_FALSE (~0 == sa0->encrypt_thread_index)) |
| 664 | { |
| 665 | /* this is the first packet to use this SA, claim the SA |
| 666 | * for this thread. this could happen simultaneously on |
| 667 | * another thread */ |
| 668 | clib_atomic_cmp_and_swap (&sa0->encrypt_thread_index, ~0, |
| 669 | ipsec_sa_assign_thread (thread_index)); |
| 670 | } |
| 671 | |
Fan Zhang | 153e41a | 2020-11-18 09:46:39 +0000 | [diff] [blame^] | 672 | if (PREDICT_FALSE (thread_index != sa0->encrypt_thread_index)) |
Neale Ranns | f62a8c0 | 2019-04-02 08:13:33 +0000 | [diff] [blame] | 673 | { |
Fan Zhang | 18f0e31 | 2020-10-19 13:08:34 +0100 | [diff] [blame] | 674 | esp_set_next_index (is_async, from, nexts, from[b - bufs], |
| 675 | &n_async_drop, |
| 676 | (is_ip6 ? ESP_ENCRYPT_NEXT_HANDOFF6 : |
| 677 | ESP_ENCRYPT_NEXT_HANDOFF4), next); |
Neale Ranns | f62a8c0 | 2019-04-02 08:13:33 +0000 | [diff] [blame] | 678 | goto trace; |
| 679 | } |
| 680 | |
Filip Tehlar | efcad1a | 2020-02-04 09:36:04 +0000 | [diff] [blame] | 681 | lb = b[0]; |
| 682 | n_bufs = vlib_buffer_chain_linearize (vm, b[0]); |
| 683 | if (n_bufs == 0) |
Damjan Marion | c59b9a2 | 2019-03-19 15:38:40 +0100 | [diff] [blame] | 684 | { |
Filip Tehlar | efcad1a | 2020-02-04 09:36:04 +0000 | [diff] [blame] | 685 | b[0]->error = node->errors[ESP_ENCRYPT_ERROR_NO_BUFFERS]; |
Fan Zhang | 18f0e31 | 2020-10-19 13:08:34 +0100 | [diff] [blame] | 686 | esp_set_next_index (is_async, from, nexts, from[b - bufs], |
| 687 | &n_async_drop, drop_next, next); |
Damjan Marion | c59b9a2 | 2019-03-19 15:38:40 +0100 | [diff] [blame] | 688 | goto trace; |
| 689 | } |
| 690 | |
Filip Tehlar | efcad1a | 2020-02-04 09:36:04 +0000 | [diff] [blame] | 691 | if (n_bufs > 1) |
| 692 | { |
Filip Tehlar | efcad1a | 2020-02-04 09:36:04 +0000 | [diff] [blame] | 693 | /* find last buffer in the chain */ |
| 694 | while (lb->flags & VLIB_BUFFER_NEXT_PRESENT) |
| 695 | lb = vlib_get_buffer (vm, lb->next_buffer); |
| 696 | } |
Filip Tehlar | efcad1a | 2020-02-04 09:36:04 +0000 | [diff] [blame] | 697 | |
Damjan Marion | c59b9a2 | 2019-03-19 15:38:40 +0100 | [diff] [blame] | 698 | if (PREDICT_FALSE (esp_seq_advance (sa0))) |
| 699 | { |
| 700 | b[0]->error = node->errors[ESP_ENCRYPT_ERROR_SEQ_CYCLED]; |
Fan Zhang | 18f0e31 | 2020-10-19 13:08:34 +0100 | [diff] [blame] | 701 | esp_set_next_index (is_async, from, nexts, from[b - bufs], |
| 702 | &n_async_drop, drop_next, next); |
Damjan Marion | c59b9a2 | 2019-03-19 15:38:40 +0100 | [diff] [blame] | 703 | goto trace; |
| 704 | } |
| 705 | |
| 706 | /* space for IV */ |
| 707 | hdr_len = iv_sz; |
| 708 | |
Damjan Marion | d709cbc | 2019-03-26 13:16:42 +0100 | [diff] [blame] | 709 | if (ipsec_sa_is_set_IS_TUNNEL (sa0)) |
Damjan Marion | c59b9a2 | 2019-03-19 15:38:40 +0100 | [diff] [blame] | 710 | { |
| 711 | payload = vlib_buffer_get_current (b[0]); |
Christian Hopps | fb7e7ed | 2019-11-03 07:02:15 -0500 | [diff] [blame] | 712 | next_hdr_ptr = esp_add_footer_and_icv (vm, &lb, esp_align, icv_sz, |
Filip Tehlar | 8cdb1a0 | 2019-11-18 22:21:37 +0000 | [diff] [blame] | 713 | next, node, |
Filip Tehlar | efcad1a | 2020-02-04 09:36:04 +0000 | [diff] [blame] | 714 | buffer_data_size, |
| 715 | vlib_buffer_length_in_chain |
| 716 | (vm, b[0])); |
Filip Tehlar | 8cdb1a0 | 2019-11-18 22:21:37 +0000 | [diff] [blame] | 717 | if (!next_hdr_ptr) |
Filip Tehlar | e4e8c6b | 2020-02-13 07:49:30 +0000 | [diff] [blame] | 718 | { |
| 719 | b[0]->error = node->errors[ESP_ENCRYPT_ERROR_NO_BUFFERS]; |
Fan Zhang | 18f0e31 | 2020-10-19 13:08:34 +0100 | [diff] [blame] | 720 | esp_set_next_index (is_async, from, nexts, from[b - bufs], |
| 721 | &n_async_drop, drop_next, next); |
Filip Tehlar | e4e8c6b | 2020-02-13 07:49:30 +0000 | [diff] [blame] | 722 | goto trace; |
| 723 | } |
Filip Tehlar | efcad1a | 2020-02-04 09:36:04 +0000 | [diff] [blame] | 724 | b[0]->flags &= ~VLIB_BUFFER_TOTAL_LENGTH_VALID; |
Filip Tehlar | 8cdb1a0 | 2019-11-18 22:21:37 +0000 | [diff] [blame] | 725 | payload_len = b[0]->current_length; |
Filip Tehlar | efcad1a | 2020-02-04 09:36:04 +0000 | [diff] [blame] | 726 | payload_len_total = vlib_buffer_length_in_chain (vm, b[0]); |
Damjan Marion | c59b9a2 | 2019-03-19 15:38:40 +0100 | [diff] [blame] | 727 | |
| 728 | /* ESP header */ |
| 729 | hdr_len += sizeof (*esp); |
| 730 | esp = (esp_header_t *) (payload - hdr_len); |
| 731 | |
| 732 | /* optional UDP header */ |
Damjan Marion | d709cbc | 2019-03-26 13:16:42 +0100 | [diff] [blame] | 733 | if (ipsec_sa_is_set_UDP_ENCAP (sa0)) |
Damjan Marion | c98275f | 2019-03-06 14:05:01 +0100 | [diff] [blame] | 734 | { |
Damjan Marion | c59b9a2 | 2019-03-19 15:38:40 +0100 | [diff] [blame] | 735 | hdr_len += sizeof (udp_header_t); |
| 736 | esp_fill_udp_hdr (sa0, (udp_header_t *) (payload - hdr_len), |
Filip Tehlar | efcad1a | 2020-02-04 09:36:04 +0000 | [diff] [blame] | 737 | payload_len_total + hdr_len); |
Damjan Marion | c59b9a2 | 2019-03-19 15:38:40 +0100 | [diff] [blame] | 738 | } |
| 739 | |
| 740 | /* IP header */ |
Damjan Marion | d709cbc | 2019-03-26 13:16:42 +0100 | [diff] [blame] | 741 | if (ipsec_sa_is_set_IS_TUNNEL_V6 (sa0)) |
Damjan Marion | c59b9a2 | 2019-03-19 15:38:40 +0100 | [diff] [blame] | 742 | { |
| 743 | ip6_header_t *ip6; |
| 744 | u16 len = sizeof (ip6_header_t); |
| 745 | hdr_len += len; |
| 746 | ip6 = (ip6_header_t *) (payload - hdr_len); |
Neale Ranns | 041add7 | 2020-01-02 04:06:10 +0000 | [diff] [blame] | 747 | clib_memcpy_fast (ip6, &sa0->ip6_hdr, sizeof (ip6_header_t)); |
| 748 | |
| 749 | if (is_ip6) |
| 750 | { |
| 751 | *next_hdr_ptr = IP_PROTOCOL_IPV6; |
| 752 | tunnel_encap_fixup_6o6 (sa0->tunnel_flags, |
| 753 | (const ip6_header_t *) payload, |
| 754 | ip6); |
| 755 | } |
| 756 | else |
| 757 | { |
| 758 | *next_hdr_ptr = IP_PROTOCOL_IP_IN_IP; |
| 759 | tunnel_encap_fixup_4o6 (sa0->tunnel_flags, |
| 760 | (const ip4_header_t *) payload, |
| 761 | ip6); |
| 762 | } |
Filip Tehlar | efcad1a | 2020-02-04 09:36:04 +0000 | [diff] [blame] | 763 | len = payload_len_total + hdr_len - len; |
Damjan Marion | c59b9a2 | 2019-03-19 15:38:40 +0100 | [diff] [blame] | 764 | ip6->payload_length = clib_net_to_host_u16 (len); |
Damjan Marion | c98275f | 2019-03-06 14:05:01 +0100 | [diff] [blame] | 765 | } |
| 766 | else |
| 767 | { |
Damjan Marion | c59b9a2 | 2019-03-19 15:38:40 +0100 | [diff] [blame] | 768 | ip4_header_t *ip4; |
| 769 | u16 len = sizeof (ip4_header_t); |
| 770 | hdr_len += len; |
| 771 | ip4 = (ip4_header_t *) (payload - hdr_len); |
Neale Ranns | 041add7 | 2020-01-02 04:06:10 +0000 | [diff] [blame] | 772 | clib_memcpy_fast (ip4, &sa0->ip4_hdr, sizeof (ip4_header_t)); |
| 773 | |
| 774 | if (is_ip6) |
| 775 | { |
| 776 | *next_hdr_ptr = IP_PROTOCOL_IPV6; |
| 777 | tunnel_encap_fixup_6o4_w_chksum (sa0->tunnel_flags, |
| 778 | (const ip6_header_t *) |
| 779 | payload, ip4); |
| 780 | } |
| 781 | else |
| 782 | { |
| 783 | *next_hdr_ptr = IP_PROTOCOL_IP_IN_IP; |
| 784 | tunnel_encap_fixup_4o4_w_chksum (sa0->tunnel_flags, |
| 785 | (const ip4_header_t *) |
| 786 | payload, ip4); |
| 787 | } |
Filip Tehlar | efcad1a | 2020-02-04 09:36:04 +0000 | [diff] [blame] | 788 | len = payload_len_total + hdr_len; |
Damjan Marion | c59b9a2 | 2019-03-19 15:38:40 +0100 | [diff] [blame] | 789 | esp_update_ip4_hdr (ip4, len, /* is_transport */ 0, 0); |
Damjan Marion | c98275f | 2019-03-06 14:05:01 +0100 | [diff] [blame] | 790 | } |
Damjan Marion | c98275f | 2019-03-06 14:05:01 +0100 | [diff] [blame] | 791 | |
Neale Ranns | 72f2a3a | 2019-06-17 15:43:38 +0000 | [diff] [blame] | 792 | dpo = &sa0->dpo; |
Neale Ranns | 25edf14 | 2019-03-22 08:12:48 +0000 | [diff] [blame] | 793 | if (!is_tun) |
| 794 | { |
| 795 | next[0] = dpo->dpoi_next_node; |
| 796 | vnet_buffer (b[0])->ip.adj_index[VLIB_TX] = dpo->dpoi_index; |
| 797 | } |
Neale Ranns | 4ec36c5 | 2020-03-31 09:21:29 -0400 | [diff] [blame] | 798 | else |
| 799 | next[0] = ESP_ENCRYPT_NEXT_INTERFACE_OUTPUT; |
Damjan Marion | c98275f | 2019-03-06 14:05:01 +0100 | [diff] [blame] | 800 | } |
Damjan Marion | c59b9a2 | 2019-03-19 15:38:40 +0100 | [diff] [blame] | 801 | else /* transport mode */ |
Damjan Marion | c98275f | 2019-03-06 14:05:01 +0100 | [diff] [blame] | 802 | { |
Damjan Marion | c59b9a2 | 2019-03-19 15:38:40 +0100 | [diff] [blame] | 803 | u8 *l2_hdr, l2_len, *ip_hdr, ip_len; |
Neale Ranns | 0295040 | 2019-12-20 00:54:57 +0000 | [diff] [blame] | 804 | ip6_ext_header_t *ext_hdr; |
Damjan Marion | c59b9a2 | 2019-03-19 15:38:40 +0100 | [diff] [blame] | 805 | udp_header_t *udp = 0; |
Alexander Chernavin | b0d2eda | 2020-03-25 10:56:52 -0400 | [diff] [blame] | 806 | u16 udp_len = 0; |
Damjan Marion | c59b9a2 | 2019-03-19 15:38:40 +0100 | [diff] [blame] | 807 | u8 *old_ip_hdr = vlib_buffer_get_current (b[0]); |
Damjan Marion | c98275f | 2019-03-06 14:05:01 +0100 | [diff] [blame] | 808 | |
Damjan Marion | c59b9a2 | 2019-03-19 15:38:40 +0100 | [diff] [blame] | 809 | ip_len = is_ip6 ? |
Neale Ranns | 0295040 | 2019-12-20 00:54:57 +0000 | [diff] [blame] | 810 | esp_get_ip6_hdr_len ((ip6_header_t *) old_ip_hdr, &ext_hdr) : |
Damjan Marion | c59b9a2 | 2019-03-19 15:38:40 +0100 | [diff] [blame] | 811 | ip4_header_bytes ((ip4_header_t *) old_ip_hdr); |
Damjan Marion | c98275f | 2019-03-06 14:05:01 +0100 | [diff] [blame] | 812 | |
Damjan Marion | c59b9a2 | 2019-03-19 15:38:40 +0100 | [diff] [blame] | 813 | vlib_buffer_advance (b[0], ip_len); |
| 814 | payload = vlib_buffer_get_current (b[0]); |
Christian Hopps | fb7e7ed | 2019-11-03 07:02:15 -0500 | [diff] [blame] | 815 | next_hdr_ptr = esp_add_footer_and_icv (vm, &lb, esp_align, icv_sz, |
Filip Tehlar | 8cdb1a0 | 2019-11-18 22:21:37 +0000 | [diff] [blame] | 816 | next, node, |
Filip Tehlar | efcad1a | 2020-02-04 09:36:04 +0000 | [diff] [blame] | 817 | buffer_data_size, |
| 818 | vlib_buffer_length_in_chain |
| 819 | (vm, b[0])); |
Filip Tehlar | 8cdb1a0 | 2019-11-18 22:21:37 +0000 | [diff] [blame] | 820 | if (!next_hdr_ptr) |
Fan Zhang | 18f0e31 | 2020-10-19 13:08:34 +0100 | [diff] [blame] | 821 | { |
| 822 | esp_set_next_index (is_async, from, nexts, from[b - bufs], |
| 823 | &n_async_drop, drop_next, next); |
| 824 | goto trace; |
| 825 | } |
Filip Tehlar | efcad1a | 2020-02-04 09:36:04 +0000 | [diff] [blame] | 826 | |
| 827 | b[0]->flags &= ~VLIB_BUFFER_TOTAL_LENGTH_VALID; |
Filip Tehlar | 8cdb1a0 | 2019-11-18 22:21:37 +0000 | [diff] [blame] | 828 | payload_len = b[0]->current_length; |
Filip Tehlar | efcad1a | 2020-02-04 09:36:04 +0000 | [diff] [blame] | 829 | payload_len_total = vlib_buffer_length_in_chain (vm, b[0]); |
Damjan Marion | c59b9a2 | 2019-03-19 15:38:40 +0100 | [diff] [blame] | 830 | |
| 831 | /* ESP header */ |
| 832 | hdr_len += sizeof (*esp); |
| 833 | esp = (esp_header_t *) (payload - hdr_len); |
| 834 | |
| 835 | /* optional UDP header */ |
Damjan Marion | d709cbc | 2019-03-26 13:16:42 +0100 | [diff] [blame] | 836 | if (ipsec_sa_is_set_UDP_ENCAP (sa0)) |
Damjan Marion | c98275f | 2019-03-06 14:05:01 +0100 | [diff] [blame] | 837 | { |
Damjan Marion | c59b9a2 | 2019-03-19 15:38:40 +0100 | [diff] [blame] | 838 | hdr_len += sizeof (udp_header_t); |
| 839 | udp = (udp_header_t *) (payload - hdr_len); |
Damjan Marion | c98275f | 2019-03-06 14:05:01 +0100 | [diff] [blame] | 840 | } |
| 841 | |
Damjan Marion | c59b9a2 | 2019-03-19 15:38:40 +0100 | [diff] [blame] | 842 | /* IP header */ |
| 843 | hdr_len += ip_len; |
| 844 | ip_hdr = payload - hdr_len; |
| 845 | |
| 846 | /* L2 header */ |
Neale Ranns | c87b66c | 2019-02-07 07:26:12 -0800 | [diff] [blame] | 847 | if (!is_tun) |
| 848 | { |
| 849 | l2_len = vnet_buffer (b[0])->ip.save_rewrite_length; |
| 850 | hdr_len += l2_len; |
| 851 | l2_hdr = payload - hdr_len; |
Damjan Marion | c59b9a2 | 2019-03-19 15:38:40 +0100 | [diff] [blame] | 852 | |
Neale Ranns | c87b66c | 2019-02-07 07:26:12 -0800 | [diff] [blame] | 853 | /* copy l2 and ip header */ |
| 854 | clib_memcpy_le32 (l2_hdr, old_ip_hdr - l2_len, l2_len); |
| 855 | } |
| 856 | else |
| 857 | l2_len = 0; |
| 858 | |
Damjan Marion | c98275f | 2019-03-06 14:05:01 +0100 | [diff] [blame] | 859 | if (is_ip6) |
Damjan Marion | c59b9a2 | 2019-03-19 15:38:40 +0100 | [diff] [blame] | 860 | { |
Neale Ranns | 0295040 | 2019-12-20 00:54:57 +0000 | [diff] [blame] | 861 | ip6_header_t *ip6 = (ip6_header_t *) (old_ip_hdr); |
| 862 | if (PREDICT_TRUE (NULL == ext_hdr)) |
| 863 | { |
| 864 | *next_hdr_ptr = ip6->protocol; |
| 865 | ip6->protocol = IP_PROTOCOL_IPSEC_ESP; |
| 866 | } |
| 867 | else |
| 868 | { |
| 869 | *next_hdr_ptr = ext_hdr->next_hdr; |
| 870 | ext_hdr->next_hdr = IP_PROTOCOL_IPSEC_ESP; |
| 871 | } |
Neale Ranns | d207fd7 | 2019-04-18 17:18:12 -0700 | [diff] [blame] | 872 | ip6->payload_length = |
Filip Tehlar | efcad1a | 2020-02-04 09:36:04 +0000 | [diff] [blame] | 873 | clib_host_to_net_u16 (payload_len_total + hdr_len - l2_len - |
Neale Ranns | 0295040 | 2019-12-20 00:54:57 +0000 | [diff] [blame] | 874 | sizeof (ip6_header_t)); |
Damjan Marion | c59b9a2 | 2019-03-19 15:38:40 +0100 | [diff] [blame] | 875 | } |
Damjan Marion | c98275f | 2019-03-06 14:05:01 +0100 | [diff] [blame] | 876 | else |
Damjan Marion | c98275f | 2019-03-06 14:05:01 +0100 | [diff] [blame] | 877 | { |
Damjan Marion | c59b9a2 | 2019-03-19 15:38:40 +0100 | [diff] [blame] | 878 | u16 len; |
Neale Ranns | 0295040 | 2019-12-20 00:54:57 +0000 | [diff] [blame] | 879 | ip4_header_t *ip4 = (ip4_header_t *) (old_ip_hdr); |
Damjan Marion | c59b9a2 | 2019-03-19 15:38:40 +0100 | [diff] [blame] | 880 | *next_hdr_ptr = ip4->protocol; |
Filip Tehlar | efcad1a | 2020-02-04 09:36:04 +0000 | [diff] [blame] | 881 | len = payload_len_total + hdr_len - l2_len; |
Damjan Marion | c59b9a2 | 2019-03-19 15:38:40 +0100 | [diff] [blame] | 882 | if (udp) |
| 883 | { |
| 884 | esp_update_ip4_hdr (ip4, len, /* is_transport */ 1, 1); |
Alexander Chernavin | b0d2eda | 2020-03-25 10:56:52 -0400 | [diff] [blame] | 885 | udp_len = len - ip_len; |
Damjan Marion | c59b9a2 | 2019-03-19 15:38:40 +0100 | [diff] [blame] | 886 | } |
| 887 | else |
| 888 | esp_update_ip4_hdr (ip4, len, /* is_transport */ 1, 0); |
Damjan Marion | c98275f | 2019-03-06 14:05:01 +0100 | [diff] [blame] | 889 | } |
Damjan Marion | c98275f | 2019-03-06 14:05:01 +0100 | [diff] [blame] | 890 | |
Neale Ranns | 0295040 | 2019-12-20 00:54:57 +0000 | [diff] [blame] | 891 | clib_memcpy_le64 (ip_hdr, old_ip_hdr, ip_len); |
| 892 | |
Alexander Chernavin | b0d2eda | 2020-03-25 10:56:52 -0400 | [diff] [blame] | 893 | if (udp) |
| 894 | { |
| 895 | esp_fill_udp_hdr (sa0, udp, udp_len); |
| 896 | } |
| 897 | |
Neale Ranns | 4ec36c5 | 2020-03-31 09:21:29 -0400 | [diff] [blame] | 898 | next[0] = ESP_ENCRYPT_NEXT_INTERFACE_OUTPUT; |
Damjan Marion | c98275f | 2019-03-06 14:05:01 +0100 | [diff] [blame] | 899 | } |
| 900 | |
PiotrX Kleski | fdca4dd | 2020-05-05 14:14:22 +0200 | [diff] [blame] | 901 | if (lb != b[0]) |
| 902 | { |
| 903 | crypto_ops = &ptd->chained_crypto_ops; |
| 904 | integ_ops = &ptd->chained_integ_ops; |
| 905 | } |
| 906 | else |
| 907 | { |
| 908 | crypto_ops = &ptd->crypto_ops; |
| 909 | integ_ops = &ptd->integ_ops; |
| 910 | } |
| 911 | |
Damjan Marion | c59b9a2 | 2019-03-19 15:38:40 +0100 | [diff] [blame] | 912 | esp->spi = spi; |
| 913 | esp->seq = clib_net_to_host_u32 (sa0->seq); |
Damjan Marion | c98275f | 2019-03-06 14:05:01 +0100 | [diff] [blame] | 914 | |
Fan Zhang | f539578 | 2020-04-29 14:00:03 +0100 | [diff] [blame] | 915 | if (is_async) |
Damjan Marion | c98275f | 2019-03-06 14:05:01 +0100 | [diff] [blame] | 916 | { |
Fan Zhang | f539578 | 2020-04-29 14:00:03 +0100 | [diff] [blame] | 917 | if (PREDICT_FALSE (sa0->crypto_async_enc_op_id == 0)) |
Fan Zhang | 18f0e31 | 2020-10-19 13:08:34 +0100 | [diff] [blame] | 918 | { |
| 919 | esp_set_next_index (is_async, from, nexts, from[b - bufs], |
| 920 | &n_async_drop, drop_next, next); |
| 921 | goto trace; |
| 922 | } |
Filip Tehlar | efcad1a | 2020-02-04 09:36:04 +0000 | [diff] [blame] | 923 | |
Fan Zhang | f539578 | 2020-04-29 14:00:03 +0100 | [diff] [blame] | 924 | if (esp_prepare_async_frame (vm, ptd, &async_frame, sa0, b[0], esp, |
| 925 | payload, payload_len, iv_sz, |
Fan Zhang | 18f0e31 | 2020-10-19 13:08:34 +0100 | [diff] [blame] | 926 | icv_sz, from[b - bufs], next[0], |
| 927 | hdr_len, async_next, lb)) |
Neale Ranns | 47feb11 | 2019-04-11 15:14:07 +0000 | [diff] [blame] | 928 | { |
Fan Zhang | 18f0e31 | 2020-10-19 13:08:34 +0100 | [diff] [blame] | 929 | /* The fail only caused by submission, free the whole frame. */ |
| 930 | if (async_frame->n_elts) |
| 931 | esp_async_recycle_failed_submit (async_frame, b, from, nexts, |
| 932 | &n_async_drop, drop_next, |
| 933 | ESP_ENCRYPT_ERROR_CRYPTO_ENGINE_ERROR); |
| 934 | b[0]->error = ESP_ENCRYPT_ERROR_CRYPTO_ENGINE_ERROR; |
| 935 | esp_set_next_index (1, from, nexts, from[b - bufs], |
| 936 | &n_async_drop, drop_next, next); |
Fan Zhang | f539578 | 2020-04-29 14:00:03 +0100 | [diff] [blame] | 937 | goto trace; |
Filip Tehlar | efcad1a | 2020-02-04 09:36:04 +0000 | [diff] [blame] | 938 | } |
Damjan Marion | c98275f | 2019-03-06 14:05:01 +0100 | [diff] [blame] | 939 | } |
Fan Zhang | f539578 | 2020-04-29 14:00:03 +0100 | [diff] [blame] | 940 | else |
Damjan Marion | c98275f | 2019-03-06 14:05:01 +0100 | [diff] [blame] | 941 | { |
Fan Zhang | f539578 | 2020-04-29 14:00:03 +0100 | [diff] [blame] | 942 | esp_prepare_sync_op (vm, ptd, crypto_ops, integ_ops, sa0, payload, |
| 943 | payload_len, iv_sz, icv_sz, bufs, b, lb, |
| 944 | hdr_len, esp, nonce++); |
Damjan Marion | c98275f | 2019-03-06 14:05:01 +0100 | [diff] [blame] | 945 | } |
| 946 | |
Damjan Marion | c59b9a2 | 2019-03-19 15:38:40 +0100 | [diff] [blame] | 947 | vlib_buffer_advance (b[0], 0LL - hdr_len); |
Damjan Marion | c98275f | 2019-03-06 14:05:01 +0100 | [diff] [blame] | 948 | |
Damjan Marion | c59b9a2 | 2019-03-19 15:38:40 +0100 | [diff] [blame] | 949 | current_sa_packets += 1; |
Filip Tehlar | efcad1a | 2020-02-04 09:36:04 +0000 | [diff] [blame] | 950 | current_sa_bytes += payload_len_total; |
Damjan Marion | c59b9a2 | 2019-03-19 15:38:40 +0100 | [diff] [blame] | 951 | |
| 952 | trace: |
| 953 | if (PREDICT_FALSE (b[0]->flags & VLIB_BUFFER_IS_TRACED)) |
Damjan Marion | c98275f | 2019-03-06 14:05:01 +0100 | [diff] [blame] | 954 | { |
Damjan Marion | c59b9a2 | 2019-03-19 15:38:40 +0100 | [diff] [blame] | 955 | esp_encrypt_trace_t *tr = vlib_add_trace (vm, node, b[0], |
| 956 | sizeof (*tr)); |
| 957 | tr->sa_index = sa_index0; |
| 958 | tr->spi = sa0->spi; |
Neale Ranns | 6afaae1 | 2019-07-17 15:07:14 +0000 | [diff] [blame] | 959 | tr->seq = sa0->seq; |
| 960 | tr->sa_seq_hi = sa0->seq_hi; |
Damjan Marion | d709cbc | 2019-03-26 13:16:42 +0100 | [diff] [blame] | 961 | tr->udp_encap = ipsec_sa_is_set_UDP_ENCAP (sa0); |
Damjan Marion | c59b9a2 | 2019-03-19 15:38:40 +0100 | [diff] [blame] | 962 | tr->crypto_alg = sa0->crypto_alg; |
| 963 | tr->integ_alg = sa0->integ_alg; |
Damjan Marion | c98275f | 2019-03-06 14:05:01 +0100 | [diff] [blame] | 964 | } |
Damjan Marion | c98275f | 2019-03-06 14:05:01 +0100 | [diff] [blame] | 965 | /* next */ |
Damjan Marion | c59b9a2 | 2019-03-19 15:38:40 +0100 | [diff] [blame] | 966 | n_left -= 1; |
Damjan Marion | c98275f | 2019-03-06 14:05:01 +0100 | [diff] [blame] | 967 | next += 1; |
Damjan Marion | c59b9a2 | 2019-03-19 15:38:40 +0100 | [diff] [blame] | 968 | b += 1; |
Damjan Marion | c98275f | 2019-03-06 14:05:01 +0100 | [diff] [blame] | 969 | } |
| 970 | |
Damjan Marion | c59b9a2 | 2019-03-19 15:38:40 +0100 | [diff] [blame] | 971 | vlib_increment_combined_counter (&ipsec_sa_counters, thread_index, |
| 972 | current_sa_index, current_sa_packets, |
| 973 | current_sa_bytes); |
Fan Zhang | f539578 | 2020-04-29 14:00:03 +0100 | [diff] [blame] | 974 | if (!is_async) |
| 975 | { |
Neale Ranns | b1fd80f | 2020-05-12 13:33:56 +0000 | [diff] [blame] | 976 | esp_process_ops (vm, node, ptd->crypto_ops, bufs, nexts, drop_next); |
Fan Zhang | f539578 | 2020-04-29 14:00:03 +0100 | [diff] [blame] | 977 | esp_process_chained_ops (vm, node, ptd->chained_crypto_ops, bufs, nexts, |
Neale Ranns | b1fd80f | 2020-05-12 13:33:56 +0000 | [diff] [blame] | 978 | ptd->chunks, drop_next); |
Filip Tehlar | efcad1a | 2020-02-04 09:36:04 +0000 | [diff] [blame] | 979 | |
Neale Ranns | b1fd80f | 2020-05-12 13:33:56 +0000 | [diff] [blame] | 980 | esp_process_ops (vm, node, ptd->integ_ops, bufs, nexts, drop_next); |
Fan Zhang | f539578 | 2020-04-29 14:00:03 +0100 | [diff] [blame] | 981 | esp_process_chained_ops (vm, node, ptd->chained_integ_ops, bufs, nexts, |
Neale Ranns | b1fd80f | 2020-05-12 13:33:56 +0000 | [diff] [blame] | 982 | ptd->chunks, drop_next); |
Fan Zhang | f539578 | 2020-04-29 14:00:03 +0100 | [diff] [blame] | 983 | } |
Fan Zhang | 18f0e31 | 2020-10-19 13:08:34 +0100 | [diff] [blame] | 984 | else |
Fan Zhang | f539578 | 2020-04-29 14:00:03 +0100 | [diff] [blame] | 985 | { |
Fan Zhang | 18f0e31 | 2020-10-19 13:08:34 +0100 | [diff] [blame] | 986 | if (async_frame && async_frame->n_elts) |
| 987 | { |
| 988 | if (vnet_crypto_async_submit_open_frame (vm, async_frame) < 0) |
| 989 | esp_async_recycle_failed_submit (async_frame, b, from, nexts, |
| 990 | &n_async_drop, drop_next, |
| 991 | ESP_ENCRYPT_ERROR_CRYPTO_ENGINE_ERROR); |
| 992 | } |
| 993 | vlib_node_increment_counter (vm, node->node_index, |
| 994 | ESP_ENCRYPT_ERROR_RX_PKTS, |
| 995 | frame->n_vectors); |
| 996 | if (n_async_drop) |
| 997 | vlib_buffer_enqueue_to_next (vm, node, from, nexts, n_async_drop); |
| 998 | |
| 999 | return frame->n_vectors; |
Fan Zhang | f539578 | 2020-04-29 14:00:03 +0100 | [diff] [blame] | 1000 | } |
Damjan Marion | c59b9a2 | 2019-03-19 15:38:40 +0100 | [diff] [blame] | 1001 | |
| 1002 | vlib_node_increment_counter (vm, node->node_index, |
| 1003 | ESP_ENCRYPT_ERROR_RX_PKTS, frame->n_vectors); |
| 1004 | |
| 1005 | vlib_buffer_enqueue_to_next (vm, node, from, nexts, frame->n_vectors); |
| 1006 | return frame->n_vectors; |
Ed Warnicke | cb9cada | 2015-12-08 15:45:58 -0700 | [diff] [blame] | 1007 | } |
| 1008 | |
Fan Zhang | f539578 | 2020-04-29 14:00:03 +0100 | [diff] [blame] | 1009 | always_inline uword |
| 1010 | esp_encrypt_post_inline (vlib_main_t * vm, vlib_node_runtime_t * node, |
| 1011 | vlib_frame_t * frame) |
| 1012 | { |
| 1013 | vlib_buffer_t *bufs[VLIB_FRAME_SIZE], **b = bufs; |
| 1014 | u16 nexts[VLIB_FRAME_SIZE], *next = nexts; |
| 1015 | u32 *from = vlib_frame_vector_args (frame); |
| 1016 | u32 n_left = frame->n_vectors; |
| 1017 | |
| 1018 | vlib_get_buffers (vm, from, b, n_left); |
| 1019 | |
| 1020 | if (n_left >= 4) |
| 1021 | { |
| 1022 | vlib_prefetch_buffer_header (b[0], LOAD); |
| 1023 | vlib_prefetch_buffer_header (b[1], LOAD); |
| 1024 | vlib_prefetch_buffer_header (b[2], LOAD); |
| 1025 | vlib_prefetch_buffer_header (b[3], LOAD); |
| 1026 | } |
| 1027 | |
| 1028 | while (n_left > 8) |
| 1029 | { |
| 1030 | vlib_prefetch_buffer_header (b[4], LOAD); |
| 1031 | vlib_prefetch_buffer_header (b[5], LOAD); |
| 1032 | vlib_prefetch_buffer_header (b[6], LOAD); |
| 1033 | vlib_prefetch_buffer_header (b[7], LOAD); |
| 1034 | |
| 1035 | next[0] = (esp_post_data (b[0]))->next_index; |
| 1036 | next[1] = (esp_post_data (b[1]))->next_index; |
| 1037 | next[2] = (esp_post_data (b[2]))->next_index; |
| 1038 | next[3] = (esp_post_data (b[3]))->next_index; |
| 1039 | |
| 1040 | if (PREDICT_FALSE (node->flags & VLIB_NODE_FLAG_TRACE)) |
| 1041 | { |
| 1042 | if (b[0]->flags & VLIB_BUFFER_IS_TRACED) |
| 1043 | { |
| 1044 | esp_encrypt_post_trace_t *tr = vlib_add_trace (vm, node, b[0], |
| 1045 | sizeof (*tr)); |
| 1046 | tr->next_index = next[0]; |
| 1047 | } |
| 1048 | if (b[1]->flags & VLIB_BUFFER_IS_TRACED) |
| 1049 | { |
| 1050 | esp_encrypt_post_trace_t *tr = vlib_add_trace (vm, node, b[1], |
| 1051 | sizeof (*tr)); |
| 1052 | tr->next_index = next[1]; |
| 1053 | } |
| 1054 | if (b[2]->flags & VLIB_BUFFER_IS_TRACED) |
| 1055 | { |
| 1056 | esp_encrypt_post_trace_t *tr = vlib_add_trace (vm, node, b[2], |
| 1057 | sizeof (*tr)); |
| 1058 | tr->next_index = next[2]; |
| 1059 | } |
| 1060 | if (b[3]->flags & VLIB_BUFFER_IS_TRACED) |
| 1061 | { |
| 1062 | esp_encrypt_post_trace_t *tr = vlib_add_trace (vm, node, b[3], |
| 1063 | sizeof (*tr)); |
| 1064 | tr->next_index = next[3]; |
| 1065 | } |
| 1066 | } |
| 1067 | |
| 1068 | b += 4; |
| 1069 | next += 4; |
| 1070 | n_left -= 4; |
| 1071 | } |
| 1072 | |
| 1073 | while (n_left > 0) |
| 1074 | { |
| 1075 | next[0] = (esp_post_data (b[0]))->next_index; |
| 1076 | if (PREDICT_FALSE (b[0]->flags & VLIB_BUFFER_IS_TRACED)) |
| 1077 | { |
| 1078 | esp_encrypt_post_trace_t *tr = vlib_add_trace (vm, node, b[0], |
| 1079 | sizeof (*tr)); |
| 1080 | tr->next_index = next[0]; |
| 1081 | } |
| 1082 | |
| 1083 | b += 1; |
| 1084 | next += 1; |
| 1085 | n_left -= 1; |
| 1086 | } |
| 1087 | |
| 1088 | vlib_node_increment_counter (vm, node->node_index, |
| 1089 | ESP_ENCRYPT_ERROR_POST_RX_PKTS, |
| 1090 | frame->n_vectors); |
| 1091 | vlib_buffer_enqueue_to_next (vm, node, from, nexts, frame->n_vectors); |
| 1092 | return frame->n_vectors; |
| 1093 | } |
| 1094 | |
Klement Sekera | b8f3544 | 2018-10-29 13:38:19 +0100 | [diff] [blame] | 1095 | VLIB_NODE_FN (esp4_encrypt_node) (vlib_main_t * vm, |
| 1096 | vlib_node_runtime_t * node, |
| 1097 | vlib_frame_t * from_frame) |
Klement Sekera | be5a5dd | 2018-10-09 16:05:48 +0200 | [diff] [blame] | 1098 | { |
Fan Zhang | f539578 | 2020-04-29 14:00:03 +0100 | [diff] [blame] | 1099 | return esp_encrypt_inline (vm, node, from_frame, 0 /* is_ip6 */ , 0, |
| 1100 | esp_encrypt_async_next.esp4_post_next); |
Klement Sekera | be5a5dd | 2018-10-09 16:05:48 +0200 | [diff] [blame] | 1101 | } |
Ed Warnicke | cb9cada | 2015-12-08 15:45:58 -0700 | [diff] [blame] | 1102 | |
Keith Burns (alagalah) | 166a9d4 | 2016-08-06 11:00:56 -0700 | [diff] [blame] | 1103 | /* *INDENT-OFF* */ |
Klement Sekera | be5a5dd | 2018-10-09 16:05:48 +0200 | [diff] [blame] | 1104 | VLIB_REGISTER_NODE (esp4_encrypt_node) = { |
Klement Sekera | be5a5dd | 2018-10-09 16:05:48 +0200 | [diff] [blame] | 1105 | .name = "esp4-encrypt", |
Ed Warnicke | cb9cada | 2015-12-08 15:45:58 -0700 | [diff] [blame] | 1106 | .vector_size = sizeof (u32), |
| 1107 | .format_trace = format_esp_encrypt_trace, |
| 1108 | .type = VLIB_NODE_TYPE_INTERNAL, |
| 1109 | |
| 1110 | .n_errors = ARRAY_LEN(esp_encrypt_error_strings), |
| 1111 | .error_strings = esp_encrypt_error_strings, |
| 1112 | |
| 1113 | .n_next_nodes = ESP_ENCRYPT_N_NEXT, |
| 1114 | .next_nodes = { |
Neale Ranns | b1fd80f | 2020-05-12 13:33:56 +0000 | [diff] [blame] | 1115 | [ESP_ENCRYPT_NEXT_DROP4] = "ip4-drop", |
| 1116 | [ESP_ENCRYPT_NEXT_DROP6] = "ip6-drop", |
| 1117 | [ESP_ENCRYPT_NEXT_HANDOFF4] = "esp4-encrypt-handoff", |
| 1118 | [ESP_ENCRYPT_NEXT_HANDOFF6] = "esp6-encrypt-handoff", |
Fan Zhang | 18f0e31 | 2020-10-19 13:08:34 +0100 | [diff] [blame] | 1119 | [ESP_ENCRYPT_NEXT_INTERFACE_OUTPUT] = "interface-output" |
Ed Warnicke | cb9cada | 2015-12-08 15:45:58 -0700 | [diff] [blame] | 1120 | }, |
| 1121 | }; |
Keith Burns (alagalah) | 166a9d4 | 2016-08-06 11:00:56 -0700 | [diff] [blame] | 1122 | /* *INDENT-ON* */ |
Ed Warnicke | cb9cada | 2015-12-08 15:45:58 -0700 | [diff] [blame] | 1123 | |
Fan Zhang | f539578 | 2020-04-29 14:00:03 +0100 | [diff] [blame] | 1124 | VLIB_NODE_FN (esp4_encrypt_post_node) (vlib_main_t * vm, |
| 1125 | vlib_node_runtime_t * node, |
| 1126 | vlib_frame_t * from_frame) |
| 1127 | { |
| 1128 | return esp_encrypt_post_inline (vm, node, from_frame); |
| 1129 | } |
| 1130 | |
| 1131 | /* *INDENT-OFF* */ |
| 1132 | VLIB_REGISTER_NODE (esp4_encrypt_post_node) = { |
| 1133 | .name = "esp4-encrypt-post", |
| 1134 | .vector_size = sizeof (u32), |
| 1135 | .format_trace = format_esp_post_encrypt_trace, |
| 1136 | .type = VLIB_NODE_TYPE_INTERNAL, |
| 1137 | .sibling_of = "esp4-encrypt", |
| 1138 | |
| 1139 | .n_errors = ARRAY_LEN(esp_encrypt_error_strings), |
| 1140 | .error_strings = esp_encrypt_error_strings, |
| 1141 | }; |
| 1142 | /* *INDENT-ON* */ |
| 1143 | |
Klement Sekera | b8f3544 | 2018-10-29 13:38:19 +0100 | [diff] [blame] | 1144 | VLIB_NODE_FN (esp6_encrypt_node) (vlib_main_t * vm, |
| 1145 | vlib_node_runtime_t * node, |
| 1146 | vlib_frame_t * from_frame) |
Klement Sekera | be5a5dd | 2018-10-09 16:05:48 +0200 | [diff] [blame] | 1147 | { |
Fan Zhang | f539578 | 2020-04-29 14:00:03 +0100 | [diff] [blame] | 1148 | return esp_encrypt_inline (vm, node, from_frame, 1 /* is_ip6 */ , 0, |
| 1149 | esp_encrypt_async_next.esp6_post_next); |
Klement Sekera | be5a5dd | 2018-10-09 16:05:48 +0200 | [diff] [blame] | 1150 | } |
| 1151 | |
| 1152 | /* *INDENT-OFF* */ |
| 1153 | VLIB_REGISTER_NODE (esp6_encrypt_node) = { |
Klement Sekera | be5a5dd | 2018-10-09 16:05:48 +0200 | [diff] [blame] | 1154 | .name = "esp6-encrypt", |
| 1155 | .vector_size = sizeof (u32), |
| 1156 | .format_trace = format_esp_encrypt_trace, |
| 1157 | .type = VLIB_NODE_TYPE_INTERNAL, |
Neale Ranns | b1fd80f | 2020-05-12 13:33:56 +0000 | [diff] [blame] | 1158 | .sibling_of = "esp4-encrypt", |
Klement Sekera | be5a5dd | 2018-10-09 16:05:48 +0200 | [diff] [blame] | 1159 | |
| 1160 | .n_errors = ARRAY_LEN(esp_encrypt_error_strings), |
| 1161 | .error_strings = esp_encrypt_error_strings, |
Klement Sekera | be5a5dd | 2018-10-09 16:05:48 +0200 | [diff] [blame] | 1162 | }; |
| 1163 | /* *INDENT-ON* */ |
| 1164 | |
Fan Zhang | f539578 | 2020-04-29 14:00:03 +0100 | [diff] [blame] | 1165 | VLIB_NODE_FN (esp6_encrypt_post_node) (vlib_main_t * vm, |
| 1166 | vlib_node_runtime_t * node, |
| 1167 | vlib_frame_t * from_frame) |
| 1168 | { |
| 1169 | return esp_encrypt_post_inline (vm, node, from_frame); |
| 1170 | } |
| 1171 | |
| 1172 | /* *INDENT-OFF* */ |
| 1173 | VLIB_REGISTER_NODE (esp6_encrypt_post_node) = { |
| 1174 | .name = "esp6-encrypt-post", |
| 1175 | .vector_size = sizeof (u32), |
| 1176 | .format_trace = format_esp_post_encrypt_trace, |
| 1177 | .type = VLIB_NODE_TYPE_INTERNAL, |
Neale Ranns | b1fd80f | 2020-05-12 13:33:56 +0000 | [diff] [blame] | 1178 | .sibling_of = "esp4-encrypt", |
Fan Zhang | f539578 | 2020-04-29 14:00:03 +0100 | [diff] [blame] | 1179 | |
| 1180 | .n_errors = ARRAY_LEN(esp_encrypt_error_strings), |
| 1181 | .error_strings = esp_encrypt_error_strings, |
| 1182 | }; |
| 1183 | /* *INDENT-ON* */ |
| 1184 | |
Neale Ranns | 25edf14 | 2019-03-22 08:12:48 +0000 | [diff] [blame] | 1185 | VLIB_NODE_FN (esp4_encrypt_tun_node) (vlib_main_t * vm, |
| 1186 | vlib_node_runtime_t * node, |
| 1187 | vlib_frame_t * from_frame) |
| 1188 | { |
Fan Zhang | f539578 | 2020-04-29 14:00:03 +0100 | [diff] [blame] | 1189 | return esp_encrypt_inline (vm, node, from_frame, 0 /* is_ip6 */ , 1, |
| 1190 | esp_encrypt_async_next.esp4_tun_post_next); |
Neale Ranns | 25edf14 | 2019-03-22 08:12:48 +0000 | [diff] [blame] | 1191 | } |
| 1192 | |
| 1193 | /* *INDENT-OFF* */ |
| 1194 | VLIB_REGISTER_NODE (esp4_encrypt_tun_node) = { |
| 1195 | .name = "esp4-encrypt-tun", |
| 1196 | .vector_size = sizeof (u32), |
| 1197 | .format_trace = format_esp_encrypt_trace, |
| 1198 | .type = VLIB_NODE_TYPE_INTERNAL, |
| 1199 | |
| 1200 | .n_errors = ARRAY_LEN(esp_encrypt_error_strings), |
| 1201 | .error_strings = esp_encrypt_error_strings, |
Neale Ranns | d7603d9 | 2019-03-28 08:56:10 +0000 | [diff] [blame] | 1202 | |
Neale Ranns | f62a8c0 | 2019-04-02 08:13:33 +0000 | [diff] [blame] | 1203 | .n_next_nodes = ESP_ENCRYPT_N_NEXT, |
Neale Ranns | d7603d9 | 2019-03-28 08:56:10 +0000 | [diff] [blame] | 1204 | .next_nodes = { |
Neale Ranns | b1fd80f | 2020-05-12 13:33:56 +0000 | [diff] [blame] | 1205 | [ESP_ENCRYPT_NEXT_DROP4] = "ip4-drop", |
| 1206 | [ESP_ENCRYPT_NEXT_DROP6] = "ip6-drop", |
| 1207 | [ESP_ENCRYPT_NEXT_HANDOFF4] = "esp4-encrypt-tun-handoff", |
| 1208 | [ESP_ENCRYPT_NEXT_HANDOFF6] = "error-drop", |
Neale Ranns | 4ec36c5 | 2020-03-31 09:21:29 -0400 | [diff] [blame] | 1209 | [ESP_ENCRYPT_NEXT_INTERFACE_OUTPUT] = "adj-midchain-tx", |
Neale Ranns | d7603d9 | 2019-03-28 08:56:10 +0000 | [diff] [blame] | 1210 | }, |
Neale Ranns | 25edf14 | 2019-03-22 08:12:48 +0000 | [diff] [blame] | 1211 | }; |
| 1212 | |
Fan Zhang | f539578 | 2020-04-29 14:00:03 +0100 | [diff] [blame] | 1213 | VLIB_NODE_FN (esp4_encrypt_tun_post_node) (vlib_main_t * vm, |
| 1214 | vlib_node_runtime_t * node, |
| 1215 | vlib_frame_t * from_frame) |
| 1216 | { |
| 1217 | return esp_encrypt_post_inline (vm, node, from_frame); |
| 1218 | } |
| 1219 | |
| 1220 | /* *INDENT-OFF* */ |
| 1221 | VLIB_REGISTER_NODE (esp4_encrypt_tun_post_node) = { |
| 1222 | .name = "esp4-encrypt-tun-post", |
| 1223 | .vector_size = sizeof (u32), |
| 1224 | .format_trace = format_esp_post_encrypt_trace, |
| 1225 | .type = VLIB_NODE_TYPE_INTERNAL, |
| 1226 | .sibling_of = "esp4-encrypt-tun", |
| 1227 | |
| 1228 | .n_errors = ARRAY_LEN(esp_encrypt_error_strings), |
| 1229 | .error_strings = esp_encrypt_error_strings, |
| 1230 | }; |
Neale Ranns | 25edf14 | 2019-03-22 08:12:48 +0000 | [diff] [blame] | 1231 | /* *INDENT-ON* */ |
| 1232 | |
| 1233 | VLIB_NODE_FN (esp6_encrypt_tun_node) (vlib_main_t * vm, |
| 1234 | vlib_node_runtime_t * node, |
| 1235 | vlib_frame_t * from_frame) |
| 1236 | { |
Fan Zhang | f539578 | 2020-04-29 14:00:03 +0100 | [diff] [blame] | 1237 | return esp_encrypt_inline (vm, node, from_frame, 1 /* is_ip6 */ , 1, |
| 1238 | esp_encrypt_async_next.esp6_tun_post_next); |
Neale Ranns | 25edf14 | 2019-03-22 08:12:48 +0000 | [diff] [blame] | 1239 | } |
| 1240 | |
| 1241 | /* *INDENT-OFF* */ |
| 1242 | VLIB_REGISTER_NODE (esp6_encrypt_tun_node) = { |
| 1243 | .name = "esp6-encrypt-tun", |
| 1244 | .vector_size = sizeof (u32), |
| 1245 | .format_trace = format_esp_encrypt_trace, |
| 1246 | .type = VLIB_NODE_TYPE_INTERNAL, |
| 1247 | |
| 1248 | .n_errors = ARRAY_LEN(esp_encrypt_error_strings), |
| 1249 | .error_strings = esp_encrypt_error_strings, |
Neale Ranns | d7603d9 | 2019-03-28 08:56:10 +0000 | [diff] [blame] | 1250 | |
Neale Ranns | f62a8c0 | 2019-04-02 08:13:33 +0000 | [diff] [blame] | 1251 | .n_next_nodes = ESP_ENCRYPT_N_NEXT, |
Neale Ranns | d7603d9 | 2019-03-28 08:56:10 +0000 | [diff] [blame] | 1252 | .next_nodes = { |
Neale Ranns | b1fd80f | 2020-05-12 13:33:56 +0000 | [diff] [blame] | 1253 | [ESP_ENCRYPT_NEXT_DROP4] = "ip4-drop", |
| 1254 | [ESP_ENCRYPT_NEXT_DROP6] = "ip6-drop", |
| 1255 | [ESP_ENCRYPT_NEXT_HANDOFF4] = "error-drop", |
| 1256 | [ESP_ENCRYPT_NEXT_HANDOFF6] = "esp6-encrypt-tun-handoff", |
Neale Ranns | 4ec36c5 | 2020-03-31 09:21:29 -0400 | [diff] [blame] | 1257 | [ESP_ENCRYPT_NEXT_INTERFACE_OUTPUT] = "adj-midchain-tx", |
Neale Ranns | d7603d9 | 2019-03-28 08:56:10 +0000 | [diff] [blame] | 1258 | }, |
Neale Ranns | 25edf14 | 2019-03-22 08:12:48 +0000 | [diff] [blame] | 1259 | }; |
| 1260 | |
Neale Ranns | 25edf14 | 2019-03-22 08:12:48 +0000 | [diff] [blame] | 1261 | /* *INDENT-ON* */ |
| 1262 | |
Fan Zhang | f539578 | 2020-04-29 14:00:03 +0100 | [diff] [blame] | 1263 | VLIB_NODE_FN (esp6_encrypt_tun_post_node) (vlib_main_t * vm, |
| 1264 | vlib_node_runtime_t * node, |
| 1265 | vlib_frame_t * from_frame) |
| 1266 | { |
| 1267 | return esp_encrypt_post_inline (vm, node, from_frame); |
| 1268 | } |
| 1269 | |
| 1270 | /* *INDENT-OFF* */ |
| 1271 | VLIB_REGISTER_NODE (esp6_encrypt_tun_post_node) = { |
| 1272 | .name = "esp6-encrypt-tun-post", |
| 1273 | .vector_size = sizeof (u32), |
| 1274 | .format_trace = format_esp_post_encrypt_trace, |
| 1275 | .type = VLIB_NODE_TYPE_INTERNAL, |
| 1276 | .sibling_of = "esp6-encrypt-tun", |
| 1277 | |
| 1278 | .n_errors = ARRAY_LEN(esp_encrypt_error_strings), |
| 1279 | .error_strings = esp_encrypt_error_strings, |
| 1280 | }; |
| 1281 | /* *INDENT-ON* */ |
| 1282 | |
Matthew Smith | 401aedf | 2019-07-08 14:45:04 -0500 | [diff] [blame] | 1283 | typedef struct |
| 1284 | { |
| 1285 | u32 sa_index; |
| 1286 | } esp_no_crypto_trace_t; |
| 1287 | |
| 1288 | static u8 * |
| 1289 | format_esp_no_crypto_trace (u8 * s, va_list * args) |
| 1290 | { |
| 1291 | CLIB_UNUSED (vlib_main_t * vm) = va_arg (*args, vlib_main_t *); |
| 1292 | CLIB_UNUSED (vlib_node_t * node) = va_arg (*args, vlib_node_t *); |
| 1293 | esp_no_crypto_trace_t *t = va_arg (*args, esp_no_crypto_trace_t *); |
| 1294 | |
| 1295 | s = format (s, "esp-no-crypto: sa-index %u", t->sa_index); |
| 1296 | |
| 1297 | return s; |
| 1298 | } |
| 1299 | |
| 1300 | enum |
| 1301 | { |
| 1302 | ESP_NO_CRYPTO_NEXT_DROP, |
| 1303 | ESP_NO_CRYPTO_N_NEXT, |
| 1304 | }; |
| 1305 | |
| 1306 | enum |
| 1307 | { |
| 1308 | ESP_NO_CRYPTO_ERROR_RX_PKTS, |
| 1309 | }; |
| 1310 | |
| 1311 | static char *esp_no_crypto_error_strings[] = { |
| 1312 | "Outbound ESP packets received", |
| 1313 | }; |
| 1314 | |
| 1315 | always_inline uword |
| 1316 | esp_no_crypto_inline (vlib_main_t * vm, vlib_node_runtime_t * node, |
| 1317 | vlib_frame_t * frame) |
| 1318 | { |
| 1319 | vlib_buffer_t *bufs[VLIB_FRAME_SIZE], **b = bufs; |
Matthew Smith | 401aedf | 2019-07-08 14:45:04 -0500 | [diff] [blame] | 1320 | u32 *from = vlib_frame_vector_args (frame); |
| 1321 | u32 n_left = frame->n_vectors; |
| 1322 | |
| 1323 | vlib_get_buffers (vm, from, b, n_left); |
| 1324 | |
| 1325 | while (n_left > 0) |
| 1326 | { |
Matthew Smith | 401aedf | 2019-07-08 14:45:04 -0500 | [diff] [blame] | 1327 | u32 sa_index0; |
| 1328 | |
| 1329 | /* packets are always going to be dropped, but get the sa_index */ |
Neale Ranns | 4ec36c5 | 2020-03-31 09:21:29 -0400 | [diff] [blame] | 1330 | sa_index0 = ipsec_tun_protect_get_sa_out |
| 1331 | (vnet_buffer (b[0])->ip.adj_index[VLIB_TX]); |
Matthew Smith | 401aedf | 2019-07-08 14:45:04 -0500 | [diff] [blame] | 1332 | |
| 1333 | if (PREDICT_FALSE (b[0]->flags & VLIB_BUFFER_IS_TRACED)) |
| 1334 | { |
| 1335 | esp_no_crypto_trace_t *tr = vlib_add_trace (vm, node, b[0], |
| 1336 | sizeof (*tr)); |
| 1337 | tr->sa_index = sa_index0; |
| 1338 | } |
| 1339 | |
| 1340 | n_left -= 1; |
Matthew Smith | 401aedf | 2019-07-08 14:45:04 -0500 | [diff] [blame] | 1341 | b += 1; |
| 1342 | } |
| 1343 | |
| 1344 | vlib_node_increment_counter (vm, node->node_index, |
| 1345 | ESP_NO_CRYPTO_ERROR_RX_PKTS, frame->n_vectors); |
| 1346 | |
Neale Ranns | 4ec36c5 | 2020-03-31 09:21:29 -0400 | [diff] [blame] | 1347 | vlib_buffer_enqueue_to_single_next (vm, node, from, |
| 1348 | ESP_NO_CRYPTO_NEXT_DROP, |
| 1349 | frame->n_vectors); |
Matthew Smith | 401aedf | 2019-07-08 14:45:04 -0500 | [diff] [blame] | 1350 | |
| 1351 | return frame->n_vectors; |
| 1352 | } |
| 1353 | |
| 1354 | VLIB_NODE_FN (esp4_no_crypto_tun_node) (vlib_main_t * vm, |
| 1355 | vlib_node_runtime_t * node, |
| 1356 | vlib_frame_t * from_frame) |
| 1357 | { |
| 1358 | return esp_no_crypto_inline (vm, node, from_frame); |
| 1359 | } |
| 1360 | |
| 1361 | /* *INDENT-OFF* */ |
| 1362 | VLIB_REGISTER_NODE (esp4_no_crypto_tun_node) = |
| 1363 | { |
| 1364 | .name = "esp4-no-crypto", |
| 1365 | .vector_size = sizeof (u32), |
| 1366 | .format_trace = format_esp_no_crypto_trace, |
| 1367 | .n_errors = ARRAY_LEN(esp_no_crypto_error_strings), |
| 1368 | .error_strings = esp_no_crypto_error_strings, |
| 1369 | .n_next_nodes = ESP_NO_CRYPTO_N_NEXT, |
| 1370 | .next_nodes = { |
| 1371 | [ESP_NO_CRYPTO_NEXT_DROP] = "ip4-drop", |
| 1372 | }, |
| 1373 | }; |
| 1374 | |
Matthew Smith | 401aedf | 2019-07-08 14:45:04 -0500 | [diff] [blame] | 1375 | VLIB_NODE_FN (esp6_no_crypto_tun_node) (vlib_main_t * vm, |
| 1376 | vlib_node_runtime_t * node, |
| 1377 | vlib_frame_t * from_frame) |
| 1378 | { |
| 1379 | return esp_no_crypto_inline (vm, node, from_frame); |
| 1380 | } |
| 1381 | |
| 1382 | /* *INDENT-OFF* */ |
| 1383 | VLIB_REGISTER_NODE (esp6_no_crypto_tun_node) = |
| 1384 | { |
| 1385 | .name = "esp6-no-crypto", |
| 1386 | .vector_size = sizeof (u32), |
| 1387 | .format_trace = format_esp_no_crypto_trace, |
| 1388 | .n_errors = ARRAY_LEN(esp_no_crypto_error_strings), |
| 1389 | .error_strings = esp_no_crypto_error_strings, |
| 1390 | .n_next_nodes = ESP_NO_CRYPTO_N_NEXT, |
| 1391 | .next_nodes = { |
| 1392 | [ESP_NO_CRYPTO_NEXT_DROP] = "ip6-drop", |
| 1393 | }, |
| 1394 | }; |
Matthew Smith | 401aedf | 2019-07-08 14:45:04 -0500 | [diff] [blame] | 1395 | /* *INDENT-ON* */ |
| 1396 | |
Keith Burns (alagalah) | 166a9d4 | 2016-08-06 11:00:56 -0700 | [diff] [blame] | 1397 | /* |
| 1398 | * fd.io coding-style-patch-verification: ON |
| 1399 | * |
| 1400 | * Local Variables: |
| 1401 | * eval: (c-set-style "gnu") |
| 1402 | * End: |
| 1403 | */ |