Ed Warnicke | cb9cada | 2015-12-08 15:45:58 -0700 | [diff] [blame] | 1 | /* |
| 2 | * esp_encrypt.c : IPSec ESP encrypt node |
| 3 | * |
| 4 | * Copyright (c) 2015 Cisco and/or its affiliates. |
| 5 | * Licensed under the Apache License, Version 2.0 (the "License"); |
| 6 | * you may not use this file except in compliance with the License. |
| 7 | * You may obtain a copy of the License at: |
| 8 | * |
| 9 | * http://www.apache.org/licenses/LICENSE-2.0 |
| 10 | * |
| 11 | * Unless required by applicable law or agreed to in writing, software |
| 12 | * distributed under the License is distributed on an "AS IS" BASIS, |
| 13 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. |
| 14 | * See the License for the specific language governing permissions and |
| 15 | * limitations under the License. |
| 16 | */ |
| 17 | |
| 18 | #include <vnet/vnet.h> |
| 19 | #include <vnet/api_errno.h> |
| 20 | #include <vnet/ip/ip.h> |
| 21 | |
Damjan Marion | 91f17dc | 2019-03-18 18:59:25 +0100 | [diff] [blame] | 22 | #include <vnet/crypto/crypto.h> |
| 23 | |
Ed Warnicke | cb9cada | 2015-12-08 15:45:58 -0700 | [diff] [blame] | 24 | #include <vnet/ipsec/ipsec.h> |
Neale Ranns | 2828721 | 2019-12-16 00:53:11 +0000 | [diff] [blame] | 25 | #include <vnet/ipsec/ipsec_tun.h> |
Ed Warnicke | cb9cada | 2015-12-08 15:45:58 -0700 | [diff] [blame] | 26 | #include <vnet/ipsec/esp.h> |
Neale Ranns | 041add7 | 2020-01-02 04:06:10 +0000 | [diff] [blame] | 27 | #include <vnet/tunnel/tunnel_dp.h> |
Ed Warnicke | cb9cada | 2015-12-08 15:45:58 -0700 | [diff] [blame] | 28 | |
Neale Ranns | 4a58e49 | 2020-12-21 13:19:10 +0000 | [diff] [blame] | 29 | #define foreach_esp_encrypt_next \ |
| 30 | _ (DROP4, "ip4-drop") \ |
| 31 | _ (DROP6, "ip6-drop") \ |
| 32 | _ (DROP_MPLS, "mpls-drop") \ |
| 33 | _ (HANDOFF4, "handoff4") \ |
| 34 | _ (HANDOFF6, "handoff6") \ |
| 35 | _ (HANDOFF_MPLS, "handoff-mpls") \ |
| 36 | _ (INTERFACE_OUTPUT, "interface-output") |
Ed Warnicke | cb9cada | 2015-12-08 15:45:58 -0700 | [diff] [blame] | 37 | |
| 38 | #define _(v, s) ESP_ENCRYPT_NEXT_##v, |
Keith Burns (alagalah) | 166a9d4 | 2016-08-06 11:00:56 -0700 | [diff] [blame] | 39 | typedef enum |
| 40 | { |
Ed Warnicke | cb9cada | 2015-12-08 15:45:58 -0700 | [diff] [blame] | 41 | foreach_esp_encrypt_next |
| 42 | #undef _ |
Keith Burns (alagalah) | 166a9d4 | 2016-08-06 11:00:56 -0700 | [diff] [blame] | 43 | ESP_ENCRYPT_N_NEXT, |
Ed Warnicke | cb9cada | 2015-12-08 15:45:58 -0700 | [diff] [blame] | 44 | } esp_encrypt_next_t; |
| 45 | |
Damjan Marion | c59b9a2 | 2019-03-19 15:38:40 +0100 | [diff] [blame] | 46 | #define foreach_esp_encrypt_error \ |
| 47 | _(RX_PKTS, "ESP pkts received") \ |
Fan Zhang | f539578 | 2020-04-29 14:00:03 +0100 | [diff] [blame] | 48 | _(POST_RX_PKTS, "ESP-post pkts received") \ |
Damjan Marion | c59b9a2 | 2019-03-19 15:38:40 +0100 | [diff] [blame] | 49 | _(SEQ_CYCLED, "sequence number cycled (packet dropped)") \ |
| 50 | _(CRYPTO_ENGINE_ERROR, "crypto engine error (packet dropped)") \ |
Fan Zhang | f539578 | 2020-04-29 14:00:03 +0100 | [diff] [blame] | 51 | _(CRYPTO_QUEUE_FULL, "crypto queue full (packet dropped)") \ |
Filip Tehlar | efcad1a | 2020-02-04 09:36:04 +0000 | [diff] [blame] | 52 | _(NO_BUFFERS, "no buffers (packet dropped)") \ |
Ed Warnicke | cb9cada | 2015-12-08 15:45:58 -0700 | [diff] [blame] | 53 | |
Keith Burns (alagalah) | 166a9d4 | 2016-08-06 11:00:56 -0700 | [diff] [blame] | 54 | typedef enum |
| 55 | { |
Ed Warnicke | cb9cada | 2015-12-08 15:45:58 -0700 | [diff] [blame] | 56 | #define _(sym,str) ESP_ENCRYPT_ERROR_##sym, |
| 57 | foreach_esp_encrypt_error |
| 58 | #undef _ |
Keith Burns (alagalah) | 166a9d4 | 2016-08-06 11:00:56 -0700 | [diff] [blame] | 59 | ESP_ENCRYPT_N_ERROR, |
Ed Warnicke | cb9cada | 2015-12-08 15:45:58 -0700 | [diff] [blame] | 60 | } esp_encrypt_error_t; |
| 61 | |
Keith Burns (alagalah) | 166a9d4 | 2016-08-06 11:00:56 -0700 | [diff] [blame] | 62 | static char *esp_encrypt_error_strings[] = { |
Ed Warnicke | cb9cada | 2015-12-08 15:45:58 -0700 | [diff] [blame] | 63 | #define _(sym,string) string, |
| 64 | foreach_esp_encrypt_error |
| 65 | #undef _ |
| 66 | }; |
| 67 | |
Keith Burns (alagalah) | 166a9d4 | 2016-08-06 11:00:56 -0700 | [diff] [blame] | 68 | typedef struct |
| 69 | { |
Neale Ranns | 8d7c502 | 2019-02-06 01:41:05 -0800 | [diff] [blame] | 70 | u32 sa_index; |
Ed Warnicke | cb9cada | 2015-12-08 15:45:58 -0700 | [diff] [blame] | 71 | u32 spi; |
| 72 | u32 seq; |
Neale Ranns | 6afaae1 | 2019-07-17 15:07:14 +0000 | [diff] [blame] | 73 | u32 sa_seq_hi; |
Klement Sekera | 4b089f2 | 2018-04-17 18:04:57 +0200 | [diff] [blame] | 74 | u8 udp_encap; |
Ed Warnicke | cb9cada | 2015-12-08 15:45:58 -0700 | [diff] [blame] | 75 | ipsec_crypto_alg_t crypto_alg; |
| 76 | ipsec_integ_alg_t integ_alg; |
| 77 | } esp_encrypt_trace_t; |
| 78 | |
Fan Zhang | f539578 | 2020-04-29 14:00:03 +0100 | [diff] [blame] | 79 | typedef struct |
| 80 | { |
| 81 | u32 next_index; |
| 82 | } esp_encrypt_post_trace_t; |
| 83 | |
Ed Warnicke | cb9cada | 2015-12-08 15:45:58 -0700 | [diff] [blame] | 84 | /* packet trace format function */ |
Keith Burns (alagalah) | 166a9d4 | 2016-08-06 11:00:56 -0700 | [diff] [blame] | 85 | static u8 * |
| 86 | format_esp_encrypt_trace (u8 * s, va_list * args) |
Ed Warnicke | cb9cada | 2015-12-08 15:45:58 -0700 | [diff] [blame] | 87 | { |
| 88 | CLIB_UNUSED (vlib_main_t * vm) = va_arg (*args, vlib_main_t *); |
| 89 | CLIB_UNUSED (vlib_node_t * node) = va_arg (*args, vlib_node_t *); |
Keith Burns (alagalah) | 166a9d4 | 2016-08-06 11:00:56 -0700 | [diff] [blame] | 90 | esp_encrypt_trace_t *t = va_arg (*args, esp_encrypt_trace_t *); |
Ed Warnicke | cb9cada | 2015-12-08 15:45:58 -0700 | [diff] [blame] | 91 | |
Guillaume Solignac | 8f818cc | 2019-05-15 12:02:33 +0200 | [diff] [blame] | 92 | s = |
| 93 | format (s, |
Neale Ranns | 6afaae1 | 2019-07-17 15:07:14 +0000 | [diff] [blame] | 94 | "esp: sa-index %d spi %u (0x%08x) seq %u sa-seq-hi %u crypto %U integrity %U%s", |
| 95 | t->sa_index, t->spi, t->spi, t->seq, t->sa_seq_hi, |
| 96 | format_ipsec_crypto_alg, |
Guillaume Solignac | 8f818cc | 2019-05-15 12:02:33 +0200 | [diff] [blame] | 97 | t->crypto_alg, format_ipsec_integ_alg, t->integ_alg, |
| 98 | t->udp_encap ? " udp-encap-enabled" : ""); |
Ed Warnicke | cb9cada | 2015-12-08 15:45:58 -0700 | [diff] [blame] | 99 | return s; |
| 100 | } |
| 101 | |
Fan Zhang | f539578 | 2020-04-29 14:00:03 +0100 | [diff] [blame] | 102 | static u8 * |
| 103 | format_esp_post_encrypt_trace (u8 * s, va_list * args) |
| 104 | { |
| 105 | CLIB_UNUSED (vlib_main_t * vm) = va_arg (*args, vlib_main_t *); |
| 106 | CLIB_UNUSED (vlib_node_t * node) = va_arg (*args, vlib_node_t *); |
| 107 | esp_encrypt_post_trace_t *t = va_arg (*args, esp_encrypt_post_trace_t *); |
| 108 | |
| 109 | s = format (s, "esp-post: next node index %u", t->next_index); |
| 110 | return s; |
| 111 | } |
| 112 | |
Damjan Marion | c59b9a2 | 2019-03-19 15:38:40 +0100 | [diff] [blame] | 113 | /* pad packet in input buffer */ |
| 114 | static_always_inline u8 * |
Filip Tehlar | e4e8c6b | 2020-02-13 07:49:30 +0000 | [diff] [blame] | 115 | esp_add_footer_and_icv (vlib_main_t * vm, vlib_buffer_t ** last, |
Christian Hopps | fb7e7ed | 2019-11-03 07:02:15 -0500 | [diff] [blame] | 116 | u8 esp_align, u8 icv_sz, |
Filip Tehlar | 8cdb1a0 | 2019-11-18 22:21:37 +0000 | [diff] [blame] | 117 | u16 * next, vlib_node_runtime_t * node, |
Filip Tehlar | efcad1a | 2020-02-04 09:36:04 +0000 | [diff] [blame] | 118 | u16 buffer_data_size, uword total_len) |
Ed Warnicke | cb9cada | 2015-12-08 15:45:58 -0700 | [diff] [blame] | 119 | { |
Damjan Marion | c59b9a2 | 2019-03-19 15:38:40 +0100 | [diff] [blame] | 120 | static const u8 pad_data[ESP_MAX_BLOCK_SIZE] = { |
| 121 | 0x01, 0x02, 0x03, 0x04, 0x05, 0x06, 0x07, 0x08, |
Milan Lenco | 7885c74 | 2020-08-20 13:23:09 +0200 | [diff] [blame] | 122 | 0x09, 0x0a, 0x0b, 0x0c, 0x0d, 0x0e, 0x0f, 0x00, |
Damjan Marion | c59b9a2 | 2019-03-19 15:38:40 +0100 | [diff] [blame] | 123 | }; |
Ed Warnicke | cb9cada | 2015-12-08 15:45:58 -0700 | [diff] [blame] | 124 | |
Filip Tehlar | efcad1a | 2020-02-04 09:36:04 +0000 | [diff] [blame] | 125 | u16 min_length = total_len + sizeof (esp_footer_t); |
Christian Hopps | fb7e7ed | 2019-11-03 07:02:15 -0500 | [diff] [blame] | 126 | u16 new_length = round_pow2 (min_length, esp_align); |
Damjan Marion | c59b9a2 | 2019-03-19 15:38:40 +0100 | [diff] [blame] | 127 | u8 pad_bytes = new_length - min_length; |
Filip Tehlar | e4e8c6b | 2020-02-13 07:49:30 +0000 | [diff] [blame] | 128 | esp_footer_t *f = (esp_footer_t *) (vlib_buffer_get_current (last[0]) + |
| 129 | last[0]->current_length + pad_bytes); |
Filip Tehlar | efcad1a | 2020-02-04 09:36:04 +0000 | [diff] [blame] | 130 | u16 tail_sz = sizeof (esp_footer_t) + pad_bytes + icv_sz; |
Ed Warnicke | cb9cada | 2015-12-08 15:45:58 -0700 | [diff] [blame] | 131 | |
Filip Tehlar | e4e8c6b | 2020-02-13 07:49:30 +0000 | [diff] [blame] | 132 | if (last[0]->current_length + tail_sz > buffer_data_size) |
Filip Tehlar | 8cdb1a0 | 2019-11-18 22:21:37 +0000 | [diff] [blame] | 133 | { |
Filip Tehlar | e4e8c6b | 2020-02-13 07:49:30 +0000 | [diff] [blame] | 134 | u32 tmp_bi = 0; |
| 135 | if (vlib_buffer_alloc (vm, &tmp_bi, 1) != 1) |
| 136 | return 0; |
Filip Tehlar | c2c1bfd | 2020-02-13 07:49:30 +0000 | [diff] [blame] | 137 | |
Filip Tehlar | e4e8c6b | 2020-02-13 07:49:30 +0000 | [diff] [blame] | 138 | vlib_buffer_t *tmp = vlib_get_buffer (vm, tmp_bi); |
| 139 | last[0]->next_buffer = tmp_bi; |
| 140 | last[0]->flags |= VLIB_BUFFER_NEXT_PRESENT; |
| 141 | f = (esp_footer_t *) (vlib_buffer_get_current (tmp) + pad_bytes); |
| 142 | tmp->current_length += tail_sz; |
| 143 | last[0] = tmp; |
| 144 | } |
| 145 | else |
| 146 | last[0]->current_length += tail_sz; |
| 147 | |
| 148 | f->pad_length = pad_bytes; |
Damjan Marion | c59b9a2 | 2019-03-19 15:38:40 +0100 | [diff] [blame] | 149 | if (pad_bytes) |
Benoît Ganne | 4505f01 | 2019-12-07 09:14:27 -0700 | [diff] [blame] | 150 | { |
| 151 | ASSERT (pad_bytes <= ESP_MAX_BLOCK_SIZE); |
| 152 | pad_bytes = clib_min (ESP_MAX_BLOCK_SIZE, pad_bytes); |
| 153 | clib_memcpy_fast ((u8 *) f - pad_bytes, pad_data, pad_bytes); |
| 154 | } |
Damjan Marion | c59b9a2 | 2019-03-19 15:38:40 +0100 | [diff] [blame] | 155 | |
Damjan Marion | c59b9a2 | 2019-03-19 15:38:40 +0100 | [diff] [blame] | 156 | return &f->next_header; |
| 157 | } |
| 158 | |
| 159 | static_always_inline void |
| 160 | esp_update_ip4_hdr (ip4_header_t * ip4, u16 len, int is_transport, int is_udp) |
| 161 | { |
Neale Ranns | 1b582b8 | 2019-04-18 19:49:13 -0700 | [diff] [blame] | 162 | ip_csum_t sum; |
| 163 | u16 old_len; |
| 164 | |
| 165 | len = clib_net_to_host_u16 (len); |
| 166 | old_len = ip4->length; |
Damjan Marion | c59b9a2 | 2019-03-19 15:38:40 +0100 | [diff] [blame] | 167 | |
| 168 | if (is_transport) |
| 169 | { |
| 170 | u8 prot = is_udp ? IP_PROTOCOL_UDP : IP_PROTOCOL_IPSEC_ESP; |
Damjan Marion | c59b9a2 | 2019-03-19 15:38:40 +0100 | [diff] [blame] | 171 | |
Neale Ranns | 1b582b8 | 2019-04-18 19:49:13 -0700 | [diff] [blame] | 172 | sum = ip_csum_update (ip4->checksum, ip4->protocol, |
| 173 | prot, ip4_header_t, protocol); |
| 174 | ip4->protocol = prot; |
| 175 | |
| 176 | sum = ip_csum_update (sum, old_len, len, ip4_header_t, length); |
| 177 | } |
| 178 | else |
| 179 | sum = ip_csum_update (ip4->checksum, old_len, len, ip4_header_t, length); |
| 180 | |
| 181 | ip4->length = len; |
Damjan Marion | c59b9a2 | 2019-03-19 15:38:40 +0100 | [diff] [blame] | 182 | ip4->checksum = ip_csum_fold (sum); |
| 183 | } |
| 184 | |
| 185 | static_always_inline void |
| 186 | esp_fill_udp_hdr (ipsec_sa_t * sa, udp_header_t * udp, u16 len) |
| 187 | { |
| 188 | clib_memcpy_fast (udp, &sa->udp_hdr, sizeof (udp_header_t)); |
| 189 | udp->length = clib_net_to_host_u16 (len); |
| 190 | } |
| 191 | |
| 192 | static_always_inline u8 |
| 193 | ext_hdr_is_pre_esp (u8 nexthdr) |
| 194 | { |
| 195 | #ifdef CLIB_HAVE_VEC128 |
| 196 | static const u8x16 ext_hdr_types = { |
| 197 | IP_PROTOCOL_IP6_HOP_BY_HOP_OPTIONS, |
| 198 | IP_PROTOCOL_IPV6_ROUTE, |
| 199 | IP_PROTOCOL_IPV6_FRAGMENTATION, |
| 200 | }; |
| 201 | |
| 202 | return !u8x16_is_all_zero (ext_hdr_types == u8x16_splat (nexthdr)); |
| 203 | #else |
| 204 | return ((nexthdr ^ IP_PROTOCOL_IP6_HOP_BY_HOP_OPTIONS) | |
| 205 | (nexthdr ^ IP_PROTOCOL_IPV6_ROUTE) | |
| 206 | (nexthdr ^ IP_PROTOCOL_IPV6_FRAGMENTATION) != 0); |
| 207 | #endif |
| 208 | } |
| 209 | |
| 210 | static_always_inline u8 |
Neale Ranns | 0295040 | 2019-12-20 00:54:57 +0000 | [diff] [blame] | 211 | esp_get_ip6_hdr_len (ip6_header_t * ip6, ip6_ext_header_t ** ext_hdr) |
Damjan Marion | c59b9a2 | 2019-03-19 15:38:40 +0100 | [diff] [blame] | 212 | { |
| 213 | /* this code assumes that HbH, route and frag headers will be before |
| 214 | others, if that is not the case, they will end up encrypted */ |
Damjan Marion | c59b9a2 | 2019-03-19 15:38:40 +0100 | [diff] [blame] | 215 | u8 len = sizeof (ip6_header_t); |
| 216 | ip6_ext_header_t *p; |
| 217 | |
| 218 | /* if next packet doesn't have ext header */ |
| 219 | if (ext_hdr_is_pre_esp (ip6->protocol) == 0) |
Neale Ranns | 0295040 | 2019-12-20 00:54:57 +0000 | [diff] [blame] | 220 | { |
| 221 | *ext_hdr = NULL; |
| 222 | return len; |
| 223 | } |
Damjan Marion | c59b9a2 | 2019-03-19 15:38:40 +0100 | [diff] [blame] | 224 | |
| 225 | p = (void *) (ip6 + 1); |
| 226 | len += ip6_ext_header_len (p); |
| 227 | |
| 228 | while (ext_hdr_is_pre_esp (p->next_hdr)) |
| 229 | { |
| 230 | len += ip6_ext_header_len (p); |
| 231 | p = ip6_ext_next_header (p); |
| 232 | } |
| 233 | |
Neale Ranns | 0295040 | 2019-12-20 00:54:57 +0000 | [diff] [blame] | 234 | *ext_hdr = p; |
Damjan Marion | c59b9a2 | 2019-03-19 15:38:40 +0100 | [diff] [blame] | 235 | return len; |
| 236 | } |
| 237 | |
Damjan Marion | c59b9a2 | 2019-03-19 15:38:40 +0100 | [diff] [blame] | 238 | static_always_inline void |
Filip Tehlar | efcad1a | 2020-02-04 09:36:04 +0000 | [diff] [blame] | 239 | esp_process_chained_ops (vlib_main_t * vm, vlib_node_runtime_t * node, |
| 240 | vnet_crypto_op_t * ops, vlib_buffer_t * b[], |
Neale Ranns | b1fd80f | 2020-05-12 13:33:56 +0000 | [diff] [blame] | 241 | u16 * nexts, vnet_crypto_op_chunk_t * chunks, |
| 242 | u16 drop_next) |
Filip Tehlar | efcad1a | 2020-02-04 09:36:04 +0000 | [diff] [blame] | 243 | { |
| 244 | u32 n_fail, n_ops = vec_len (ops); |
| 245 | vnet_crypto_op_t *op = ops; |
| 246 | |
| 247 | if (n_ops == 0) |
| 248 | return; |
| 249 | |
| 250 | n_fail = n_ops - vnet_crypto_process_chained_ops (vm, op, chunks, n_ops); |
| 251 | |
| 252 | while (n_fail) |
| 253 | { |
| 254 | ASSERT (op - ops < n_ops); |
| 255 | |
| 256 | if (op->status != VNET_CRYPTO_OP_STATUS_COMPLETED) |
| 257 | { |
| 258 | u32 bi = op->user_data; |
| 259 | b[bi]->error = node->errors[ESP_ENCRYPT_ERROR_CRYPTO_ENGINE_ERROR]; |
Neale Ranns | b1fd80f | 2020-05-12 13:33:56 +0000 | [diff] [blame] | 260 | nexts[bi] = drop_next; |
Filip Tehlar | efcad1a | 2020-02-04 09:36:04 +0000 | [diff] [blame] | 261 | n_fail--; |
| 262 | } |
| 263 | op++; |
| 264 | } |
| 265 | } |
| 266 | |
| 267 | static_always_inline void |
Damjan Marion | c59b9a2 | 2019-03-19 15:38:40 +0100 | [diff] [blame] | 268 | esp_process_ops (vlib_main_t * vm, vlib_node_runtime_t * node, |
Neale Ranns | b1fd80f | 2020-05-12 13:33:56 +0000 | [diff] [blame] | 269 | vnet_crypto_op_t * ops, vlib_buffer_t * b[], u16 * nexts, |
| 270 | u16 drop_next) |
Damjan Marion | c59b9a2 | 2019-03-19 15:38:40 +0100 | [diff] [blame] | 271 | { |
| 272 | u32 n_fail, n_ops = vec_len (ops); |
| 273 | vnet_crypto_op_t *op = ops; |
| 274 | |
| 275 | if (n_ops == 0) |
Ed Warnicke | cb9cada | 2015-12-08 15:45:58 -0700 | [diff] [blame] | 276 | return; |
| 277 | |
Damjan Marion | c59b9a2 | 2019-03-19 15:38:40 +0100 | [diff] [blame] | 278 | n_fail = n_ops - vnet_crypto_process_ops (vm, op, n_ops); |
Ed Warnicke | cb9cada | 2015-12-08 15:45:58 -0700 | [diff] [blame] | 279 | |
Damjan Marion | c59b9a2 | 2019-03-19 15:38:40 +0100 | [diff] [blame] | 280 | while (n_fail) |
| 281 | { |
| 282 | ASSERT (op - ops < n_ops); |
| 283 | |
| 284 | if (op->status != VNET_CRYPTO_OP_STATUS_COMPLETED) |
| 285 | { |
| 286 | u32 bi = op->user_data; |
| 287 | b[bi]->error = node->errors[ESP_ENCRYPT_ERROR_CRYPTO_ENGINE_ERROR]; |
Neale Ranns | b1fd80f | 2020-05-12 13:33:56 +0000 | [diff] [blame] | 288 | nexts[bi] = drop_next; |
Damjan Marion | c59b9a2 | 2019-03-19 15:38:40 +0100 | [diff] [blame] | 289 | n_fail--; |
| 290 | } |
| 291 | op++; |
| 292 | } |
Ed Warnicke | cb9cada | 2015-12-08 15:45:58 -0700 | [diff] [blame] | 293 | } |
| 294 | |
Fan Zhang | f539578 | 2020-04-29 14:00:03 +0100 | [diff] [blame] | 295 | static_always_inline u32 |
| 296 | esp_encrypt_chain_crypto (vlib_main_t * vm, ipsec_per_thread_data_t * ptd, |
| 297 | ipsec_sa_t * sa0, vlib_buffer_t * b, |
| 298 | vlib_buffer_t * lb, u8 icv_sz, u8 * start, |
| 299 | u32 start_len, u16 * n_ch) |
| 300 | { |
| 301 | vnet_crypto_op_chunk_t *ch; |
| 302 | vlib_buffer_t *cb = b; |
| 303 | u32 n_chunks = 1; |
| 304 | u32 total_len; |
| 305 | vec_add2 (ptd->chunks, ch, 1); |
| 306 | total_len = ch->len = start_len; |
| 307 | ch->src = ch->dst = start; |
| 308 | cb = vlib_get_buffer (vm, cb->next_buffer); |
| 309 | |
| 310 | while (1) |
| 311 | { |
| 312 | vec_add2 (ptd->chunks, ch, 1); |
| 313 | n_chunks += 1; |
| 314 | if (lb == cb) |
| 315 | total_len += ch->len = cb->current_length - icv_sz; |
| 316 | else |
| 317 | total_len += ch->len = cb->current_length; |
| 318 | ch->src = ch->dst = vlib_buffer_get_current (cb); |
| 319 | |
| 320 | if (!(cb->flags & VLIB_BUFFER_NEXT_PRESENT)) |
| 321 | break; |
| 322 | |
| 323 | cb = vlib_get_buffer (vm, cb->next_buffer); |
| 324 | } |
| 325 | |
| 326 | if (n_ch) |
| 327 | *n_ch = n_chunks; |
| 328 | |
| 329 | return total_len; |
| 330 | } |
| 331 | |
| 332 | static_always_inline u32 |
| 333 | esp_encrypt_chain_integ (vlib_main_t * vm, ipsec_per_thread_data_t * ptd, |
| 334 | ipsec_sa_t * sa0, vlib_buffer_t * b, |
| 335 | vlib_buffer_t * lb, u8 icv_sz, u8 * start, |
| 336 | u32 start_len, u8 * digest, u16 * n_ch) |
| 337 | { |
| 338 | vnet_crypto_op_chunk_t *ch; |
| 339 | vlib_buffer_t *cb = b; |
| 340 | u32 n_chunks = 1; |
| 341 | u32 total_len; |
| 342 | vec_add2 (ptd->chunks, ch, 1); |
| 343 | total_len = ch->len = start_len; |
| 344 | ch->src = start; |
| 345 | cb = vlib_get_buffer (vm, cb->next_buffer); |
| 346 | |
| 347 | while (1) |
| 348 | { |
| 349 | vec_add2 (ptd->chunks, ch, 1); |
| 350 | n_chunks += 1; |
| 351 | if (lb == cb) |
| 352 | { |
| 353 | total_len += ch->len = cb->current_length - icv_sz; |
| 354 | if (ipsec_sa_is_set_USE_ESN (sa0)) |
| 355 | { |
| 356 | u32 seq_hi = clib_net_to_host_u32 (sa0->seq_hi); |
| 357 | clib_memcpy_fast (digest, &seq_hi, sizeof (seq_hi)); |
| 358 | ch->len += sizeof (seq_hi); |
| 359 | total_len += sizeof (seq_hi); |
| 360 | } |
| 361 | } |
| 362 | else |
| 363 | total_len += ch->len = cb->current_length; |
| 364 | ch->src = vlib_buffer_get_current (cb); |
| 365 | |
| 366 | if (!(cb->flags & VLIB_BUFFER_NEXT_PRESENT)) |
| 367 | break; |
| 368 | |
| 369 | cb = vlib_get_buffer (vm, cb->next_buffer); |
| 370 | } |
| 371 | |
| 372 | if (n_ch) |
| 373 | *n_ch = n_chunks; |
| 374 | |
| 375 | return total_len; |
| 376 | } |
| 377 | |
| 378 | always_inline void |
Benoît Ganne | 490b927 | 2021-01-22 18:03:09 +0100 | [diff] [blame] | 379 | esp_prepare_sync_op (vlib_main_t *vm, ipsec_per_thread_data_t *ptd, |
| 380 | vnet_crypto_op_t **crypto_ops, |
| 381 | vnet_crypto_op_t **integ_ops, ipsec_sa_t *sa0, |
| 382 | u8 *payload, u16 payload_len, u8 iv_sz, u8 icv_sz, |
| 383 | vlib_buffer_t **bufs, vlib_buffer_t **b, |
| 384 | vlib_buffer_t *lb, u32 hdr_len, esp_header_t *esp) |
Fan Zhang | f539578 | 2020-04-29 14:00:03 +0100 | [diff] [blame] | 385 | { |
| 386 | if (sa0->crypto_enc_op_id) |
| 387 | { |
| 388 | vnet_crypto_op_t *op; |
| 389 | vec_add2_aligned (crypto_ops[0], op, 1, CLIB_CACHE_LINE_BYTES); |
| 390 | vnet_crypto_op_init (op, sa0->crypto_enc_op_id); |
| 391 | |
| 392 | op->src = op->dst = payload; |
| 393 | op->key_index = sa0->crypto_key_index; |
| 394 | op->len = payload_len - icv_sz; |
| 395 | op->user_data = b - bufs; |
| 396 | |
Benoît Ganne | 490b927 | 2021-01-22 18:03:09 +0100 | [diff] [blame] | 397 | if (ipsec_sa_is_set_IS_CTR (sa0)) |
Fan Zhang | f539578 | 2020-04-29 14:00:03 +0100 | [diff] [blame] | 398 | { |
Benoît Ganne | 490b927 | 2021-01-22 18:03:09 +0100 | [diff] [blame] | 399 | ASSERT (sizeof (u64) == iv_sz); |
| 400 | /* construct nonce in a scratch space in front of the IP header */ |
| 401 | esp_ctr_nonce_t *nonce = |
| 402 | (esp_ctr_nonce_t *) (payload - sizeof (u64) - hdr_len - |
| 403 | sizeof (*nonce)); |
| 404 | u64 *pkt_iv = (u64 *) (payload - sizeof (u64)); |
Fan Zhang | f539578 | 2020-04-29 14:00:03 +0100 | [diff] [blame] | 405 | |
Benoît Ganne | 490b927 | 2021-01-22 18:03:09 +0100 | [diff] [blame] | 406 | if (ipsec_sa_is_set_IS_AEAD (sa0)) |
| 407 | { |
| 408 | /* constuct aad in a scratch space in front of the nonce */ |
| 409 | op->aad = (u8 *) nonce - sizeof (esp_aead_t); |
| 410 | op->aad_len = esp_aad_fill (op->aad, esp, sa0); |
| 411 | op->tag = payload + op->len; |
| 412 | op->tag_len = 16; |
| 413 | } |
| 414 | else |
| 415 | { |
| 416 | nonce->ctr = clib_host_to_net_u32 (1); |
| 417 | } |
Fan Zhang | f539578 | 2020-04-29 14:00:03 +0100 | [diff] [blame] | 418 | |
Fan Zhang | f539578 | 2020-04-29 14:00:03 +0100 | [diff] [blame] | 419 | nonce->salt = sa0->salt; |
Benoît Ganne | 490b927 | 2021-01-22 18:03:09 +0100 | [diff] [blame] | 420 | nonce->iv = *pkt_iv = clib_host_to_net_u64 (sa0->ctr_iv_counter++); |
Fan Zhang | f539578 | 2020-04-29 14:00:03 +0100 | [diff] [blame] | 421 | op->iv = (u8 *) nonce; |
| 422 | } |
| 423 | else |
| 424 | { |
| 425 | op->iv = payload - iv_sz; |
| 426 | op->flags = VNET_CRYPTO_OP_FLAG_INIT_IV; |
| 427 | } |
| 428 | |
| 429 | if (lb != b[0]) |
| 430 | { |
| 431 | /* is chained */ |
| 432 | op->flags |= VNET_CRYPTO_OP_FLAG_CHAINED_BUFFERS; |
| 433 | op->chunk_index = vec_len (ptd->chunks); |
| 434 | op->tag = vlib_buffer_get_tail (lb) - icv_sz; |
| 435 | esp_encrypt_chain_crypto (vm, ptd, sa0, b[0], lb, icv_sz, payload, |
| 436 | payload_len, &op->n_chunks); |
| 437 | } |
| 438 | } |
| 439 | |
| 440 | if (sa0->integ_op_id) |
| 441 | { |
| 442 | vnet_crypto_op_t *op; |
| 443 | vec_add2_aligned (integ_ops[0], op, 1, CLIB_CACHE_LINE_BYTES); |
| 444 | vnet_crypto_op_init (op, sa0->integ_op_id); |
| 445 | op->src = payload - iv_sz - sizeof (esp_header_t); |
| 446 | op->digest = payload + payload_len - icv_sz; |
| 447 | op->key_index = sa0->integ_key_index; |
| 448 | op->digest_len = icv_sz; |
| 449 | op->len = payload_len - icv_sz + iv_sz + sizeof (esp_header_t); |
| 450 | op->user_data = b - bufs; |
| 451 | |
| 452 | if (lb != b[0]) |
| 453 | { |
| 454 | /* is chained */ |
| 455 | op->flags |= VNET_CRYPTO_OP_FLAG_CHAINED_BUFFERS; |
| 456 | op->chunk_index = vec_len (ptd->chunks); |
| 457 | op->digest = vlib_buffer_get_tail (lb) - icv_sz; |
| 458 | |
| 459 | esp_encrypt_chain_integ (vm, ptd, sa0, b[0], lb, icv_sz, |
| 460 | payload - iv_sz - sizeof (esp_header_t), |
| 461 | payload_len + iv_sz + |
| 462 | sizeof (esp_header_t), op->digest, |
| 463 | &op->n_chunks); |
| 464 | } |
| 465 | else if (ipsec_sa_is_set_USE_ESN (sa0)) |
| 466 | { |
| 467 | u32 seq_hi = clib_net_to_host_u32 (sa0->seq_hi); |
| 468 | clib_memcpy_fast (op->digest, &seq_hi, sizeof (seq_hi)); |
| 469 | op->len += sizeof (seq_hi); |
| 470 | } |
| 471 | } |
| 472 | } |
| 473 | |
| 474 | static_always_inline int |
| 475 | esp_prepare_async_frame (vlib_main_t * vm, ipsec_per_thread_data_t * ptd, |
| 476 | vnet_crypto_async_frame_t ** async_frame, |
| 477 | ipsec_sa_t * sa, vlib_buffer_t * b, |
| 478 | esp_header_t * esp, u8 * payload, u32 payload_len, |
Fan Zhang | 18f0e31 | 2020-10-19 13:08:34 +0100 | [diff] [blame] | 479 | u8 iv_sz, u8 icv_sz, u32 bi, u16 next, u32 hdr_len, |
Fan Zhang | f539578 | 2020-04-29 14:00:03 +0100 | [diff] [blame] | 480 | u16 async_next, vlib_buffer_t * lb) |
| 481 | { |
| 482 | esp_post_data_t *post = esp_post_data (b); |
| 483 | u8 *tag, *iv, *aad = 0; |
| 484 | u8 flag = 0; |
| 485 | u32 key_index; |
| 486 | i16 crypto_start_offset, integ_start_offset = 0; |
| 487 | u16 crypto_total_len, integ_total_len; |
| 488 | |
Fan Zhang | 18f0e31 | 2020-10-19 13:08:34 +0100 | [diff] [blame] | 489 | post->next_index = next; |
Fan Zhang | f539578 | 2020-04-29 14:00:03 +0100 | [diff] [blame] | 490 | |
| 491 | /* crypto */ |
| 492 | crypto_start_offset = payload - b->data; |
| 493 | crypto_total_len = integ_total_len = payload_len - icv_sz; |
| 494 | tag = payload + crypto_total_len; |
| 495 | |
Fan Zhang | f539578 | 2020-04-29 14:00:03 +0100 | [diff] [blame] | 496 | key_index = sa->linked_key_index; |
| 497 | |
Benoît Ganne | 490b927 | 2021-01-22 18:03:09 +0100 | [diff] [blame] | 498 | if (ipsec_sa_is_set_IS_CTR (sa)) |
Fan Zhang | f539578 | 2020-04-29 14:00:03 +0100 | [diff] [blame] | 499 | { |
Benoît Ganne | 490b927 | 2021-01-22 18:03:09 +0100 | [diff] [blame] | 500 | ASSERT (sizeof (u64) == iv_sz); |
| 501 | /* construct nonce in a scratch space in front of the IP header */ |
| 502 | esp_ctr_nonce_t *nonce = (esp_ctr_nonce_t *) (payload - sizeof (u64) - |
| 503 | hdr_len - sizeof (*nonce)); |
| 504 | u64 *pkt_iv = (u64 *) (payload - sizeof (u64)); |
| 505 | |
| 506 | if (ipsec_sa_is_set_IS_AEAD (sa)) |
| 507 | { |
| 508 | /* constuct aad in a scratch space in front of the nonce */ |
| 509 | aad = (u8 *) nonce - sizeof (esp_aead_t); |
| 510 | esp_aad_fill (aad, esp, sa); |
| 511 | key_index = sa->crypto_key_index; |
| 512 | } |
| 513 | else |
| 514 | { |
| 515 | nonce->ctr = clib_host_to_net_u32 (1); |
| 516 | } |
| 517 | |
| 518 | nonce->salt = sa->salt; |
| 519 | nonce->iv = *pkt_iv = clib_host_to_net_u64 (sa->ctr_iv_counter++); |
| 520 | iv = (u8 *) nonce; |
Fan Zhang | f539578 | 2020-04-29 14:00:03 +0100 | [diff] [blame] | 521 | } |
Benoît Ganne | 490b927 | 2021-01-22 18:03:09 +0100 | [diff] [blame] | 522 | else |
Fan Zhang | f539578 | 2020-04-29 14:00:03 +0100 | [diff] [blame] | 523 | { |
Benoît Ganne | 490b927 | 2021-01-22 18:03:09 +0100 | [diff] [blame] | 524 | iv = payload - iv_sz; |
| 525 | flag |= VNET_CRYPTO_OP_FLAG_INIT_IV; |
Fan Zhang | f539578 | 2020-04-29 14:00:03 +0100 | [diff] [blame] | 526 | } |
| 527 | |
Benoît Ganne | 490b927 | 2021-01-22 18:03:09 +0100 | [diff] [blame] | 528 | if (lb != b) |
| 529 | { |
| 530 | /* chain */ |
| 531 | flag |= VNET_CRYPTO_OP_FLAG_CHAINED_BUFFERS; |
| 532 | tag = vlib_buffer_get_tail (lb) - icv_sz; |
| 533 | crypto_total_len = esp_encrypt_chain_crypto (vm, ptd, sa, b, lb, icv_sz, |
| 534 | payload, payload_len, 0); |
| 535 | } |
| 536 | |
| 537 | if (sa->integ_op_id) |
| 538 | { |
| 539 | integ_start_offset = crypto_start_offset - iv_sz - sizeof (esp_header_t); |
| 540 | integ_total_len += iv_sz + sizeof (esp_header_t); |
| 541 | |
| 542 | if (b != lb) |
| 543 | { |
| 544 | integ_total_len = esp_encrypt_chain_integ ( |
| 545 | vm, ptd, sa, b, lb, icv_sz, |
| 546 | payload - iv_sz - sizeof (esp_header_t), |
| 547 | payload_len + iv_sz + sizeof (esp_header_t), tag, 0); |
| 548 | } |
| 549 | else if (ipsec_sa_is_set_USE_ESN (sa)) |
| 550 | { |
| 551 | u32 seq_hi = clib_net_to_host_u32 (sa->seq_hi); |
| 552 | clib_memcpy_fast (tag, &seq_hi, sizeof (seq_hi)); |
| 553 | integ_total_len += sizeof (seq_hi); |
| 554 | } |
| 555 | } |
| 556 | |
Fan Zhang | f539578 | 2020-04-29 14:00:03 +0100 | [diff] [blame] | 557 | return vnet_crypto_async_add_to_frame (vm, async_frame, key_index, |
| 558 | crypto_total_len, |
| 559 | integ_total_len - crypto_total_len, |
| 560 | crypto_start_offset, |
| 561 | integ_start_offset, bi, async_next, |
| 562 | iv, tag, aad, flag); |
| 563 | } |
| 564 | |
Klement Sekera | be5a5dd | 2018-10-09 16:05:48 +0200 | [diff] [blame] | 565 | always_inline uword |
Neale Ranns | 4a58e49 | 2020-12-21 13:19:10 +0000 | [diff] [blame] | 566 | esp_encrypt_inline (vlib_main_t *vm, vlib_node_runtime_t *node, |
| 567 | vlib_frame_t *frame, vnet_link_t lt, int is_tun, |
Fan Zhang | f539578 | 2020-04-29 14:00:03 +0100 | [diff] [blame] | 568 | u16 async_next) |
Ed Warnicke | cb9cada | 2015-12-08 15:45:58 -0700 | [diff] [blame] | 569 | { |
Ed Warnicke | cb9cada | 2015-12-08 15:45:58 -0700 | [diff] [blame] | 570 | ipsec_main_t *im = &ipsec_main; |
Damjan Marion | c59b9a2 | 2019-03-19 15:38:40 +0100 | [diff] [blame] | 571 | ipsec_per_thread_data_t *ptd = vec_elt_at_index (im->ptd, vm->thread_index); |
| 572 | u32 *from = vlib_frame_vector_args (frame); |
| 573 | u32 n_left = frame->n_vectors; |
| 574 | vlib_buffer_t *bufs[VLIB_FRAME_SIZE], **b = bufs; |
Damjan Marion | c98275f | 2019-03-06 14:05:01 +0100 | [diff] [blame] | 575 | u16 nexts[VLIB_FRAME_SIZE], *next = nexts; |
Damjan Marion | c59b9a2 | 2019-03-19 15:38:40 +0100 | [diff] [blame] | 576 | u32 thread_index = vm->thread_index; |
| 577 | u16 buffer_data_size = vlib_buffer_get_default_data_size (vm); |
| 578 | u32 current_sa_index = ~0, current_sa_packets = 0; |
| 579 | u32 current_sa_bytes = 0, spi = 0; |
Christian Hopps | fb7e7ed | 2019-11-03 07:02:15 -0500 | [diff] [blame] | 580 | u8 esp_align = 4, iv_sz = 0, icv_sz = 0; |
Damjan Marion | c59b9a2 | 2019-03-19 15:38:40 +0100 | [diff] [blame] | 581 | ipsec_sa_t *sa0 = 0; |
Filip Tehlar | efcad1a | 2020-02-04 09:36:04 +0000 | [diff] [blame] | 582 | vlib_buffer_t *lb; |
| 583 | vnet_crypto_op_t **crypto_ops = &ptd->crypto_ops; |
| 584 | vnet_crypto_op_t **integ_ops = &ptd->integ_ops; |
Fan Zhang | f539578 | 2020-04-29 14:00:03 +0100 | [diff] [blame] | 585 | vnet_crypto_async_frame_t *async_frame = 0; |
| 586 | int is_async = im->async_mode; |
| 587 | vnet_crypto_async_op_id_t last_async_op = ~0; |
Neale Ranns | 4a58e49 | 2020-12-21 13:19:10 +0000 | [diff] [blame] | 588 | u16 drop_next = |
| 589 | (lt == VNET_LINK_IP6 ? ESP_ENCRYPT_NEXT_DROP6 : |
| 590 | (lt == VNET_LINK_IP4 ? ESP_ENCRYPT_NEXT_DROP4 : |
| 591 | ESP_ENCRYPT_NEXT_DROP_MPLS)); |
| 592 | u16 handoff_next = (lt == VNET_LINK_IP6 ? |
| 593 | ESP_ENCRYPT_NEXT_HANDOFF6 : |
| 594 | (lt == VNET_LINK_IP4 ? ESP_ENCRYPT_NEXT_HANDOFF4 : |
| 595 | ESP_ENCRYPT_NEXT_HANDOFF_MPLS)); |
Fan Zhang | 18f0e31 | 2020-10-19 13:08:34 +0100 | [diff] [blame] | 596 | u16 n_async_drop = 0; |
Ed Warnicke | cb9cada | 2015-12-08 15:45:58 -0700 | [diff] [blame] | 597 | |
Damjan Marion | c59b9a2 | 2019-03-19 15:38:40 +0100 | [diff] [blame] | 598 | vlib_get_buffers (vm, from, b, n_left); |
Fan Zhang | f539578 | 2020-04-29 14:00:03 +0100 | [diff] [blame] | 599 | if (!is_async) |
| 600 | { |
| 601 | vec_reset_length (ptd->crypto_ops); |
| 602 | vec_reset_length (ptd->integ_ops); |
| 603 | vec_reset_length (ptd->chained_crypto_ops); |
| 604 | vec_reset_length (ptd->chained_integ_ops); |
| 605 | } |
Filip Tehlar | efcad1a | 2020-02-04 09:36:04 +0000 | [diff] [blame] | 606 | vec_reset_length (ptd->chunks); |
Damjan Marion | c59b9a2 | 2019-03-19 15:38:40 +0100 | [diff] [blame] | 607 | |
| 608 | while (n_left > 0) |
Keith Burns (alagalah) | 166a9d4 | 2016-08-06 11:00:56 -0700 | [diff] [blame] | 609 | { |
Zhiyong Yang | 1cff643 | 2019-04-30 05:33:53 -0400 | [diff] [blame] | 610 | u32 sa_index0; |
Damjan Marion | c59b9a2 | 2019-03-19 15:38:40 +0100 | [diff] [blame] | 611 | dpo_id_t *dpo; |
| 612 | esp_header_t *esp; |
| 613 | u8 *payload, *next_hdr_ptr; |
Filip Tehlar | efcad1a | 2020-02-04 09:36:04 +0000 | [diff] [blame] | 614 | u16 payload_len, payload_len_total, n_bufs; |
Neale Ranns | 4ec36c5 | 2020-03-31 09:21:29 -0400 | [diff] [blame] | 615 | u32 hdr_len; |
Ed Warnicke | cb9cada | 2015-12-08 15:45:58 -0700 | [diff] [blame] | 616 | |
Damjan Marion | c59b9a2 | 2019-03-19 15:38:40 +0100 | [diff] [blame] | 617 | if (n_left > 2) |
Keith Burns (alagalah) | 166a9d4 | 2016-08-06 11:00:56 -0700 | [diff] [blame] | 618 | { |
Damjan Marion | c59b9a2 | 2019-03-19 15:38:40 +0100 | [diff] [blame] | 619 | u8 *p; |
| 620 | vlib_prefetch_buffer_header (b[2], LOAD); |
| 621 | p = vlib_buffer_get_current (b[1]); |
| 622 | CLIB_PREFETCH (p, CLIB_CACHE_LINE_BYTES, LOAD); |
| 623 | p -= CLIB_CACHE_LINE_BYTES; |
| 624 | CLIB_PREFETCH (p, CLIB_CACHE_LINE_BYTES, LOAD); |
Neale Ranns | 123b5eb | 2020-10-16 14:03:55 +0000 | [diff] [blame] | 625 | /* speculate that the trailer goes in the first buffer */ |
| 626 | CLIB_PREFETCH (vlib_buffer_get_tail (b[1]), |
| 627 | CLIB_CACHE_LINE_BYTES, LOAD); |
Keith Burns (alagalah) | 166a9d4 | 2016-08-06 11:00:56 -0700 | [diff] [blame] | 628 | } |
Ed Warnicke | cb9cada | 2015-12-08 15:45:58 -0700 | [diff] [blame] | 629 | |
Neale Ranns | 25edf14 | 2019-03-22 08:12:48 +0000 | [diff] [blame] | 630 | if (is_tun) |
| 631 | { |
| 632 | /* we are on a ipsec tunnel's feature arc */ |
Neale Ranns | 2828721 | 2019-12-16 00:53:11 +0000 | [diff] [blame] | 633 | vnet_buffer (b[0])->ipsec.sad_index = |
| 634 | sa_index0 = ipsec_tun_protect_get_sa_out |
| 635 | (vnet_buffer (b[0])->ip.adj_index[VLIB_TX]); |
Neale Ranns | 25edf14 | 2019-03-22 08:12:48 +0000 | [diff] [blame] | 636 | } |
| 637 | else |
| 638 | sa_index0 = vnet_buffer (b[0])->ipsec.sad_index; |
| 639 | |
| 640 | if (sa_index0 != current_sa_index) |
Damjan Marion | c59b9a2 | 2019-03-19 15:38:40 +0100 | [diff] [blame] | 641 | { |
Damjan Marion | 21f265f | 2019-06-05 15:45:50 +0200 | [diff] [blame] | 642 | if (current_sa_packets) |
| 643 | vlib_increment_combined_counter (&ipsec_sa_counters, thread_index, |
| 644 | current_sa_index, |
| 645 | current_sa_packets, |
| 646 | current_sa_bytes); |
| 647 | current_sa_packets = current_sa_bytes = 0; |
| 648 | |
Neale Ranns | c5fe57d | 2021-02-25 16:01:28 +0000 | [diff] [blame] | 649 | sa0 = ipsec_sa_get (sa_index0); |
Neale Ranns | 123b5eb | 2020-10-16 14:03:55 +0000 | [diff] [blame] | 650 | |
| 651 | /* fetch the second cacheline ASAP */ |
| 652 | CLIB_PREFETCH (sa0->cacheline1, CLIB_CACHE_LINE_BYTES, LOAD); |
| 653 | |
Damjan Marion | c59b9a2 | 2019-03-19 15:38:40 +0100 | [diff] [blame] | 654 | current_sa_index = sa_index0; |
Damjan Marion | c59b9a2 | 2019-03-19 15:38:40 +0100 | [diff] [blame] | 655 | spi = clib_net_to_host_u32 (sa0->spi); |
Christian Hopps | fb7e7ed | 2019-11-03 07:02:15 -0500 | [diff] [blame] | 656 | esp_align = sa0->esp_block_align; |
Damjan Marion | 7c22ff7 | 2019-04-04 12:25:44 +0200 | [diff] [blame] | 657 | icv_sz = sa0->integ_icv_size; |
Damjan Marion | c59b9a2 | 2019-03-19 15:38:40 +0100 | [diff] [blame] | 658 | iv_sz = sa0->crypto_iv_size; |
Fan Zhang | f539578 | 2020-04-29 14:00:03 +0100 | [diff] [blame] | 659 | |
| 660 | /* submit frame when op_id is different then the old one */ |
| 661 | if (is_async && sa0->crypto_async_enc_op_id != last_async_op) |
| 662 | { |
| 663 | if (async_frame && async_frame->n_elts) |
| 664 | { |
Fan Zhang | 18f0e31 | 2020-10-19 13:08:34 +0100 | [diff] [blame] | 665 | if (vnet_crypto_async_submit_open_frame (vm, async_frame)) |
| 666 | esp_async_recycle_failed_submit (async_frame, b, from, |
| 667 | nexts, &n_async_drop, |
| 668 | drop_next, |
| 669 | ESP_ENCRYPT_ERROR_CRYPTO_ENGINE_ERROR); |
Fan Zhang | f539578 | 2020-04-29 14:00:03 +0100 | [diff] [blame] | 670 | } |
| 671 | async_frame = |
| 672 | vnet_crypto_async_get_frame (vm, sa0->crypto_async_enc_op_id); |
| 673 | last_async_op = sa0->crypto_async_enc_op_id; |
| 674 | } |
Damjan Marion | c59b9a2 | 2019-03-19 15:38:40 +0100 | [diff] [blame] | 675 | } |
| 676 | |
Neale Ranns | 1a52d37 | 2021-02-04 11:33:32 +0000 | [diff] [blame] | 677 | if (PREDICT_FALSE (~0 == sa0->thread_index)) |
Neale Ranns | f62a8c0 | 2019-04-02 08:13:33 +0000 | [diff] [blame] | 678 | { |
| 679 | /* this is the first packet to use this SA, claim the SA |
| 680 | * for this thread. this could happen simultaneously on |
| 681 | * another thread */ |
Neale Ranns | 1a52d37 | 2021-02-04 11:33:32 +0000 | [diff] [blame] | 682 | clib_atomic_cmp_and_swap (&sa0->thread_index, ~0, |
Neale Ranns | f62a8c0 | 2019-04-02 08:13:33 +0000 | [diff] [blame] | 683 | ipsec_sa_assign_thread (thread_index)); |
| 684 | } |
| 685 | |
Neale Ranns | 1a52d37 | 2021-02-04 11:33:32 +0000 | [diff] [blame] | 686 | if (PREDICT_FALSE (thread_index != sa0->thread_index)) |
Neale Ranns | f62a8c0 | 2019-04-02 08:13:33 +0000 | [diff] [blame] | 687 | { |
Neale Ranns | aa7d766 | 2021-02-10 08:42:49 +0000 | [diff] [blame] | 688 | vnet_buffer (b[0])->ipsec.thread_index = sa0->thread_index; |
Fan Zhang | 18f0e31 | 2020-10-19 13:08:34 +0100 | [diff] [blame] | 689 | esp_set_next_index (is_async, from, nexts, from[b - bufs], |
Neale Ranns | 4a58e49 | 2020-12-21 13:19:10 +0000 | [diff] [blame] | 690 | &n_async_drop, handoff_next, next); |
Neale Ranns | f62a8c0 | 2019-04-02 08:13:33 +0000 | [diff] [blame] | 691 | goto trace; |
| 692 | } |
| 693 | |
Filip Tehlar | efcad1a | 2020-02-04 09:36:04 +0000 | [diff] [blame] | 694 | lb = b[0]; |
| 695 | n_bufs = vlib_buffer_chain_linearize (vm, b[0]); |
| 696 | if (n_bufs == 0) |
Damjan Marion | c59b9a2 | 2019-03-19 15:38:40 +0100 | [diff] [blame] | 697 | { |
Filip Tehlar | efcad1a | 2020-02-04 09:36:04 +0000 | [diff] [blame] | 698 | b[0]->error = node->errors[ESP_ENCRYPT_ERROR_NO_BUFFERS]; |
Fan Zhang | 18f0e31 | 2020-10-19 13:08:34 +0100 | [diff] [blame] | 699 | esp_set_next_index (is_async, from, nexts, from[b - bufs], |
| 700 | &n_async_drop, drop_next, next); |
Damjan Marion | c59b9a2 | 2019-03-19 15:38:40 +0100 | [diff] [blame] | 701 | goto trace; |
| 702 | } |
| 703 | |
Filip Tehlar | efcad1a | 2020-02-04 09:36:04 +0000 | [diff] [blame] | 704 | if (n_bufs > 1) |
| 705 | { |
Filip Tehlar | efcad1a | 2020-02-04 09:36:04 +0000 | [diff] [blame] | 706 | /* find last buffer in the chain */ |
| 707 | while (lb->flags & VLIB_BUFFER_NEXT_PRESENT) |
| 708 | lb = vlib_get_buffer (vm, lb->next_buffer); |
| 709 | } |
Filip Tehlar | efcad1a | 2020-02-04 09:36:04 +0000 | [diff] [blame] | 710 | |
Damjan Marion | c59b9a2 | 2019-03-19 15:38:40 +0100 | [diff] [blame] | 711 | if (PREDICT_FALSE (esp_seq_advance (sa0))) |
| 712 | { |
| 713 | b[0]->error = node->errors[ESP_ENCRYPT_ERROR_SEQ_CYCLED]; |
Fan Zhang | 18f0e31 | 2020-10-19 13:08:34 +0100 | [diff] [blame] | 714 | esp_set_next_index (is_async, from, nexts, from[b - bufs], |
| 715 | &n_async_drop, drop_next, next); |
Damjan Marion | c59b9a2 | 2019-03-19 15:38:40 +0100 | [diff] [blame] | 716 | goto trace; |
| 717 | } |
| 718 | |
| 719 | /* space for IV */ |
| 720 | hdr_len = iv_sz; |
| 721 | |
Damjan Marion | d709cbc | 2019-03-26 13:16:42 +0100 | [diff] [blame] | 722 | if (ipsec_sa_is_set_IS_TUNNEL (sa0)) |
Damjan Marion | c59b9a2 | 2019-03-19 15:38:40 +0100 | [diff] [blame] | 723 | { |
| 724 | payload = vlib_buffer_get_current (b[0]); |
Christian Hopps | fb7e7ed | 2019-11-03 07:02:15 -0500 | [diff] [blame] | 725 | next_hdr_ptr = esp_add_footer_and_icv (vm, &lb, esp_align, icv_sz, |
Filip Tehlar | 8cdb1a0 | 2019-11-18 22:21:37 +0000 | [diff] [blame] | 726 | next, node, |
Filip Tehlar | efcad1a | 2020-02-04 09:36:04 +0000 | [diff] [blame] | 727 | buffer_data_size, |
| 728 | vlib_buffer_length_in_chain |
| 729 | (vm, b[0])); |
Filip Tehlar | 8cdb1a0 | 2019-11-18 22:21:37 +0000 | [diff] [blame] | 730 | if (!next_hdr_ptr) |
Filip Tehlar | e4e8c6b | 2020-02-13 07:49:30 +0000 | [diff] [blame] | 731 | { |
| 732 | b[0]->error = node->errors[ESP_ENCRYPT_ERROR_NO_BUFFERS]; |
Fan Zhang | 18f0e31 | 2020-10-19 13:08:34 +0100 | [diff] [blame] | 733 | esp_set_next_index (is_async, from, nexts, from[b - bufs], |
| 734 | &n_async_drop, drop_next, next); |
Filip Tehlar | e4e8c6b | 2020-02-13 07:49:30 +0000 | [diff] [blame] | 735 | goto trace; |
| 736 | } |
Filip Tehlar | efcad1a | 2020-02-04 09:36:04 +0000 | [diff] [blame] | 737 | b[0]->flags &= ~VLIB_BUFFER_TOTAL_LENGTH_VALID; |
Filip Tehlar | 8cdb1a0 | 2019-11-18 22:21:37 +0000 | [diff] [blame] | 738 | payload_len = b[0]->current_length; |
Filip Tehlar | efcad1a | 2020-02-04 09:36:04 +0000 | [diff] [blame] | 739 | payload_len_total = vlib_buffer_length_in_chain (vm, b[0]); |
Damjan Marion | c59b9a2 | 2019-03-19 15:38:40 +0100 | [diff] [blame] | 740 | |
| 741 | /* ESP header */ |
| 742 | hdr_len += sizeof (*esp); |
| 743 | esp = (esp_header_t *) (payload - hdr_len); |
| 744 | |
| 745 | /* optional UDP header */ |
Damjan Marion | d709cbc | 2019-03-26 13:16:42 +0100 | [diff] [blame] | 746 | if (ipsec_sa_is_set_UDP_ENCAP (sa0)) |
Damjan Marion | c98275f | 2019-03-06 14:05:01 +0100 | [diff] [blame] | 747 | { |
Damjan Marion | c59b9a2 | 2019-03-19 15:38:40 +0100 | [diff] [blame] | 748 | hdr_len += sizeof (udp_header_t); |
| 749 | esp_fill_udp_hdr (sa0, (udp_header_t *) (payload - hdr_len), |
Filip Tehlar | efcad1a | 2020-02-04 09:36:04 +0000 | [diff] [blame] | 750 | payload_len_total + hdr_len); |
Damjan Marion | c59b9a2 | 2019-03-19 15:38:40 +0100 | [diff] [blame] | 751 | } |
| 752 | |
| 753 | /* IP header */ |
Damjan Marion | d709cbc | 2019-03-26 13:16:42 +0100 | [diff] [blame] | 754 | if (ipsec_sa_is_set_IS_TUNNEL_V6 (sa0)) |
Damjan Marion | c59b9a2 | 2019-03-19 15:38:40 +0100 | [diff] [blame] | 755 | { |
| 756 | ip6_header_t *ip6; |
| 757 | u16 len = sizeof (ip6_header_t); |
| 758 | hdr_len += len; |
| 759 | ip6 = (ip6_header_t *) (payload - hdr_len); |
Neale Ranns | 041add7 | 2020-01-02 04:06:10 +0000 | [diff] [blame] | 760 | clib_memcpy_fast (ip6, &sa0->ip6_hdr, sizeof (ip6_header_t)); |
| 761 | |
Neale Ranns | 4a58e49 | 2020-12-21 13:19:10 +0000 | [diff] [blame] | 762 | if (VNET_LINK_IP6 == lt) |
Neale Ranns | 041add7 | 2020-01-02 04:06:10 +0000 | [diff] [blame] | 763 | { |
| 764 | *next_hdr_ptr = IP_PROTOCOL_IPV6; |
| 765 | tunnel_encap_fixup_6o6 (sa0->tunnel_flags, |
| 766 | (const ip6_header_t *) payload, |
| 767 | ip6); |
| 768 | } |
Neale Ranns | 4a58e49 | 2020-12-21 13:19:10 +0000 | [diff] [blame] | 769 | else if (VNET_LINK_IP4 == lt) |
Neale Ranns | 041add7 | 2020-01-02 04:06:10 +0000 | [diff] [blame] | 770 | { |
| 771 | *next_hdr_ptr = IP_PROTOCOL_IP_IN_IP; |
Neale Ranns | a91cb45 | 2021-02-04 11:02:52 +0000 | [diff] [blame] | 772 | tunnel_encap_fixup_4o6 (sa0->tunnel_flags, b[0], |
| 773 | (const ip4_header_t *) payload, ip6); |
Neale Ranns | 041add7 | 2020-01-02 04:06:10 +0000 | [diff] [blame] | 774 | } |
Neale Ranns | 4a58e49 | 2020-12-21 13:19:10 +0000 | [diff] [blame] | 775 | else if (VNET_LINK_MPLS == lt) |
| 776 | { |
| 777 | *next_hdr_ptr = IP_PROTOCOL_MPLS_IN_IP; |
| 778 | tunnel_encap_fixup_mplso6 ( |
Neale Ranns | a91cb45 | 2021-02-04 11:02:52 +0000 | [diff] [blame] | 779 | sa0->tunnel_flags, b[0], |
| 780 | (const mpls_unicast_header_t *) payload, ip6); |
Neale Ranns | 4a58e49 | 2020-12-21 13:19:10 +0000 | [diff] [blame] | 781 | } |
| 782 | else |
| 783 | ASSERT (0); |
| 784 | |
Filip Tehlar | efcad1a | 2020-02-04 09:36:04 +0000 | [diff] [blame] | 785 | len = payload_len_total + hdr_len - len; |
Damjan Marion | c59b9a2 | 2019-03-19 15:38:40 +0100 | [diff] [blame] | 786 | ip6->payload_length = clib_net_to_host_u16 (len); |
Neale Ranns | 45d6d83 | 2021-01-19 13:38:47 +0000 | [diff] [blame] | 787 | b[0]->flags |= VNET_BUFFER_F_LOCALLY_ORIGINATED; |
Damjan Marion | c98275f | 2019-03-06 14:05:01 +0100 | [diff] [blame] | 788 | } |
| 789 | else |
| 790 | { |
Damjan Marion | c59b9a2 | 2019-03-19 15:38:40 +0100 | [diff] [blame] | 791 | ip4_header_t *ip4; |
| 792 | u16 len = sizeof (ip4_header_t); |
| 793 | hdr_len += len; |
| 794 | ip4 = (ip4_header_t *) (payload - hdr_len); |
Neale Ranns | 041add7 | 2020-01-02 04:06:10 +0000 | [diff] [blame] | 795 | clib_memcpy_fast (ip4, &sa0->ip4_hdr, sizeof (ip4_header_t)); |
| 796 | |
Neale Ranns | 4a58e49 | 2020-12-21 13:19:10 +0000 | [diff] [blame] | 797 | if (VNET_LINK_IP6 == lt) |
Neale Ranns | 041add7 | 2020-01-02 04:06:10 +0000 | [diff] [blame] | 798 | { |
| 799 | *next_hdr_ptr = IP_PROTOCOL_IPV6; |
| 800 | tunnel_encap_fixup_6o4_w_chksum (sa0->tunnel_flags, |
| 801 | (const ip6_header_t *) |
| 802 | payload, ip4); |
| 803 | } |
Neale Ranns | 4a58e49 | 2020-12-21 13:19:10 +0000 | [diff] [blame] | 804 | else if (VNET_LINK_IP4 == lt) |
Neale Ranns | 041add7 | 2020-01-02 04:06:10 +0000 | [diff] [blame] | 805 | { |
| 806 | *next_hdr_ptr = IP_PROTOCOL_IP_IN_IP; |
| 807 | tunnel_encap_fixup_4o4_w_chksum (sa0->tunnel_flags, |
| 808 | (const ip4_header_t *) |
| 809 | payload, ip4); |
| 810 | } |
Neale Ranns | 4a58e49 | 2020-12-21 13:19:10 +0000 | [diff] [blame] | 811 | else if (VNET_LINK_MPLS == lt) |
| 812 | { |
| 813 | *next_hdr_ptr = IP_PROTOCOL_MPLS_IN_IP; |
| 814 | tunnel_encap_fixup_mplso4_w_chksum ( |
| 815 | sa0->tunnel_flags, (const mpls_unicast_header_t *) payload, |
| 816 | ip4); |
| 817 | } |
| 818 | else |
| 819 | ASSERT (0); |
| 820 | |
Filip Tehlar | efcad1a | 2020-02-04 09:36:04 +0000 | [diff] [blame] | 821 | len = payload_len_total + hdr_len; |
Damjan Marion | c59b9a2 | 2019-03-19 15:38:40 +0100 | [diff] [blame] | 822 | esp_update_ip4_hdr (ip4, len, /* is_transport */ 0, 0); |
Damjan Marion | c98275f | 2019-03-06 14:05:01 +0100 | [diff] [blame] | 823 | } |
Damjan Marion | c98275f | 2019-03-06 14:05:01 +0100 | [diff] [blame] | 824 | |
Neale Ranns | 72f2a3a | 2019-06-17 15:43:38 +0000 | [diff] [blame] | 825 | dpo = &sa0->dpo; |
Neale Ranns | 25edf14 | 2019-03-22 08:12:48 +0000 | [diff] [blame] | 826 | if (!is_tun) |
| 827 | { |
| 828 | next[0] = dpo->dpoi_next_node; |
| 829 | vnet_buffer (b[0])->ip.adj_index[VLIB_TX] = dpo->dpoi_index; |
| 830 | } |
Neale Ranns | 4ec36c5 | 2020-03-31 09:21:29 -0400 | [diff] [blame] | 831 | else |
| 832 | next[0] = ESP_ENCRYPT_NEXT_INTERFACE_OUTPUT; |
Neale Ranns | 9ec846c | 2021-02-09 14:04:02 +0000 | [diff] [blame] | 833 | b[0]->flags |= VNET_BUFFER_F_LOCALLY_ORIGINATED; |
Damjan Marion | c98275f | 2019-03-06 14:05:01 +0100 | [diff] [blame] | 834 | } |
Damjan Marion | c59b9a2 | 2019-03-19 15:38:40 +0100 | [diff] [blame] | 835 | else /* transport mode */ |
Damjan Marion | c98275f | 2019-03-06 14:05:01 +0100 | [diff] [blame] | 836 | { |
Damjan Marion | c59b9a2 | 2019-03-19 15:38:40 +0100 | [diff] [blame] | 837 | u8 *l2_hdr, l2_len, *ip_hdr, ip_len; |
Neale Ranns | 0295040 | 2019-12-20 00:54:57 +0000 | [diff] [blame] | 838 | ip6_ext_header_t *ext_hdr; |
Damjan Marion | c59b9a2 | 2019-03-19 15:38:40 +0100 | [diff] [blame] | 839 | udp_header_t *udp = 0; |
Alexander Chernavin | b0d2eda | 2020-03-25 10:56:52 -0400 | [diff] [blame] | 840 | u16 udp_len = 0; |
Damjan Marion | c59b9a2 | 2019-03-19 15:38:40 +0100 | [diff] [blame] | 841 | u8 *old_ip_hdr = vlib_buffer_get_current (b[0]); |
Damjan Marion | c98275f | 2019-03-06 14:05:01 +0100 | [diff] [blame] | 842 | |
Neale Ranns | 4a58e49 | 2020-12-21 13:19:10 +0000 | [diff] [blame] | 843 | ip_len = |
| 844 | (VNET_LINK_IP6 == lt ? |
| 845 | esp_get_ip6_hdr_len ((ip6_header_t *) old_ip_hdr, &ext_hdr) : |
| 846 | ip4_header_bytes ((ip4_header_t *) old_ip_hdr)); |
Damjan Marion | c98275f | 2019-03-06 14:05:01 +0100 | [diff] [blame] | 847 | |
Damjan Marion | c59b9a2 | 2019-03-19 15:38:40 +0100 | [diff] [blame] | 848 | vlib_buffer_advance (b[0], ip_len); |
| 849 | payload = vlib_buffer_get_current (b[0]); |
Christian Hopps | fb7e7ed | 2019-11-03 07:02:15 -0500 | [diff] [blame] | 850 | next_hdr_ptr = esp_add_footer_and_icv (vm, &lb, esp_align, icv_sz, |
Filip Tehlar | 8cdb1a0 | 2019-11-18 22:21:37 +0000 | [diff] [blame] | 851 | next, node, |
Filip Tehlar | efcad1a | 2020-02-04 09:36:04 +0000 | [diff] [blame] | 852 | buffer_data_size, |
| 853 | vlib_buffer_length_in_chain |
| 854 | (vm, b[0])); |
Filip Tehlar | 8cdb1a0 | 2019-11-18 22:21:37 +0000 | [diff] [blame] | 855 | if (!next_hdr_ptr) |
Fan Zhang | 18f0e31 | 2020-10-19 13:08:34 +0100 | [diff] [blame] | 856 | { |
| 857 | esp_set_next_index (is_async, from, nexts, from[b - bufs], |
| 858 | &n_async_drop, drop_next, next); |
| 859 | goto trace; |
| 860 | } |
Filip Tehlar | efcad1a | 2020-02-04 09:36:04 +0000 | [diff] [blame] | 861 | |
| 862 | b[0]->flags &= ~VLIB_BUFFER_TOTAL_LENGTH_VALID; |
Filip Tehlar | 8cdb1a0 | 2019-11-18 22:21:37 +0000 | [diff] [blame] | 863 | payload_len = b[0]->current_length; |
Filip Tehlar | efcad1a | 2020-02-04 09:36:04 +0000 | [diff] [blame] | 864 | payload_len_total = vlib_buffer_length_in_chain (vm, b[0]); |
Damjan Marion | c59b9a2 | 2019-03-19 15:38:40 +0100 | [diff] [blame] | 865 | |
| 866 | /* ESP header */ |
| 867 | hdr_len += sizeof (*esp); |
| 868 | esp = (esp_header_t *) (payload - hdr_len); |
| 869 | |
| 870 | /* optional UDP header */ |
Damjan Marion | d709cbc | 2019-03-26 13:16:42 +0100 | [diff] [blame] | 871 | if (ipsec_sa_is_set_UDP_ENCAP (sa0)) |
Damjan Marion | c98275f | 2019-03-06 14:05:01 +0100 | [diff] [blame] | 872 | { |
Damjan Marion | c59b9a2 | 2019-03-19 15:38:40 +0100 | [diff] [blame] | 873 | hdr_len += sizeof (udp_header_t); |
| 874 | udp = (udp_header_t *) (payload - hdr_len); |
Damjan Marion | c98275f | 2019-03-06 14:05:01 +0100 | [diff] [blame] | 875 | } |
| 876 | |
Damjan Marion | c59b9a2 | 2019-03-19 15:38:40 +0100 | [diff] [blame] | 877 | /* IP header */ |
| 878 | hdr_len += ip_len; |
| 879 | ip_hdr = payload - hdr_len; |
| 880 | |
| 881 | /* L2 header */ |
Neale Ranns | c87b66c | 2019-02-07 07:26:12 -0800 | [diff] [blame] | 882 | if (!is_tun) |
| 883 | { |
| 884 | l2_len = vnet_buffer (b[0])->ip.save_rewrite_length; |
| 885 | hdr_len += l2_len; |
| 886 | l2_hdr = payload - hdr_len; |
Damjan Marion | c59b9a2 | 2019-03-19 15:38:40 +0100 | [diff] [blame] | 887 | |
Neale Ranns | c87b66c | 2019-02-07 07:26:12 -0800 | [diff] [blame] | 888 | /* copy l2 and ip header */ |
| 889 | clib_memcpy_le32 (l2_hdr, old_ip_hdr - l2_len, l2_len); |
| 890 | } |
| 891 | else |
| 892 | l2_len = 0; |
| 893 | |
Neale Ranns | 4a58e49 | 2020-12-21 13:19:10 +0000 | [diff] [blame] | 894 | if (VNET_LINK_IP6 == lt) |
Damjan Marion | c59b9a2 | 2019-03-19 15:38:40 +0100 | [diff] [blame] | 895 | { |
Neale Ranns | 0295040 | 2019-12-20 00:54:57 +0000 | [diff] [blame] | 896 | ip6_header_t *ip6 = (ip6_header_t *) (old_ip_hdr); |
| 897 | if (PREDICT_TRUE (NULL == ext_hdr)) |
| 898 | { |
| 899 | *next_hdr_ptr = ip6->protocol; |
| 900 | ip6->protocol = IP_PROTOCOL_IPSEC_ESP; |
| 901 | } |
| 902 | else |
| 903 | { |
| 904 | *next_hdr_ptr = ext_hdr->next_hdr; |
| 905 | ext_hdr->next_hdr = IP_PROTOCOL_IPSEC_ESP; |
| 906 | } |
Neale Ranns | d207fd7 | 2019-04-18 17:18:12 -0700 | [diff] [blame] | 907 | ip6->payload_length = |
Filip Tehlar | efcad1a | 2020-02-04 09:36:04 +0000 | [diff] [blame] | 908 | clib_host_to_net_u16 (payload_len_total + hdr_len - l2_len - |
Neale Ranns | 0295040 | 2019-12-20 00:54:57 +0000 | [diff] [blame] | 909 | sizeof (ip6_header_t)); |
Damjan Marion | c59b9a2 | 2019-03-19 15:38:40 +0100 | [diff] [blame] | 910 | } |
Neale Ranns | 4a58e49 | 2020-12-21 13:19:10 +0000 | [diff] [blame] | 911 | else if (VNET_LINK_IP4 == lt) |
Damjan Marion | c98275f | 2019-03-06 14:05:01 +0100 | [diff] [blame] | 912 | { |
Damjan Marion | c59b9a2 | 2019-03-19 15:38:40 +0100 | [diff] [blame] | 913 | u16 len; |
Neale Ranns | 0295040 | 2019-12-20 00:54:57 +0000 | [diff] [blame] | 914 | ip4_header_t *ip4 = (ip4_header_t *) (old_ip_hdr); |
Damjan Marion | c59b9a2 | 2019-03-19 15:38:40 +0100 | [diff] [blame] | 915 | *next_hdr_ptr = ip4->protocol; |
Filip Tehlar | efcad1a | 2020-02-04 09:36:04 +0000 | [diff] [blame] | 916 | len = payload_len_total + hdr_len - l2_len; |
Damjan Marion | c59b9a2 | 2019-03-19 15:38:40 +0100 | [diff] [blame] | 917 | if (udp) |
| 918 | { |
| 919 | esp_update_ip4_hdr (ip4, len, /* is_transport */ 1, 1); |
Alexander Chernavin | b0d2eda | 2020-03-25 10:56:52 -0400 | [diff] [blame] | 920 | udp_len = len - ip_len; |
Damjan Marion | c59b9a2 | 2019-03-19 15:38:40 +0100 | [diff] [blame] | 921 | } |
| 922 | else |
| 923 | esp_update_ip4_hdr (ip4, len, /* is_transport */ 1, 0); |
Damjan Marion | c98275f | 2019-03-06 14:05:01 +0100 | [diff] [blame] | 924 | } |
Damjan Marion | c98275f | 2019-03-06 14:05:01 +0100 | [diff] [blame] | 925 | |
Neale Ranns | 0295040 | 2019-12-20 00:54:57 +0000 | [diff] [blame] | 926 | clib_memcpy_le64 (ip_hdr, old_ip_hdr, ip_len); |
| 927 | |
Alexander Chernavin | b0d2eda | 2020-03-25 10:56:52 -0400 | [diff] [blame] | 928 | if (udp) |
| 929 | { |
| 930 | esp_fill_udp_hdr (sa0, udp, udp_len); |
| 931 | } |
| 932 | |
Neale Ranns | 4ec36c5 | 2020-03-31 09:21:29 -0400 | [diff] [blame] | 933 | next[0] = ESP_ENCRYPT_NEXT_INTERFACE_OUTPUT; |
Damjan Marion | c98275f | 2019-03-06 14:05:01 +0100 | [diff] [blame] | 934 | } |
| 935 | |
PiotrX Kleski | fdca4dd | 2020-05-05 14:14:22 +0200 | [diff] [blame] | 936 | if (lb != b[0]) |
| 937 | { |
| 938 | crypto_ops = &ptd->chained_crypto_ops; |
| 939 | integ_ops = &ptd->chained_integ_ops; |
| 940 | } |
| 941 | else |
| 942 | { |
| 943 | crypto_ops = &ptd->crypto_ops; |
| 944 | integ_ops = &ptd->integ_ops; |
| 945 | } |
| 946 | |
Damjan Marion | c59b9a2 | 2019-03-19 15:38:40 +0100 | [diff] [blame] | 947 | esp->spi = spi; |
| 948 | esp->seq = clib_net_to_host_u32 (sa0->seq); |
Damjan Marion | c98275f | 2019-03-06 14:05:01 +0100 | [diff] [blame] | 949 | |
Fan Zhang | f539578 | 2020-04-29 14:00:03 +0100 | [diff] [blame] | 950 | if (is_async) |
Damjan Marion | c98275f | 2019-03-06 14:05:01 +0100 | [diff] [blame] | 951 | { |
Fan Zhang | f539578 | 2020-04-29 14:00:03 +0100 | [diff] [blame] | 952 | if (PREDICT_FALSE (sa0->crypto_async_enc_op_id == 0)) |
Fan Zhang | 18f0e31 | 2020-10-19 13:08:34 +0100 | [diff] [blame] | 953 | { |
| 954 | esp_set_next_index (is_async, from, nexts, from[b - bufs], |
| 955 | &n_async_drop, drop_next, next); |
| 956 | goto trace; |
| 957 | } |
Filip Tehlar | efcad1a | 2020-02-04 09:36:04 +0000 | [diff] [blame] | 958 | |
Fan Zhang | f539578 | 2020-04-29 14:00:03 +0100 | [diff] [blame] | 959 | if (esp_prepare_async_frame (vm, ptd, &async_frame, sa0, b[0], esp, |
| 960 | payload, payload_len, iv_sz, |
Fan Zhang | 18f0e31 | 2020-10-19 13:08:34 +0100 | [diff] [blame] | 961 | icv_sz, from[b - bufs], next[0], |
| 962 | hdr_len, async_next, lb)) |
Neale Ranns | 47feb11 | 2019-04-11 15:14:07 +0000 | [diff] [blame] | 963 | { |
Fan Zhang | 18f0e31 | 2020-10-19 13:08:34 +0100 | [diff] [blame] | 964 | /* The fail only caused by submission, free the whole frame. */ |
| 965 | if (async_frame->n_elts) |
| 966 | esp_async_recycle_failed_submit (async_frame, b, from, nexts, |
| 967 | &n_async_drop, drop_next, |
| 968 | ESP_ENCRYPT_ERROR_CRYPTO_ENGINE_ERROR); |
| 969 | b[0]->error = ESP_ENCRYPT_ERROR_CRYPTO_ENGINE_ERROR; |
| 970 | esp_set_next_index (1, from, nexts, from[b - bufs], |
| 971 | &n_async_drop, drop_next, next); |
Fan Zhang | f539578 | 2020-04-29 14:00:03 +0100 | [diff] [blame] | 972 | goto trace; |
Filip Tehlar | efcad1a | 2020-02-04 09:36:04 +0000 | [diff] [blame] | 973 | } |
Damjan Marion | c98275f | 2019-03-06 14:05:01 +0100 | [diff] [blame] | 974 | } |
Fan Zhang | f539578 | 2020-04-29 14:00:03 +0100 | [diff] [blame] | 975 | else |
Damjan Marion | c98275f | 2019-03-06 14:05:01 +0100 | [diff] [blame] | 976 | { |
Fan Zhang | f539578 | 2020-04-29 14:00:03 +0100 | [diff] [blame] | 977 | esp_prepare_sync_op (vm, ptd, crypto_ops, integ_ops, sa0, payload, |
| 978 | payload_len, iv_sz, icv_sz, bufs, b, lb, |
Benoît Ganne | 490b927 | 2021-01-22 18:03:09 +0100 | [diff] [blame] | 979 | hdr_len, esp); |
Damjan Marion | c98275f | 2019-03-06 14:05:01 +0100 | [diff] [blame] | 980 | } |
| 981 | |
Damjan Marion | c59b9a2 | 2019-03-19 15:38:40 +0100 | [diff] [blame] | 982 | vlib_buffer_advance (b[0], 0LL - hdr_len); |
Damjan Marion | c98275f | 2019-03-06 14:05:01 +0100 | [diff] [blame] | 983 | |
Damjan Marion | c59b9a2 | 2019-03-19 15:38:40 +0100 | [diff] [blame] | 984 | current_sa_packets += 1; |
Filip Tehlar | efcad1a | 2020-02-04 09:36:04 +0000 | [diff] [blame] | 985 | current_sa_bytes += payload_len_total; |
Damjan Marion | c59b9a2 | 2019-03-19 15:38:40 +0100 | [diff] [blame] | 986 | |
| 987 | trace: |
| 988 | if (PREDICT_FALSE (b[0]->flags & VLIB_BUFFER_IS_TRACED)) |
Damjan Marion | c98275f | 2019-03-06 14:05:01 +0100 | [diff] [blame] | 989 | { |
Damjan Marion | c59b9a2 | 2019-03-19 15:38:40 +0100 | [diff] [blame] | 990 | esp_encrypt_trace_t *tr = vlib_add_trace (vm, node, b[0], |
| 991 | sizeof (*tr)); |
| 992 | tr->sa_index = sa_index0; |
| 993 | tr->spi = sa0->spi; |
Neale Ranns | 6afaae1 | 2019-07-17 15:07:14 +0000 | [diff] [blame] | 994 | tr->seq = sa0->seq; |
| 995 | tr->sa_seq_hi = sa0->seq_hi; |
Damjan Marion | d709cbc | 2019-03-26 13:16:42 +0100 | [diff] [blame] | 996 | tr->udp_encap = ipsec_sa_is_set_UDP_ENCAP (sa0); |
Damjan Marion | c59b9a2 | 2019-03-19 15:38:40 +0100 | [diff] [blame] | 997 | tr->crypto_alg = sa0->crypto_alg; |
| 998 | tr->integ_alg = sa0->integ_alg; |
Damjan Marion | c98275f | 2019-03-06 14:05:01 +0100 | [diff] [blame] | 999 | } |
Damjan Marion | c98275f | 2019-03-06 14:05:01 +0100 | [diff] [blame] | 1000 | /* next */ |
Damjan Marion | c59b9a2 | 2019-03-19 15:38:40 +0100 | [diff] [blame] | 1001 | n_left -= 1; |
Damjan Marion | c98275f | 2019-03-06 14:05:01 +0100 | [diff] [blame] | 1002 | next += 1; |
Damjan Marion | c59b9a2 | 2019-03-19 15:38:40 +0100 | [diff] [blame] | 1003 | b += 1; |
Damjan Marion | c98275f | 2019-03-06 14:05:01 +0100 | [diff] [blame] | 1004 | } |
| 1005 | |
Damjan Marion | c59b9a2 | 2019-03-19 15:38:40 +0100 | [diff] [blame] | 1006 | vlib_increment_combined_counter (&ipsec_sa_counters, thread_index, |
| 1007 | current_sa_index, current_sa_packets, |
| 1008 | current_sa_bytes); |
Fan Zhang | f539578 | 2020-04-29 14:00:03 +0100 | [diff] [blame] | 1009 | if (!is_async) |
| 1010 | { |
Neale Ranns | b1fd80f | 2020-05-12 13:33:56 +0000 | [diff] [blame] | 1011 | esp_process_ops (vm, node, ptd->crypto_ops, bufs, nexts, drop_next); |
Fan Zhang | f539578 | 2020-04-29 14:00:03 +0100 | [diff] [blame] | 1012 | esp_process_chained_ops (vm, node, ptd->chained_crypto_ops, bufs, nexts, |
Neale Ranns | b1fd80f | 2020-05-12 13:33:56 +0000 | [diff] [blame] | 1013 | ptd->chunks, drop_next); |
Filip Tehlar | efcad1a | 2020-02-04 09:36:04 +0000 | [diff] [blame] | 1014 | |
Neale Ranns | b1fd80f | 2020-05-12 13:33:56 +0000 | [diff] [blame] | 1015 | esp_process_ops (vm, node, ptd->integ_ops, bufs, nexts, drop_next); |
Fan Zhang | f539578 | 2020-04-29 14:00:03 +0100 | [diff] [blame] | 1016 | esp_process_chained_ops (vm, node, ptd->chained_integ_ops, bufs, nexts, |
Neale Ranns | b1fd80f | 2020-05-12 13:33:56 +0000 | [diff] [blame] | 1017 | ptd->chunks, drop_next); |
Fan Zhang | f539578 | 2020-04-29 14:00:03 +0100 | [diff] [blame] | 1018 | } |
Fan Zhang | 18f0e31 | 2020-10-19 13:08:34 +0100 | [diff] [blame] | 1019 | else |
Fan Zhang | f539578 | 2020-04-29 14:00:03 +0100 | [diff] [blame] | 1020 | { |
Fan Zhang | 18f0e31 | 2020-10-19 13:08:34 +0100 | [diff] [blame] | 1021 | if (async_frame && async_frame->n_elts) |
| 1022 | { |
| 1023 | if (vnet_crypto_async_submit_open_frame (vm, async_frame) < 0) |
| 1024 | esp_async_recycle_failed_submit (async_frame, b, from, nexts, |
| 1025 | &n_async_drop, drop_next, |
| 1026 | ESP_ENCRYPT_ERROR_CRYPTO_ENGINE_ERROR); |
| 1027 | } |
| 1028 | vlib_node_increment_counter (vm, node->node_index, |
| 1029 | ESP_ENCRYPT_ERROR_RX_PKTS, |
| 1030 | frame->n_vectors); |
| 1031 | if (n_async_drop) |
| 1032 | vlib_buffer_enqueue_to_next (vm, node, from, nexts, n_async_drop); |
| 1033 | |
| 1034 | return frame->n_vectors; |
Fan Zhang | f539578 | 2020-04-29 14:00:03 +0100 | [diff] [blame] | 1035 | } |
Damjan Marion | c59b9a2 | 2019-03-19 15:38:40 +0100 | [diff] [blame] | 1036 | |
| 1037 | vlib_node_increment_counter (vm, node->node_index, |
| 1038 | ESP_ENCRYPT_ERROR_RX_PKTS, frame->n_vectors); |
| 1039 | |
| 1040 | vlib_buffer_enqueue_to_next (vm, node, from, nexts, frame->n_vectors); |
| 1041 | return frame->n_vectors; |
Ed Warnicke | cb9cada | 2015-12-08 15:45:58 -0700 | [diff] [blame] | 1042 | } |
| 1043 | |
Fan Zhang | f539578 | 2020-04-29 14:00:03 +0100 | [diff] [blame] | 1044 | always_inline uword |
| 1045 | esp_encrypt_post_inline (vlib_main_t * vm, vlib_node_runtime_t * node, |
| 1046 | vlib_frame_t * frame) |
| 1047 | { |
| 1048 | vlib_buffer_t *bufs[VLIB_FRAME_SIZE], **b = bufs; |
| 1049 | u16 nexts[VLIB_FRAME_SIZE], *next = nexts; |
| 1050 | u32 *from = vlib_frame_vector_args (frame); |
| 1051 | u32 n_left = frame->n_vectors; |
| 1052 | |
| 1053 | vlib_get_buffers (vm, from, b, n_left); |
| 1054 | |
| 1055 | if (n_left >= 4) |
| 1056 | { |
| 1057 | vlib_prefetch_buffer_header (b[0], LOAD); |
| 1058 | vlib_prefetch_buffer_header (b[1], LOAD); |
| 1059 | vlib_prefetch_buffer_header (b[2], LOAD); |
| 1060 | vlib_prefetch_buffer_header (b[3], LOAD); |
| 1061 | } |
| 1062 | |
| 1063 | while (n_left > 8) |
| 1064 | { |
| 1065 | vlib_prefetch_buffer_header (b[4], LOAD); |
| 1066 | vlib_prefetch_buffer_header (b[5], LOAD); |
| 1067 | vlib_prefetch_buffer_header (b[6], LOAD); |
| 1068 | vlib_prefetch_buffer_header (b[7], LOAD); |
| 1069 | |
| 1070 | next[0] = (esp_post_data (b[0]))->next_index; |
| 1071 | next[1] = (esp_post_data (b[1]))->next_index; |
| 1072 | next[2] = (esp_post_data (b[2]))->next_index; |
| 1073 | next[3] = (esp_post_data (b[3]))->next_index; |
| 1074 | |
| 1075 | if (PREDICT_FALSE (node->flags & VLIB_NODE_FLAG_TRACE)) |
| 1076 | { |
| 1077 | if (b[0]->flags & VLIB_BUFFER_IS_TRACED) |
| 1078 | { |
| 1079 | esp_encrypt_post_trace_t *tr = vlib_add_trace (vm, node, b[0], |
| 1080 | sizeof (*tr)); |
| 1081 | tr->next_index = next[0]; |
| 1082 | } |
| 1083 | if (b[1]->flags & VLIB_BUFFER_IS_TRACED) |
| 1084 | { |
| 1085 | esp_encrypt_post_trace_t *tr = vlib_add_trace (vm, node, b[1], |
| 1086 | sizeof (*tr)); |
| 1087 | tr->next_index = next[1]; |
| 1088 | } |
| 1089 | if (b[2]->flags & VLIB_BUFFER_IS_TRACED) |
| 1090 | { |
| 1091 | esp_encrypt_post_trace_t *tr = vlib_add_trace (vm, node, b[2], |
| 1092 | sizeof (*tr)); |
| 1093 | tr->next_index = next[2]; |
| 1094 | } |
| 1095 | if (b[3]->flags & VLIB_BUFFER_IS_TRACED) |
| 1096 | { |
| 1097 | esp_encrypt_post_trace_t *tr = vlib_add_trace (vm, node, b[3], |
| 1098 | sizeof (*tr)); |
| 1099 | tr->next_index = next[3]; |
| 1100 | } |
| 1101 | } |
| 1102 | |
| 1103 | b += 4; |
| 1104 | next += 4; |
| 1105 | n_left -= 4; |
| 1106 | } |
| 1107 | |
| 1108 | while (n_left > 0) |
| 1109 | { |
| 1110 | next[0] = (esp_post_data (b[0]))->next_index; |
| 1111 | if (PREDICT_FALSE (b[0]->flags & VLIB_BUFFER_IS_TRACED)) |
| 1112 | { |
| 1113 | esp_encrypt_post_trace_t *tr = vlib_add_trace (vm, node, b[0], |
| 1114 | sizeof (*tr)); |
| 1115 | tr->next_index = next[0]; |
| 1116 | } |
| 1117 | |
| 1118 | b += 1; |
| 1119 | next += 1; |
| 1120 | n_left -= 1; |
| 1121 | } |
| 1122 | |
| 1123 | vlib_node_increment_counter (vm, node->node_index, |
| 1124 | ESP_ENCRYPT_ERROR_POST_RX_PKTS, |
| 1125 | frame->n_vectors); |
| 1126 | vlib_buffer_enqueue_to_next (vm, node, from, nexts, frame->n_vectors); |
| 1127 | return frame->n_vectors; |
| 1128 | } |
| 1129 | |
Klement Sekera | b8f3544 | 2018-10-29 13:38:19 +0100 | [diff] [blame] | 1130 | VLIB_NODE_FN (esp4_encrypt_node) (vlib_main_t * vm, |
| 1131 | vlib_node_runtime_t * node, |
| 1132 | vlib_frame_t * from_frame) |
Klement Sekera | be5a5dd | 2018-10-09 16:05:48 +0200 | [diff] [blame] | 1133 | { |
Neale Ranns | 4a58e49 | 2020-12-21 13:19:10 +0000 | [diff] [blame] | 1134 | return esp_encrypt_inline (vm, node, from_frame, VNET_LINK_IP4, 0, |
Fan Zhang | f539578 | 2020-04-29 14:00:03 +0100 | [diff] [blame] | 1135 | esp_encrypt_async_next.esp4_post_next); |
Klement Sekera | be5a5dd | 2018-10-09 16:05:48 +0200 | [diff] [blame] | 1136 | } |
Ed Warnicke | cb9cada | 2015-12-08 15:45:58 -0700 | [diff] [blame] | 1137 | |
Keith Burns (alagalah) | 166a9d4 | 2016-08-06 11:00:56 -0700 | [diff] [blame] | 1138 | /* *INDENT-OFF* */ |
Klement Sekera | be5a5dd | 2018-10-09 16:05:48 +0200 | [diff] [blame] | 1139 | VLIB_REGISTER_NODE (esp4_encrypt_node) = { |
Klement Sekera | be5a5dd | 2018-10-09 16:05:48 +0200 | [diff] [blame] | 1140 | .name = "esp4-encrypt", |
Ed Warnicke | cb9cada | 2015-12-08 15:45:58 -0700 | [diff] [blame] | 1141 | .vector_size = sizeof (u32), |
| 1142 | .format_trace = format_esp_encrypt_trace, |
| 1143 | .type = VLIB_NODE_TYPE_INTERNAL, |
| 1144 | |
Neale Ranns | 4a58e49 | 2020-12-21 13:19:10 +0000 | [diff] [blame] | 1145 | .n_errors = ARRAY_LEN (esp_encrypt_error_strings), |
Ed Warnicke | cb9cada | 2015-12-08 15:45:58 -0700 | [diff] [blame] | 1146 | .error_strings = esp_encrypt_error_strings, |
| 1147 | |
| 1148 | .n_next_nodes = ESP_ENCRYPT_N_NEXT, |
Neale Ranns | 4a58e49 | 2020-12-21 13:19:10 +0000 | [diff] [blame] | 1149 | .next_nodes = { [ESP_ENCRYPT_NEXT_DROP4] = "ip4-drop", |
| 1150 | [ESP_ENCRYPT_NEXT_DROP6] = "ip6-drop", |
| 1151 | [ESP_ENCRYPT_NEXT_DROP_MPLS] = "mpls-drop", |
| 1152 | [ESP_ENCRYPT_NEXT_HANDOFF4] = "esp4-encrypt-handoff", |
| 1153 | [ESP_ENCRYPT_NEXT_HANDOFF6] = "esp6-encrypt-handoff", |
| 1154 | [ESP_ENCRYPT_NEXT_HANDOFF_MPLS] = "error-drop", |
| 1155 | [ESP_ENCRYPT_NEXT_INTERFACE_OUTPUT] = "interface-output" }, |
Ed Warnicke | cb9cada | 2015-12-08 15:45:58 -0700 | [diff] [blame] | 1156 | }; |
Keith Burns (alagalah) | 166a9d4 | 2016-08-06 11:00:56 -0700 | [diff] [blame] | 1157 | /* *INDENT-ON* */ |
Ed Warnicke | cb9cada | 2015-12-08 15:45:58 -0700 | [diff] [blame] | 1158 | |
Fan Zhang | f539578 | 2020-04-29 14:00:03 +0100 | [diff] [blame] | 1159 | VLIB_NODE_FN (esp4_encrypt_post_node) (vlib_main_t * vm, |
| 1160 | vlib_node_runtime_t * node, |
| 1161 | vlib_frame_t * from_frame) |
| 1162 | { |
| 1163 | return esp_encrypt_post_inline (vm, node, from_frame); |
| 1164 | } |
| 1165 | |
| 1166 | /* *INDENT-OFF* */ |
| 1167 | VLIB_REGISTER_NODE (esp4_encrypt_post_node) = { |
| 1168 | .name = "esp4-encrypt-post", |
| 1169 | .vector_size = sizeof (u32), |
| 1170 | .format_trace = format_esp_post_encrypt_trace, |
| 1171 | .type = VLIB_NODE_TYPE_INTERNAL, |
| 1172 | .sibling_of = "esp4-encrypt", |
| 1173 | |
| 1174 | .n_errors = ARRAY_LEN(esp_encrypt_error_strings), |
| 1175 | .error_strings = esp_encrypt_error_strings, |
| 1176 | }; |
| 1177 | /* *INDENT-ON* */ |
| 1178 | |
Klement Sekera | b8f3544 | 2018-10-29 13:38:19 +0100 | [diff] [blame] | 1179 | VLIB_NODE_FN (esp6_encrypt_node) (vlib_main_t * vm, |
| 1180 | vlib_node_runtime_t * node, |
| 1181 | vlib_frame_t * from_frame) |
Klement Sekera | be5a5dd | 2018-10-09 16:05:48 +0200 | [diff] [blame] | 1182 | { |
Neale Ranns | 4a58e49 | 2020-12-21 13:19:10 +0000 | [diff] [blame] | 1183 | return esp_encrypt_inline (vm, node, from_frame, VNET_LINK_IP6, 0, |
Fan Zhang | f539578 | 2020-04-29 14:00:03 +0100 | [diff] [blame] | 1184 | esp_encrypt_async_next.esp6_post_next); |
Klement Sekera | be5a5dd | 2018-10-09 16:05:48 +0200 | [diff] [blame] | 1185 | } |
| 1186 | |
| 1187 | /* *INDENT-OFF* */ |
| 1188 | VLIB_REGISTER_NODE (esp6_encrypt_node) = { |
Klement Sekera | be5a5dd | 2018-10-09 16:05:48 +0200 | [diff] [blame] | 1189 | .name = "esp6-encrypt", |
| 1190 | .vector_size = sizeof (u32), |
| 1191 | .format_trace = format_esp_encrypt_trace, |
| 1192 | .type = VLIB_NODE_TYPE_INTERNAL, |
Neale Ranns | b1fd80f | 2020-05-12 13:33:56 +0000 | [diff] [blame] | 1193 | .sibling_of = "esp4-encrypt", |
Klement Sekera | be5a5dd | 2018-10-09 16:05:48 +0200 | [diff] [blame] | 1194 | |
| 1195 | .n_errors = ARRAY_LEN(esp_encrypt_error_strings), |
| 1196 | .error_strings = esp_encrypt_error_strings, |
Klement Sekera | be5a5dd | 2018-10-09 16:05:48 +0200 | [diff] [blame] | 1197 | }; |
| 1198 | /* *INDENT-ON* */ |
| 1199 | |
Fan Zhang | f539578 | 2020-04-29 14:00:03 +0100 | [diff] [blame] | 1200 | VLIB_NODE_FN (esp6_encrypt_post_node) (vlib_main_t * vm, |
| 1201 | vlib_node_runtime_t * node, |
| 1202 | vlib_frame_t * from_frame) |
| 1203 | { |
| 1204 | return esp_encrypt_post_inline (vm, node, from_frame); |
| 1205 | } |
| 1206 | |
| 1207 | /* *INDENT-OFF* */ |
| 1208 | VLIB_REGISTER_NODE (esp6_encrypt_post_node) = { |
| 1209 | .name = "esp6-encrypt-post", |
| 1210 | .vector_size = sizeof (u32), |
| 1211 | .format_trace = format_esp_post_encrypt_trace, |
| 1212 | .type = VLIB_NODE_TYPE_INTERNAL, |
Neale Ranns | b1fd80f | 2020-05-12 13:33:56 +0000 | [diff] [blame] | 1213 | .sibling_of = "esp4-encrypt", |
Fan Zhang | f539578 | 2020-04-29 14:00:03 +0100 | [diff] [blame] | 1214 | |
| 1215 | .n_errors = ARRAY_LEN(esp_encrypt_error_strings), |
| 1216 | .error_strings = esp_encrypt_error_strings, |
| 1217 | }; |
| 1218 | /* *INDENT-ON* */ |
| 1219 | |
Neale Ranns | 25edf14 | 2019-03-22 08:12:48 +0000 | [diff] [blame] | 1220 | VLIB_NODE_FN (esp4_encrypt_tun_node) (vlib_main_t * vm, |
| 1221 | vlib_node_runtime_t * node, |
| 1222 | vlib_frame_t * from_frame) |
| 1223 | { |
Neale Ranns | 4a58e49 | 2020-12-21 13:19:10 +0000 | [diff] [blame] | 1224 | return esp_encrypt_inline (vm, node, from_frame, VNET_LINK_IP4, 1, |
Fan Zhang | f539578 | 2020-04-29 14:00:03 +0100 | [diff] [blame] | 1225 | esp_encrypt_async_next.esp4_tun_post_next); |
Neale Ranns | 25edf14 | 2019-03-22 08:12:48 +0000 | [diff] [blame] | 1226 | } |
| 1227 | |
| 1228 | /* *INDENT-OFF* */ |
| 1229 | VLIB_REGISTER_NODE (esp4_encrypt_tun_node) = { |
| 1230 | .name = "esp4-encrypt-tun", |
| 1231 | .vector_size = sizeof (u32), |
| 1232 | .format_trace = format_esp_encrypt_trace, |
| 1233 | .type = VLIB_NODE_TYPE_INTERNAL, |
| 1234 | |
| 1235 | .n_errors = ARRAY_LEN(esp_encrypt_error_strings), |
| 1236 | .error_strings = esp_encrypt_error_strings, |
Neale Ranns | d7603d9 | 2019-03-28 08:56:10 +0000 | [diff] [blame] | 1237 | |
Neale Ranns | f62a8c0 | 2019-04-02 08:13:33 +0000 | [diff] [blame] | 1238 | .n_next_nodes = ESP_ENCRYPT_N_NEXT, |
Neale Ranns | d7603d9 | 2019-03-28 08:56:10 +0000 | [diff] [blame] | 1239 | .next_nodes = { |
Neale Ranns | b1fd80f | 2020-05-12 13:33:56 +0000 | [diff] [blame] | 1240 | [ESP_ENCRYPT_NEXT_DROP4] = "ip4-drop", |
| 1241 | [ESP_ENCRYPT_NEXT_DROP6] = "ip6-drop", |
Neale Ranns | 4a58e49 | 2020-12-21 13:19:10 +0000 | [diff] [blame] | 1242 | [ESP_ENCRYPT_NEXT_DROP_MPLS] = "mpls-drop", |
Neale Ranns | b1fd80f | 2020-05-12 13:33:56 +0000 | [diff] [blame] | 1243 | [ESP_ENCRYPT_NEXT_HANDOFF4] = "esp4-encrypt-tun-handoff", |
Neale Ranns | 4a58e49 | 2020-12-21 13:19:10 +0000 | [diff] [blame] | 1244 | [ESP_ENCRYPT_NEXT_HANDOFF6] = "esp6-encrypt-tun-handoff", |
| 1245 | [ESP_ENCRYPT_NEXT_HANDOFF_MPLS] = "esp-mpls-encrypt-tun-handoff", |
Neale Ranns | 4ec36c5 | 2020-03-31 09:21:29 -0400 | [diff] [blame] | 1246 | [ESP_ENCRYPT_NEXT_INTERFACE_OUTPUT] = "adj-midchain-tx", |
Neale Ranns | d7603d9 | 2019-03-28 08:56:10 +0000 | [diff] [blame] | 1247 | }, |
Neale Ranns | 25edf14 | 2019-03-22 08:12:48 +0000 | [diff] [blame] | 1248 | }; |
| 1249 | |
Fan Zhang | f539578 | 2020-04-29 14:00:03 +0100 | [diff] [blame] | 1250 | VLIB_NODE_FN (esp4_encrypt_tun_post_node) (vlib_main_t * vm, |
| 1251 | vlib_node_runtime_t * node, |
| 1252 | vlib_frame_t * from_frame) |
| 1253 | { |
| 1254 | return esp_encrypt_post_inline (vm, node, from_frame); |
| 1255 | } |
| 1256 | |
| 1257 | /* *INDENT-OFF* */ |
| 1258 | VLIB_REGISTER_NODE (esp4_encrypt_tun_post_node) = { |
| 1259 | .name = "esp4-encrypt-tun-post", |
| 1260 | .vector_size = sizeof (u32), |
| 1261 | .format_trace = format_esp_post_encrypt_trace, |
| 1262 | .type = VLIB_NODE_TYPE_INTERNAL, |
| 1263 | .sibling_of = "esp4-encrypt-tun", |
| 1264 | |
| 1265 | .n_errors = ARRAY_LEN(esp_encrypt_error_strings), |
| 1266 | .error_strings = esp_encrypt_error_strings, |
| 1267 | }; |
Neale Ranns | 25edf14 | 2019-03-22 08:12:48 +0000 | [diff] [blame] | 1268 | /* *INDENT-ON* */ |
| 1269 | |
| 1270 | VLIB_NODE_FN (esp6_encrypt_tun_node) (vlib_main_t * vm, |
| 1271 | vlib_node_runtime_t * node, |
| 1272 | vlib_frame_t * from_frame) |
| 1273 | { |
Neale Ranns | 4a58e49 | 2020-12-21 13:19:10 +0000 | [diff] [blame] | 1274 | return esp_encrypt_inline (vm, node, from_frame, VNET_LINK_IP6, 1, |
Fan Zhang | f539578 | 2020-04-29 14:00:03 +0100 | [diff] [blame] | 1275 | esp_encrypt_async_next.esp6_tun_post_next); |
Neale Ranns | 25edf14 | 2019-03-22 08:12:48 +0000 | [diff] [blame] | 1276 | } |
| 1277 | |
| 1278 | /* *INDENT-OFF* */ |
| 1279 | VLIB_REGISTER_NODE (esp6_encrypt_tun_node) = { |
| 1280 | .name = "esp6-encrypt-tun", |
| 1281 | .vector_size = sizeof (u32), |
| 1282 | .format_trace = format_esp_encrypt_trace, |
| 1283 | .type = VLIB_NODE_TYPE_INTERNAL, |
| 1284 | |
| 1285 | .n_errors = ARRAY_LEN(esp_encrypt_error_strings), |
| 1286 | .error_strings = esp_encrypt_error_strings, |
Neale Ranns | d7603d9 | 2019-03-28 08:56:10 +0000 | [diff] [blame] | 1287 | |
Neale Ranns | f62a8c0 | 2019-04-02 08:13:33 +0000 | [diff] [blame] | 1288 | .n_next_nodes = ESP_ENCRYPT_N_NEXT, |
Neale Ranns | d7603d9 | 2019-03-28 08:56:10 +0000 | [diff] [blame] | 1289 | .next_nodes = { |
Neale Ranns | b1fd80f | 2020-05-12 13:33:56 +0000 | [diff] [blame] | 1290 | [ESP_ENCRYPT_NEXT_DROP4] = "ip4-drop", |
| 1291 | [ESP_ENCRYPT_NEXT_DROP6] = "ip6-drop", |
Neale Ranns | 4a58e49 | 2020-12-21 13:19:10 +0000 | [diff] [blame] | 1292 | [ESP_ENCRYPT_NEXT_DROP_MPLS] = "mpls-drop", |
| 1293 | [ESP_ENCRYPT_NEXT_HANDOFF4] = "esp4-encrypt-tun-handoff", |
Neale Ranns | b1fd80f | 2020-05-12 13:33:56 +0000 | [diff] [blame] | 1294 | [ESP_ENCRYPT_NEXT_HANDOFF6] = "esp6-encrypt-tun-handoff", |
Neale Ranns | 4a58e49 | 2020-12-21 13:19:10 +0000 | [diff] [blame] | 1295 | [ESP_ENCRYPT_NEXT_HANDOFF_MPLS] = "esp-mpls-encrypt-tun-handoff", |
Neale Ranns | 4ec36c5 | 2020-03-31 09:21:29 -0400 | [diff] [blame] | 1296 | [ESP_ENCRYPT_NEXT_INTERFACE_OUTPUT] = "adj-midchain-tx", |
Neale Ranns | d7603d9 | 2019-03-28 08:56:10 +0000 | [diff] [blame] | 1297 | }, |
Neale Ranns | 25edf14 | 2019-03-22 08:12:48 +0000 | [diff] [blame] | 1298 | }; |
| 1299 | |
Neale Ranns | 25edf14 | 2019-03-22 08:12:48 +0000 | [diff] [blame] | 1300 | /* *INDENT-ON* */ |
| 1301 | |
Fan Zhang | f539578 | 2020-04-29 14:00:03 +0100 | [diff] [blame] | 1302 | VLIB_NODE_FN (esp6_encrypt_tun_post_node) (vlib_main_t * vm, |
| 1303 | vlib_node_runtime_t * node, |
| 1304 | vlib_frame_t * from_frame) |
| 1305 | { |
| 1306 | return esp_encrypt_post_inline (vm, node, from_frame); |
| 1307 | } |
| 1308 | |
| 1309 | /* *INDENT-OFF* */ |
| 1310 | VLIB_REGISTER_NODE (esp6_encrypt_tun_post_node) = { |
| 1311 | .name = "esp6-encrypt-tun-post", |
| 1312 | .vector_size = sizeof (u32), |
| 1313 | .format_trace = format_esp_post_encrypt_trace, |
| 1314 | .type = VLIB_NODE_TYPE_INTERNAL, |
Neale Ranns | 4a58e49 | 2020-12-21 13:19:10 +0000 | [diff] [blame] | 1315 | .sibling_of = "esp-mpls-encrypt-tun", |
Fan Zhang | f539578 | 2020-04-29 14:00:03 +0100 | [diff] [blame] | 1316 | |
Neale Ranns | 4a58e49 | 2020-12-21 13:19:10 +0000 | [diff] [blame] | 1317 | .n_errors = ARRAY_LEN (esp_encrypt_error_strings), |
Fan Zhang | f539578 | 2020-04-29 14:00:03 +0100 | [diff] [blame] | 1318 | .error_strings = esp_encrypt_error_strings, |
| 1319 | }; |
| 1320 | /* *INDENT-ON* */ |
| 1321 | |
Neale Ranns | 4a58e49 | 2020-12-21 13:19:10 +0000 | [diff] [blame] | 1322 | VLIB_NODE_FN (esp_mpls_encrypt_tun_node) |
| 1323 | (vlib_main_t *vm, vlib_node_runtime_t *node, vlib_frame_t *from_frame) |
| 1324 | { |
| 1325 | return esp_encrypt_inline (vm, node, from_frame, VNET_LINK_MPLS, 1, |
| 1326 | esp_encrypt_async_next.esp_mpls_tun_post_next); |
| 1327 | } |
| 1328 | |
| 1329 | VLIB_REGISTER_NODE (esp_mpls_encrypt_tun_node) = { |
| 1330 | .name = "esp-mpls-encrypt-tun", |
| 1331 | .vector_size = sizeof (u32), |
| 1332 | .format_trace = format_esp_encrypt_trace, |
| 1333 | .type = VLIB_NODE_TYPE_INTERNAL, |
| 1334 | |
| 1335 | .n_errors = ARRAY_LEN(esp_encrypt_error_strings), |
| 1336 | .error_strings = esp_encrypt_error_strings, |
| 1337 | |
| 1338 | .n_next_nodes = ESP_ENCRYPT_N_NEXT, |
| 1339 | .next_nodes = { |
| 1340 | [ESP_ENCRYPT_NEXT_DROP4] = "ip4-drop", |
| 1341 | [ESP_ENCRYPT_NEXT_DROP6] = "ip6-drop", |
| 1342 | [ESP_ENCRYPT_NEXT_DROP_MPLS] = "mpls-drop", |
| 1343 | [ESP_ENCRYPT_NEXT_HANDOFF4] = "esp4-encrypt-tun-handoff", |
| 1344 | [ESP_ENCRYPT_NEXT_HANDOFF6] = "esp6-encrypt-tun-handoff", |
| 1345 | [ESP_ENCRYPT_NEXT_HANDOFF_MPLS] = "esp-mpls-encrypt-tun-handoff", |
| 1346 | [ESP_ENCRYPT_NEXT_INTERFACE_OUTPUT] = "adj-midchain-tx", |
| 1347 | }, |
| 1348 | }; |
| 1349 | |
| 1350 | VLIB_NODE_FN (esp_mpls_encrypt_tun_post_node) |
| 1351 | (vlib_main_t *vm, vlib_node_runtime_t *node, vlib_frame_t *from_frame) |
| 1352 | { |
| 1353 | return esp_encrypt_post_inline (vm, node, from_frame); |
| 1354 | } |
| 1355 | |
| 1356 | VLIB_REGISTER_NODE (esp_mpls_encrypt_tun_post_node) = { |
| 1357 | .name = "esp-mpls-encrypt-tun-post", |
| 1358 | .vector_size = sizeof (u32), |
| 1359 | .format_trace = format_esp_post_encrypt_trace, |
| 1360 | .type = VLIB_NODE_TYPE_INTERNAL, |
| 1361 | .sibling_of = "esp-mpls-encrypt-tun", |
| 1362 | |
| 1363 | .n_errors = ARRAY_LEN (esp_encrypt_error_strings), |
| 1364 | .error_strings = esp_encrypt_error_strings, |
| 1365 | }; |
| 1366 | |
Matthew Smith | 401aedf | 2019-07-08 14:45:04 -0500 | [diff] [blame] | 1367 | typedef struct |
| 1368 | { |
| 1369 | u32 sa_index; |
| 1370 | } esp_no_crypto_trace_t; |
| 1371 | |
| 1372 | static u8 * |
| 1373 | format_esp_no_crypto_trace (u8 * s, va_list * args) |
| 1374 | { |
| 1375 | CLIB_UNUSED (vlib_main_t * vm) = va_arg (*args, vlib_main_t *); |
| 1376 | CLIB_UNUSED (vlib_node_t * node) = va_arg (*args, vlib_node_t *); |
| 1377 | esp_no_crypto_trace_t *t = va_arg (*args, esp_no_crypto_trace_t *); |
| 1378 | |
| 1379 | s = format (s, "esp-no-crypto: sa-index %u", t->sa_index); |
| 1380 | |
| 1381 | return s; |
| 1382 | } |
| 1383 | |
| 1384 | enum |
| 1385 | { |
| 1386 | ESP_NO_CRYPTO_NEXT_DROP, |
| 1387 | ESP_NO_CRYPTO_N_NEXT, |
| 1388 | }; |
| 1389 | |
| 1390 | enum |
| 1391 | { |
| 1392 | ESP_NO_CRYPTO_ERROR_RX_PKTS, |
| 1393 | }; |
| 1394 | |
| 1395 | static char *esp_no_crypto_error_strings[] = { |
| 1396 | "Outbound ESP packets received", |
| 1397 | }; |
| 1398 | |
| 1399 | always_inline uword |
| 1400 | esp_no_crypto_inline (vlib_main_t * vm, vlib_node_runtime_t * node, |
| 1401 | vlib_frame_t * frame) |
| 1402 | { |
| 1403 | vlib_buffer_t *bufs[VLIB_FRAME_SIZE], **b = bufs; |
Matthew Smith | 401aedf | 2019-07-08 14:45:04 -0500 | [diff] [blame] | 1404 | u32 *from = vlib_frame_vector_args (frame); |
| 1405 | u32 n_left = frame->n_vectors; |
| 1406 | |
| 1407 | vlib_get_buffers (vm, from, b, n_left); |
| 1408 | |
| 1409 | while (n_left > 0) |
| 1410 | { |
Matthew Smith | 401aedf | 2019-07-08 14:45:04 -0500 | [diff] [blame] | 1411 | u32 sa_index0; |
| 1412 | |
| 1413 | /* packets are always going to be dropped, but get the sa_index */ |
Neale Ranns | 4ec36c5 | 2020-03-31 09:21:29 -0400 | [diff] [blame] | 1414 | sa_index0 = ipsec_tun_protect_get_sa_out |
| 1415 | (vnet_buffer (b[0])->ip.adj_index[VLIB_TX]); |
Matthew Smith | 401aedf | 2019-07-08 14:45:04 -0500 | [diff] [blame] | 1416 | |
| 1417 | if (PREDICT_FALSE (b[0]->flags & VLIB_BUFFER_IS_TRACED)) |
| 1418 | { |
| 1419 | esp_no_crypto_trace_t *tr = vlib_add_trace (vm, node, b[0], |
| 1420 | sizeof (*tr)); |
| 1421 | tr->sa_index = sa_index0; |
| 1422 | } |
| 1423 | |
| 1424 | n_left -= 1; |
Matthew Smith | 401aedf | 2019-07-08 14:45:04 -0500 | [diff] [blame] | 1425 | b += 1; |
| 1426 | } |
| 1427 | |
| 1428 | vlib_node_increment_counter (vm, node->node_index, |
| 1429 | ESP_NO_CRYPTO_ERROR_RX_PKTS, frame->n_vectors); |
| 1430 | |
Neale Ranns | 4ec36c5 | 2020-03-31 09:21:29 -0400 | [diff] [blame] | 1431 | vlib_buffer_enqueue_to_single_next (vm, node, from, |
| 1432 | ESP_NO_CRYPTO_NEXT_DROP, |
| 1433 | frame->n_vectors); |
Matthew Smith | 401aedf | 2019-07-08 14:45:04 -0500 | [diff] [blame] | 1434 | |
| 1435 | return frame->n_vectors; |
| 1436 | } |
| 1437 | |
| 1438 | VLIB_NODE_FN (esp4_no_crypto_tun_node) (vlib_main_t * vm, |
| 1439 | vlib_node_runtime_t * node, |
| 1440 | vlib_frame_t * from_frame) |
| 1441 | { |
| 1442 | return esp_no_crypto_inline (vm, node, from_frame); |
| 1443 | } |
| 1444 | |
| 1445 | /* *INDENT-OFF* */ |
| 1446 | VLIB_REGISTER_NODE (esp4_no_crypto_tun_node) = |
| 1447 | { |
| 1448 | .name = "esp4-no-crypto", |
| 1449 | .vector_size = sizeof (u32), |
| 1450 | .format_trace = format_esp_no_crypto_trace, |
| 1451 | .n_errors = ARRAY_LEN(esp_no_crypto_error_strings), |
| 1452 | .error_strings = esp_no_crypto_error_strings, |
| 1453 | .n_next_nodes = ESP_NO_CRYPTO_N_NEXT, |
| 1454 | .next_nodes = { |
| 1455 | [ESP_NO_CRYPTO_NEXT_DROP] = "ip4-drop", |
| 1456 | }, |
| 1457 | }; |
| 1458 | |
Matthew Smith | 401aedf | 2019-07-08 14:45:04 -0500 | [diff] [blame] | 1459 | VLIB_NODE_FN (esp6_no_crypto_tun_node) (vlib_main_t * vm, |
| 1460 | vlib_node_runtime_t * node, |
| 1461 | vlib_frame_t * from_frame) |
| 1462 | { |
| 1463 | return esp_no_crypto_inline (vm, node, from_frame); |
| 1464 | } |
| 1465 | |
| 1466 | /* *INDENT-OFF* */ |
| 1467 | VLIB_REGISTER_NODE (esp6_no_crypto_tun_node) = |
| 1468 | { |
| 1469 | .name = "esp6-no-crypto", |
| 1470 | .vector_size = sizeof (u32), |
| 1471 | .format_trace = format_esp_no_crypto_trace, |
| 1472 | .n_errors = ARRAY_LEN(esp_no_crypto_error_strings), |
| 1473 | .error_strings = esp_no_crypto_error_strings, |
| 1474 | .n_next_nodes = ESP_NO_CRYPTO_N_NEXT, |
| 1475 | .next_nodes = { |
| 1476 | [ESP_NO_CRYPTO_NEXT_DROP] = "ip6-drop", |
| 1477 | }, |
| 1478 | }; |
Matthew Smith | 401aedf | 2019-07-08 14:45:04 -0500 | [diff] [blame] | 1479 | /* *INDENT-ON* */ |
| 1480 | |
Neale Ranns | 2d49830 | 2021-02-25 08:38:58 +0000 | [diff] [blame] | 1481 | #ifndef CLIB_MARCH_VARIANT |
| 1482 | |
| 1483 | static clib_error_t * |
| 1484 | esp_encrypt_init (vlib_main_t *vm) |
| 1485 | { |
| 1486 | ipsec_main_t *im = &ipsec_main; |
| 1487 | |
| 1488 | im->esp4_enc_fq_index = |
| 1489 | vlib_frame_queue_main_init (esp4_encrypt_node.index, 0); |
| 1490 | im->esp6_enc_fq_index = |
| 1491 | vlib_frame_queue_main_init (esp6_encrypt_node.index, 0); |
| 1492 | im->esp4_enc_tun_fq_index = |
| 1493 | vlib_frame_queue_main_init (esp4_encrypt_tun_node.index, 0); |
| 1494 | im->esp6_enc_tun_fq_index = |
| 1495 | vlib_frame_queue_main_init (esp6_encrypt_tun_node.index, 0); |
| 1496 | im->esp_mpls_enc_tun_fq_index = |
| 1497 | vlib_frame_queue_main_init (esp_mpls_encrypt_tun_node.index, 0); |
| 1498 | |
| 1499 | return 0; |
| 1500 | } |
| 1501 | |
| 1502 | VLIB_INIT_FUNCTION (esp_encrypt_init); |
| 1503 | |
| 1504 | #endif |
| 1505 | |
Keith Burns (alagalah) | 166a9d4 | 2016-08-06 11:00:56 -0700 | [diff] [blame] | 1506 | /* |
| 1507 | * fd.io coding-style-patch-verification: ON |
| 1508 | * |
| 1509 | * Local Variables: |
| 1510 | * eval: (c-set-style "gnu") |
| 1511 | * End: |
| 1512 | */ |