blob: 94f3204b51ffe5d839b6869b4d33ee3015f227da [file] [log] [blame]
Ed Warnickecb9cada2015-12-08 15:45:58 -07001/*
2 * esp_decrypt.c : IPSec ESP decrypt node
3 *
4 * Copyright (c) 2015 Cisco and/or its affiliates.
5 * Licensed under the Apache License, Version 2.0 (the "License");
6 * you may not use this file except in compliance with the License.
7 * You may obtain a copy of the License at:
8 *
9 * http://www.apache.org/licenses/LICENSE-2.0
10 *
11 * Unless required by applicable law or agreed to in writing, software
12 * distributed under the License is distributed on an "AS IS" BASIS,
13 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
14 * See the License for the specific language governing permissions and
15 * limitations under the License.
16 */
Ed Warnickecb9cada2015-12-08 15:45:58 -070017#include <vnet/vnet.h>
18#include <vnet/api_errno.h>
19#include <vnet/ip/ip.h>
John Lo90430b62020-01-31 23:48:30 -050020#include <vnet/l2/l2_input.h>
Ed Warnickecb9cada2015-12-08 15:45:58 -070021
22#include <vnet/ipsec/ipsec.h>
23#include <vnet/ipsec/esp.h>
Neale Ranns918c1612019-02-21 23:34:59 -080024#include <vnet/ipsec/ipsec_io.h>
Neale Rannsc87b66c2019-02-07 07:26:12 -080025#include <vnet/ipsec/ipsec_tun.h>
Ed Warnickecb9cada2015-12-08 15:45:58 -070026
Neale Ranns119c0d72020-11-26 13:12:37 +000027#include <vnet/gre/packet.h>
Neale Ranns568acbb2019-12-18 05:54:40 +000028
Neale Ranns4a58e492020-12-21 13:19:10 +000029#define foreach_esp_decrypt_next \
30 _ (DROP, "error-drop") \
31 _ (IP4_INPUT, "ip4-input-no-checksum") \
32 _ (IP6_INPUT, "ip6-input") \
33 _ (L2_INPUT, "l2-input") \
34 _ (MPLS_INPUT, "mpls-input") \
35 _ (HANDOFF, "handoff")
Ed Warnickecb9cada2015-12-08 15:45:58 -070036
37#define _(v, s) ESP_DECRYPT_NEXT_##v,
Keith Burns (alagalah)166a9d42016-08-06 11:00:56 -070038typedef enum
39{
Ed Warnickecb9cada2015-12-08 15:45:58 -070040 foreach_esp_decrypt_next
41#undef _
Keith Burns (alagalah)166a9d42016-08-06 11:00:56 -070042 ESP_DECRYPT_N_NEXT,
Ed Warnickecb9cada2015-12-08 15:45:58 -070043} esp_decrypt_next_t;
44
Neale Ranns4a58e492020-12-21 13:19:10 +000045#define foreach_esp_decrypt_post_next \
46 _ (DROP, "error-drop") \
47 _ (IP4_INPUT, "ip4-input-no-checksum") \
48 _ (IP6_INPUT, "ip6-input") \
49 _ (MPLS_INPUT, "mpls-input") \
50 _ (L2_INPUT, "l2-input")
Fan Zhangf5395782020-04-29 14:00:03 +010051
52#define _(v, s) ESP_DECRYPT_POST_NEXT_##v,
53typedef enum
54{
55 foreach_esp_decrypt_post_next
56#undef _
57 ESP_DECRYPT_POST_N_NEXT,
58} esp_decrypt_post_next_t;
Ed Warnickecb9cada2015-12-08 15:45:58 -070059
Keith Burns (alagalah)166a9d42016-08-06 11:00:56 -070060typedef struct
61{
Damjan Marionb4fff3a2019-03-25 15:54:40 +010062 u32 seq;
Neale Ranns6afaae12019-07-17 15:07:14 +000063 u32 sa_seq;
64 u32 sa_seq_hi;
Neale Ranns5b891102021-06-28 13:31:28 +000065 u32 pkt_seq_hi;
Ed Warnickecb9cada2015-12-08 15:45:58 -070066 ipsec_crypto_alg_t crypto_alg;
67 ipsec_integ_alg_t integ_alg;
68} esp_decrypt_trace_t;
69
Neale Ranns93688d72022-08-09 03:34:51 +000070typedef vl_counter_esp_decrypt_enum_t esp_decrypt_error_t;
71
Neale Ranns5b891102021-06-28 13:31:28 +000072/* The number of byres in the hisequence number */
73#define N_HI_ESN_BYTES 4
74
Ed Warnickecb9cada2015-12-08 15:45:58 -070075/* packet trace format function */
Keith Burns (alagalah)166a9d42016-08-06 11:00:56 -070076static u8 *
77format_esp_decrypt_trace (u8 * s, va_list * args)
Ed Warnickecb9cada2015-12-08 15:45:58 -070078{
79 CLIB_UNUSED (vlib_main_t * vm) = va_arg (*args, vlib_main_t *);
80 CLIB_UNUSED (vlib_node_t * node) = va_arg (*args, vlib_node_t *);
Keith Burns (alagalah)166a9d42016-08-06 11:00:56 -070081 esp_decrypt_trace_t *t = va_arg (*args, esp_decrypt_trace_t *);
Ed Warnickecb9cada2015-12-08 15:45:58 -070082
Neale Ranns5b891102021-06-28 13:31:28 +000083 s = format (s,
84 "esp: crypto %U integrity %U pkt-seq %d sa-seq %u sa-seq-hi %u "
85 "pkt-seq-hi %u",
86 format_ipsec_crypto_alg, t->crypto_alg, format_ipsec_integ_alg,
87 t->integ_alg, t->seq, t->sa_seq, t->sa_seq_hi, t->pkt_seq_hi);
Ed Warnickecb9cada2015-12-08 15:45:58 -070088 return s;
89}
90
Damjan Marionb4fff3a2019-03-25 15:54:40 +010091#define ESP_ENCRYPT_PD_F_FD_TRANSPORT (1 << 2)
Ed Warnickecb9cada2015-12-08 15:45:58 -070092
Filip Tehlarefcad1a2020-02-04 09:36:04 +000093static_always_inline void
94esp_process_ops (vlib_main_t * vm, vlib_node_runtime_t * node,
95 vnet_crypto_op_t * ops, vlib_buffer_t * b[], u16 * nexts,
96 int e)
97{
98 vnet_crypto_op_t *op = ops;
99 u32 n_fail, n_ops = vec_len (ops);
100
101 if (n_ops == 0)
102 return;
103
104 n_fail = n_ops - vnet_crypto_process_ops (vm, op, n_ops);
105
106 while (n_fail)
107 {
108 ASSERT (op - ops < n_ops);
109 if (op->status != VNET_CRYPTO_OP_STATUS_COMPLETED)
110 {
111 u32 err, bi = op->user_data;
112 if (op->status == VNET_CRYPTO_OP_STATUS_FAIL_BAD_HMAC)
113 err = e;
114 else
115 err = ESP_DECRYPT_ERROR_CRYPTO_ENGINE_ERROR;
Arthur de Kerhorad95b062022-11-16 19:12:05 +0100116 esp_decrypt_set_next_index (b[bi], node, vm->thread_index, err, bi,
117 nexts, ESP_DECRYPT_NEXT_DROP,
118 vnet_buffer (b[bi])->ipsec.sad_index);
Filip Tehlarefcad1a2020-02-04 09:36:04 +0000119 n_fail--;
120 }
121 op++;
122 }
123}
124
125static_always_inline void
126esp_process_chained_ops (vlib_main_t * vm, vlib_node_runtime_t * node,
127 vnet_crypto_op_t * ops, vlib_buffer_t * b[],
128 u16 * nexts, vnet_crypto_op_chunk_t * chunks, int e)
129{
130
131 vnet_crypto_op_t *op = ops;
132 u32 n_fail, n_ops = vec_len (ops);
133
Neale Rannsf16e9a52021-02-25 19:09:24 +0000134 if (PREDICT_TRUE (n_ops == 0))
Filip Tehlarefcad1a2020-02-04 09:36:04 +0000135 return;
136
137 n_fail = n_ops - vnet_crypto_process_chained_ops (vm, op, chunks, n_ops);
138
139 while (n_fail)
140 {
141 ASSERT (op - ops < n_ops);
142 if (op->status != VNET_CRYPTO_OP_STATUS_COMPLETED)
143 {
144 u32 err, bi = op->user_data;
145 if (op->status == VNET_CRYPTO_OP_STATUS_FAIL_BAD_HMAC)
146 err = e;
147 else
148 err = ESP_DECRYPT_ERROR_CRYPTO_ENGINE_ERROR;
Arthur de Kerhorad95b062022-11-16 19:12:05 +0100149 esp_decrypt_set_next_index (b[bi], node, vm->thread_index, err, bi,
150 nexts, ESP_DECRYPT_NEXT_DROP,
151 vnet_buffer (b[bi])->ipsec.sad_index);
Filip Tehlarefcad1a2020-02-04 09:36:04 +0000152 n_fail--;
153 }
154 op++;
155 }
156}
157
158always_inline void
159esp_remove_tail (vlib_main_t * vm, vlib_buffer_t * b, vlib_buffer_t * last,
160 u16 tail)
161{
162 vlib_buffer_t *before_last = b;
163
Arthur de Kerhor8fce5462023-06-16 09:48:52 +0200164 if (b != last)
165 b->total_length_not_including_first_buffer -= tail;
166
Filip Tehlarefcad1a2020-02-04 09:36:04 +0000167 if (last->current_length > tail)
168 {
169 last->current_length -= tail;
170 return;
171 }
172 ASSERT (b->flags & VLIB_BUFFER_NEXT_PRESENT);
173
174 while (b->flags & VLIB_BUFFER_NEXT_PRESENT)
175 {
176 before_last = b;
177 b = vlib_get_buffer (vm, b->next_buffer);
178 }
179 before_last->current_length -= tail - last->current_length;
180 vlib_buffer_free_one (vm, before_last->next_buffer);
181 before_last->flags &= ~VLIB_BUFFER_NEXT_PRESENT;
182}
183
Arthur de Kerhor8fce5462023-06-16 09:48:52 +0200184always_inline void
185esp_remove_tail_and_tfc_padding (vlib_main_t *vm, vlib_node_runtime_t *node,
186 const esp_decrypt_packet_data_t *pd,
187 vlib_buffer_t *b, vlib_buffer_t *last,
188 u16 *next, u16 tail, int is_ip6)
189{
190 const u16 total_buffer_length = vlib_buffer_length_in_chain (vm, b);
191 u16 ip_packet_length;
192 if (is_ip6)
193 {
194 const ip6_header_t *ip6 = vlib_buffer_get_current (b);
195 ip_packet_length =
196 clib_net_to_host_u16 (ip6->payload_length) + sizeof (ip6_header_t);
197 }
198 else
199 {
200 const ip4_header_t *ip4 = vlib_buffer_get_current (b);
201 ip_packet_length = clib_net_to_host_u16 (ip4->length);
202 }
203 /* In case of TFC padding, the size of the buffer data needs to be adjusted
204 * to the ip packet length */
205 if (PREDICT_FALSE (total_buffer_length < ip_packet_length + tail))
206 {
207 esp_decrypt_set_next_index (b, node, vm->thread_index,
208 ESP_DECRYPT_ERROR_NO_TAIL_SPACE, 0, next,
209 ESP_DECRYPT_NEXT_DROP, pd->sa_index);
210 return;
211 }
212 esp_remove_tail (vm, b, last, total_buffer_length - ip_packet_length);
213}
214
Filip Tehlarefcad1a2020-02-04 09:36:04 +0000215/* ICV is splitted in last two buffers so move it to the last buffer and
216 return pointer to it */
217static_always_inline u8 *
218esp_move_icv (vlib_main_t * vm, vlib_buffer_t * first,
PiotrX Kleskia9585fd2020-12-11 15:10:31 +0000219 esp_decrypt_packet_data_t * pd,
Fan Zhangf5395782020-04-29 14:00:03 +0100220 esp_decrypt_packet_data2_t * pd2, u16 icv_sz, u16 * dif)
Filip Tehlarefcad1a2020-02-04 09:36:04 +0000221{
222 vlib_buffer_t *before_last, *bp;
Fan Zhangf5395782020-04-29 14:00:03 +0100223 u16 last_sz = pd2->lb->current_length;
Filip Tehlarefcad1a2020-02-04 09:36:04 +0000224 u16 first_sz = icv_sz - last_sz;
225
226 bp = before_last = first;
227 while (bp->flags & VLIB_BUFFER_NEXT_PRESENT)
228 {
229 before_last = bp;
230 bp = vlib_get_buffer (vm, bp->next_buffer);
231 }
232
Fan Zhangf5395782020-04-29 14:00:03 +0100233 u8 *lb_curr = vlib_buffer_get_current (pd2->lb);
Filip Tehlarefcad1a2020-02-04 09:36:04 +0000234 memmove (lb_curr + first_sz, lb_curr, last_sz);
235 clib_memcpy_fast (lb_curr, vlib_buffer_get_tail (before_last) - first_sz,
236 first_sz);
237 before_last->current_length -= first_sz;
PiotrX Kleskia9585fd2020-12-11 15:10:31 +0000238 if (before_last == first)
239 pd->current_length -= first_sz;
Arthur de Kerhor8fce5462023-06-16 09:48:52 +0200240 else
241 first->total_length_not_including_first_buffer -= first_sz;
Fan Zhangf5395782020-04-29 14:00:03 +0100242 clib_memset (vlib_buffer_get_tail (before_last), 0, first_sz);
Filip Tehlare4e8c6b2020-02-13 07:49:30 +0000243 if (dif)
244 dif[0] = first_sz;
Arthur de Kerhor8fce5462023-06-16 09:48:52 +0200245 first->total_length_not_including_first_buffer -= last_sz;
Fan Zhangf5395782020-04-29 14:00:03 +0100246 pd2->lb = before_last;
247 pd2->icv_removed = 1;
248 pd2->free_buffer_index = before_last->next_buffer;
Filip Tehlarefcad1a2020-02-04 09:36:04 +0000249 before_last->flags &= ~VLIB_BUFFER_NEXT_PRESENT;
250 return lb_curr;
251}
252
Neale Ranns5b891102021-06-28 13:31:28 +0000253static_always_inline u16
254esp_insert_esn (vlib_main_t *vm, ipsec_sa_t *sa, esp_decrypt_packet_data_t *pd,
255 esp_decrypt_packet_data2_t *pd2, u32 *data_len, u8 **digest,
256 u16 *len, vlib_buffer_t *b, u8 *payload)
Filip Tehlare4e8c6b2020-02-13 07:49:30 +0000257{
258 if (!ipsec_sa_is_set_USE_ESN (sa))
Fan Zhangf5395782020-04-29 14:00:03 +0100259 return 0;
Filip Tehlare4e8c6b2020-02-13 07:49:30 +0000260 /* shift ICV by 4 bytes to insert ESN */
Neale Ranns5b891102021-06-28 13:31:28 +0000261 u32 seq_hi = clib_host_to_net_u32 (pd->seq_hi);
262 u8 tmp[ESP_MAX_ICV_SIZE];
Filip Tehlare4e8c6b2020-02-13 07:49:30 +0000263
Fan Zhangf5395782020-04-29 14:00:03 +0100264 if (pd2->icv_removed)
Filip Tehlare4e8c6b2020-02-13 07:49:30 +0000265 {
Fan Zhangf5395782020-04-29 14:00:03 +0100266 u16 space_left = vlib_buffer_space_left_at_end (vm, pd2->lb);
Neale Ranns5b891102021-06-28 13:31:28 +0000267 if (space_left >= N_HI_ESN_BYTES)
Filip Tehlare4e8c6b2020-02-13 07:49:30 +0000268 {
Neale Ranns5b891102021-06-28 13:31:28 +0000269 clib_memcpy_fast (vlib_buffer_get_tail (pd2->lb), &seq_hi,
270 N_HI_ESN_BYTES);
271 *data_len += N_HI_ESN_BYTES;
Filip Tehlare4e8c6b2020-02-13 07:49:30 +0000272 }
273 else
Neale Ranns5b891102021-06-28 13:31:28 +0000274 return N_HI_ESN_BYTES;
Filip Tehlare4e8c6b2020-02-13 07:49:30 +0000275
276 len[0] = b->current_length;
277 }
278 else
279 {
280 clib_memcpy_fast (tmp, payload + len[0], ESP_MAX_ICV_SIZE);
Neale Ranns5b891102021-06-28 13:31:28 +0000281 clib_memcpy_fast (payload + len[0], &seq_hi, N_HI_ESN_BYTES);
282 clib_memcpy_fast (payload + len[0] + N_HI_ESN_BYTES, tmp,
283 ESP_MAX_ICV_SIZE);
284 *data_len += N_HI_ESN_BYTES;
285 *digest += N_HI_ESN_BYTES;
Filip Tehlare4e8c6b2020-02-13 07:49:30 +0000286 }
Neale Ranns5b891102021-06-28 13:31:28 +0000287 return N_HI_ESN_BYTES;
Filip Tehlare4e8c6b2020-02-13 07:49:30 +0000288}
289
290static_always_inline u8 *
291esp_move_icv_esn (vlib_main_t * vm, vlib_buffer_t * first,
PiotrX Kleskia9585fd2020-12-11 15:10:31 +0000292 esp_decrypt_packet_data_t * pd,
Fan Zhangf5395782020-04-29 14:00:03 +0100293 esp_decrypt_packet_data2_t * pd2, u16 icv_sz,
294 ipsec_sa_t * sa, u8 * extra_esn, u32 * len)
Filip Tehlare4e8c6b2020-02-13 07:49:30 +0000295{
296 u16 dif = 0;
PiotrX Kleskia9585fd2020-12-11 15:10:31 +0000297 u8 *digest = esp_move_icv (vm, first, pd, pd2, icv_sz, &dif);
Filip Tehlare4e8c6b2020-02-13 07:49:30 +0000298 if (dif)
Fan Zhangf5395782020-04-29 14:00:03 +0100299 *len -= dif;
Filip Tehlare4e8c6b2020-02-13 07:49:30 +0000300
301 if (ipsec_sa_is_set_USE_ESN (sa))
302 {
Neale Ranns5b891102021-06-28 13:31:28 +0000303 u32 seq_hi = clib_host_to_net_u32 (pd->seq_hi);
Fan Zhangf5395782020-04-29 14:00:03 +0100304 u16 space_left = vlib_buffer_space_left_at_end (vm, pd2->lb);
Filip Tehlare4e8c6b2020-02-13 07:49:30 +0000305
Neale Ranns5b891102021-06-28 13:31:28 +0000306 if (space_left >= N_HI_ESN_BYTES)
Filip Tehlare4e8c6b2020-02-13 07:49:30 +0000307 {
Neale Ranns5b891102021-06-28 13:31:28 +0000308 clib_memcpy_fast (vlib_buffer_get_tail (pd2->lb), &seq_hi,
309 N_HI_ESN_BYTES);
310 *len += N_HI_ESN_BYTES;
Filip Tehlare4e8c6b2020-02-13 07:49:30 +0000311 }
312 else
313 {
314 /* no space for ESN at the tail, use the next buffer
315 * (with ICV data) */
Fan Zhangf5395782020-04-29 14:00:03 +0100316 ASSERT (pd2->icv_removed);
317 vlib_buffer_t *tmp = vlib_get_buffer (vm, pd2->free_buffer_index);
Neale Ranns5b891102021-06-28 13:31:28 +0000318 clib_memcpy_fast (vlib_buffer_get_current (tmp) - N_HI_ESN_BYTES,
319 &seq_hi, N_HI_ESN_BYTES);
Filip Tehlare4e8c6b2020-02-13 07:49:30 +0000320 extra_esn[0] = 1;
321 }
322 }
323 return digest;
324}
325
Fan Zhangf5395782020-04-29 14:00:03 +0100326static_always_inline int
Neale Ranns5b891102021-06-28 13:31:28 +0000327esp_decrypt_chain_integ (vlib_main_t *vm, ipsec_per_thread_data_t *ptd,
328 const esp_decrypt_packet_data_t *pd,
329 esp_decrypt_packet_data2_t *pd2, ipsec_sa_t *sa0,
330 vlib_buffer_t *b, u8 icv_sz, u8 *start_src,
331 u32 start_len, u8 **digest, u16 *n_ch,
332 u32 *integ_total_len)
Fan Zhangf5395782020-04-29 14:00:03 +0100333{
334 vnet_crypto_op_chunk_t *ch;
335 vlib_buffer_t *cb = vlib_get_buffer (vm, b->next_buffer);
336 u16 n_chunks = 1;
337 u32 total_len;
338 vec_add2 (ptd->chunks, ch, 1);
339 total_len = ch->len = start_len;
340 ch->src = start_src;
341
342 while (1)
343 {
344 vec_add2 (ptd->chunks, ch, 1);
345 n_chunks += 1;
346 ch->src = vlib_buffer_get_current (cb);
347 if (pd2->lb == cb)
348 {
349 if (pd2->icv_removed)
350 ch->len = cb->current_length;
351 else
352 ch->len = cb->current_length - icv_sz;
353 if (ipsec_sa_is_set_USE_ESN (sa0))
354 {
Neale Ranns5b891102021-06-28 13:31:28 +0000355 u32 seq_hi = clib_host_to_net_u32 (pd->seq_hi);
356 u8 tmp[ESP_MAX_ICV_SIZE];
Fan Zhangf5395782020-04-29 14:00:03 +0100357 u8 *esn;
358 vlib_buffer_t *tmp_b;
359 u16 space_left = vlib_buffer_space_left_at_end (vm, pd2->lb);
Neale Ranns5b891102021-06-28 13:31:28 +0000360 if (space_left < N_HI_ESN_BYTES)
Fan Zhangf5395782020-04-29 14:00:03 +0100361 {
362 if (pd2->icv_removed)
363 {
364 /* use pre-data area from the last bufer
365 that was removed from the chain */
366 tmp_b = vlib_get_buffer (vm, pd2->free_buffer_index);
Neale Ranns5b891102021-06-28 13:31:28 +0000367 esn = tmp_b->data - N_HI_ESN_BYTES;
Fan Zhangf5395782020-04-29 14:00:03 +0100368 }
369 else
370 {
371 /* no space, need to allocate new buffer */
372 u32 tmp_bi = 0;
373 if (vlib_buffer_alloc (vm, &tmp_bi, 1) != 1)
374 return -1;
375 tmp_b = vlib_get_buffer (vm, tmp_bi);
376 esn = tmp_b->data;
377 pd2->free_buffer_index = tmp_bi;
378 }
Neale Ranns5b891102021-06-28 13:31:28 +0000379 clib_memcpy_fast (esn, &seq_hi, N_HI_ESN_BYTES);
Fan Zhangf5395782020-04-29 14:00:03 +0100380
381 vec_add2 (ptd->chunks, ch, 1);
382 n_chunks += 1;
383 ch->src = esn;
Neale Ranns5b891102021-06-28 13:31:28 +0000384 ch->len = N_HI_ESN_BYTES;
Fan Zhangf5395782020-04-29 14:00:03 +0100385 }
386 else
387 {
388 if (pd2->icv_removed)
389 {
Neale Ranns5b891102021-06-28 13:31:28 +0000390 clib_memcpy_fast (vlib_buffer_get_tail (pd2->lb),
391 &seq_hi, N_HI_ESN_BYTES);
Fan Zhangf5395782020-04-29 14:00:03 +0100392 }
393 else
394 {
395 clib_memcpy_fast (tmp, *digest, ESP_MAX_ICV_SIZE);
Neale Ranns5b891102021-06-28 13:31:28 +0000396 clib_memcpy_fast (*digest, &seq_hi, N_HI_ESN_BYTES);
397 clib_memcpy_fast (*digest + N_HI_ESN_BYTES, tmp,
398 ESP_MAX_ICV_SIZE);
399 *digest += N_HI_ESN_BYTES;
Fan Zhangf5395782020-04-29 14:00:03 +0100400 }
Neale Ranns5b891102021-06-28 13:31:28 +0000401 ch->len += N_HI_ESN_BYTES;
Fan Zhangf5395782020-04-29 14:00:03 +0100402 }
403 }
404 total_len += ch->len;
405 break;
406 }
407 else
408 total_len += ch->len = cb->current_length;
409
410 if (!(cb->flags & VLIB_BUFFER_NEXT_PRESENT))
411 break;
412
413 cb = vlib_get_buffer (vm, cb->next_buffer);
414 }
415
416 if (n_ch)
417 *n_ch = n_chunks;
418 if (integ_total_len)
419 *integ_total_len = total_len;
420
421 return 0;
422}
423
424static_always_inline u32
425esp_decrypt_chain_crypto (vlib_main_t * vm, ipsec_per_thread_data_t * ptd,
PiotrX Kleskia9585fd2020-12-11 15:10:31 +0000426 esp_decrypt_packet_data_t * pd,
Fan Zhangf5395782020-04-29 14:00:03 +0100427 esp_decrypt_packet_data2_t * pd2,
428 ipsec_sa_t * sa0, vlib_buffer_t * b, u8 icv_sz,
429 u8 * start, u32 start_len, u8 ** tag, u16 * n_ch)
430{
431 vnet_crypto_op_chunk_t *ch;
432 vlib_buffer_t *cb = b;
433 u16 n_chunks = 1;
434 u32 total_len;
435 vec_add2 (ptd->chunks, ch, 1);
436 total_len = ch->len = start_len;
437 ch->src = ch->dst = start;
438 cb = vlib_get_buffer (vm, cb->next_buffer);
439 n_chunks = 1;
440
441 while (1)
442 {
443 vec_add2 (ptd->chunks, ch, 1);
444 n_chunks += 1;
445 ch->src = ch->dst = vlib_buffer_get_current (cb);
446 if (pd2->lb == cb)
447 {
448 if (ipsec_sa_is_set_IS_AEAD (sa0))
449 {
450 if (pd2->lb->current_length < icv_sz)
451 {
452 u16 dif = 0;
PiotrX Kleskia9585fd2020-12-11 15:10:31 +0000453 *tag = esp_move_icv (vm, b, pd, pd2, icv_sz, &dif);
Fan Zhangf5395782020-04-29 14:00:03 +0100454
455 /* this chunk does not contain crypto data */
456 n_chunks -= 1;
457 /* and fix previous chunk's length as it might have
458 been changed */
459 ASSERT (n_chunks > 0);
460 if (pd2->lb == b)
461 {
462 total_len -= dif;
463 ch[-1].len -= dif;
464 }
465 else
466 {
467 total_len = total_len + pd2->lb->current_length -
468 ch[-1].len;
469 ch[-1].len = pd2->lb->current_length;
470 }
471 break;
472 }
473 else
474 *tag = vlib_buffer_get_tail (pd2->lb) - icv_sz;
475 }
476
477 if (pd2->icv_removed)
478 total_len += ch->len = cb->current_length;
479 else
480 total_len += ch->len = cb->current_length - icv_sz;
481 }
482 else
483 total_len += ch->len = cb->current_length;
484
485 if (!(cb->flags & VLIB_BUFFER_NEXT_PRESENT))
486 break;
487
488 cb = vlib_get_buffer (vm, cb->next_buffer);
489 }
490
491 if (n_ch)
492 *n_ch = n_chunks;
493
494 return total_len;
495}
496
Fan Zhanga4df00f2024-03-13 02:49:31 +0000497static_always_inline esp_decrypt_error_t
498esp_decrypt_prepare_sync_op (vlib_main_t *vm, ipsec_per_thread_data_t *ptd,
499 ipsec_sa_t *sa0, u8 *payload, u16 len, u8 icv_sz,
500 u8 iv_sz, esp_decrypt_packet_data_t *pd,
501 esp_decrypt_packet_data2_t *pd2, vlib_buffer_t *b,
502 u32 index)
Fan Zhangf5395782020-04-29 14:00:03 +0100503{
Fan Zhanga4df00f2024-03-13 02:49:31 +0000504 vnet_crypto_op_t **crypto_ops;
505 vnet_crypto_op_t **integ_ops;
506 vnet_crypto_op_t _op, *op = &_op;
Fan Zhangf5395782020-04-29 14:00:03 +0100507 const u8 esp_sz = sizeof (esp_header_t);
508
509 if (PREDICT_TRUE (sa0->integ_op_id != VNET_CRYPTO_OP_NONE))
510 {
511 vnet_crypto_op_init (op, sa0->integ_op_id);
512 op->key_index = sa0->integ_key_index;
513 op->src = payload;
514 op->flags = VNET_CRYPTO_OP_FLAG_HMAC_CHECK;
515 op->user_data = index;
516 op->digest = payload + len;
517 op->digest_len = icv_sz;
518 op->len = len;
519
520 if (pd->is_chain)
521 {
522 /* buffer is chained */
Fan Zhanga4df00f2024-03-13 02:49:31 +0000523 integ_ops = &ptd->chained_integ_ops;
524
Fan Zhangf5395782020-04-29 14:00:03 +0100525 op->len = pd->current_length;
526
527 /* special case when ICV is splitted and needs to be reassembled
528 * first -> move it to the last buffer. Also take into account
529 * that ESN needs to be added after encrypted data and may or
530 * may not fit in the tail.*/
531 if (pd2->lb->current_length < icv_sz)
532 {
533 u8 extra_esn = 0;
534 op->digest =
PiotrX Kleskia9585fd2020-12-11 15:10:31 +0000535 esp_move_icv_esn (vm, b, pd, pd2, icv_sz, sa0,
Fan Zhangf5395782020-04-29 14:00:03 +0100536 &extra_esn, &op->len);
537
538 if (extra_esn)
539 {
540 /* esn is in the last buffer, that was unlinked from
541 * the chain */
542 op->len = b->current_length;
543 }
544 else
545 {
546 if (pd2->lb == b)
547 {
548 /* we now have a single buffer of crypto data, adjust
549 * the length (second buffer contains only ICV) */
Fan Zhanga4df00f2024-03-13 02:49:31 +0000550 integ_ops = &ptd->integ_ops;
Fan Zhangf5395782020-04-29 14:00:03 +0100551 len = b->current_length;
552 goto out;
553 }
554 }
555 }
556 else
557 op->digest = vlib_buffer_get_tail (pd2->lb) - icv_sz;
558
559 op->flags |= VNET_CRYPTO_OP_FLAG_CHAINED_BUFFERS;
560 op->chunk_index = vec_len (ptd->chunks);
Neale Ranns5b891102021-06-28 13:31:28 +0000561 if (esp_decrypt_chain_integ (vm, ptd, pd, pd2, sa0, b, icv_sz,
Fan Zhangf5395782020-04-29 14:00:03 +0100562 payload, pd->current_length,
563 &op->digest, &op->n_chunks, 0) < 0)
Fan Zhanga4df00f2024-03-13 02:49:31 +0000564 return ESP_DECRYPT_ERROR_NO_BUFFERS;
Fan Zhangf5395782020-04-29 14:00:03 +0100565 }
566 else
Fan Zhanga4df00f2024-03-13 02:49:31 +0000567 {
568 integ_ops = &ptd->integ_ops;
569 esp_insert_esn (vm, sa0, pd, pd2, &op->len, &op->digest, &len, b,
570 payload);
571 }
Fan Zhangf5395782020-04-29 14:00:03 +0100572 out:
Fan Zhanga4df00f2024-03-13 02:49:31 +0000573 vec_add_aligned (*integ_ops, op, 1, CLIB_CACHE_LINE_BYTES);
Fan Zhangf5395782020-04-29 14:00:03 +0100574 }
575
576 payload += esp_sz;
577 len -= esp_sz;
578
579 if (sa0->crypto_dec_op_id != VNET_CRYPTO_OP_NONE)
580 {
581 vnet_crypto_op_init (op, sa0->crypto_dec_op_id);
582 op->key_index = sa0->crypto_key_index;
583 op->iv = payload;
584
Benoît Ganne490b9272021-01-22 18:03:09 +0100585 if (ipsec_sa_is_set_IS_CTR (sa0))
Fan Zhangf5395782020-04-29 14:00:03 +0100586 {
Benoît Ganne490b9272021-01-22 18:03:09 +0100587 /* construct nonce in a scratch space in front of the IP header */
588 esp_ctr_nonce_t *nonce =
589 (esp_ctr_nonce_t *) (payload - esp_sz - pd->hdr_sz -
590 sizeof (*nonce));
591 if (ipsec_sa_is_set_IS_AEAD (sa0))
592 {
593 /* constuct aad in a scratch space in front of the nonce */
594 esp_header_t *esp0 = (esp_header_t *) (payload - esp_sz);
595 op->aad = (u8 *) nonce - sizeof (esp_aead_t);
Neale Ranns5b891102021-06-28 13:31:28 +0000596 op->aad_len = esp_aad_fill (op->aad, esp0, sa0, pd->seq_hi);
Benoît Ganne490b9272021-01-22 18:03:09 +0100597 op->tag = payload + len;
598 op->tag_len = 16;
Benoît Ganne84e66582023-03-10 17:33:03 +0100599 if (PREDICT_FALSE (ipsec_sa_is_set_IS_NULL_GMAC (sa0)))
600 {
601 /* RFC-4543 ENCR_NULL_AUTH_AES_GMAC: IV is part of AAD */
602 payload -= iv_sz;
603 len += iv_sz;
604 }
Benoît Ganne490b9272021-01-22 18:03:09 +0100605 }
606 else
607 {
608 nonce->ctr = clib_host_to_net_u32 (1);
609 }
610 nonce->salt = sa0->salt;
611 ASSERT (sizeof (u64) == iv_sz);
612 nonce->iv = *(u64 *) op->iv;
613 op->iv = (u8 *) nonce;
Fan Zhangf5395782020-04-29 14:00:03 +0100614 }
615 op->src = op->dst = payload += iv_sz;
616 op->len = len - iv_sz;
617 op->user_data = index;
618
619 if (pd->is_chain && (pd2->lb != b))
620 {
621 /* buffer is chained */
622 op->flags |= VNET_CRYPTO_OP_FLAG_CHAINED_BUFFERS;
623 op->chunk_index = vec_len (ptd->chunks);
PiotrX Kleskia9585fd2020-12-11 15:10:31 +0000624 esp_decrypt_chain_crypto (vm, ptd, pd, pd2, sa0, b, icv_sz,
Fan Zhangf5395782020-04-29 14:00:03 +0100625 payload, len - pd->iv_sz + pd->icv_sz,
626 &op->tag, &op->n_chunks);
Fan Zhanga4df00f2024-03-13 02:49:31 +0000627 crypto_ops = &ptd->chained_crypto_ops;
628 }
629 else
630 {
631 crypto_ops = &ptd->crypto_ops;
Fan Zhangf5395782020-04-29 14:00:03 +0100632 }
633
Fan Zhanga4df00f2024-03-13 02:49:31 +0000634 vec_add_aligned (*crypto_ops, op, 1, CLIB_CACHE_LINE_BYTES);
Fan Zhangf5395782020-04-29 14:00:03 +0100635 }
Fan Zhanga4df00f2024-03-13 02:49:31 +0000636
637 return ESP_DECRYPT_ERROR_RX_PKTS;
Fan Zhangf5395782020-04-29 14:00:03 +0100638}
639
Neale Rannsfc811342021-02-26 10:35:33 +0000640static_always_inline esp_decrypt_error_t
Fan Zhanga4df00f2024-03-13 02:49:31 +0000641esp_decrypt_prepare_async_frame (vlib_main_t *vm, ipsec_per_thread_data_t *ptd,
Neale Rannsfc811342021-02-26 10:35:33 +0000642 vnet_crypto_async_frame_t *f, ipsec_sa_t *sa0,
643 u8 *payload, u16 len, u8 icv_sz, u8 iv_sz,
644 esp_decrypt_packet_data_t *pd,
645 esp_decrypt_packet_data2_t *pd2, u32 bi,
Fan Zhanga4df00f2024-03-13 02:49:31 +0000646 vlib_buffer_t *b, u16 async_next)
Fan Zhangf5395782020-04-29 14:00:03 +0100647{
648 const u8 esp_sz = sizeof (esp_header_t);
Fan Zhangf5395782020-04-29 14:00:03 +0100649 esp_decrypt_packet_data_t *async_pd = &(esp_post_data (b))->decrypt_data;
650 esp_decrypt_packet_data2_t *async_pd2 = esp_post_data2 (b);
651 u8 *tag = payload + len, *iv = payload + esp_sz, *aad = 0;
Benoît Ganne5527a782022-01-18 15:56:41 +0100652 const u32 key_index = sa0->crypto_key_index;
Fan Zhangf5395782020-04-29 14:00:03 +0100653 u32 crypto_len, integ_len = 0;
654 i16 crypto_start_offset, integ_start_offset = 0;
655 u8 flags = 0;
656
657 if (!ipsec_sa_is_set_IS_AEAD (sa0))
658 {
659 /* linked algs */
Fan Zhangf5395782020-04-29 14:00:03 +0100660 integ_start_offset = payload - b->data;
661 integ_len = len;
Benoît Ganne48524a92021-01-22 18:11:37 +0100662 if (PREDICT_TRUE (sa0->integ_op_id != VNET_CRYPTO_OP_NONE))
663 flags |= VNET_CRYPTO_OP_FLAG_HMAC_CHECK;
Fan Zhangf5395782020-04-29 14:00:03 +0100664
665 if (pd->is_chain)
666 {
667 /* buffer is chained */
668 integ_len = pd->current_length;
669
670 /* special case when ICV is splitted and needs to be reassembled
671 * first -> move it to the last buffer. Also take into account
672 * that ESN needs to be added after encrypted data and may or
673 * may not fit in the tail.*/
674 if (pd2->lb->current_length < icv_sz)
675 {
676 u8 extra_esn = 0;
PiotrX Kleskia9585fd2020-12-11 15:10:31 +0000677 tag = esp_move_icv_esn (vm, b, pd, pd2, icv_sz, sa0,
Fan Zhangf5395782020-04-29 14:00:03 +0100678 &extra_esn, &integ_len);
679
680 if (extra_esn)
681 {
682 /* esn is in the last buffer, that was unlinked from
683 * the chain */
684 integ_len = b->current_length;
685 }
686 else
687 {
688 if (pd2->lb == b)
689 {
690 /* we now have a single buffer of crypto data, adjust
691 * the length (second buffer contains only ICV) */
692 len = b->current_length;
693 goto out;
694 }
695 }
696 }
697 else
698 tag = vlib_buffer_get_tail (pd2->lb) - icv_sz;
699
700 flags |= VNET_CRYPTO_OP_FLAG_CHAINED_BUFFERS;
Neale Ranns5b891102021-06-28 13:31:28 +0000701 if (esp_decrypt_chain_integ (vm, ptd, pd, pd2, sa0, b, icv_sz,
702 payload, pd->current_length, &tag, 0,
703 &integ_len) < 0)
Fan Zhangf5395782020-04-29 14:00:03 +0100704 {
705 /* allocate buffer failed, will not add to frame and drop */
Neale Rannsfc811342021-02-26 10:35:33 +0000706 return (ESP_DECRYPT_ERROR_NO_BUFFERS);
Fan Zhangf5395782020-04-29 14:00:03 +0100707 }
708 }
709 else
Neale Ranns5b891102021-06-28 13:31:28 +0000710 esp_insert_esn (vm, sa0, pd, pd2, &integ_len, &tag, &len, b, payload);
Fan Zhangf5395782020-04-29 14:00:03 +0100711 }
Fan Zhangf5395782020-04-29 14:00:03 +0100712
713out:
714 /* crypto */
715 payload += esp_sz;
716 len -= esp_sz;
717 iv = payload;
718
Benoît Ganne490b9272021-01-22 18:03:09 +0100719 if (ipsec_sa_is_set_IS_CTR (sa0))
Fan Zhangf5395782020-04-29 14:00:03 +0100720 {
Benoît Ganne490b9272021-01-22 18:03:09 +0100721 /* construct nonce in a scratch space in front of the IP header */
722 esp_ctr_nonce_t *nonce =
723 (esp_ctr_nonce_t *) (payload - esp_sz - pd->hdr_sz - sizeof (*nonce));
724 if (ipsec_sa_is_set_IS_AEAD (sa0))
725 {
726 /* constuct aad in a scratch space in front of the nonce */
727 esp_header_t *esp0 = (esp_header_t *) (payload - esp_sz);
728 aad = (u8 *) nonce - sizeof (esp_aead_t);
Neale Ranns5b891102021-06-28 13:31:28 +0000729 esp_aad_fill (aad, esp0, sa0, pd->seq_hi);
Benoît Ganne490b9272021-01-22 18:03:09 +0100730 tag = payload + len;
Benoît Ganne84e66582023-03-10 17:33:03 +0100731 if (PREDICT_FALSE (ipsec_sa_is_set_IS_NULL_GMAC (sa0)))
732 {
733 /* RFC-4543 ENCR_NULL_AUTH_AES_GMAC: IV is part of AAD */
734 payload -= iv_sz;
735 len += iv_sz;
736 }
Benoît Ganne490b9272021-01-22 18:03:09 +0100737 }
738 else
739 {
740 nonce->ctr = clib_host_to_net_u32 (1);
741 }
742 nonce->salt = sa0->salt;
743 ASSERT (sizeof (u64) == iv_sz);
744 nonce->iv = *(u64 *) iv;
745 iv = (u8 *) nonce;
Fan Zhangf5395782020-04-29 14:00:03 +0100746 }
747
748 crypto_start_offset = (payload += iv_sz) - b->data;
749 crypto_len = len - iv_sz;
750
751 if (pd->is_chain && (pd2->lb != b))
752 {
753 /* buffer is chained */
754 flags |= VNET_CRYPTO_OP_FLAG_CHAINED_BUFFERS;
755
PiotrX Kleskia9585fd2020-12-11 15:10:31 +0000756 crypto_len = esp_decrypt_chain_crypto (vm, ptd, pd, pd2, sa0, b, icv_sz,
Fan Zhangf5395782020-04-29 14:00:03 +0100757 payload,
758 len - pd->iv_sz + pd->icv_sz,
759 &tag, 0);
760 }
761
762 *async_pd = *pd;
763 *async_pd2 = *pd2;
Fan Zhangf5395782020-04-29 14:00:03 +0100764
765 /* for AEAD integ_len - crypto_len will be negative, it is ok since it
766 * is ignored by the engine. */
Neale Rannsfc811342021-02-26 10:35:33 +0000767 vnet_crypto_async_add_to_frame (
768 vm, f, key_index, crypto_len, integ_len - crypto_len, crypto_start_offset,
769 integ_start_offset, bi, async_next, iv, tag, aad, flags);
770
771 return (ESP_DECRYPT_ERROR_RX_PKTS);
Fan Zhangf5395782020-04-29 14:00:03 +0100772}
773
774static_always_inline void
Arthur de Kerhorad95b062022-11-16 19:12:05 +0100775esp_decrypt_post_crypto (vlib_main_t *vm, vlib_node_runtime_t *node,
Benoît Ganne98ca76a2022-04-11 18:51:25 +0200776 const u16 *next_by_next_header,
Neale Rannse11203e2021-09-21 12:34:19 +0000777 const esp_decrypt_packet_data_t *pd,
778 const esp_decrypt_packet_data2_t *pd2,
779 vlib_buffer_t *b, u16 *next, int is_ip6, int is_tun,
780 int is_async)
Fan Zhangf5395782020-04-29 14:00:03 +0100781{
Neale Rannsc5fe57d2021-02-25 16:01:28 +0000782 ipsec_sa_t *sa0 = ipsec_sa_get (pd->sa_index);
Fan Zhangf5395782020-04-29 14:00:03 +0100783 vlib_buffer_t *lb = b;
784 const u8 esp_sz = sizeof (esp_header_t);
785 const u8 tun_flags = IPSEC_SA_FLAG_IS_TUNNEL | IPSEC_SA_FLAG_IS_TUNNEL_V6;
786 u8 pad_length = 0, next_header = 0;
787 u16 icv_sz;
Maxime Peim0e2f1882022-12-22 11:26:57 +0000788 u64 n_lost;
Fan Zhangf5395782020-04-29 14:00:03 +0100789
790 /*
791 * redo the anti-reply check
792 * in this frame say we have sequence numbers, s, s+1, s+1, s+1
793 * and s and s+1 are in the window. When we did the anti-replay
794 * check above we did so against the state of the window (W),
795 * after packet s-1. So each of the packets in the sequence will be
796 * accepted.
Maxime Peim0e2f1882022-12-22 11:26:57 +0000797 * This time s will be cheked against Ws-1, s+1 checked against Ws
798 * (i.e. the window state is updated/advanced)
799 * so this time the successive s+1 packet will be dropped.
Fan Zhangf5395782020-04-29 14:00:03 +0100800 * This is a consequence of batching the decrypts. If the
Maxime Peim0e2f1882022-12-22 11:26:57 +0000801 * check-decrypt-advance process was done for each packet it would
Fan Zhangf5395782020-04-29 14:00:03 +0100802 * be fine. But we batch the decrypts because it's much more efficient
803 * to do so in SW and if we offload to HW and the process is async.
804 *
805 * You're probably thinking, but this means an attacker can send the
Maxime Peim0e2f1882022-12-22 11:26:57 +0000806 * above sequence and cause VPP to perform decrypts that will fail,
Fan Zhangf5395782020-04-29 14:00:03 +0100807 * and that's true. But if the attacker can determine s (a valid
808 * sequence number in the window) which is non-trivial, it can generate
809 * a sequence s, s+1, s+2, s+3, ... s+n and nothing will prevent any
810 * implementation, sequential or batching, from decrypting these.
811 */
Maxime Peim0e2f1882022-12-22 11:26:57 +0000812 if (PREDICT_FALSE (ipsec_sa_is_set_ANTI_REPLAY_HUGE (sa0)))
Fan Zhangf5395782020-04-29 14:00:03 +0100813 {
Maxime Peim0e2f1882022-12-22 11:26:57 +0000814 if (ipsec_sa_anti_replay_and_sn_advance (sa0, pd->seq, pd->seq_hi, true,
815 NULL, true))
816 {
817 esp_decrypt_set_next_index (b, node, vm->thread_index,
818 ESP_DECRYPT_ERROR_REPLAY, 0, next,
819 ESP_DECRYPT_NEXT_DROP, pd->sa_index);
820 return;
821 }
822 n_lost = ipsec_sa_anti_replay_advance (sa0, vm->thread_index, pd->seq,
823 pd->seq_hi, true);
Fan Zhangf5395782020-04-29 14:00:03 +0100824 }
Maxime Peim0e2f1882022-12-22 11:26:57 +0000825 else
826 {
827 if (ipsec_sa_anti_replay_and_sn_advance (sa0, pd->seq, pd->seq_hi, true,
828 NULL, false))
829 {
830 esp_decrypt_set_next_index (b, node, vm->thread_index,
831 ESP_DECRYPT_ERROR_REPLAY, 0, next,
832 ESP_DECRYPT_NEXT_DROP, pd->sa_index);
833 return;
834 }
835 n_lost = ipsec_sa_anti_replay_advance (sa0, vm->thread_index, pd->seq,
836 pd->seq_hi, false);
837 }
Neale Rannse11203e2021-09-21 12:34:19 +0000838
Arthur de Kerhorad95b062022-11-16 19:12:05 +0100839 vlib_prefetch_simple_counter (&ipsec_sa_err_counters[IPSEC_SA_ERROR_LOST],
840 vm->thread_index, pd->sa_index);
Fan Zhangf5395782020-04-29 14:00:03 +0100841
842 if (pd->is_chain)
843 {
844 lb = pd2->lb;
845 icv_sz = pd2->icv_removed ? 0 : pd->icv_sz;
846 if (pd2->free_buffer_index)
847 {
848 vlib_buffer_free_one (vm, pd2->free_buffer_index);
849 lb->next_buffer = 0;
850 }
851 if (lb->current_length < sizeof (esp_footer_t) + icv_sz)
852 {
853 /* esp footer is either splitted in two buffers or in the before
854 * last buffer */
855
856 vlib_buffer_t *before_last = b, *bp = b;
857 while (bp->flags & VLIB_BUFFER_NEXT_PRESENT)
858 {
859 before_last = bp;
860 bp = vlib_get_buffer (vm, bp->next_buffer);
861 }
862 u8 *bt = vlib_buffer_get_tail (before_last);
863
864 if (lb->current_length == icv_sz)
865 {
866 esp_footer_t *f = (esp_footer_t *) (bt - sizeof (*f));
867 pad_length = f->pad_length;
868 next_header = f->next_header;
869 }
870 else
871 {
872 pad_length = (bt - 1)[0];
873 next_header = ((u8 *) vlib_buffer_get_current (lb))[0];
874 }
875 }
876 else
877 {
878 esp_footer_t *f =
879 (esp_footer_t *) (lb->data + lb->current_data +
880 lb->current_length - sizeof (esp_footer_t) -
881 icv_sz);
882 pad_length = f->pad_length;
883 next_header = f->next_header;
884 }
885 }
886 else
887 {
888 icv_sz = pd->icv_sz;
889 esp_footer_t *f =
890 (esp_footer_t *) (lb->data + lb->current_data + lb->current_length -
891 sizeof (esp_footer_t) - icv_sz);
892 pad_length = f->pad_length;
893 next_header = f->next_header;
894 }
895
896 u16 adv = pd->iv_sz + esp_sz;
897 u16 tail = sizeof (esp_footer_t) + pad_length + icv_sz;
898 u16 tail_orig = sizeof (esp_footer_t) + pad_length + pd->icv_sz;
Frédéric Perrina4157ae2023-07-14 11:13:42 +0100899 b->flags &=
Arthur de Kerhor8fce5462023-06-16 09:48:52 +0200900 ~(VNET_BUFFER_F_L4_CHECKSUM_COMPUTED | VNET_BUFFER_F_L4_CHECKSUM_CORRECT);
Fan Zhangf5395782020-04-29 14:00:03 +0100901
902 if ((pd->flags & tun_flags) == 0 && !is_tun) /* transport mode */
903 {
904 u8 udp_sz = (is_ip6 == 0 && pd->flags & IPSEC_SA_FLAG_UDP_ENCAP) ?
905 sizeof (udp_header_t) : 0;
906 u16 ip_hdr_sz = pd->hdr_sz - udp_sz;
907 u8 *old_ip = b->data + pd->current_data - ip_hdr_sz - udp_sz;
908 u8 *ip = old_ip + adv + udp_sz;
909
910 if (is_ip6 && ip_hdr_sz > 64)
911 memmove (ip, old_ip, ip_hdr_sz);
912 else
913 clib_memcpy_le64 (ip, old_ip, ip_hdr_sz);
914
915 b->current_data = pd->current_data + adv - ip_hdr_sz;
916 b->current_length += ip_hdr_sz - adv;
917 esp_remove_tail (vm, b, lb, tail);
918
919 if (is_ip6)
920 {
921 ip6_header_t *ip6 = (ip6_header_t *) ip;
922 u16 len = clib_net_to_host_u16 (ip6->payload_length);
923 len -= adv + tail_orig;
924 ip6->payload_length = clib_host_to_net_u16 (len);
925 ip6->protocol = next_header;
926 next[0] = ESP_DECRYPT_NEXT_IP6_INPUT;
927 }
928 else
929 {
930 ip4_header_t *ip4 = (ip4_header_t *) ip;
931 ip_csum_t sum = ip4->checksum;
932 u16 len = clib_net_to_host_u16 (ip4->length);
933 len = clib_host_to_net_u16 (len - adv - tail_orig - udp_sz);
934 sum = ip_csum_update (sum, ip4->protocol, next_header,
935 ip4_header_t, protocol);
936 sum = ip_csum_update (sum, ip4->length, len, ip4_header_t, length);
937 ip4->checksum = ip_csum_fold (sum);
938 ip4->protocol = next_header;
939 ip4->length = len;
940 next[0] = ESP_DECRYPT_NEXT_IP4_INPUT;
941 }
942 }
943 else
944 {
945 if (PREDICT_TRUE (next_header == IP_PROTOCOL_IP_IN_IP))
946 {
947 next[0] = ESP_DECRYPT_NEXT_IP4_INPUT;
948 b->current_data = pd->current_data + adv;
949 b->current_length = pd->current_length - adv;
Arthur de Kerhor8fce5462023-06-16 09:48:52 +0200950 esp_remove_tail_and_tfc_padding (vm, node, pd, b, lb, next, tail,
951 false);
Fan Zhangf5395782020-04-29 14:00:03 +0100952 }
953 else if (next_header == IP_PROTOCOL_IPV6)
954 {
955 next[0] = ESP_DECRYPT_NEXT_IP6_INPUT;
956 b->current_data = pd->current_data + adv;
957 b->current_length = pd->current_length - adv;
Arthur de Kerhor8fce5462023-06-16 09:48:52 +0200958 esp_remove_tail_and_tfc_padding (vm, node, pd, b, lb, next, tail,
959 true);
Fan Zhangf5395782020-04-29 14:00:03 +0100960 }
Neale Ranns4a58e492020-12-21 13:19:10 +0000961 else if (next_header == IP_PROTOCOL_MPLS_IN_IP)
962 {
963 next[0] = ESP_DECRYPT_NEXT_MPLS_INPUT;
964 b->current_data = pd->current_data + adv;
965 b->current_length = pd->current_length - adv;
966 esp_remove_tail (vm, b, lb, tail);
967 }
Benoît Ganne98ca76a2022-04-11 18:51:25 +0200968 else if (is_tun && next_header == IP_PROTOCOL_GRE)
Fan Zhangf5395782020-04-29 14:00:03 +0100969 {
Benoît Ganne98ca76a2022-04-11 18:51:25 +0200970 gre_header_t *gre;
971
972 b->current_data = pd->current_data + adv;
973 b->current_length = pd->current_length - adv - tail;
974
975 gre = vlib_buffer_get_current (b);
976
977 vlib_buffer_advance (b, sizeof (*gre));
978
979 switch (clib_net_to_host_u16 (gre->protocol))
Fan Zhangf5395782020-04-29 14:00:03 +0100980 {
Benoît Ganne98ca76a2022-04-11 18:51:25 +0200981 case GRE_PROTOCOL_teb:
982 vnet_update_l2_len (b);
983 next[0] = ESP_DECRYPT_NEXT_L2_INPUT;
984 break;
985 case GRE_PROTOCOL_ip4:
986 next[0] = ESP_DECRYPT_NEXT_IP4_INPUT;
987 break;
988 case GRE_PROTOCOL_ip6:
989 next[0] = ESP_DECRYPT_NEXT_IP6_INPUT;
990 break;
991 default:
Arthur de Kerhorad95b062022-11-16 19:12:05 +0100992 esp_decrypt_set_next_index (
993 b, node, vm->thread_index, ESP_DECRYPT_ERROR_UNSUP_PAYLOAD, 0,
994 next, ESP_DECRYPT_NEXT_DROP, pd->sa_index);
Benoît Ganne98ca76a2022-04-11 18:51:25 +0200995 break;
Fan Zhangf5395782020-04-29 14:00:03 +0100996 }
997 }
Benoît Ganne98ca76a2022-04-11 18:51:25 +0200998 else if ((next[0] = vec_elt (next_by_next_header, next_header)) !=
999 (u16) ~0)
1000 {
1001 b->current_data = pd->current_data + adv;
1002 b->current_length = pd->current_length - adv;
1003 esp_remove_tail (vm, b, lb, tail);
1004 }
1005 else
1006 {
Arthur de Kerhorad95b062022-11-16 19:12:05 +01001007 esp_decrypt_set_next_index (b, node, vm->thread_index,
1008 ESP_DECRYPT_ERROR_UNSUP_PAYLOAD, 0, next,
1009 ESP_DECRYPT_NEXT_DROP, pd->sa_index);
Benoît Ganne98ca76a2022-04-11 18:51:25 +02001010 return;
1011 }
1012
Fan Zhangf5395782020-04-29 14:00:03 +01001013 if (is_tun)
1014 {
1015 if (ipsec_sa_is_set_IS_PROTECT (sa0))
1016 {
1017 /*
1018 * There are two encap possibilities
1019 * 1) the tunnel and ths SA are prodiving encap, i.e. it's
1020 * MAC | SA-IP | TUN-IP | ESP | PAYLOAD
1021 * implying the SA is in tunnel mode (on a tunnel interface)
1022 * 2) only the tunnel provides encap
1023 * MAC | TUN-IP | ESP | PAYLOAD
1024 * implying the SA is in transport mode.
1025 *
1026 * For 2) we need only strip the tunnel encap and we're good.
1027 * since the tunnel and crypto ecnap (int the tun=protect
1028 * object) are the same and we verified above that these match
1029 * for 1) we need to strip the SA-IP outer headers, to
1030 * reveal the tunnel IP and then check that this matches
1031 * the configured tunnel.
1032 */
1033 const ipsec_tun_protect_t *itp;
1034
Neale Ranns5b891102021-06-28 13:31:28 +00001035 itp =
1036 ipsec_tun_protect_get (vnet_buffer (b)->ipsec.protect_index);
Fan Zhangf5395782020-04-29 14:00:03 +01001037
1038 if (PREDICT_TRUE (next_header == IP_PROTOCOL_IP_IN_IP))
1039 {
1040 const ip4_header_t *ip4;
1041
1042 ip4 = vlib_buffer_get_current (b);
1043
1044 if (!ip46_address_is_equal_v4 (&itp->itp_tun.src,
1045 &ip4->dst_address) ||
1046 !ip46_address_is_equal_v4 (&itp->itp_tun.dst,
1047 &ip4->src_address))
1048 {
Arthur de Kerhorad95b062022-11-16 19:12:05 +01001049 esp_decrypt_set_next_index (
1050 b, node, vm->thread_index,
1051 ESP_DECRYPT_ERROR_TUN_NO_PROTO, 0, next,
1052 ESP_DECRYPT_NEXT_DROP, pd->sa_index);
Fan Zhangf5395782020-04-29 14:00:03 +01001053 }
1054 }
1055 else if (next_header == IP_PROTOCOL_IPV6)
1056 {
1057 const ip6_header_t *ip6;
1058
1059 ip6 = vlib_buffer_get_current (b);
1060
1061 if (!ip46_address_is_equal_v6 (&itp->itp_tun.src,
1062 &ip6->dst_address) ||
1063 !ip46_address_is_equal_v6 (&itp->itp_tun.dst,
1064 &ip6->src_address))
1065 {
Arthur de Kerhorad95b062022-11-16 19:12:05 +01001066 esp_decrypt_set_next_index (
1067 b, node, vm->thread_index,
1068 ESP_DECRYPT_ERROR_TUN_NO_PROTO, 0, next,
1069 ESP_DECRYPT_NEXT_DROP, pd->sa_index);
Fan Zhangf5395782020-04-29 14:00:03 +01001070 }
1071 }
1072 }
1073 }
1074 }
Neale Rannse11203e2021-09-21 12:34:19 +00001075
1076 if (PREDICT_FALSE (n_lost))
Arthur de Kerhorad95b062022-11-16 19:12:05 +01001077 vlib_increment_simple_counter (&ipsec_sa_err_counters[IPSEC_SA_ERROR_LOST],
1078 vm->thread_index, pd->sa_index, n_lost);
Fan Zhangf5395782020-04-29 14:00:03 +01001079}
1080
Klement Sekerabe5a5dd2018-10-09 16:05:48 +02001081always_inline uword
Neale Rannsf16e9a52021-02-25 19:09:24 +00001082esp_decrypt_inline (vlib_main_t *vm, vlib_node_runtime_t *node,
1083 vlib_frame_t *from_frame, int is_ip6, int is_tun,
1084 u16 async_next_node)
Ed Warnickecb9cada2015-12-08 15:45:58 -07001085{
Ed Warnickecb9cada2015-12-08 15:45:58 -07001086 ipsec_main_t *im = &ipsec_main;
Benoît Ganne98ca76a2022-04-11 18:51:25 +02001087 const u16 *next_by_next_header = im->next_header_registrations;
Damjan Marionb4fff3a2019-03-25 15:54:40 +01001088 u32 thread_index = vm->thread_index;
Damjan Marionb4fff3a2019-03-25 15:54:40 +01001089 u16 len;
1090 ipsec_per_thread_data_t *ptd = vec_elt_at_index (im->ptd, thread_index);
Damjan Marionc98275f2019-03-06 14:05:01 +01001091 u32 *from = vlib_frame_vector_args (from_frame);
Filip Tehlarefcad1a2020-02-04 09:36:04 +00001092 u32 n_left = from_frame->n_vectors;
Damjan Marionb4fff3a2019-03-25 15:54:40 +01001093 vlib_buffer_t *bufs[VLIB_FRAME_SIZE], **b = bufs;
Neale Rannsf16e9a52021-02-25 19:09:24 +00001094 vlib_buffer_t *sync_bufs[VLIB_FRAME_SIZE];
1095 u16 sync_nexts[VLIB_FRAME_SIZE], *sync_next = sync_nexts, n_sync = 0;
Matthew Smith51d56ba2021-06-04 09:18:37 -05001096 u16 async_nexts[VLIB_FRAME_SIZE], *async_next = async_nexts;
Damjan Mariondd298e82022-10-12 16:02:18 +02001097 u16 noop_nexts[VLIB_FRAME_SIZE], n_noop = 0;
Neale Rannsf16e9a52021-02-25 19:09:24 +00001098 u32 sync_bi[VLIB_FRAME_SIZE];
1099 u32 noop_bi[VLIB_FRAME_SIZE];
Damjan Marionb4fff3a2019-03-25 15:54:40 +01001100 esp_decrypt_packet_data_t pkt_data[VLIB_FRAME_SIZE], *pd = pkt_data;
Fan Zhangf5395782020-04-29 14:00:03 +01001101 esp_decrypt_packet_data2_t pkt_data2[VLIB_FRAME_SIZE], *pd2 = pkt_data2;
Damjan Marionb4fff3a2019-03-25 15:54:40 +01001102 esp_decrypt_packet_data_t cpd = { };
1103 u32 current_sa_index = ~0, current_sa_bytes = 0, current_sa_pkts = 0;
1104 const u8 esp_sz = sizeof (esp_header_t);
1105 ipsec_sa_t *sa0 = 0;
Maxime Peim0e2f1882022-12-22 11:26:57 +00001106 bool anti_replay_result;
Fan Zhangf5395782020-04-29 14:00:03 +01001107 int is_async = im->async_mode;
Neale Rannsfc811342021-02-26 10:35:33 +00001108 vnet_crypto_async_op_id_t async_op = ~0;
Neale Rannsfc811342021-02-26 10:35:33 +00001109 vnet_crypto_async_frame_t *async_frames[VNET_CRYPTO_ASYNC_OP_N_IDS];
Neale Rannsf16e9a52021-02-25 19:09:24 +00001110 esp_decrypt_error_t err;
Ed Warnickecb9cada2015-12-08 15:45:58 -07001111
Damjan Marionb4fff3a2019-03-25 15:54:40 +01001112 vlib_get_buffers (vm, from, b, n_left);
Fan Zhangf5395782020-04-29 14:00:03 +01001113 if (!is_async)
1114 {
1115 vec_reset_length (ptd->crypto_ops);
1116 vec_reset_length (ptd->integ_ops);
1117 vec_reset_length (ptd->chained_crypto_ops);
1118 vec_reset_length (ptd->chained_integ_ops);
1119 }
Neale Rannsfc811342021-02-26 10:35:33 +00001120 vec_reset_length (ptd->async_frames);
Filip Tehlarefcad1a2020-02-04 09:36:04 +00001121 vec_reset_length (ptd->chunks);
Neale Rannsf16e9a52021-02-25 19:09:24 +00001122 clib_memset (sync_nexts, -1, sizeof (sync_nexts));
Neale Rannsfc811342021-02-26 10:35:33 +00001123 clib_memset (async_frames, 0, sizeof (async_frames));
Damjan Marionb4fff3a2019-03-25 15:54:40 +01001124
1125 while (n_left > 0)
Keith Burns (alagalah)166a9d42016-08-06 11:00:56 -07001126 {
Damjan Marionb4fff3a2019-03-25 15:54:40 +01001127 u8 *payload;
Ed Warnickecb9cada2015-12-08 15:45:58 -07001128
Neale Rannsf16e9a52021-02-25 19:09:24 +00001129 err = ESP_DECRYPT_ERROR_RX_PKTS;
Damjan Marionb4fff3a2019-03-25 15:54:40 +01001130 if (n_left > 2)
Keith Burns (alagalah)166a9d42016-08-06 11:00:56 -07001131 {
Damjan Marionb4fff3a2019-03-25 15:54:40 +01001132 u8 *p;
1133 vlib_prefetch_buffer_header (b[2], LOAD);
1134 p = vlib_buffer_get_current (b[1]);
Damjan Marionaf7fb042021-07-15 11:54:41 +02001135 clib_prefetch_load (p);
Damjan Marionb4fff3a2019-03-25 15:54:40 +01001136 p -= CLIB_CACHE_LINE_BYTES;
Damjan Marionaf7fb042021-07-15 11:54:41 +02001137 clib_prefetch_load (p);
Damjan Marionb4fff3a2019-03-25 15:54:40 +01001138 }
1139
Filip Tehlarefcad1a2020-02-04 09:36:04 +00001140 u32 n_bufs = vlib_buffer_chain_linearize (vm, b[0]);
1141 if (n_bufs == 0)
Damjan Marionb4fff3a2019-03-25 15:54:40 +01001142 {
Neale Rannsf16e9a52021-02-25 19:09:24 +00001143 err = ESP_DECRYPT_ERROR_NO_BUFFERS;
Arthur de Kerhorad95b062022-11-16 19:12:05 +01001144 esp_decrypt_set_next_index (b[0], node, thread_index, err, n_noop,
1145 noop_nexts, ESP_DECRYPT_NEXT_DROP,
1146 vnet_buffer (b[0])->ipsec.sad_index);
Damjan Marionb4fff3a2019-03-25 15:54:40 +01001147 goto next;
Keith Burns (alagalah)166a9d42016-08-06 11:00:56 -07001148 }
Damjan Marionc98275f2019-03-06 14:05:01 +01001149
Damjan Marionb4fff3a2019-03-25 15:54:40 +01001150 if (vnet_buffer (b[0])->ipsec.sad_index != current_sa_index)
Damjan Marionc98275f2019-03-06 14:05:01 +01001151 {
Damjan Marion867dfdd2019-06-05 15:42:54 +02001152 if (current_sa_pkts)
1153 vlib_increment_combined_counter (&ipsec_sa_counters, thread_index,
Arthur de Kerhorad95b062022-11-16 19:12:05 +01001154 current_sa_index, current_sa_pkts,
Damjan Marion867dfdd2019-06-05 15:42:54 +02001155 current_sa_bytes);
1156 current_sa_bytes = current_sa_pkts = 0;
1157
Damjan Marionb4fff3a2019-03-25 15:54:40 +01001158 current_sa_index = vnet_buffer (b[0])->ipsec.sad_index;
Arthur de Kerhorad95b062022-11-16 19:12:05 +01001159 vlib_prefetch_combined_counter (&ipsec_sa_counters, thread_index,
1160 current_sa_index);
Neale Rannsc5fe57d2021-02-25 16:01:28 +00001161 sa0 = ipsec_sa_get (current_sa_index);
Neale Ranns123b5eb2020-10-16 14:03:55 +00001162
1163 /* fetch the second cacheline ASAP */
Damjan Marionaf7fb042021-07-15 11:54:41 +02001164 clib_prefetch_load (sa0->cacheline1);
Damjan Marion7c22ff72019-04-04 12:25:44 +02001165 cpd.icv_sz = sa0->integ_icv_size;
Damjan Marionb4fff3a2019-03-25 15:54:40 +01001166 cpd.iv_sz = sa0->crypto_iv_size;
1167 cpd.flags = sa0->flags;
1168 cpd.sa_index = current_sa_index;
Neale Rannsf16e9a52021-02-25 19:09:24 +00001169 is_async = im->async_mode | ipsec_sa_is_set_IS_ASYNC (sa0);
Neale Rannsfc811342021-02-26 10:35:33 +00001170 }
Fan Zhangf5395782020-04-29 14:00:03 +01001171
Benoît Ganne5527a782022-01-18 15:56:41 +01001172 if (PREDICT_FALSE ((u16) ~0 == sa0->thread_index))
Neale Rannsf62a8c02019-04-02 08:13:33 +00001173 {
1174 /* this is the first packet to use this SA, claim the SA
1175 * for this thread. this could happen simultaneously on
1176 * another thread */
Neale Ranns1a52d372021-02-04 11:33:32 +00001177 clib_atomic_cmp_and_swap (&sa0->thread_index, ~0,
Neale Rannsf62a8c02019-04-02 08:13:33 +00001178 ipsec_sa_assign_thread (thread_index));
1179 }
1180
Neale Ranns1a52d372021-02-04 11:33:32 +00001181 if (PREDICT_FALSE (thread_index != sa0->thread_index))
Neale Rannsf62a8c02019-04-02 08:13:33 +00001182 {
Neale Rannsaa7d7662021-02-10 08:42:49 +00001183 vnet_buffer (b[0])->ipsec.thread_index = sa0->thread_index;
Neale Rannsf16e9a52021-02-25 19:09:24 +00001184 err = ESP_DECRYPT_ERROR_HANDOFF;
Arthur de Kerhorad95b062022-11-16 19:12:05 +01001185 esp_decrypt_set_next_index (b[0], node, thread_index, err, n_noop,
1186 noop_nexts, ESP_DECRYPT_NEXT_HANDOFF,
1187 current_sa_index);
Neale Rannsf62a8c02019-04-02 08:13:33 +00001188 goto next;
1189 }
1190
Damjan Marionb4fff3a2019-03-25 15:54:40 +01001191 /* store packet data for next round for easier prefetch */
1192 pd->sa_data = cpd.sa_data;
1193 pd->current_data = b[0]->current_data;
Damjan Marionb4fff3a2019-03-25 15:54:40 +01001194 pd->hdr_sz = pd->current_data - vnet_buffer (b[0])->l3_hdr_offset;
1195 payload = b[0]->data + pd->current_data;
Neale Ranns6afaae12019-07-17 15:07:14 +00001196 pd->seq = clib_host_to_net_u32 (((esp_header_t *) payload)->seq);
Fan Zhangf5395782020-04-29 14:00:03 +01001197 pd->is_chain = 0;
1198 pd2->lb = b[0];
1199 pd2->free_buffer_index = 0;
1200 pd2->icv_removed = 0;
Filip Tehlarefcad1a2020-02-04 09:36:04 +00001201
Filip Tehlarefcad1a2020-02-04 09:36:04 +00001202 if (n_bufs > 1)
1203 {
Fan Zhangf5395782020-04-29 14:00:03 +01001204 pd->is_chain = 1;
Filip Tehlarefcad1a2020-02-04 09:36:04 +00001205 /* find last buffer in the chain */
Fan Zhangf5395782020-04-29 14:00:03 +01001206 while (pd2->lb->flags & VLIB_BUFFER_NEXT_PRESENT)
1207 pd2->lb = vlib_get_buffer (vm, pd2->lb->next_buffer);
Benoît Gannee631ece2021-05-27 18:49:42 +02001208 }
Fan Zhangf5395782020-04-29 14:00:03 +01001209
Filip Tehlarefcad1a2020-02-04 09:36:04 +00001210 pd->current_length = b[0]->current_length;
Damjan Marionb4fff3a2019-03-25 15:54:40 +01001211
Damjan Marionb4fff3a2019-03-25 15:54:40 +01001212 /* anti-reply check */
Maxime Peim0e2f1882022-12-22 11:26:57 +00001213 if (PREDICT_FALSE (ipsec_sa_is_set_ANTI_REPLAY_HUGE (sa0)))
1214 {
1215 anti_replay_result = ipsec_sa_anti_replay_and_sn_advance (
1216 sa0, pd->seq, ~0, false, &pd->seq_hi, true);
1217 }
1218 else
1219 {
1220 anti_replay_result = ipsec_sa_anti_replay_and_sn_advance (
1221 sa0, pd->seq, ~0, false, &pd->seq_hi, false);
1222 }
1223
1224 if (anti_replay_result)
Damjan Marionb4fff3a2019-03-25 15:54:40 +01001225 {
Neale Rannsf16e9a52021-02-25 19:09:24 +00001226 err = ESP_DECRYPT_ERROR_REPLAY;
Arthur de Kerhorad95b062022-11-16 19:12:05 +01001227 esp_decrypt_set_next_index (b[0], node, thread_index, err, n_noop,
1228 noop_nexts, ESP_DECRYPT_NEXT_DROP,
1229 current_sa_index);
Damjan Marionb4fff3a2019-03-25 15:54:40 +01001230 goto next;
1231 }
1232
Damjan Mariona829b132019-04-24 23:39:16 +02001233 if (pd->current_length < cpd.icv_sz + esp_sz + cpd.iv_sz)
1234 {
Neale Rannsf16e9a52021-02-25 19:09:24 +00001235 err = ESP_DECRYPT_ERROR_RUNT;
Arthur de Kerhorad95b062022-11-16 19:12:05 +01001236 esp_decrypt_set_next_index (b[0], node, thread_index, err, n_noop,
1237 noop_nexts, ESP_DECRYPT_NEXT_DROP,
1238 current_sa_index);
Damjan Mariona829b132019-04-24 23:39:16 +02001239 goto next;
1240 }
1241
Damjan Marionb4fff3a2019-03-25 15:54:40 +01001242 len = pd->current_length - cpd.icv_sz;
1243 current_sa_pkts += 1;
Filip Tehlarefcad1a2020-02-04 09:36:04 +00001244 current_sa_bytes += vlib_buffer_length_in_chain (vm, b[0]);
Damjan Marionb4fff3a2019-03-25 15:54:40 +01001245
Fan Zhangf5395782020-04-29 14:00:03 +01001246 if (is_async)
Damjan Marionb4fff3a2019-03-25 15:54:40 +01001247 {
Matthew Smith51d56ba2021-06-04 09:18:37 -05001248 async_op = sa0->crypto_async_dec_op_id;
1249
1250 /* get a frame for this op if we don't yet have one or it's full
1251 */
1252 if (NULL == async_frames[async_op] ||
1253 vnet_crypto_async_frame_is_full (async_frames[async_op]))
1254 {
1255 async_frames[async_op] =
1256 vnet_crypto_async_get_frame (vm, async_op);
gaoginskxf441b5d2021-06-07 12:07:01 +01001257 if (PREDICT_FALSE (!async_frames[async_op]))
1258 {
1259 err = ESP_DECRYPT_ERROR_NO_AVAIL_FRAME;
1260 esp_decrypt_set_next_index (
1261 b[0], node, thread_index, err, n_noop, noop_nexts,
1262 ESP_DECRYPT_NEXT_DROP, current_sa_index);
1263 goto next;
1264 }
1265
Matthew Smith51d56ba2021-06-04 09:18:37 -05001266 /* Save the frame to the list we'll submit at the end */
1267 vec_add1 (ptd->async_frames, async_frames[async_op]);
1268 }
Neale Rannsfc811342021-02-26 10:35:33 +00001269
1270 err = esp_decrypt_prepare_async_frame (
Fan Zhanga4df00f2024-03-13 02:49:31 +00001271 vm, ptd, async_frames[async_op], sa0, payload, len, cpd.icv_sz,
1272 cpd.iv_sz, pd, pd2, from[b - bufs], b[0], async_next_node);
Neale Rannsfc811342021-02-26 10:35:33 +00001273 if (ESP_DECRYPT_ERROR_RX_PKTS != err)
Filip Tehlarefcad1a2020-02-04 09:36:04 +00001274 {
Arthur de Kerhorad95b062022-11-16 19:12:05 +01001275 esp_decrypt_set_next_index (
1276 b[0], node, thread_index, err, n_noop, noop_nexts,
1277 ESP_DECRYPT_NEXT_DROP, current_sa_index);
Filip Tehlarefcad1a2020-02-04 09:36:04 +00001278 }
Damjan Marionc98275f2019-03-06 14:05:01 +01001279 }
Fan Zhangf5395782020-04-29 14:00:03 +01001280 else
Fan Zhanga4df00f2024-03-13 02:49:31 +00001281 {
1282 err = esp_decrypt_prepare_sync_op (vm, ptd, sa0, payload, len,
1283 cpd.icv_sz, cpd.iv_sz, pd, pd2,
1284 b[0], n_sync);
1285 if (err != ESP_DECRYPT_ERROR_RX_PKTS)
1286 {
1287 esp_decrypt_set_next_index (b[0], node, thread_index, err, 0,
1288 sync_next, ESP_DECRYPT_NEXT_DROP,
1289 current_sa_index);
1290 }
1291 }
Damjan Marionc98275f2019-03-06 14:05:01 +01001292 /* next */
Damjan Marionb4fff3a2019-03-25 15:54:40 +01001293 next:
Neale Rannsf16e9a52021-02-25 19:09:24 +00001294 if (ESP_DECRYPT_ERROR_RX_PKTS != err)
1295 {
1296 noop_bi[n_noop] = from[b - bufs];
1297 n_noop++;
Neale Rannsf16e9a52021-02-25 19:09:24 +00001298 }
1299 else if (!is_async)
1300 {
1301 sync_bi[n_sync] = from[b - bufs];
1302 sync_bufs[n_sync] = b[0];
1303 n_sync++;
1304 sync_next++;
1305 pd += 1;
1306 pd2 += 1;
1307 }
1308 else
Matthew Smith51d56ba2021-06-04 09:18:37 -05001309 async_next++;
1310
Damjan Marionb4fff3a2019-03-25 15:54:40 +01001311 n_left -= 1;
Damjan Marionb4fff3a2019-03-25 15:54:40 +01001312 b += 1;
Ed Warnickecb9cada2015-12-08 15:45:58 -07001313 }
Damjan Marionc98275f2019-03-06 14:05:01 +01001314
Neale Ranns02950402019-12-20 00:54:57 +00001315 if (PREDICT_TRUE (~0 != current_sa_index))
1316 vlib_increment_combined_counter (&ipsec_sa_counters, thread_index,
1317 current_sa_index, current_sa_pkts,
1318 current_sa_bytes);
Klement Sekerabe5a5dd2018-10-09 16:05:48 +02001319
Matthew Smith51d56ba2021-06-04 09:18:37 -05001320 /* submit or free all of the open frames */
1321 vnet_crypto_async_frame_t **async_frame;
Neale Rannsfc811342021-02-26 10:35:33 +00001322
Matthew Smith51d56ba2021-06-04 09:18:37 -05001323 vec_foreach (async_frame, ptd->async_frames)
1324 {
1325 /* free frame and move on if no ops were successfully added */
1326 if (PREDICT_FALSE (!(*async_frame)->n_elts))
Fan Zhangf5395782020-04-29 14:00:03 +01001327 {
Matthew Smith51d56ba2021-06-04 09:18:37 -05001328 vnet_crypto_async_free_frame (vm, *async_frame);
1329 continue;
1330 }
1331 if (vnet_crypto_async_submit_open_frame (vm, *async_frame) < 0)
1332 {
1333 n_noop += esp_async_recycle_failed_submit (
1334 vm, *async_frame, node, ESP_DECRYPT_ERROR_CRYPTO_ENGINE_ERROR,
Arthur de Kerhorad95b062022-11-16 19:12:05 +01001335 IPSEC_SA_ERROR_CRYPTO_ENGINE_ERROR, n_noop, noop_bi, noop_nexts,
Xiaoming Jiang0c1454c2023-05-05 02:28:20 +00001336 ESP_DECRYPT_NEXT_DROP, false);
Matthew Smith51d56ba2021-06-04 09:18:37 -05001337 vnet_crypto_async_reset_frame (*async_frame);
1338 vnet_crypto_async_free_frame (vm, *async_frame);
Fan Zhangf5395782020-04-29 14:00:03 +01001339 }
Fan Zhangf5395782020-04-29 14:00:03 +01001340 }
Fan Zhangf5395782020-04-29 14:00:03 +01001341
Neale Rannsf16e9a52021-02-25 19:09:24 +00001342 if (n_sync)
1343 {
1344 esp_process_ops (vm, node, ptd->integ_ops, sync_bufs, sync_nexts,
1345 ESP_DECRYPT_ERROR_INTEG_ERROR);
1346 esp_process_chained_ops (vm, node, ptd->chained_integ_ops, sync_bufs,
1347 sync_nexts, ptd->chunks,
1348 ESP_DECRYPT_ERROR_INTEG_ERROR);
1349
1350 esp_process_ops (vm, node, ptd->crypto_ops, sync_bufs, sync_nexts,
Fan Zhangf5395782020-04-29 14:00:03 +01001351 ESP_DECRYPT_ERROR_DECRYPTION_FAILED);
Neale Rannsf16e9a52021-02-25 19:09:24 +00001352 esp_process_chained_ops (vm, node, ptd->chained_crypto_ops, sync_bufs,
1353 sync_nexts, ptd->chunks,
Fan Zhangf5395782020-04-29 14:00:03 +01001354 ESP_DECRYPT_ERROR_DECRYPTION_FAILED);
1355 }
Damjan Marionb4fff3a2019-03-25 15:54:40 +01001356
1357 /* Post decryption ronud - adjust packet data start and length and next
1358 node */
1359
Neale Rannsf16e9a52021-02-25 19:09:24 +00001360 n_left = n_sync;
1361 sync_next = sync_nexts;
Damjan Marionb4fff3a2019-03-25 15:54:40 +01001362 pd = pkt_data;
Fan Zhangf5395782020-04-29 14:00:03 +01001363 pd2 = pkt_data2;
Neale Rannsf16e9a52021-02-25 19:09:24 +00001364 b = sync_bufs;
Damjan Marionb4fff3a2019-03-25 15:54:40 +01001365
1366 while (n_left)
1367 {
Damjan Marionb4fff3a2019-03-25 15:54:40 +01001368 if (n_left >= 2)
1369 {
1370 void *data = b[1]->data + pd[1].current_data;
1371
1372 /* buffer metadata */
1373 vlib_prefetch_buffer_header (b[1], LOAD);
1374
1375 /* esp_footer_t */
1376 CLIB_PREFETCH (data + pd[1].current_length - pd[1].icv_sz - 2,
1377 CLIB_CACHE_LINE_BYTES, LOAD);
1378
1379 /* packet headers */
1380 CLIB_PREFETCH (data - CLIB_CACHE_LINE_BYTES,
1381 CLIB_CACHE_LINE_BYTES * 2, LOAD);
1382 }
1383
Christian Hoppsd570e532020-08-25 12:40:40 -04001384 /* save the sa_index as GRE_teb post_crypto changes L2 opaque */
1385 if (PREDICT_FALSE (b[0]->flags & VLIB_BUFFER_IS_TRACED))
1386 current_sa_index = vnet_buffer (b[0])->ipsec.sad_index;
1387
Neale Rannsf16e9a52021-02-25 19:09:24 +00001388 if (sync_next[0] >= ESP_DECRYPT_N_NEXT)
Benoît Ganne98ca76a2022-04-11 18:51:25 +02001389 esp_decrypt_post_crypto (vm, node, next_by_next_header, pd, pd2, b[0],
1390 sync_next, is_ip6, is_tun, 0);
Damjan Marionb4fff3a2019-03-25 15:54:40 +01001391
Fan Zhangf5395782020-04-29 14:00:03 +01001392 /* trace: */
Damjan Marionb4fff3a2019-03-25 15:54:40 +01001393 if (PREDICT_FALSE (b[0]->flags & VLIB_BUFFER_IS_TRACED))
1394 {
1395 esp_decrypt_trace_t *tr;
Damjan Marionb4fff3a2019-03-25 15:54:40 +01001396 tr = vlib_add_trace (vm, node, b[0], sizeof (*tr));
Neale Rannsc5fe57d2021-02-25 16:01:28 +00001397 sa0 = ipsec_sa_get (current_sa_index);
Damjan Marionb4fff3a2019-03-25 15:54:40 +01001398 tr->crypto_alg = sa0->crypto_alg;
1399 tr->integ_alg = sa0->integ_alg;
Neale Ranns6afaae12019-07-17 15:07:14 +00001400 tr->seq = pd->seq;
Neale Ranns5b891102021-06-28 13:31:28 +00001401 tr->sa_seq = sa0->seq;
Neale Ranns6afaae12019-07-17 15:07:14 +00001402 tr->sa_seq_hi = sa0->seq_hi;
Neale Ranns5b891102021-06-28 13:31:28 +00001403 tr->pkt_seq_hi = pd->seq_hi;
Damjan Marionb4fff3a2019-03-25 15:54:40 +01001404 }
1405
1406 /* next */
1407 n_left -= 1;
Neale Rannsf16e9a52021-02-25 19:09:24 +00001408 sync_next += 1;
Damjan Marionb4fff3a2019-03-25 15:54:40 +01001409 pd += 1;
Fan Zhangf5395782020-04-29 14:00:03 +01001410 pd2 += 1;
Damjan Marionb4fff3a2019-03-25 15:54:40 +01001411 b += 1;
1412 }
1413
Neale Rannsf16e9a52021-02-25 19:09:24 +00001414 vlib_node_increment_counter (vm, node->node_index, ESP_DECRYPT_ERROR_RX_PKTS,
1415 from_frame->n_vectors);
Damjan Marionb4fff3a2019-03-25 15:54:40 +01001416
Neale Rannsf16e9a52021-02-25 19:09:24 +00001417 if (n_sync)
1418 vlib_buffer_enqueue_to_next (vm, node, sync_bi, sync_nexts, n_sync);
Damjan Marionb4fff3a2019-03-25 15:54:40 +01001419
Neale Rannsf16e9a52021-02-25 19:09:24 +00001420 if (n_noop)
1421 vlib_buffer_enqueue_to_next (vm, node, noop_bi, noop_nexts, n_noop);
1422
1423 return (from_frame->n_vectors);
Ed Warnickecb9cada2015-12-08 15:45:58 -07001424}
1425
Fan Zhangf5395782020-04-29 14:00:03 +01001426always_inline uword
1427esp_decrypt_post_inline (vlib_main_t * vm,
1428 vlib_node_runtime_t * node,
1429 vlib_frame_t * from_frame, int is_ip6, int is_tun)
1430{
Benoît Ganne98ca76a2022-04-11 18:51:25 +02001431 const ipsec_main_t *im = &ipsec_main;
1432 const u16 *next_by_next_header = im->next_header_registrations;
Fan Zhangf5395782020-04-29 14:00:03 +01001433 u32 *from = vlib_frame_vector_args (from_frame);
1434 u32 n_left = from_frame->n_vectors;
1435 vlib_buffer_t *bufs[VLIB_FRAME_SIZE], **b = bufs;
1436 u16 nexts[VLIB_FRAME_SIZE], *next = nexts;
1437 vlib_get_buffers (vm, from, b, n_left);
1438
1439 while (n_left > 0)
1440 {
1441 esp_decrypt_packet_data_t *pd = &(esp_post_data (b[0]))->decrypt_data;
1442
1443 if (n_left > 2)
1444 {
1445 vlib_prefetch_buffer_header (b[2], LOAD);
1446 vlib_prefetch_buffer_header (b[1], LOAD);
1447 }
1448
1449 if (!pd->is_chain)
Benoît Ganne98ca76a2022-04-11 18:51:25 +02001450 esp_decrypt_post_crypto (vm, node, next_by_next_header, pd, 0, b[0],
1451 next, is_ip6, is_tun, 1);
Fan Zhangf5395782020-04-29 14:00:03 +01001452 else
1453 {
1454 esp_decrypt_packet_data2_t *pd2 = esp_post_data2 (b[0]);
Benoît Ganne98ca76a2022-04-11 18:51:25 +02001455 esp_decrypt_post_crypto (vm, node, next_by_next_header, pd, pd2,
1456 b[0], next, is_ip6, is_tun, 1);
Fan Zhangf5395782020-04-29 14:00:03 +01001457 }
1458
1459 /*trace: */
1460 if (PREDICT_FALSE (b[0]->flags & VLIB_BUFFER_IS_TRACED))
1461 {
Neale Rannsc5fe57d2021-02-25 16:01:28 +00001462 ipsec_sa_t *sa0 = ipsec_sa_get (pd->sa_index);
Fan Zhangf5395782020-04-29 14:00:03 +01001463 esp_decrypt_trace_t *tr;
1464 esp_decrypt_packet_data_t *async_pd =
1465 &(esp_post_data (b[0]))->decrypt_data;
1466 tr = vlib_add_trace (vm, node, b[0], sizeof (*tr));
Neale Rannsc5fe57d2021-02-25 16:01:28 +00001467 sa0 = ipsec_sa_get (async_pd->sa_index);
Fan Zhangf5395782020-04-29 14:00:03 +01001468
1469 tr->crypto_alg = sa0->crypto_alg;
1470 tr->integ_alg = sa0->integ_alg;
1471 tr->seq = pd->seq;
Neale Ranns5b891102021-06-28 13:31:28 +00001472 tr->sa_seq = sa0->seq;
Fan Zhangf5395782020-04-29 14:00:03 +01001473 tr->sa_seq_hi = sa0->seq_hi;
1474 }
1475
1476 n_left--;
1477 next++;
1478 b++;
1479 }
1480
1481 n_left = from_frame->n_vectors;
1482 vlib_node_increment_counter (vm, node->node_index,
1483 ESP_DECRYPT_ERROR_RX_POST_PKTS, n_left);
1484
1485 vlib_buffer_enqueue_to_next (vm, node, from, nexts, n_left);
1486
1487 return n_left;
1488}
1489
Klement Sekerab8f35442018-10-29 13:38:19 +01001490VLIB_NODE_FN (esp4_decrypt_node) (vlib_main_t * vm,
1491 vlib_node_runtime_t * node,
1492 vlib_frame_t * from_frame)
Klement Sekerabe5a5dd2018-10-09 16:05:48 +02001493{
Fan Zhangf5395782020-04-29 14:00:03 +01001494 return esp_decrypt_inline (vm, node, from_frame, 0, 0,
1495 esp_decrypt_async_next.esp4_post_next);
1496}
1497
1498VLIB_NODE_FN (esp4_decrypt_post_node) (vlib_main_t * vm,
1499 vlib_node_runtime_t * node,
1500 vlib_frame_t * from_frame)
1501{
1502 return esp_decrypt_post_inline (vm, node, from_frame, 0, 0);
Neale Rannsc87b66c2019-02-07 07:26:12 -08001503}
1504
1505VLIB_NODE_FN (esp4_decrypt_tun_node) (vlib_main_t * vm,
1506 vlib_node_runtime_t * node,
1507 vlib_frame_t * from_frame)
1508{
Fan Zhangf5395782020-04-29 14:00:03 +01001509 return esp_decrypt_inline (vm, node, from_frame, 0, 1,
1510 esp_decrypt_async_next.esp4_tun_post_next);
1511}
1512
1513VLIB_NODE_FN (esp4_decrypt_tun_post_node) (vlib_main_t * vm,
1514 vlib_node_runtime_t * node,
1515 vlib_frame_t * from_frame)
1516{
1517 return esp_decrypt_post_inline (vm, node, from_frame, 0, 1);
Neale Rannsc87b66c2019-02-07 07:26:12 -08001518}
1519
1520VLIB_NODE_FN (esp6_decrypt_node) (vlib_main_t * vm,
1521 vlib_node_runtime_t * node,
1522 vlib_frame_t * from_frame)
1523{
Fan Zhangf5395782020-04-29 14:00:03 +01001524 return esp_decrypt_inline (vm, node, from_frame, 1, 0,
1525 esp_decrypt_async_next.esp6_post_next);
1526}
1527
1528VLIB_NODE_FN (esp6_decrypt_post_node) (vlib_main_t * vm,
1529 vlib_node_runtime_t * node,
1530 vlib_frame_t * from_frame)
1531{
1532 return esp_decrypt_post_inline (vm, node, from_frame, 1, 0);
Neale Rannsc87b66c2019-02-07 07:26:12 -08001533}
1534
1535VLIB_NODE_FN (esp6_decrypt_tun_node) (vlib_main_t * vm,
1536 vlib_node_runtime_t * node,
1537 vlib_frame_t * from_frame)
1538{
Fan Zhangf5395782020-04-29 14:00:03 +01001539 return esp_decrypt_inline (vm, node, from_frame, 1, 1,
1540 esp_decrypt_async_next.esp6_tun_post_next);
Klement Sekerabe5a5dd2018-10-09 16:05:48 +02001541}
Ed Warnickecb9cada2015-12-08 15:45:58 -07001542
Fan Zhangf5395782020-04-29 14:00:03 +01001543VLIB_NODE_FN (esp6_decrypt_tun_post_node) (vlib_main_t * vm,
1544 vlib_node_runtime_t * node,
1545 vlib_frame_t * from_frame)
1546{
1547 return esp_decrypt_post_inline (vm, node, from_frame, 1, 1);
1548}
1549
Klement Sekerabe5a5dd2018-10-09 16:05:48 +02001550VLIB_REGISTER_NODE (esp4_decrypt_node) = {
Klement Sekerabe5a5dd2018-10-09 16:05:48 +02001551 .name = "esp4-decrypt",
Ed Warnickecb9cada2015-12-08 15:45:58 -07001552 .vector_size = sizeof (u32),
1553 .format_trace = format_esp_decrypt_trace,
1554 .type = VLIB_NODE_TYPE_INTERNAL,
1555
Neale Ranns93688d72022-08-09 03:34:51 +00001556 .n_errors = ESP_DECRYPT_N_ERROR,
1557 .error_counters = esp_decrypt_error_counters,
Ed Warnickecb9cada2015-12-08 15:45:58 -07001558
1559 .n_next_nodes = ESP_DECRYPT_N_NEXT,
1560 .next_nodes = {
Neale Rannsf62a8c02019-04-02 08:13:33 +00001561 [ESP_DECRYPT_NEXT_DROP] = "ip4-drop",
1562 [ESP_DECRYPT_NEXT_IP4_INPUT] = "ip4-input-no-checksum",
1563 [ESP_DECRYPT_NEXT_IP6_INPUT] = "ip6-input",
Neale Ranns4a58e492020-12-21 13:19:10 +00001564 [ESP_DECRYPT_NEXT_MPLS_INPUT] = "mpls-drop",
Neale Ranns568acbb2019-12-18 05:54:40 +00001565 [ESP_DECRYPT_NEXT_L2_INPUT] = "l2-input",
Neale Rannsf62a8c02019-04-02 08:13:33 +00001566 [ESP_DECRYPT_NEXT_HANDOFF] = "esp4-decrypt-handoff",
Ed Warnickecb9cada2015-12-08 15:45:58 -07001567 },
1568};
1569
Fan Zhangf5395782020-04-29 14:00:03 +01001570VLIB_REGISTER_NODE (esp4_decrypt_post_node) = {
1571 .name = "esp4-decrypt-post",
1572 .vector_size = sizeof (u32),
1573 .format_trace = format_esp_decrypt_trace,
1574 .type = VLIB_NODE_TYPE_INTERNAL,
1575
Neale Ranns93688d72022-08-09 03:34:51 +00001576 .n_errors = ESP_DECRYPT_N_ERROR,
1577 .error_counters = esp_decrypt_error_counters,
Fan Zhangf5395782020-04-29 14:00:03 +01001578
1579 .sibling_of = "esp4-decrypt",
1580};
1581
Klement Sekerabe5a5dd2018-10-09 16:05:48 +02001582VLIB_REGISTER_NODE (esp6_decrypt_node) = {
Klement Sekerabe5a5dd2018-10-09 16:05:48 +02001583 .name = "esp6-decrypt",
1584 .vector_size = sizeof (u32),
1585 .format_trace = format_esp_decrypt_trace,
1586 .type = VLIB_NODE_TYPE_INTERNAL,
1587
Neale Ranns93688d72022-08-09 03:34:51 +00001588 .n_errors = ESP_DECRYPT_N_ERROR,
1589 .error_counters = esp_decrypt_error_counters,
Klement Sekerabe5a5dd2018-10-09 16:05:48 +02001590
1591 .n_next_nodes = ESP_DECRYPT_N_NEXT,
1592 .next_nodes = {
Neale Rannsf62a8c02019-04-02 08:13:33 +00001593 [ESP_DECRYPT_NEXT_DROP] = "ip6-drop",
1594 [ESP_DECRYPT_NEXT_IP4_INPUT] = "ip4-input-no-checksum",
1595 [ESP_DECRYPT_NEXT_IP6_INPUT] = "ip6-input",
Neale Ranns4a58e492020-12-21 13:19:10 +00001596 [ESP_DECRYPT_NEXT_MPLS_INPUT] = "mpls-drop",
Neale Ranns568acbb2019-12-18 05:54:40 +00001597 [ESP_DECRYPT_NEXT_L2_INPUT] = "l2-input",
Neale Rannsf62a8c02019-04-02 08:13:33 +00001598 [ESP_DECRYPT_NEXT_HANDOFF]= "esp6-decrypt-handoff",
Klement Sekerabe5a5dd2018-10-09 16:05:48 +02001599 },
1600};
Neale Rannsc87b66c2019-02-07 07:26:12 -08001601
Fan Zhangf5395782020-04-29 14:00:03 +01001602VLIB_REGISTER_NODE (esp6_decrypt_post_node) = {
1603 .name = "esp6-decrypt-post",
1604 .vector_size = sizeof (u32),
1605 .format_trace = format_esp_decrypt_trace,
1606 .type = VLIB_NODE_TYPE_INTERNAL,
1607
Neale Ranns93688d72022-08-09 03:34:51 +00001608 .n_errors = ESP_DECRYPT_N_ERROR,
1609 .error_counters = esp_decrypt_error_counters,
Fan Zhangf5395782020-04-29 14:00:03 +01001610
1611 .sibling_of = "esp6-decrypt",
1612};
1613
Neale Rannsc87b66c2019-02-07 07:26:12 -08001614VLIB_REGISTER_NODE (esp4_decrypt_tun_node) = {
1615 .name = "esp4-decrypt-tun",
1616 .vector_size = sizeof (u32),
1617 .format_trace = format_esp_decrypt_trace,
1618 .type = VLIB_NODE_TYPE_INTERNAL,
Neale Ranns93688d72022-08-09 03:34:51 +00001619 .n_errors = ESP_DECRYPT_N_ERROR,
1620 .error_counters = esp_decrypt_error_counters,
Neale Rannsf62a8c02019-04-02 08:13:33 +00001621 .n_next_nodes = ESP_DECRYPT_N_NEXT,
1622 .next_nodes = {
1623 [ESP_DECRYPT_NEXT_DROP] = "ip4-drop",
1624 [ESP_DECRYPT_NEXT_IP4_INPUT] = "ip4-input-no-checksum",
1625 [ESP_DECRYPT_NEXT_IP6_INPUT] = "ip6-input",
Neale Ranns4a58e492020-12-21 13:19:10 +00001626 [ESP_DECRYPT_NEXT_MPLS_INPUT] = "mpls-input",
Neale Ranns568acbb2019-12-18 05:54:40 +00001627 [ESP_DECRYPT_NEXT_L2_INPUT] = "l2-input",
Neale Ranns4a56f4e2019-12-23 04:10:25 +00001628 [ESP_DECRYPT_NEXT_HANDOFF] = "esp4-decrypt-tun-handoff",
Neale Rannsf62a8c02019-04-02 08:13:33 +00001629 },
Neale Rannsc87b66c2019-02-07 07:26:12 -08001630};
1631
Fan Zhangf5395782020-04-29 14:00:03 +01001632VLIB_REGISTER_NODE (esp4_decrypt_tun_post_node) = {
1633 .name = "esp4-decrypt-tun-post",
1634 .vector_size = sizeof (u32),
1635 .format_trace = format_esp_decrypt_trace,
1636 .type = VLIB_NODE_TYPE_INTERNAL,
1637
Neale Ranns93688d72022-08-09 03:34:51 +00001638 .n_errors = ESP_DECRYPT_N_ERROR,
1639 .error_counters = esp_decrypt_error_counters,
Fan Zhangf5395782020-04-29 14:00:03 +01001640
1641 .sibling_of = "esp4-decrypt-tun",
1642};
1643
Neale Rannsc87b66c2019-02-07 07:26:12 -08001644VLIB_REGISTER_NODE (esp6_decrypt_tun_node) = {
1645 .name = "esp6-decrypt-tun",
1646 .vector_size = sizeof (u32),
1647 .format_trace = format_esp_decrypt_trace,
1648 .type = VLIB_NODE_TYPE_INTERNAL,
Neale Ranns93688d72022-08-09 03:34:51 +00001649 .n_errors = ESP_DECRYPT_N_ERROR,
1650 .error_counters = esp_decrypt_error_counters,
Neale Rannsf62a8c02019-04-02 08:13:33 +00001651 .n_next_nodes = ESP_DECRYPT_N_NEXT,
1652 .next_nodes = {
1653 [ESP_DECRYPT_NEXT_DROP] = "ip6-drop",
1654 [ESP_DECRYPT_NEXT_IP4_INPUT] = "ip4-input-no-checksum",
1655 [ESP_DECRYPT_NEXT_IP6_INPUT] = "ip6-input",
Neale Ranns4a58e492020-12-21 13:19:10 +00001656 [ESP_DECRYPT_NEXT_MPLS_INPUT] = "mpls-input",
Neale Ranns568acbb2019-12-18 05:54:40 +00001657 [ESP_DECRYPT_NEXT_L2_INPUT] = "l2-input",
Neale Ranns4a56f4e2019-12-23 04:10:25 +00001658 [ESP_DECRYPT_NEXT_HANDOFF]= "esp6-decrypt-tun-handoff",
Neale Rannsf62a8c02019-04-02 08:13:33 +00001659 },
Neale Rannsc87b66c2019-02-07 07:26:12 -08001660};
Fan Zhangf5395782020-04-29 14:00:03 +01001661
1662VLIB_REGISTER_NODE (esp6_decrypt_tun_post_node) = {
1663 .name = "esp6-decrypt-tun-post",
1664 .vector_size = sizeof (u32),
1665 .format_trace = format_esp_decrypt_trace,
1666 .type = VLIB_NODE_TYPE_INTERNAL,
1667
Neale Ranns93688d72022-08-09 03:34:51 +00001668 .n_errors = ESP_DECRYPT_N_ERROR,
1669 .error_counters = esp_decrypt_error_counters,
Fan Zhangf5395782020-04-29 14:00:03 +01001670
1671 .sibling_of = "esp6-decrypt-tun",
1672};
Klement Sekerabe5a5dd2018-10-09 16:05:48 +02001673
Neale Ranns2d498302021-02-25 08:38:58 +00001674#ifndef CLIB_MARCH_VARIANT
1675
1676static clib_error_t *
1677esp_decrypt_init (vlib_main_t *vm)
1678{
1679 ipsec_main_t *im = &ipsec_main;
1680
1681 im->esp4_dec_fq_index =
1682 vlib_frame_queue_main_init (esp4_decrypt_node.index, 0);
1683 im->esp6_dec_fq_index =
1684 vlib_frame_queue_main_init (esp6_decrypt_node.index, 0);
1685 im->esp4_dec_tun_fq_index =
1686 vlib_frame_queue_main_init (esp4_decrypt_tun_node.index, 0);
1687 im->esp6_dec_tun_fq_index =
1688 vlib_frame_queue_main_init (esp6_decrypt_tun_node.index, 0);
1689
1690 return 0;
1691}
1692
1693VLIB_INIT_FUNCTION (esp_decrypt_init);
1694
1695#endif
1696
Keith Burns (alagalah)166a9d42016-08-06 11:00:56 -07001697/*
1698 * fd.io coding-style-patch-verification: ON
1699 *
1700 * Local Variables:
1701 * eval: (c-set-style "gnu")
1702 * End:
1703 */