Mohsin Kazmi | b7e4e6d | 2021-12-13 18:32:42 +0000 | [diff] [blame] | 1 | /* |
| 2 | *------------------------------------------------------------------ |
| 3 | * Copyright (c) 2021 Cisco and/or its affiliates. |
| 4 | * Licensed under the Apache License, Version 2.0 (the "License"); |
| 5 | * you may not use this file except in compliance with the License. |
| 6 | * You may obtain a copy of the License at: |
| 7 | * |
| 8 | * http://www.apache.org/licenses/LICENSE-2.0 |
| 9 | * |
| 10 | * Unless required by applicable law or agreed to in writing, software |
| 11 | * distributed under the License is distributed on an "AS IS" BASIS, |
| 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. |
| 13 | * See the License for the specific language governing permissions and |
| 14 | * limitations under the License. |
| 15 | *------------------------------------------------------------------ |
| 16 | */ |
| 17 | |
| 18 | #include <vlib/vlib.h> |
| 19 | #include <vnet/gso/gro_func.h> |
| 20 | #include <vnet/interface/tx_queue_funcs.h> |
| 21 | #include <vnet/devices/virtio/virtio.h> |
| 22 | #include <vnet/devices/virtio/virtio_inline.h> |
| 23 | |
| 24 | static_always_inline uword |
Mohsin Kazmi | 0f8912f | 2022-02-01 18:35:59 +0000 | [diff] [blame] | 25 | virtio_pre_input_inline (vlib_main_t *vm, vnet_virtio_vring_t *txq_vring, |
Mohsin Kazmi | b7e4e6d | 2021-12-13 18:32:42 +0000 | [diff] [blame] | 26 | vnet_hw_if_tx_queue_t *txq, u8 packet_coalesce, |
| 27 | u8 packet_buffering) |
| 28 | { |
| 29 | if (txq->shared_queue) |
| 30 | { |
| 31 | if (clib_spinlock_trylock (&txq_vring->lockp)) |
| 32 | { |
| 33 | if (virtio_txq_is_scheduled (txq_vring)) |
Mohsin Kazmi | 049dee9 | 2022-10-20 12:53:28 +0000 | [diff] [blame^] | 34 | goto unlock; |
Mohsin Kazmi | b7e4e6d | 2021-12-13 18:32:42 +0000 | [diff] [blame] | 35 | if (packet_coalesce) |
| 36 | vnet_gro_flow_table_schedule_node_on_dispatcher ( |
| 37 | vm, txq, txq_vring->flow_table); |
| 38 | else if (packet_buffering) |
| 39 | virtio_vring_buffering_schedule_node_on_dispatcher ( |
| 40 | vm, txq, txq_vring->buffering); |
| 41 | virtio_txq_set_scheduled (txq_vring); |
Mohsin Kazmi | 049dee9 | 2022-10-20 12:53:28 +0000 | [diff] [blame^] | 42 | unlock: |
Mohsin Kazmi | b7e4e6d | 2021-12-13 18:32:42 +0000 | [diff] [blame] | 43 | clib_spinlock_unlock (&txq_vring->lockp); |
| 44 | } |
| 45 | } |
| 46 | else |
| 47 | { |
| 48 | if (packet_coalesce) |
| 49 | vnet_gro_flow_table_schedule_node_on_dispatcher ( |
| 50 | vm, txq, txq_vring->flow_table); |
| 51 | else if (packet_buffering) |
| 52 | virtio_vring_buffering_schedule_node_on_dispatcher ( |
| 53 | vm, txq, txq_vring->buffering); |
| 54 | } |
| 55 | return 0; |
| 56 | } |
| 57 | |
| 58 | static uword |
| 59 | virtio_pre_input (vlib_main_t *vm, vlib_node_runtime_t *node, |
| 60 | vlib_frame_t *frame) |
| 61 | { |
| 62 | virtio_main_t *vim = &virtio_main; |
| 63 | vnet_main_t *vnm = vnet_get_main (); |
| 64 | virtio_if_t *vif; |
| 65 | |
| 66 | pool_foreach (vif, vim->interfaces) |
| 67 | { |
| 68 | if (vif->packet_coalesce || vif->packet_buffering) |
| 69 | { |
Mohsin Kazmi | 0f8912f | 2022-02-01 18:35:59 +0000 | [diff] [blame] | 70 | vnet_virtio_vring_t *txq_vring; |
Mohsin Kazmi | b7e4e6d | 2021-12-13 18:32:42 +0000 | [diff] [blame] | 71 | vec_foreach (txq_vring, vif->txq_vrings) |
| 72 | { |
| 73 | vnet_hw_if_tx_queue_t *txq = |
| 74 | vnet_hw_if_get_tx_queue (vnm, txq_vring->queue_index); |
| 75 | if (clib_bitmap_get (txq->threads, vm->thread_index) == 1) |
| 76 | virtio_pre_input_inline (vm, txq_vring, txq, |
| 77 | vif->packet_coalesce, |
| 78 | vif->packet_buffering); |
| 79 | } |
| 80 | } |
| 81 | } |
| 82 | |
| 83 | return 0; |
| 84 | } |
| 85 | |
| 86 | /** |
| 87 | * virtio interfaces support packet coalescing and buffering which |
| 88 | * depends on timer expiry to flush the stored packets periodically. |
| 89 | * Previously, virtio input node checked timer expiry and scheduled |
| 90 | * tx queue accordingly. |
| 91 | * |
| 92 | * In poll mode, timer expiry was handled naturally, as input node |
| 93 | * runs periodically. In interrupt mode, virtio input node was dependent |
| 94 | * on the interrupts send from backend. Stored packets could starve, |
| 95 | * if there would not be interrupts to input node. |
| 96 | * |
| 97 | * This problem had been solved through a dedicated process node which |
| 98 | * periodically sends interrupt to virtio input node given coalescing |
| 99 | * or buffering feature were enabled on an interface. |
| 100 | * |
| 101 | * But that approach worked with following limitations: |
| 102 | * 1) Each VPP thread should have (atleast) 1 rx queue of an interface |
| 103 | * (with buffering enabled). And rxqs and txqs should be placed on the |
| 104 | * same thread. |
| 105 | * |
| 106 | * New design provides solution to above problem(s) without any limitation |
| 107 | * through (dedicated) pre-input node running on each VPP thread when |
| 108 | * atleast 1 virtio interface is enabled with coalescing or buffering. |
| 109 | */ |
| 110 | VLIB_REGISTER_NODE (virtio_pre_input_node) = { |
| 111 | .function = virtio_pre_input, |
| 112 | .type = VLIB_NODE_TYPE_PRE_INPUT, |
| 113 | .name = "virtio-pre-input", |
| 114 | .state = VLIB_NODE_STATE_DISABLED, |
| 115 | }; |
| 116 | |
| 117 | void |
| 118 | virtio_pre_input_node_enable (vlib_main_t *vm, virtio_if_t *vif) |
| 119 | { |
| 120 | virtio_main_t *vim = &virtio_main; |
| 121 | if (vif->packet_coalesce || vif->packet_buffering) |
| 122 | { |
| 123 | vim->gro_or_buffering_if_count++; |
| 124 | if (vim->gro_or_buffering_if_count == 1) |
| 125 | { |
| 126 | foreach_vlib_main () |
| 127 | { |
| 128 | vlib_node_set_state (this_vlib_main, virtio_pre_input_node.index, |
| 129 | VLIB_NODE_STATE_POLLING); |
| 130 | } |
| 131 | } |
| 132 | } |
| 133 | } |
| 134 | |
| 135 | void |
| 136 | virtio_pre_input_node_disable (vlib_main_t *vm, virtio_if_t *vif) |
| 137 | { |
| 138 | virtio_main_t *vim = &virtio_main; |
| 139 | if (vif->packet_coalesce || vif->packet_buffering) |
| 140 | { |
| 141 | if (vim->gro_or_buffering_if_count > 0) |
| 142 | vim->gro_or_buffering_if_count--; |
| 143 | if (vim->gro_or_buffering_if_count == 0) |
| 144 | { |
| 145 | foreach_vlib_main () |
| 146 | { |
| 147 | vlib_node_set_state (this_vlib_main, virtio_pre_input_node.index, |
| 148 | VLIB_NODE_STATE_DISABLED); |
| 149 | } |
| 150 | } |
| 151 | } |
| 152 | } |
| 153 | |
| 154 | /* |
| 155 | * fd.io coding-style-patch-verification: ON |
| 156 | * |
| 157 | * Local Variables: |
| 158 | * eval: (c-set-style "gnu") |
| 159 | * End: |
| 160 | */ |