diff --git a/src/vnet/ipsec/ipsec_if.c b/src/vnet/ipsec/ipsec_if.c
index 562f40e..43997bc 100644
--- a/src/vnet/ipsec/ipsec_if.c
+++ b/src/vnet/ipsec/ipsec_if.c
@@ -284,7 +284,7 @@
 
   if (!is_ip6)
     {
-      key4.remote_ip = args->remote_ip.ip4.as_u32;
+      key4.remote_ip.as_u32 = args->remote_ip.ip4.as_u32;
       key4.spi = clib_host_to_net_u32 (args->remote_spi);
       p = hash_get (im->ipsec4_if_pool_index_by_key, key4.as_u64);
     }
@@ -375,8 +375,14 @@
 	hash_set_mem_alloc (&im->ipsec6_if_pool_index_by_key, &key6,
 			    t - im->tunnel_interfaces);
       else
-	hash_set (im->ipsec4_if_pool_index_by_key, key4.as_u64,
-		  t - im->tunnel_interfaces);
+	{
+	  hash_set (im->ipsec4_if_pool_index_by_key, key4.as_u64,
+		    t - im->tunnel_interfaces);
+	  if (1 == hash_elts (im->ipsec4_if_pool_index_by_key))
+	    udp_register_dst_port (vlib_get_main (),
+				   UDP_DST_PORT_ipsec,
+				   ipsec4_if_input_node.index, 1);
+	}
 
       hw_if_index = vnet_register_interface (vnm, ipsec_device_class.index,
 					     t - im->tunnel_interfaces,
@@ -427,9 +433,13 @@
       if (is_ip6)
 	hash_unset_mem_free (&im->ipsec6_if_pool_index_by_key, &key6);
       else
-	hash_unset (im->ipsec4_if_pool_index_by_key, key4.as_u64);
-
+	{
+	  hash_unset (im->ipsec4_if_pool_index_by_key, key4.as_u64);
+	  if (0 == hash_elts (im->ipsec4_if_pool_index_by_key))
+	    udp_unregister_dst_port (vlib_get_main (), UDP_DST_PORT_ipsec, 1);
+	}
       hash_unset (im->ipsec_if_real_dev_by_show_dev, t->show_instance);
+
       im->ipsec_if_by_sw_if_index[t->sw_if_index] = ~0;
 
       /* delete input and output SA */
@@ -506,7 +516,7 @@
 	  ipsec4_tunnel_key_t key;
 
 	  /* unset old inbound hash entry. packets should stop arriving */
-	  key.remote_ip = old_sa->tunnel_src_addr.ip4.as_u32;
+	  key.remote_ip.as_u32 = old_sa->tunnel_src_addr.ip4.as_u32;
 	  key.spi = clib_host_to_net_u32 (old_sa->spi);
 
 	  p = hash_get (im->ipsec4_if_pool_index_by_key, key.as_u64);
@@ -515,7 +525,7 @@
 
 	  /* set new inbound SA, then set new hash entry */
 	  t->input_sa_index = sa_index;
-	  key.remote_ip = sa->tunnel_src_addr.ip4.as_u32;
+	  key.remote_ip.as_u32 = sa->tunnel_src_addr.ip4.as_u32;
 	  key.spi = clib_host_to_net_u32 (sa->spi);
 
 	  hash_set (im->ipsec4_if_pool_index_by_key, key.as_u64,
@@ -572,9 +582,6 @@
 						     sizeof (uword));
   im->ipsec_if_real_dev_by_show_dev = hash_create (0, sizeof (uword));
 
-  udp_register_dst_port (vm, UDP_DST_PORT_ipsec, ipsec4_if_input_node.index,
-			 1);
-
   /* set up feature nodes to drop outbound packets with no crypto alg set */
   ipsec_add_feature ("ip4-output", "esp4-no-crypto",
 		     &im->esp4_no_crypto_tun_feature_index);