Fixes for subscription merge release cases
 - Code fixes
 - UT cases fixes and improvements

Change-Id: I97c295d51029888b41ae0dacac76d366eaeb3c54
Signed-off-by: Anssi Mannila <anssi.mannila@nokia.com>
diff --git a/pkg/control/control.go b/pkg/control/control.go
index 4c09ee2..68688e2 100755
--- a/pkg/control/control.go
+++ b/pkg/control/control.go
@@ -25,6 +25,7 @@
 	"os"
 	"strconv"
 	"strings"
+	"sync"
 	"time"
 
 	"gerrit.o-ran-sc.org/r/ric-plt/e2ap/pkg/e2ap"
@@ -213,6 +214,8 @@
 //
 //-------------------------------------------------------------------
 func (c *Control) ReadRESTSubscriptions() error {
+
+	xapp.Logger.Debug("ReadRESTSubscriptions()")
 	var err error
 	var restSubscriptions map[string]*RESTSubscription
 	for i := 0; dbRetryForever == "true" || i < dbTryCount; i++ {
@@ -222,6 +225,12 @@
 			xapp.Logger.Error("%v", err)
 			<-time.After(1 * time.Second)
 		} else {
+			// Fix REST subscriptions ongoing status after restart
+			for restSubId, restSubscription := range restSubscriptions {
+				restSubscription.SubReqOngoing = false
+				restSubscription.SubDelReqOngoing = false
+				c.WriteRESTSubscriptionToSdl(restSubId, restSubscription)
+			}
 			c.registry.restSubscriptions = restSubscriptions
 			return nil
 		}
@@ -522,7 +531,7 @@
 	clientEndpoint *models.SubscriptionParamsClientEndpoint, meid *string, restSubId *string, xAppRmrEndpoint string, md5sum string, e2SubscriptionDirectives *E2SubscriptionDirectives) {
 
 	c.SubscriptionProcessingStartDelay()
-	xapp.Logger.Debug("E2 SubscriptionRequest count =%v ", len(subReqList.E2APSubscriptionRequests))
+	xapp.Logger.Debug("E2 SubscriptionRequest count = %v ", len(subReqList.E2APSubscriptionRequests))
 
 	var xAppEventInstanceID int64
 	var e2EventInstanceID int64
@@ -551,6 +560,10 @@
 		trans.Release()
 
 		if err != nil {
+			if err.Error() == "TEST: restart event received" {
+				// This is just for UT cases. Stop here subscription processing
+				return
+			}
 			c.sendUnsuccesfullResponseNotification(restSubId, restSubscription, xAppEventInstanceID, err, clientEndpoint, trans, errorInfo)
 		} else {
 			e2EventInstanceID = (int64)(subRespMsg.RequestId.InstanceId)
@@ -601,7 +614,7 @@
 	// Wake subs request
 	//
 	subs.OngoingReqCount++
-	go c.handleSubscriptionCreate(subs, trans, e2SubscriptionDirectives)
+	go c.handleSubscriptionCreate(subs, trans, e2SubscriptionDirectives, 0)
 	event, _ := trans.WaitEvent(0) //blocked wait as timeout is handled in subs side
 	subs.OngoingReqCount--
 
@@ -627,6 +640,10 @@
 		case *SDLWriteErrortEvent:
 			err = fmt.Errorf("SDL write failure")
 			errorInfo = themsg.ErrorInfo
+		case *SubmgrRestartTestEvent:
+			err = fmt.Errorf("TEST: restart event received")
+			xapp.Logger.Debug("%s", err)
+			return nil, &errorInfo, err
 		default:
 			err = fmt.Errorf("Unexpected E2 subscription response received")
 			errorInfo.SetInfo(err.Error(), models.SubscriptionInstanceErrorSourceE2Node, "")
@@ -759,7 +776,7 @@
 	go func() {
 		xapp.Logger.Debug("Deleteting handler: processing instances = %v", restSubscription.InstanceIds)
 		for _, instanceId := range restSubscription.InstanceIds {
-			xAppEventInstanceID, err := c.SubscriptionDeleteHandler(&restSubId, &xAppRmrEndPoint, &restSubscription.Meid, instanceId)
+			xAppEventInstanceID, err := c.SubscriptionDeleteHandler(&restSubId, &xAppRmrEndPoint, &restSubscription.Meid, instanceId, 0)
 
 			if err != nil {
 				xapp.Logger.Error("%s", err.Error())
@@ -780,7 +797,7 @@
 //-------------------------------------------------------------------
 //
 //-------------------------------------------------------------------
-func (c *Control) SubscriptionDeleteHandler(restSubId *string, endPoint *string, meid *string, instanceId uint32) (int64, error) {
+func (c *Control) SubscriptionDeleteHandler(restSubId *string, endPoint *string, meid *string, instanceId uint32, waitRouteCleanupTime time.Duration) (int64, error) {
 
 	var xAppEventInstanceID int64
 	subs, err := c.registry.GetSubscriptionFirstMatch([]uint32{instanceId})
@@ -808,7 +825,7 @@
 	// Wake subs delete
 	//
 	subs.OngoingDelCount++
-	go c.handleSubscriptionDelete(subs, trans)
+	go c.handleSubscriptionDelete(subs, trans, waitRouteCleanupTime)
 	trans.WaitEvent(0) //blocked wait as timeout is handled in subs side
 	subs.OngoingDelCount--
 
@@ -990,7 +1007,7 @@
 
 	e2SubscriptionDirectives, _ := c.GetE2SubscriptionDirectives(nil)
 	subs.OngoingReqCount++
-	go c.handleSubscriptionCreate(subs, trans, e2SubscriptionDirectives)
+	go c.handleSubscriptionCreate(subs, trans, e2SubscriptionDirectives, waitRouteCleanup_ms)
 	event, _ := trans.WaitEvent(0) //blocked wait as timeout is handled in subs side
 	subs.OngoingReqCount--
 	var err error
@@ -1061,7 +1078,7 @@
 	// Wake subs delete
 	//
 	subs.OngoingDelCount++
-	go c.handleSubscriptionDelete(subs, trans)
+	go c.handleSubscriptionDelete(subs, trans, waitRouteCleanup_ms)
 	trans.WaitEvent(0) //blocked wait as timeout is handled in subs side
 	subs.OngoingDelCount--
 
@@ -1091,7 +1108,7 @@
 //-------------------------------------------------------------------
 // SUBS CREATE Handling
 //-------------------------------------------------------------------
-func (c *Control) handleSubscriptionCreate(subs *Subscription, parentTrans *TransactionXapp, e2SubscriptionDirectives *E2SubscriptionDirectives) {
+func (c *Control) handleSubscriptionCreate(subs *Subscription, parentTrans *TransactionXapp, e2SubscriptionDirectives *E2SubscriptionDirectives, waitRouteCleanupTime time.Duration) {
 
 	var event interface{} = nil
 	var removeSubscriptionFromDb bool = false
@@ -1110,38 +1127,43 @@
 			subRfMsg, valid = subs.SetCachedResponse(event, true)
 			subs.SubRespRcvd = true
 		case *e2ap.E2APSubscriptionFailure:
-			removeSubscriptionFromDb = true
 			subRfMsg, valid = subs.SetCachedResponse(event, false)
 			xapp.Logger.Debug("SUBS-SubReq: internal delete due failure event(%s) %s", typeofSubsMessage(event), idstring(nil, trans, subs, parentTrans))
 		case *SubmgrRestartTestEvent:
-			// This simulates that no response has been received and after restart subscriptions are restored from db
+			// This is used to simulate that no response has been received and after restart, subscriptions are restored from db
 			xapp.Logger.Debug("Test restart flag is active. Dropping this transaction to test restart case")
+			subRfMsg, valid = subs.SetCachedResponse(event, false)
+			parentTrans.SendEvent(subRfMsg, 0)
+			return
 		case *PackSubscriptionRequestErrortEvent, *SDLWriteErrortEvent:
-			removeSubscriptionFromDb = true
 			subRfMsg, valid = subs.SetCachedResponse(event, false)
 		default:
 			// Timer expiry
 			if subs.PolicyUpdate == false {
 				xapp.Logger.Debug("SUBS-SubReq: internal delete due default event(%s) %s", typeofSubsMessage(event), idstring(nil, trans, subs, parentTrans))
-				removeSubscriptionFromDb = true
 				subRfMsg, valid = subs.SetCachedResponse(nil, false)
 				c.sendE2TSubscriptionDeleteRequest(subs, trans, parentTrans)
+			} else {
+				subRfMsg, valid = subs.SetCachedResponse(nil, true)
 			}
 		}
 		xapp.Logger.Debug("SUBS-SubReq: Handling (e2t response %s) %s", typeofSubsMessage(subRfMsg), idstring(nil, trans, subs, parentTrans))
 	} else {
 		xapp.Logger.Debug("SUBS-SubReq: Handling (cached response %s) %s", typeofSubsMessage(subRfMsg), idstring(nil, trans, subs, parentTrans))
 	}
+	if valid == false {
+		removeSubscriptionFromDb = true
+	}
 
 	err := c.UpdateSubscriptionInDB(subs, removeSubscriptionFromDb)
 	if err != nil {
-		subRfMsg, valid = subs.SetCachedResponse(event, false)
+		valid = false
 		c.sendE2TSubscriptionDeleteRequest(subs, trans, parentTrans)
 	}
 
 	//Now RemoveFromSubscription in here to avoid race conditions (mostly concerns delete)
 	if valid == false {
-		c.registry.RemoveFromSubscription(subs, parentTrans, waitRouteCleanup_ms, c)
+		c.registry.RemoveFromSubscription(subs, parentTrans, waitRouteCleanupTime, c)
 	}
 
 	parentTrans.SendEvent(subRfMsg, 0)
@@ -1151,7 +1173,7 @@
 // SUBS DELETE Handling
 //-------------------------------------------------------------------
 
-func (c *Control) handleSubscriptionDelete(subs *Subscription, parentTrans *TransactionXapp) {
+func (c *Control) handleSubscriptionDelete(subs *Subscription, parentTrans *TransactionXapp, waitRouteCleanupTime time.Duration) {
 
 	trans := c.tracker.NewSubsTransaction(subs)
 	subs.WaitTransactionTurn(trans)
@@ -1172,8 +1194,7 @@
 	//Now RemoveFromSubscription in here to avoid race conditions (mostly concerns delete)
 	//  If parallel deletes ongoing both might pass earlier sendE2TSubscriptionDeleteRequest(...) if
 	//  RemoveFromSubscription locates in caller side (now in handleXAPPSubscriptionDeleteRequest(...))
-	c.registry.RemoveFromSubscription(subs, parentTrans, waitRouteCleanup_ms, c)
-	c.registry.UpdateSubscriptionToDb(subs, c)
+	c.registry.RemoveFromSubscription(subs, parentTrans, waitRouteCleanupTime, c)
 	parentTrans.SendEvent(nil, 0)
 }
 
@@ -1298,6 +1319,7 @@
 		xapp.Logger.Error("MSG-SubResp: %s", idstring(err, params, subs))
 		return
 	}
+	xapp.Logger.Debug("SUBS-SubResp: Sending event, trans= %v", trans)
 	sendOk, timedOut := trans.SendEvent(subRespMsg, e2tRecvMsgTimeout)
 	if sendOk == false {
 		err = fmt.Errorf("Passing event to transaction failed: sendOk(%t) timedOut(%t)", sendOk, timedOut)
@@ -1501,6 +1523,11 @@
 
 func (c *Control) SendSubscriptionDeleteReq(subs *Subscription) {
 
+	if c.UTTesting == true {
+		// Reqistry mutex is not locked after real restart but it can be when restart is simulated in unit tests
+		c.registry.mutex = new(sync.Mutex)
+	}
+
 	const ricRequestorId = 123
 	xapp.Logger.Debug("Sending subscription delete due to restart. subId = %v", subs.ReqId.InstanceId)
 
diff --git a/pkg/control/registry.go b/pkg/control/registry.go
index 07affbd..36e9b4c 100644
--- a/pkg/control/registry.go
+++ b/pkg/control/registry.go
@@ -89,7 +89,7 @@
 }
 
 type Registry struct {
-	mutex             sync.Mutex
+	mutex             *sync.Mutex
 	register          map[uint32]*Subscription
 	subIds            []uint32
 	rtmgrClient       *RtmgrClient
@@ -97,6 +97,7 @@
 }
 
 func (r *Registry) Initialize() {
+	r.mutex = new(sync.Mutex)
 	r.register = make(map[uint32]*Subscription)
 	r.restSubscriptions = make(map[string]*RESTSubscription)
 
@@ -396,9 +397,9 @@
 	return e2ap.E2AP_ActionTypeInvalid, fmt.Errorf("Invalid action type in RICactions-ToBeSetup-List")
 }
 
-// TODO: Works with concurrent calls, but check if can be improved
 func (r *Registry) RemoveFromSubscription(subs *Subscription, trans *TransactionXapp, waitRouteClean time.Duration, c *Control) error {
 
+	xapp.Logger.Debug("RemoveFromSubscription %s", idstring(nil, trans, subs, trans))
 	r.mutex.Lock()
 	defer r.mutex.Unlock()
 	subs.mutex.Lock()
@@ -406,56 +407,59 @@
 
 	delStatus := subs.EpList.DelEndpoint(trans.GetEndpoint())
 	epamount := subs.EpList.Size()
+
 	subId := subs.ReqId.InstanceId
 	if delStatus == false {
 		return nil
 	}
 
-	go func() {
-		if waitRouteClean > 0 {
-			xapp.Logger.Debug("Pending %v in order to wait route cleanup", waitRouteClean)
-			time.Sleep(waitRouteClean)
+	if waitRouteClean > 0 {
+		// Wait here that response is delivered to xApp via RMR before route is cleaned
+		xapp.Logger.Debug("Pending %v in order to wait route cleanup", waitRouteClean)
+		time.Sleep(waitRouteClean)
+	}
+
+	xapp.Logger.Debug("CLEAN %s", subs.String())
+
+	if epamount == 0 {
+		//
+		// Subscription route delete
+		//
+		if subs.RMRRouteCreated == true {
+			r.RouteDelete(subs, trans, c)
 		}
 
-		subs.mutex.Lock()
-		defer subs.mutex.Unlock()
-		xapp.Logger.Debug("CLEAN %s", subs.String())
+		// Not merged subscription is being deleted
+		xapp.Logger.Debug("Subscription route delete RemoveSubscriptionFromDb")
+		c.RemoveSubscriptionFromDb(subs)
 
-		if epamount == 0 {
-			//
-			// Subscription route delete
-			//
-			if subs.RMRRouteCreated == true {
-				r.RouteDelete(subs, trans, c)
-			}
+		//
+		// Subscription release
+		//
 
-			//
-			// Subscription release
-			//
-			r.mutex.Lock()
-			defer r.mutex.Unlock()
-
-			if _, ok := r.register[subId]; ok {
-				xapp.Logger.Debug("RELEASE %s", subs.String())
-				delete(r.register, subId)
-				xapp.Logger.Debug("Registry: substable=%v", r.register)
-			}
-			r.subIds = append(r.subIds, subId)
-		} else if subs.EpList.Size() > 0 {
-			//
-			// Subscription route update
-			//
-			if subs.RMRRouteCreated == true {
-				r.RouteDeleteUpdate(subs, c)
-			}
+		if _, ok := r.register[subId]; ok {
+			xapp.Logger.Debug("RELEASE %s", subs.String())
+			delete(r.register, subId)
+			xapp.Logger.Debug("Registry: substable=%v", r.register)
 		}
-	}()
+		r.subIds = append(r.subIds, subId)
+	} else if subs.EpList.Size() > 0 {
+		//
+		// Subscription route update
+		//
+		if subs.RMRRouteCreated == true {
+			r.RouteDeleteUpdate(subs, c)
+		}
 
+		// Endpoint of merged subscription is being deleted
+		xapp.Logger.Debug("Subscription route update WriteSubscriptionToDb")
+		c.WriteSubscriptionToDb(subs)
+		c.UpdateCounter(cUnmergedSubscriptions)
+	}
 	return nil
 }
 
 func (r *Registry) RouteDelete(subs *Subscription, trans *TransactionXapp, c *Control) {
-
 	tmpList := xapp.RmrEndpointList{}
 	tmpList.AddEndpoint(trans.GetEndpoint())
 	subRouteAction := SubRouteInfo{tmpList, uint16(subs.ReqId.InstanceId)}
@@ -471,26 +475,6 @@
 	}
 }
 
-func (r *Registry) UpdateSubscriptionToDb(subs *Subscription, c *Control) {
-	r.mutex.Lock()
-	defer r.mutex.Unlock()
-	subs.mutex.Lock()
-	defer subs.mutex.Unlock()
-
-	epamount := subs.EpList.Size()
-	if epamount == 0 {
-		if _, ok := r.register[subs.ReqId.InstanceId]; ok {
-			// Not merged subscription is being deleted
-			c.RemoveSubscriptionFromDb(subs)
-
-		}
-	} else if subs.EpList.Size() > 0 {
-		// Endpoint of merged subscription is being deleted
-		c.WriteSubscriptionToDb(subs)
-		c.UpdateCounter(cUnmergedSubscriptions)
-	}
-}
-
 func (r *Registry) GetSubscription(subId uint32) *Subscription {
 	r.mutex.Lock()
 	defer r.mutex.Unlock()
diff --git a/pkg/control/ut_ctrl_submgr_test.go b/pkg/control/ut_ctrl_submgr_test.go
index ea3ffd9..31eed31 100644
--- a/pkg/control/ut_ctrl_submgr_test.go
+++ b/pkg/control/ut_ctrl_submgr_test.go
@@ -76,37 +76,30 @@
 
 func (mc *testingSubmgrControl) SimulateRestart(t *testing.T) {
 	mc.TestLog(t, "Simulating submgr restart")
-	mainCtrl.c.registry.subIds = nil
+
 	// Initialize subIds slice and subscription map
+	mainCtrl.c.registry.subIds = nil
 	mainCtrl.c.registry.Initialize()
 	mainCtrl.c.restDuplicateCtrl.Init()
+
 	// Read subIds and subscriptions from database
-	subIds, register, err := mainCtrl.c.ReadAllSubscriptionsFromSdl()
-	if err != nil {
-		mc.TestError(t, "%v", err)
-	} else {
-		mainCtrl.c.registry.subIds = subIds
-		mainCtrl.c.registry.register = register
-		mc.TestLog(t, "mainCtrl.c.registry.register:")
+	go mainCtrl.c.ReadE2Subscriptions() // This needs to be run in own go routine when called from here <<--- improve this
+	mc.TestLog(t, "mainCtrl.c.registry.register:")
+	/*
 		for subId, subs := range mainCtrl.c.registry.register {
 			mc.TestLog(t, "  subId=%v", subId)
 			mc.TestLog(t, "  subs.SubRespRcvd=%v", subs.SubRespRcvd)
 			mc.TestLog(t, "  subs=%v\n", subs)
 		}
+	*/
+	// Read REST subIds and REST subscriptions from database
+	mainCtrl.c.ReadRESTSubscriptions()
+	mc.TestLog(t, "mainCtrl.c.registry.restSubscriptions:")
+	for restSubId, restSubs := range mainCtrl.c.registry.restSubscriptions {
+		mc.TestLog(t, "  restSubId=%v", restSubId)
+		mc.TestLog(t, "  restSubs=%v\n", restSubs)
 	}
-	restSubscriptions, err := mainCtrl.c.ReadAllRESTSubscriptionsFromSdl()
-	if err != nil {
-		mc.TestError(t, "%v", err)
-	} else {
-		mainCtrl.c.registry.restSubscriptions = restSubscriptions
-		mc.TestLog(t, "mainCtrl.c.registry.restSubscriptions:")
-		for restSubId, restSubs := range mainCtrl.c.registry.restSubscriptions {
-			mc.TestLog(t, "  restSubId=%v", restSubId)
-			mc.TestLog(t, "  restSubs=%v\n", restSubs)
-		}
-	}
-
-	go mainCtrl.c.HandleUncompletedSubscriptions(mainCtrl.c.registry.register)
+	//go mainCtrl.c.HandleUncompletedSubscriptions(mainCtrl.c.registry.register) // This needs to be run in own go routine when called from here
 }
 
 func (mc *testingSubmgrControl) MakeTransactionNil(t *testing.T, subId uint32) {
@@ -288,7 +281,26 @@
 }
 
 func (mc *testingSubmgrControl) VerifyAllClean(t *testing.T) {
-	// Verify that all resources are freed
+
+	// Verify that all resources are freed. Wait cleaning up to 10 seconds
+	for i := 0; i < 100; i++ {
+		if len(mainCtrl.c.registry.register) == 0 && len(mainCtrl.c.registry.restSubscriptions) == 0 {
+			RESTKeyCount, err := mainCtrl.c.GetRESTKeyCount()
+			if err != nil {
+				t.Errorf("TEST: %s", err.Error())
+			}
+			E2KeyCount, err := mainCtrl.c.GetE2KeyCount()
+			if err != nil {
+				t.Errorf("TEST: %s", err.Error())
+			}
+			if RESTKeyCount == 0 && E2KeyCount == 0 {
+				break
+			}
+		}
+		<-time.After(time.Millisecond * 100)
+		mc.TestLog(t, "VerifyAllClean delay plus 100ms")
+	}
+
 	assert.Equal(t, 0, len(mainCtrl.c.registry.register))
 	if len(mainCtrl.c.registry.register) > 0 {
 		fmt.Printf("registry.register: %v\n", mainCtrl.c.registry.register)
@@ -369,7 +381,7 @@
 		}
 	}
 
-	// Check that not any unexpected counter are added (this is not working correctly!)
+	// Check that not any unexpected counter are added
 	// Get current values of all counters
 	currentCountersMap = mc.GetCurrentCounterValues(t, allCountersMap)
 	for _, currentCounter := range currentCountersMap {
diff --git a/pkg/control/ut_messaging_test.go b/pkg/control/ut_messaging_test.go
index 6ce7b51..7d7f67b 100644
--- a/pkg/control/ut_messaging_test.go
+++ b/pkg/control/ut_messaging_test.go
@@ -97,7 +97,7 @@
 
 	mainCtrl.VerifyCounterValues(t)
 	mainCtrl.VerifyAllClean(t)
-	//os.Exit(1)
+	//os.Exit(0)
 }
 
 //-----------------------------------------------------------------------------
@@ -2375,9 +2375,6 @@
 func TestSubReqNokAndSubDelOkWithRestartInMiddle(t *testing.T) {
 	CaseBegin("TestSubReqNokAndSubDelOkWithRestartInMiddle")
 
-	// Remove possible existing subscrition
-	mainCtrl.removeExistingSubscriptions(t)
-
 	mainCtrl.SetResetTestFlag(t, true) // subs.DoNotWaitSubResp will be set TRUE for the subscription
 	xappConn1.SendSubsReq(t, nil, nil)
 	e2termConn1.RecvSubsReq(t)
@@ -2459,6 +2456,10 @@
 	mainCtrl.SimulateRestart(t)
 	xapp.Logger.Debug("mainCtrl.SimulateRestart done")
 
+	// ReadE2Subscriptions() for testing is running in own go routine (go mainCtrl.c.ReadE2Subscriptions())
+	// That needs to be completed before successful subscription query is possible
+	<-time.After(time.Second * 1)
+
 	// Check that subscription is restored correctly after restart
 	resp, _ = xapp.Subscription.QuerySubscriptions()
 	assert.Equal(t, resp[0].SubscriptionID, int64(e2SubsId))
@@ -2553,7 +2554,7 @@
 	e2SubsId2 := xappConn2.RecvSubsResp(t, cretrans2)
 
 	// Check subscription
-	resp, _ := xapp.Subscription.QuerySubscriptions() ////////////////////////////////
+	resp, _ := xapp.Subscription.QuerySubscriptions()
 	assert.Equal(t, resp[0].SubscriptionID, int64(e2SubsId1))
 	assert.Equal(t, resp[0].Meid, "RAN_NAME_1")
 	assert.Equal(t, resp[0].ClientEndpoint, []string{"localhost:13560", "localhost:13660"})
@@ -2561,6 +2562,10 @@
 	mainCtrl.SimulateRestart(t)
 	xapp.Logger.Debug("mainCtrl.SimulateRestart done")
 
+	// ReadE2Subscriptions() for testing is running in own go routine (go mainCtrl.c.ReadE2Subscriptions())
+	// That needs to be completed before successful subscription query is possible
+	<-time.After(time.Second * 1)
+
 	// Check that subscription is restored correctly after restart
 	resp, _ = xapp.Subscription.QuerySubscriptions()
 	assert.Equal(t, resp[0].SubscriptionID, int64(e2SubsId1))
@@ -2576,6 +2581,10 @@
 	mainCtrl.SimulateRestart(t)
 	xapp.Logger.Debug("mainCtrl.SimulateRestart done")
 
+	// ReadE2Subscriptions() for testing is running in own go routine (go mainCtrl.c.ReadE2Subscriptions())
+	// Submgr need be ready before successful subscription deletion is possible
+	<-time.After(time.Second * 1)
+
 	//Del2
 	deltrans2 := xappConn2.SendSubsDelReq(t, nil, e2SubsId2)
 	delreq2, delmsg2 := e2termConn1.RecvSubsDelReq(t)
@@ -4358,7 +4367,6 @@
 	mainCtrl.VerifyAllClean(t)
 }
 
-/*
 //-----------------------------------------------------------------------------
 // TestRESTSubReqAndSubDelNoAnswerSameActionParallel
 //
@@ -4421,6 +4429,7 @@
 		Counter{cSubReqTimerExpiry, 2},
 		Counter{cSubReReqToE2, 1},
 		Counter{cRestSubFailNotifToXapp, 2},
+		Counter{cUnmergedSubscriptions, 1},
 		Counter{cRestSubDelReqFromXapp, 2},
 		Counter{cSubDelReqToE2, 1},
 		Counter{cSubDelRespFromE2, 1},
@@ -4522,6 +4531,7 @@
 		Counter{cSubReqToE2, 1},
 		Counter{cSubFailFromE2, 1},
 		Counter{cRestSubFailNotifToXapp, 2},
+		Counter{cUnmergedSubscriptions, 1},
 		Counter{cRestSubDelReqFromXapp, 2},
 		Counter{cRestSubDelRespToXapp, 2},
 	})
@@ -4563,7 +4573,7 @@
 	mainCtrl.VerifyCounterValues(t)
 	mainCtrl.VerifyAllClean(t)
 }
-*/
+
 func TestRESTSubReqPolicyAndSubDelOk(t *testing.T) {
 	CaseBegin("TestRESTSubReqPolicyAndSubDelOk")
 
@@ -5030,16 +5040,12 @@
 		Counter{cSubDelReqFromXapp, 1},
 		Counter{cSubDelReqToE2, 1},
 		Counter{cSubDelRespFromE2, 1},
-		Counter{cRestSubFailNotifToXapp, 1},
 		Counter{cRestSubDelReqFromXapp, 1},
 		Counter{cRestSubDelRespToXapp, 1},
 	})
 
 	const subReqCount int = 1
 
-	// Remove possible existing subscription
-	mainCtrl.removeExistingSubscriptions(t)
-
 	params := xappConn1.GetRESTSubsReqReportParams(subReqCount)
 
 	//Req
@@ -5122,12 +5128,9 @@
 		Counter{cSubDelRespFromE2, 1},
 		Counter{cRestSubDelRespToXapp, 1},
 	})
-	// Remove possible existing subscription
-	mainCtrl.removeExistingSubscriptions(t)
-
-	var params *teststube2ap.RESTSubsReqParams = nil
 
 	// Create subscription
+	var params *teststube2ap.RESTSubsReqParams = nil
 	restSubId, e2SubsId := createSubscription(t, xappConn1, e2termConn1, params)
 	xapp.Logger.Debug("Send REST subscriber request for subscriber : %v", restSubId)
 
@@ -5137,6 +5140,10 @@
 	mainCtrl.SimulateRestart(t)
 	xapp.Logger.Debug("mainCtrl.SimulateRestart done")
 
+	// ReadE2Subscriptions() for testing is running in own go routine (go mainCtrl.c.ReadE2Subscriptions())
+	// That needs to be completed before successful subscription query is possible
+	<-time.After(time.Second * 1)
+
 	// Check subscription
 	queryXappSubscription(t, int64(e2SubsId), "RAN_NAME_1", []string{"localhost:13560"})
 
@@ -5204,7 +5211,6 @@
 //     |             |                 |              |
 //
 //-----------------------------------------------------------------------------
-
 func TestRESTSubReqAndSubDelOkSameActionWithRestartsInMiddle(t *testing.T) {
 	CaseBegin("TestRESTSubReqAndSubDelOkSameActionWithRestartsInMiddle")
 
@@ -5222,12 +5228,8 @@
 		Counter{cRestSubDelRespToXapp, 2},
 	})
 
-	// Remove possible existing subscription
-	mainCtrl.removeExistingSubscriptions(t)
-
-	var params *teststube2ap.RESTSubsReqParams = nil
-
 	// Create subscription 1
+	var params *teststube2ap.RESTSubsReqParams = nil
 	restSubId1, e2SubsId1 := createSubscription(t, xappConn1, e2termConn1, params)
 	xapp.Logger.Debug("Send REST subscriber request for subscriber 1 : %v", restSubId1)
 
@@ -5244,15 +5246,27 @@
 	queryXappSubscription(t, int64(e2SubsId1), "RAN_NAME_1", []string{"localhost:13560", "localhost:13660"})
 
 	mainCtrl.SimulateRestart(t)
-	xapp.Logger.Debug("mainCtrl.SimulateRestart done")
+	xapp.Logger.Debug("mainCtrl.SimulateRestart done 1")
+
+	// ReadE2Subscriptions() for testing is running in own go routine (go mainCtrl.c.ReadE2Subscriptions())
+	// That needs to be completed before successful subscription delete is possible
+	<-time.After(time.Second * 1)
 
 	// Delete subscription 1, and wait until it has removed the first endpoint
 	subepcnt := mainCtrl.get_subs_entrypoint_cnt(t, e2SubsId1)
 	xappConn1.SendRESTSubsDelReq(t, &restSubId1)
 	mainCtrl.wait_subs_entrypoint_cnt_change(t, e2SubsId1, subepcnt, 10)
 
+	// Above wait does not work correctly anymore as this delay makes this test case work
+	//<-time.After(time.Second * 1)
+
 	mainCtrl.SimulateRestart(t)
-	xapp.Logger.Debug("mainCtrl.SimulateRestart done")
+	xapp.Logger.Debug("mainCtrl.SimulateRestart done 2")
+
+	// ReadE2Subscriptions() for testing is running in own go routine (go mainCtrl.c.ReadE2Subscriptions())
+	// That needs to be completed before successful subscription query is possible
+	<-time.After(time.Second * 1)
+
 	queryXappSubscription(t, int64(e2SubsId1), "RAN_NAME_1", []string{"localhost:13660"})
 
 	// Delete subscription 2
@@ -5360,7 +5374,7 @@
 }
 
 /*
-func TestRESTPolicySubReqAndSubDelOk(t *testing.T) {
+func TestRESTPolicySubReqAndSubDelOk(t *testing.T) {  was in comments already
 	CaseBegin("TestRESTPolicySubReqAndSubDelOk")
 
 	subReqCount := 2
@@ -6970,16 +6984,12 @@
 		Counter{cSubReqToE2, 2},
 		Counter{cSubRespFromE2, 1},
 		Counter{cRestSubNotifToXapp, 1},
-		Counter{cRestSubFailNotifToXapp, 1},
 		Counter{cRestSubDelReqFromXapp, 1},
 		Counter{cSubDelReqToE2, 1},
 		Counter{cSubDelRespFromE2, 1},
 		Counter{cRestSubDelRespToXapp, 1},
 	})
 
-	// Remove possible existing subscription
-	mainCtrl.removeExistingSubscriptions(t)
-
 	const e2Timeout int64 = 1
 	const e2RetryCount int64 = 0
 	const routingNeeded bool = false
@@ -7017,6 +7027,10 @@
 	mainCtrl.SimulateRestart(t)
 	xapp.Logger.Debug("mainCtrl.SimulateRestart done")
 
+	// ReadE2Subscriptions() for testing is running in own go routine (go mainCtrl.c.ReadE2Subscriptions())
+	// That needs to be completed before successful subscription query is possible
+	<-time.After(time.Second * 1)
+
 	// Check subscription
 	queryXappSubscription(t, int64(e2SubsId), "RAN_NAME_1", []string{"localhost:13560"})
 
@@ -7123,9 +7137,9 @@
 
 func queryXappSubscription(t *testing.T, e2SubsId int64, meid string, endpoint []string) {
 	resp, _ := xapp.Subscription.QuerySubscriptions()
-	assert.Equal(t, resp[0].SubscriptionID, e2SubsId)
-	assert.Equal(t, resp[0].Meid, meid)
-	assert.Equal(t, resp[0].ClientEndpoint, endpoint)
+	assert.Equal(t, e2SubsId, resp[0].SubscriptionID)
+	assert.Equal(t, meid, resp[0].Meid)
+	assert.Equal(t, endpoint, resp[0].ClientEndpoint)
 }
 
 func waitSubsCleanup(t *testing.T, e2SubsId uint32, timeout int) {