[RICPLT-2165] Add rnibDataService to support retries

Change-Id: Ia9dc8bbeead1d1f4fd0f970789bcd4b9af2f0540
Signed-off-by: Amichai <amichai.sichel@intl.att.com>
diff --git a/E2Manager/services/receivers/rmr_service_receiver_test.go b/E2Manager/services/receivers/rmr_service_receiver_test.go
index 15e9857..1482085 100644
--- a/E2Manager/services/receivers/rmr_service_receiver_test.go
+++ b/E2Manager/services/receivers/rmr_service_receiver_test.go
@@ -58,6 +58,8 @@
 }
 
 func getRmrServiceReceiver(rmrMessengerMock *mocks.RmrMessengerMock, logger *logger.Logger) *RmrServiceReceiver {
+	config := &configuration.Configuration{RnibRetryIntervalMs: 10, MaxRnibConnectionAttempts: 3}
+
 	readerMock := &mocks.RnibReaderMock{}
 	rnibReaderProvider := func() reader.RNibReader {
 		return readerMock
@@ -67,10 +69,11 @@
 		return writerMock
 	}
 
+	rnibDataService := services.NewRnibDataService(logger, config, rnibReaderProvider, rnibWriterProvider)
 	rmrService := getRmrService(rmrMessengerMock, logger)
-	ranSetupManager := managers.NewRanSetupManager(logger, rmrService, rNibWriter.GetRNibWriter)
-	ranReconnectionManager := managers.NewRanReconnectionManager(logger, configuration.ParseConfiguration(), rnibReaderProvider, rnibWriterProvider, ranSetupManager)
-	nManager := notificationmanager.NewNotificationManager(rnibReaderProvider, rnibWriterProvider, ranReconnectionManager)
+	ranSetupManager := managers.NewRanSetupManager(logger, rmrService, rnibDataService)
+	ranReconnectionManager := managers.NewRanReconnectionManager(logger, configuration.ParseConfiguration(), rnibDataService, ranSetupManager)
+	nManager := notificationmanager.NewNotificationManager(rnibDataService, ranReconnectionManager)
 
 	return NewRmrServiceReceiver(*rmrService, nManager)
 }
diff --git a/E2Manager/services/rnib_data_service.go b/E2Manager/services/rnib_data_service.go
new file mode 100644
index 0000000..8a91894
--- /dev/null
+++ b/E2Manager/services/rnib_data_service.go
@@ -0,0 +1,128 @@
+package services
+
+import (
+	"e2mgr/configuration"
+	"e2mgr/logger"
+	"e2mgr/rNibWriter"
+	"gerrit.o-ran-sc.org/r/ric-plt/nodeb-rnib.git/common"
+	"gerrit.o-ran-sc.org/r/ric-plt/nodeb-rnib.git/entities"
+	"gerrit.o-ran-sc.org/r/ric-plt/nodeb-rnib.git/reader"
+	"net"
+	"time"
+)
+
+type RNibDataService interface {
+	SaveNodeb(nbIdentity *entities.NbIdentity, nb *entities.NodebInfo) error
+	UpdateNodebInfo(nodebInfo *entities.NodebInfo) error
+	SaveRanLoadInformation(inventoryName string, ranLoadInformation *entities.RanLoadInformation) error
+	GetNodeb(ranName string) (*entities.NodebInfo, error)
+	GetListNodebIds() ([]*entities.NbIdentity, error)
+}
+
+type rNibDataService struct {
+	logger             *logger.Logger
+	rnibReaderProvider func() reader.RNibReader
+	rnibWriterProvider func() rNibWriter.RNibWriter
+	maxAttempts        int
+	retryInterval      time.Duration
+}
+
+func NewRnibDataService(logger *logger.Logger, config *configuration.Configuration, rnibReaderProvider func() reader.RNibReader, rnibWriterProvider func() rNibWriter.RNibWriter) *rNibDataService {
+	return &rNibDataService{
+		logger:             logger,
+		rnibReaderProvider: rnibReaderProvider,
+		rnibWriterProvider: rnibWriterProvider,
+		maxAttempts:        config.MaxRnibConnectionAttempts,
+		retryInterval:      time.Duration(config.RnibRetryIntervalMs) * time.Millisecond,
+	}
+}
+
+func (w *rNibDataService) UpdateNodebInfo(nodebInfo *entities.NodebInfo) error {
+	w.logger.Infof("#RnibDataService.UpdateNodebInfo - nodebInfo: %s", nodebInfo)
+
+	err := w.retry("UpdateNodebInfo", func() (err error) {
+		err = w.rnibWriterProvider().UpdateNodebInfo(nodebInfo)
+		return
+	})
+
+	return err
+}
+
+func (w *rNibDataService) SaveNodeb(nbIdentity *entities.NbIdentity, nb *entities.NodebInfo) error {
+	w.logger.Infof("#RnibDataService.SaveNodeb - nbIdentity: %s, nodebInfo: %s", nbIdentity, nb)
+
+	err := w.retry("SaveNodeb", func() (err error) {
+		err = w.rnibWriterProvider().SaveNodeb(nbIdentity, nb)
+		return
+	})
+
+	return err
+}
+
+func (w *rNibDataService) SaveRanLoadInformation(inventoryName string, ranLoadInformation *entities.RanLoadInformation) error {
+	w.logger.Infof("#RnibDataService.SaveRanLoadInformation - inventoryName: %s, ranLoadInformation: %s", inventoryName, ranLoadInformation)
+
+	err := w.retry("SaveRanLoadInformation", func() (err error) {
+		err = w.rnibWriterProvider().SaveRanLoadInformation(inventoryName, ranLoadInformation)
+		return
+	})
+
+	return err
+}
+
+func (w *rNibDataService) GetNodeb(ranName string) (*entities.NodebInfo, error) {
+	w.logger.Infof("#RnibDataService.GetNodeb - ranName: %s", ranName)
+
+	var nodeb *entities.NodebInfo = nil
+
+	err := w.retry("GetNodeb", func() (err error) {
+		nodeb, err = w.rnibReaderProvider().GetNodeb(ranName)
+		return
+	})
+
+	return nodeb, err
+}
+
+func (w *rNibDataService) GetListNodebIds() ([]*entities.NbIdentity, error) {
+	w.logger.Infof("#RnibDataService.GetListNodebIds")
+
+	var nodeIds []*entities.NbIdentity = nil
+
+	err := w.retry("GetListNodebIds", func() (err error) {
+		nodeIds, err = w.rnibReaderProvider().GetListNodebIds()
+		return
+	})
+
+	return nodeIds, err
+}
+
+func (w *rNibDataService) retry(rnibFunc string, f func() error) (err error) {
+	attempts := w.maxAttempts
+
+	for i := 1; ; i++ {
+		err = f()
+		if err == nil {
+			return
+		}
+		if !w.isConnError(err) {
+			return err
+		}
+		if i >= attempts {
+			w.logger.Errorf("#RnibDataService.retry - after %d attempts of %s, last error: %s", attempts, rnibFunc, err)
+			return err
+		}
+		time.Sleep(w.retryInterval)
+
+		w.logger.Infof("#RnibDataService.retry - retrying %d %s after error: %s", i, rnibFunc, err)
+	}
+}
+
+func (w *rNibDataService) isConnError(err error) bool {
+	internalErr, ok := err.(common.InternalError)
+	if !ok {
+		return false
+	}
+	_, ok = internalErr.Err.(*net.OpError)
+
+	return ok
+}
diff --git a/E2Manager/services/rnib_data_service_test.go b/E2Manager/services/rnib_data_service_test.go
new file mode 100644
index 0000000..7a96310
--- /dev/null
+++ b/E2Manager/services/rnib_data_service_test.go
@@ -0,0 +1,186 @@
+package services
+
+import (
+	"e2mgr/configuration"
+	"e2mgr/logger"
+	"e2mgr/mocks"
+	"e2mgr/rNibWriter"
+	"fmt"
+	"gerrit.o-ran-sc.org/r/ric-plt/nodeb-rnib.git/common"
+	"gerrit.o-ran-sc.org/r/ric-plt/nodeb-rnib.git/entities"
+	"gerrit.o-ran-sc.org/r/ric-plt/nodeb-rnib.git/reader"
+	"github.com/stretchr/testify/assert"
+	"net"
+	"strings"
+	"testing"
+)
+
+func setupTest(t *testing.T) (*rNibDataService, *mocks.RnibReaderMock, *mocks.RnibWriterMock) {
+	logger, err := logger.InitLogger(logger.DebugLevel)
+	if err != nil {
+		t.Errorf("#... - failed to initialize logger, error: %s", err)
+	}
+
+	config := &configuration.Configuration{RnibRetryIntervalMs: 10, MaxRnibConnectionAttempts: 3}
+
+	readerMock := &mocks.RnibReaderMock{}
+	rnibReaderProvider := func() reader.RNibReader {
+		return readerMock
+	}
+
+	writerMock := &mocks.RnibWriterMock{}
+	rnibWriterProvider := func() rNibWriter.RNibWriter {
+		return writerMock
+	}
+
+	rnibDataService := NewRnibDataService(logger, config, rnibReaderProvider, rnibWriterProvider)
+	assert.NotNil(t, rnibDataService)
+
+	return rnibDataService, readerMock, writerMock
+}
+
+func TestSuccessfulSaveNodeb(t *testing.T) {
+	rnibDataService, _, writerMock := setupTest(t)
+
+	nodebInfo := &entities.NodebInfo{}
+	nbIdentity := &entities.NbIdentity{}
+	writerMock.On("SaveNodeb", nbIdentity, nodebInfo).Return(nil)
+
+	rnibDataService.SaveNodeb(nbIdentity, nodebInfo)
+	writerMock.AssertNumberOfCalls(t, "SaveNodeb", 1)
+}
+
+func TestConnFailureSaveNodeb(t *testing.T) {
+	rnibDataService, _, writerMock := setupTest(t)
+
+	nodebInfo := &entities.NodebInfo{}
+	nbIdentity := &entities.NbIdentity{}
+	mockErr := common.InternalError{Err: &net.OpError{Err: fmt.Errorf("connection error")}}
+	writerMock.On("SaveNodeb", nbIdentity, nodebInfo).Return(mockErr)
+
+	rnibDataService.SaveNodeb(nbIdentity, nodebInfo)
+	writerMock.AssertNumberOfCalls(t, "SaveNodeb", 3)
+}
+
+func TestNonConnFailureSaveNodeb(t *testing.T) {
+	rnibDataService, _, writerMock := setupTest(t)
+
+	nodebInfo := &entities.NodebInfo{}
+	nbIdentity := &entities.NbIdentity{}
+	mockErr := common.InternalError{Err: fmt.Errorf("non connection failure")}
+	writerMock.On("SaveNodeb", nbIdentity, nodebInfo).Return(mockErr)
+
+	rnibDataService.SaveNodeb(nbIdentity, nodebInfo)
+	writerMock.AssertNumberOfCalls(t, "SaveNodeb", 1)
+}
+
+func TestSuccessfulUpdateNodebInfo(t *testing.T) {
+	rnibDataService, _, writerMock := setupTest(t)
+
+	nodebInfo := &entities.NodebInfo{}
+	writerMock.On("UpdateNodebInfo", nodebInfo).Return(nil)
+
+	rnibDataService.UpdateNodebInfo(nodebInfo)
+	writerMock.AssertNumberOfCalls(t, "UpdateNodebInfo", 1)
+}
+
+func TestConnFailureUpdateNodebInfo(t *testing.T) {
+	rnibDataService, _, writerMock := setupTest(t)
+
+	nodebInfo := &entities.NodebInfo{}
+	mockErr := common.InternalError{Err: &net.OpError{Err: fmt.Errorf("connection error")}}
+	writerMock.On("UpdateNodebInfo", nodebInfo).Return(mockErr)
+
+	rnibDataService.UpdateNodebInfo(nodebInfo)
+	writerMock.AssertNumberOfCalls(t, "UpdateNodebInfo", 3)
+}
+
+func TestSuccessfulSaveRanLoadInformation(t *testing.T) {
+	rnibDataService, _, writerMock := setupTest(t)
+
+	var ranName string = "abcd"
+	ranLoadInformation := &entities.RanLoadInformation{}
+	writerMock.On("SaveRanLoadInformation", ranName, ranLoadInformation).Return(nil)
+
+	rnibDataService.SaveRanLoadInformation(ranName, ranLoadInformation)
+	writerMock.AssertNumberOfCalls(t, "SaveRanLoadInformation", 1)
+}
+
+func TestConnFailureSaveRanLoadInformation(t *testing.T) {
+	rnibDataService, _, writerMock := setupTest(t)
+
+	var ranName string = "abcd"
+	ranLoadInformation := &entities.RanLoadInformation{}
+	mockErr := common.InternalError{Err: &net.OpError{Err: fmt.Errorf("connection error")}}
+	writerMock.On("SaveRanLoadInformation", ranName, ranLoadInformation).Return(mockErr)
+
+	rnibDataService.SaveRanLoadInformation(ranName, ranLoadInformation)
+	writerMock.AssertNumberOfCalls(t, "SaveRanLoadInformation", 3)
+}
+
+func TestSuccessfulGetNodeb(t *testing.T) {
+	rnibDataService, readerMock, _ := setupTest(t)
+
+	invName := "abcd"
+	nodebInfo := &entities.NodebInfo{}
+	readerMock.On("GetNodeb", invName).Return(nodebInfo, nil)
+
+	res, err := rnibDataService.GetNodeb(invName)
+	readerMock.AssertNumberOfCalls(t, "GetNodeb", 1)
+	assert.Equal(t, nodebInfo, res)
+	assert.Nil(t, err)
+}
+
+func TestConnFailureGetNodeb(t *testing.T) {
+	rnibDataService, readerMock, _ := setupTest(t)
+
+	invName := "abcd"
+	var nodeb *entities.NodebInfo = nil
+	mockErr := common.InternalError{Err: &net.OpError{Err: fmt.Errorf("connection error")}}
+	readerMock.On("GetNodeb", invName).Return(nodeb, mockErr)
+
+	res, err := rnibDataService.GetNodeb(invName)
+	readerMock.AssertNumberOfCalls(t, "GetNodeb", 3)
+	assert.True(t, strings.Contains(err.Error(), "connection error", ))
+	assert.Equal(t, nodeb, res)
+}
+
+func TestSuccessfulGetNodebIdList(t *testing.T) {
+	rnibDataService, readerMock, _ := setupTest(t)
+
+	nodeIds := []*entities.NbIdentity{}
+	readerMock.On("GetListNodebIds").Return(nodeIds, nil)
+
+	res, err := rnibDataService.GetListNodebIds()
+	readerMock.AssertNumberOfCalls(t, "GetListNodebIds", 1)
+	assert.Equal(t, nodeIds, res)
+	assert.Nil(t, err)
+}
+
+func TestConnFailureGetNodebIdList(t *testing.T) {
+	rnibDataService, readerMock, _ := setupTest(t)
+
+	var nodeIds []*entities.NbIdentity = nil
+	mockErr := common.InternalError{Err: &net.OpError{Err: fmt.Errorf("connection error")}}
+	readerMock.On("GetListNodebIds").Return(nodeIds, mockErr)
+
+	res, err := rnibDataService.GetListNodebIds()
+	readerMock.AssertNumberOfCalls(t, "GetListNodebIds", 3)
+	assert.True(t, strings.Contains(err.Error(), "connection error", ))
+	assert.Equal(t, nodeIds, res)
+}
+
+//func TestConnFailureThenSuccessGetNodebIdList(t *testing.T) {
+//	rnibDataService, readerMock, _ := setupTest(t)
+//
+//	var nilNodeIds []*entities.NbIdentity = nil
+//	nodeIds := []*entities.NbIdentity{}
+//	mockErr := common.InternalError{Err: &net.OpError{Err:fmt.Errorf("connection error")}}
+//	//readerMock.On("GetListNodebIds").Return(nilNodeIds, mockErr)
+//	//readerMock.On("GetListNodebIds").Return(nodeIds, nil)
+//
+//	res, err := rnibDataService.GetListNodebIds()
+//	readerMock.AssertNumberOfCalls(t, "GetListNodebIds", 2)
+//	assert.True(t, strings.Contains(err.Error(),"connection failure", ))
+//	assert.Equal(t, nodeIds, res)
+//}
diff --git a/E2Manager/services/rnib_reader_service.go b/E2Manager/services/rnib_reader_service.go
deleted file mode 100644
index 4b014d7..0000000
--- a/E2Manager/services/rnib_reader_service.go
+++ /dev/null
@@ -1,41 +0,0 @@
-//
-// Copyright 2019 AT&T Intellectual Property
-// Copyright 2019 Nokia
-//
-// Licensed under the Apache License, Version 2.0 (the "License");
-// you may not use this file except in compliance with the License.
-// You may obtain a copy of the License at
-//
-//      http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing, software
-// distributed under the License is distributed on an "AS IS" BASIS,
-// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-// See the License for the specific language governing permissions and
-// limitations under the License.
-//
-
-package services
-
-import (
-	"gerrit.o-ran-sc.org/r/ric-plt/nodeb-rnib.git/entities"
-	"gerrit.o-ran-sc.org/r/ric-plt/nodeb-rnib.git/reader"
-)
-
-type RnibReaderService struct {
-	rnibReaderProvider func() reader.RNibReader
-}
-
-func NewRnibReaderService(rnibReaderProvider func() reader.RNibReader) *RnibReaderService{
-	return &RnibReaderService{rnibReaderProvider}
-}
-
-func (s RnibReaderService) GetNodeb(ranName string) (*entities.NodebInfo, error) {
-	return s.rnibReaderProvider().GetNodeb(ranName)
-}
-
-func (s  RnibReaderService) GetNodebIdList()([]*entities.NbIdentity, error) {
-	return s.rnibReaderProvider().GetListNodebIds()
-}
-
-