SDN-C Multi-site High-availability - Auto-failover
Change-Id: I4e028b31ce7a60154d2c04c63431c5ea996de8f8
Signed-off-by: Mohammadreza Pasandideh <mohammadreza.pasandideh@amdocs.com>
Issue-ID: SDNC-213
diff --git a/kubernetes/sdnc/resources/geo/bin/sdnc.cluster b/kubernetes/sdnc/resources/geo/bin/sdnc.cluster
index d59718f..87cdeff 100755
--- a/kubernetes/sdnc/resources/geo/bin/sdnc.cluster
+++ b/kubernetes/sdnc/resources/geo/bin/sdnc.cluster
@@ -1,6 +1,18 @@
#!/bin/bash
-OOM_HOME=${OOM_HOME:-$HOME}
+# Copyright © 2018 Amdocs
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
if ! [ "$(command -v jq)" ]; then
echo "Error: jq is not installed."
@@ -8,7 +20,8 @@
exit 1
fi
-IS_PRIMARY_CLUSTER=`./sdnc.isPrimaryCluster`
+dir=$( dirname $0 )
+IS_PRIMARY_CLUSTER=$( $dir/sdnc.isPrimaryCluster )
case $IS_PRIMARY_CLUSTER in
true)
@@ -18,21 +31,30 @@
MEMBER_NUMBER=4
;;
*)
- echo "Error: isPrimaryODLCluster not defined in ${OOM_HOME}/oom/kubernetes/sdnc/values.yaml."
+ echo "Error: isPrimaryCluster not defined in $dir/../../../values.yaml."
exit 1
;;
esac
+USERNAME=admin
+PASSWORD=admin
+
for pod_number in {0..2}
do
- curl "http://localhost:3026$((${pod_number} + 1))" > /dev/null 2>&1
- if [ "$?" = "7" ]; then
+
+ response=`curl -s -u $USERNAME:$PASSWORD -H "Content-Type: application/json" -H "Accept: application/json" -X GET http://localhost:3026$((${pod_number} + 1))/jolokia/read/org.opendaylight.controller:Category=Shards,name=member-$((${MEMBER_NUMBER} + ${pod_number}))-shard-default-config,type=DistributedConfigDatastore`
+
+ if [ $? -ne 0 ]; then
continue
fi
- VOTING_RESULT=`curl -u admin:admin -H "Content-Type: application/json" -H "Accept: application/json" -X GET http://localhost:3026$((${pod_number} + 1))/jolokia/read/org.opendaylight.controller:Category=Shards,name=member-$((${MEMBER_NUMBER} + ${pod_number}))-shard-default-config,type=DistributedConfigDatastore 2>/dev/null | jq '.value.Voting'`
+ status=$( echo -E "$response" | jq -r ".status" )
+ if [ "$status" != "200" ]; then
+ continue
+ fi
- case $VOTING_RESULT in
+ voting=$( echo -E "$response" | jq -r ".value.Voting" )
+ case $voting in
true)
echo "active"
exit 0
diff --git a/kubernetes/sdnc/resources/geo/bin/sdnc.failover b/kubernetes/sdnc/resources/geo/bin/sdnc.failover
deleted file mode 100755
index 961a5cb..0000000
--- a/kubernetes/sdnc/resources/geo/bin/sdnc.failover
+++ /dev/null
@@ -1,65 +0,0 @@
-#!/usr/bin/perl -s
-use strict;
-
-my $keyWord_standby = "standby";
-my $keyWord_active = "active";
-my $keyWord_true = "true";
-my $keyWord_false = "false";
-my $keyWord_success = "success";
-my $keyWord_failure = "failure";
-my $file_cluster = "sdnc.cluster";
-my $file_switchVoting = "switchVoting.sh";
-my $file_isPrimaryCluster = "sdnc.isPrimaryCluster";
-
-if ((!(-e $file_cluster)) || (!(-e $file_switchVoting))|| (!(-e $file_isPrimaryCluster))) {
- # file not exist.
- print qq|$keyWord_failure\n|;
- exit 1;
-}
-
-my $roleRes = qx("./$file_isPrimaryCluster");
-my $clusterRes = qx("./$file_cluster");
-
-if ( index ($clusterRes, $keyWord_standby) != -1) {
- # We are at standby side
- if ( index ($roleRes, $keyWord_false) != -1) {
- # We are at Secondary cluster
- sub_activate_secondary();
- } elsif ( index ($roleRes, $keyWord_true) != -1) {
- # We are at Primary cluster
- sub_activate_primary();
- } else {
- # Error.
- print qq|$keyWord_failure\n|;
- exit 1;
- }
-} elsif ( index ($clusterRes, $keyWord_active) != -1) {
- # We are at active side
- if ( index ($roleRes, $keyWord_false) != -1) {
- # We are at Secondary cluster
- sub_activate_primary();
- } elsif ( index ($roleRes, $keyWord_true) != -1) {
- # We are at Primary cluster
- sub_activate_secondary();
- } else {
- # Error.
- print qq|$keyWord_failure\n|;
- exit 1;
- }
-} else {
- # Error.
- print qq|$keyWord_failure\n|;
- exit 1;
-}
-
-sub sub_activate_primary {
- #Switching voting in Primary cluster
- system("./$file_switchVoting primary");
- print qq|$keyWord_success\n|;
-}
-
-sub sub_activate_secondary {
- #Switching voting in secondary cluster
- system("./$file_switchVoting secondary");
- print qq|$keyWord_success\n|;
-}
diff --git a/kubernetes/sdnc/resources/geo/bin/sdnc.isPrimaryCluster b/kubernetes/sdnc/resources/geo/bin/sdnc.isPrimaryCluster
index 77fc65f..7a4f6a7 100755
--- a/kubernetes/sdnc/resources/geo/bin/sdnc.isPrimaryCluster
+++ b/kubernetes/sdnc/resources/geo/bin/sdnc.isPrimaryCluster
@@ -1,8 +1,22 @@
#!/bin/bash
-OOM_HOME=${OOM_HOME:-$HOME}
+# Copyright © 2018 Amdocs
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
-IS_PRIMARY_CLUSTER=`awk '/isPrimaryCluster/ {print $2}' ${OOM_HOME}/oom/kubernetes/sdnc/values.yaml`
+dir=$( dirname $0 )
+
+IS_PRIMARY_CLUSTER=`awk '/isPrimaryCluster/ {print $2}' $dir/../../../values.yaml`
if [ "$?" -eq "2" ]; then
echo "Make sure you are ubuntu user." >&2
diff --git a/kubernetes/sdnc/resources/geo/bin/sdnc.makeActive b/kubernetes/sdnc/resources/geo/bin/sdnc.makeActive
new file mode 100755
index 0000000..76eca48
--- /dev/null
+++ b/kubernetes/sdnc/resources/geo/bin/sdnc.makeActive
@@ -0,0 +1,45 @@
+#!/bin/sh
+
+# Copyright © 2018 Amdocs
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+if [ $# -lt 1 ];then
+ echo "Usage: makeactive <release> [namespace]"
+ exit 1
+fi
+
+RELEASE=$1
+NAMESPACE=onap
+if [ -n "$2" ];then
+ NAMESPACE=$2
+fi
+
+dir=$(dirname $0)
+isPrimary=$( $dir/sdnc.isPrimaryCluster)
+if [ "$isPrimary" = "true" ];then
+ SITE_NAME="sdnc01"
+elif [ "$isPrimary" = "false" ];then
+ SITE_NAME="sdnc02"
+else
+ echo "sdnc.isPrimaryCluster returned unexpected value \"$isPrimary\""
+ exit 1
+fi
+
+pod=$( kubectl -n $NAMESPACE get pods -l app=sdnc-prom,release=$RELEASE | grep Running | cut -f1 -d' ' )
+if [ -z "$pod" ];then
+ echo "prom pod not found - is prom running?"
+ exit 1
+fi
+
+kubectl -n $NAMESPACE exec $pod -- /app/promoverride.py --id $SITE_NAME --config /app/config/config.json
diff --git a/kubernetes/sdnc/resources/geo/bin/sdnc.monitor b/kubernetes/sdnc/resources/geo/bin/sdnc.monitor
new file mode 100755
index 0000000..b14bd73
--- /dev/null
+++ b/kubernetes/sdnc/resources/geo/bin/sdnc.monitor
@@ -0,0 +1,39 @@
+#!/bin/sh
+
+# Copyright © 2018 Amdocs
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+if [ $# -lt 1 ];then
+ echo "Usage: $(basename $0) [--debug] <release> [namespace]"
+ exit 1
+fi
+
+if [ "$1" = "--debug" -o "$1" = "-debug" -o "$1" = "-d" ];then
+ DEBUG="--debug"
+ shift
+fi
+
+RELEASE=$1
+NAMESPACE=onap
+if [ -n "$2" ];then
+ NAMESPACE=$2
+fi
+
+pod=$( kubectl -n $NAMESPACE get pods -l app=sdnc-prom,release=$RELEASE | grep Running | cut -f1 -d' ' )
+if [ -z "$pod" ];then
+ echo "prom pod not found - is prom running?"
+ exit 1
+fi
+
+kubectl -n $NAMESPACE exec $pod -- /app/bin/sdnc.monitor $DEBUG
diff --git a/kubernetes/sdnc/resources/geo/bin/switchVoting.sh b/kubernetes/sdnc/resources/geo/bin/switchVoting.sh
index 27e4ead..7a1c193 100755
--- a/kubernetes/sdnc/resources/geo/bin/switchVoting.sh
+++ b/kubernetes/sdnc/resources/geo/bin/switchVoting.sh
@@ -1,5 +1,19 @@
#!/bin/bash
+# Copyright © 2018 Amdocs
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
function usage()
{
echo usage: switchVoting.sh primary\|secondary
@@ -10,18 +24,26 @@
usage
fi
-partition=$1
+dir=$( dirname $0 )
+USERNAME=admin
+PASSWORD=`awk '/odlPassword/ {print $2}' $dir/../../../values.yaml | head -1`
-if [ "$partition" == "primary" ]; then
- curl -u admin:{{.Values.config.odlPassword}} -H "Content-Type: application/json" -H "Accept: application/json" -X POST http://localhost:30202/restconf/operations/cluster-admin:change-member-voting-states-for-all-shards -d '{ "input" : { "member-voting-state" : [ { "member-name" : "member-1", "voting":true}, { "member-name" : "member-2", "voting":true}, { "member-name" : "member-3", "voting":true},{ "member-name" : "member-4", "voting":false},{ "member-name" : "member-5", "voting":false},{ "member-name" : "member-6", "voting":false}] } }' > switch_voting_resp.json 2>/dev/null
- echo "" >> switch_voting_resp.json
- exit 0
+case "$1" in
+
+primary)
+ status=$(curl -u $USERNAME:$PASSWORD -o /dev/null -H "Content-Type: application/json" -H "Accept: application/json" -X POST http://localhost:30202/restconf/operations/cluster-admin:change-member-voting-states-for-all-shards -d '{ "input" : { "member-voting-state" : [ { "member-name" : "member-1", "voting":true}, { "member-name" : "member-2", "voting":true}, { "member-name" : "member-3", "voting":true},{ "member-name" : "member-4", "voting":false},{ "member-name" : "member-5", "voting":false},{ "member-name" : "member-6", "voting":false}] } }' -w "%{http_code}\n" $url 2> /dev/null)
+;;
+
+secondary)
+ status=$(curl -u $USERNAME:$PASSWORD -o /dev/null -H "Content-Type: application/json" -H "Accept: application/json" -X POST http://localhost:30202/restconf/operations/cluster-admin:change-member-voting-states-for-all-shards -d '{ "input" : { "member-voting-state" : [ { "member-name" : "member-1", "voting":false}, { "member-name" : "member-2", "voting":false}, { "member-name" : "member-3", "voting":false},{ "member-name" : "member-4", "voting":true},{ "member-name" : "member-5", "voting":true},{ "member-name" : "member-6", "voting":true}] } }' -w "%{http_code}\n" $url 2> /dev/null)
+;;
+
+*)
+ usage
+esac
+
+if [ $status -ne 200 ];then
+ echo "failure"
+else
+ echo "success"
fi
-
-if [ "$partition" == "secondary" ]; then
- curl -u admin:{{.Values.config.odlPassword}} -H "Content-Type: application/json" -H "Accept: application/json" -X POST http://localhost:30202/restconf/operations/cluster-admin:change-member-voting-states-for-all-shards -d '{ "input" : { "member-voting-state" : [ { "member-name" : "member-1", "voting":false}, { "member-name" : "member-2", "voting":false}, { "member-name" : "member-3", "voting":false},{ "member-name" : "member-4", "voting":true},{ "member-name" : "member-5", "voting":true},{ "member-name" : "member-6", "voting":true}] } }' > switch_voting_resp.json 2>/dev/null
- echo "" >> switch_voting_resp.json
- exit 0
-fi
-
-usage