kubernetes/common/cassandra/templates/backup/cronjob.yaml

{{/*
# Copyright © 2019 Amdocs, Bell Canada
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#       http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
*/}}
{{- if .Values.backup.enabled }}
apiVersion: batch/v1beta1
kind: CronJob
metadata:
  name: {{ include "common.fullname" . }}-backup
  namespace: {{ include "common.namespace" . }}
  labels:
    app: {{ include "common.fullname" . }}
    chart: {{ .Chart.Name }}-{{ .Chart.Version | replace "+" "_" }}
    release: {{ .Release.Name }}
    heritage: {{ .Release.Service }}
spec:
  schedule: {{ .Values.backup.cron | quote }}
  concurrencyPolicy: Forbid
  startingDeadlineSeconds: 120
  jobTemplate:
    spec:
      template:
        spec:
          restartPolicy: Never
          initContainers:
            - command:
              - /root/ready.py
              args:
              - --container-name
              - {{ include "common.name" . }}
              env:
              - name: NAMESPACE
                valueFrom:
                  fieldRef:
                    apiVersion: v1
                    fieldPath: metadata.namespace
              image: "{{ .Values.global.readinessRepository }}/{{ .Values.global.readinessImage }}"
              imagePullPolicy: {{ .Values.global.pullPolicy | default .Values.pullPolicy }}
              name: {{ include "common.name" . }}-readiness
            - name: "cassandra-backup-init"
              image: "{{ .Values.global.readinessRepository }}/{{ .Values.global.readinessImage }}"
              imagePullPolicy: {{ .Values.global.pullPolicy | default .Values.pullPolicy }}
              command:
              - /bin/bash
              - -c
              - |
                clearSnapshot(){
                  curr_time=$1
                  echo "Clearing snapshots!!!"
                  command="nodetool clearsnapshot -t $curr_time"
                  /root/exec.py -p "cassandra" -c "$command"
                }
                {{ $root := . }}
                curr_time=`date +%s`
                pids=""
                set -x

                echo "Copying data"
                {{ range $i, $e := until (int .Values.replicaCount) }}
                  target_dir=/backup/temp/cassandra-{{ $i }}
                  mkdir -p $target_dir
                  cp -Ra /onap-data/cassandra-{{ $i }}/data/ $target_dir/
                {{- end }}

                echo "Executing cleanup!!"
                command="nodetool cleanup"
                /root/exec.py -p "cassandra" -c "$command"
                echo "Cleaned Node!! Backing up database now!!!"

                command="nodetool snapshot -t $curr_time"
                /root/exec.py -p "cassandra" -c "$command"
                retCode=$?
                if [ $retCode -ne 0 ]; then
                  echo "Backup Failed!!!"
                  rm -rf /backup/temp
                  clearSnapshot $curr_time
                  echo "Failed" > /backup/backup.log
                  exit 0
                fi

                backup_dir=/backup/temp
                {{ range $i, $e := until (int .Values.replicaCount) }}
                  for d in $backup_dir/cassandra-{{ $i }}/data/*/ ; do
                    d=$(echo $d | sed 's:/*$::')
                    keyspace_name=$(echo "$d" | awk -F/ '{ print $NF }')
                    if [ 1 ] {{- range $t, $keyspace := $root.Values.backup.keyspacesToSkip }} && [ "{{ $keyspace.name }}" != "$keyspace_name" ] {{- end }}; then
                      /root/restore.sh -b $backup_dir/cassandra-{{ $i }}/data -s  /onap-data/cassandra-{{ $i }}/data/$keyspace_name -k $keyspace_name -t $curr_time &
                      pids="$pids $!"
                    fi
                  done
                {{- end }}

                for p in $pids; do
                  wait $p
                  if [ $? -ne 0 ]; then
                    rm -rf /backup/temp
                    echo "Creation of Backup Failed!!!"
                    clearSnapshot $curr_time
                    echo "Failed" > /backup/backup.log
                    exit 0
                  fi
                done

                clearSnapshot $curr_time

                exit_code=$?
                if [ $exit_code -ne 0 ]; then
                  rm -rf /backup/temp
                  echo "Backup Failed!!!"
                  echo "Failed" > /backup/backup.log
                  exit 0
                fi

                mv /backup/temp /backup/backup-${curr_time}
                echo "Success" > /backup/backup.log
                echo "Cassandra Backup Succeeded"
              env:
              - name: NAMESPACE
                valueFrom:
                  fieldRef:
                    apiVersion: v1
                    fieldPath: metadata.namespace
              volumeMounts:
              - mountPath: /etc/localtime
                name: localtime
                readOnly: true
              {{- range $i := until (int .Values.replicaCount)}}
              - mountPath: /onap-data/cassandra-{{ $i }}
                name: data-dir-{{ $i }}
              {{- end }}
              - mountPath: /backup
                name: backup-dir
              - name: scripts
                mountPath: /root/restore.sh
                subPath: restore.sh
              - name: scripts
                mountPath: /root/exec.py
                subPath: exec.py
          containers:
            - name: cassandra-backup-validate
              image: "{{ .Values.image }}"
              imagePullPolicy: {{ .Values.global.pullPolicy | default .Values.pullPolicy }}
              command:
              - /bin/bash
              - -c
              - |
                remove_dir(){
                  dirToRemove=$1
                  rm -rf $dirToRemove
                }

                backup_result=`cat /backup/backup.log`
                rm -rf /backup/backup.log

                if [ "$backup_result" == "Failed" ]; then
                  echo "Backup Failed!!! So Validation Failed!!!";
                  exit 0
                fi

                target_dir=$(ls -td -- /backup/*/ | head -n 1)
                chown -R cassandra.cassandra $target_dir
                {{- $root := . -}}
                {{ range $i, $e := until (int .Values.replicaCount) }}
                  dbSize=$(du -ks $target_dir/cassandra-{{ $i }}/data|awk -F " " '{ printf $1 }')
                  minDbSize={{ (int $root.Values.backup.dbSize) }}
                  if [ $dbSize -lt $minDbSize ]; then
                    remove_dir $target_dir
                    echo "Validation Failed!!! dbSize ($dbSize) is less than minimum size (1)!!!"
                    exit 0
                  fi
                  rm -rf /var/lib/cassandra/*
                  cp -Ra $target_dir/cassandra-{{ $i }}/data /var/lib/cassandra
                  export CASSANDRA_LISTEN_ADDRESS="127.0.0.1"
                  /docker-entrypoint.sh -Dcassandra.ignore_dc=true -Dcassandra.ignore_rack=true &
                  CASS_PID=$!
                  sleep 45

                  for d in $target_dir/cassandra-{{ $i }}/data/*/; do
                      d=$(echo $d | sed 's:/*$::')
                      keyspace_name=$(echo "$d" | awk -F/ '{ print $NF }')
                      if [ 1 ] {{- range $t, $keyspace := $root.Values.backup.keyspacesToSkip }} && [ "{{ $keyspace.name }}" != "$keyspace_name" ] {{- end }}; then
                        echo "Verifying the data for  $keyspace_name "
                        nodetool verify -e $keyspace_name
                        ret=$?
                        if [ $ret -ne 0 ]; then
                          remove_dir $target_dir
                          echo "Validation Failed!!!"
                          exit 0
                        fi
                      fi
                  done
                  kill -9 $CASS_PID
                {{- end }}
                echo "Validation Successful!!!"
                cd /backup
                totalFiles=`ls -t | grep "backup-" | wc -l`
                if [ $totalFiles -gt {{ .Values.backup.retentionPeriod }} ]; then
                  filestoDelete=`expr $totalFiles - {{ .Values.backup.retentionPeriod }}`
                  ls -tr | grep backup | head -$filestoDelete | xargs rm -rf
                fi
              env:
              - name: CASSANDRA_CLUSTER_NAME
                value: {{ .Values.config.clusterName }}
              - name: MAX_HEAP_SIZE
                value: {{ .Values.config.heap.max }}
              - name: HEAP_NEWSIZE
                value: {{ .Values.config.heap.min }}
              - name: HOST_IP
                valueFrom:
                  fieldRef:
                    fieldPath: status.podIP
              volumeMounts:
              - name: backup-dir
                mountPath: /backup
              - name: localtime
                mountPath: /etc/localtime
                readOnly: true
          volumes:
          - name: localtime
            hostPath:
              path: /etc/localtime
          - name: scripts
            configMap:
              name: {{ include "common.fullname" . }}-configmap
              defaultMode: 0755
          {{- range $i := until (int .Values.replicaCount)}}
          - name: data-dir-{{ $i }}
            persistentVolumeClaim:
              claimName: {{ include "common.fullname" . }}-data-{{ $i }}
          {{- end }}
          - name: backup-dir
            persistentVolumeClaim:
              claimName: {{ include "common.fullname" . }}-backup-data
{{- end -}}