[POLICY] Enhance liveness probes and metrics
- added logback prometheus support for xacml and drools.
- enabled enhanced healthchecks.
- enabled enhanced liveness probes for drools pdp to detect stuck PDP
application and restart it.
- add prometheus servicemonitor for drools.
- set latest docker image versions for drools-applications and xacml
Issue-ID: POLICY-3386
Issue-ID: POLICY-4089
Signed-off-by: jhh <jorge.hernandez-herrero@att.com>
Change-Id: I0ffd9bf93452f8a7a63deb33f2103f9b80476d07
Signed-off-by: jhh <jorge.hernandez-herrero@att.com>
diff --git a/kubernetes/policy/components/policy-drools-pdp/resources/configmaps/base.conf b/kubernetes/policy/components/policy-drools-pdp/resources/configmaps/base.conf
index ff532ab..57fa29b 100755
--- a/kubernetes/policy/components/policy-drools-pdp/resources/configmaps/base.conf
+++ b/kubernetes/policy/components/policy-drools-pdp/resources/configmaps/base.conf
@@ -1,6 +1,6 @@
{{/*
# Copyright © 2017-2018 Amdocs, Bell Canada.
-# Modifications Copyright (C) 2018-2020 AT&T Intellectual Property.
+# Modifications Copyright (C) 2018-2020, 2022 AT&T Intellectual Property.
# Modifications Copyright (C) 2021 Bell Canada. All rights reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License");
@@ -43,6 +43,9 @@
SQL_HOST={{ .Values.db.name }}
SQL_PORT=3306
+# Liveness
+LIVENESS_CONTROLLERS=*
+
# AAF
AAF={{.Values.aaf.enabled}}
diff --git a/kubernetes/policy/components/policy-drools-pdp/resources/configmaps/logback.xml b/kubernetes/policy/components/policy-drools-pdp/resources/configmaps/logback.xml
index 2fc08e4..a25a7e1 100755
--- a/kubernetes/policy/components/policy-drools-pdp/resources/configmaps/logback.xml
+++ b/kubernetes/policy/components/policy-drools-pdp/resources/configmaps/logback.xml
@@ -1,7 +1,7 @@
<!--
============LICENSE_START=======================================================
Copyright (C) 2020 Bell Canada. All rights reserved.
- Modifications Copyright (C) 2021 AT&T Intellectual Property. All rights reserved.
+ Modifications Copyright (C) 2021-2022 AT&T Intellectual Property. All rights reserved.
================================================================================
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
@@ -150,6 +150,8 @@
<appender-ref ref="AsyncStdOut" />
</logger>
+ <appender name="PromLogback" class="io.prometheus.client.logback.InstrumentedAppender"/>
+
<root level="INFO">
<appender-ref ref="AsyncDebugOut" />
<appender-ref ref="AsyncErrorOut" />
@@ -158,6 +160,7 @@
<appender-ref ref="AsyncStdOut" />
<appender-ref ref="AsyncMetricStdOut" />
<appender-ref ref="AsyncTransactionStdOut" />
+ <appender-ref ref="PromLogback" />
</root>
</configuration>
diff --git a/kubernetes/policy/components/policy-drools-pdp/templates/serviceMonitor.yaml b/kubernetes/policy/components/policy-drools-pdp/templates/serviceMonitor.yaml
new file mode 100644
index 0000000..1c04296
--- /dev/null
+++ b/kubernetes/policy/components/policy-drools-pdp/templates/serviceMonitor.yaml
@@ -0,0 +1,23 @@
+{{/*
+# ============LICENSE_START=======================================================
+# Copyright (c) 2022 AT&T Intellectual Property
+# ================================================================================
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+
+# http://www.apache.org/licenses/LICENSE-2.0
+
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+# SPDX-License-Identifier: Apache-2.0
+# ============LICENSE_END=========================================================
+*/}}
+
+{{- if .Values.prometheus.enabled }}
+{{ include "common.serviceMonitor" . }}
+{{- end }}
diff --git a/kubernetes/policy/components/policy-drools-pdp/templates/statefulset.yaml b/kubernetes/policy/components/policy-drools-pdp/templates/statefulset.yaml
index d389246..7dee453 100755
--- a/kubernetes/policy/components/policy-drools-pdp/templates/statefulset.yaml
+++ b/kubernetes/policy/components/policy-drools-pdp/templates/statefulset.yaml
@@ -1,6 +1,6 @@
{{/*
# Copyright © 2017 Amdocs, Bell Canada
-# Modifications Copyright © 2018-2020 AT&T Intellectual Property
+# Modifications Copyright © 2018-2020, 2022 AT&T Intellectual Property
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
@@ -83,10 +83,12 @@
- containerPort: {{ .Values.service.externalPort2 }}
{{- if eq .Values.liveness.enabled true }}
livenessProbe:
- tcpSocket:
- port: {{ .Values.service.externalPort }}
+ httpGet:
+ path: /healthcheck/controllers
+ port: 6968
initialDelaySeconds: {{ .Values.liveness.initialDelaySeconds }}
periodSeconds: {{ .Values.liveness.periodSeconds }}
+ timeoutSeconds: {{ .Values.liveness.timeoutSeconds }}
{{- end }}
readinessProbe:
tcpSocket:
diff --git a/kubernetes/policy/components/policy-drools-pdp/values.yaml b/kubernetes/policy/components/policy-drools-pdp/values.yaml
index 2ce7503..344a600 100755
--- a/kubernetes/policy/components/policy-drools-pdp/values.yaml
+++ b/kubernetes/policy/components/policy-drools-pdp/values.yaml
@@ -1,6 +1,6 @@
# Copyright © 2017 Amdocs
# Copyright © 2017, 2021 Bell Canada
-# Modifications Copyright © 2018-2021 AT&T Intellectual Property
+# Modifications Copyright © 2018-2022 AT&T Intellectual Property
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
@@ -30,12 +30,18 @@
login: '{{ .Values.db.user }}'
password: '{{ .Values.db.password }}'
passwordPolicy: required
+ - uid: telemetry-creds
+ type: basicAuth
+ externalSecret: '{{ tpl (default "" .Values.telemetry.credsExternalSecret) . }}'
+ login: '{{ .Values.telemetry.user }}'
+ password: '{{ .Values.telemetry.password }}'
+ passwordPolicy: required
#################################################################
# Application configuration defaults.
#################################################################
# application image
-image: onap/policy-pdpd-cl:1.10.1
+image: onap/policy-pdpd-cl:1.10.2
pullPolicy: Always
# flag to enable debugging - application support required
@@ -51,7 +57,8 @@
# probe configuration parameters
liveness:
initialDelaySeconds: 180
- periodSeconds: 10
+ periodSeconds: 60
+ timeoutSeconds: 10
# necessary to disable liveness probe when setting breakpoints
# in debugger so K8s doesn't restart unresponsive container
enabled: true
@@ -195,3 +202,27 @@
nameOverride: policy-drools-pdp
roles:
- read
+
+prometheus:
+ enabled: true
+
+metrics:
+ serviceMonitor:
+ # Override the labels based on the Prometheus config parameter: serviceMonitorSelector.
+ # The default operator for prometheus enforces the below label.
+ labels:
+ release: prometheus
+ enabled: true
+ port: policy-drools-pdp-9696
+ interval: 60s
+ isHttps: true
+ basicAuth:
+ enabled: true
+ externalSecretNameSuffix: policy-drools-pdp-telemetry-creds
+ externalSecretUserKey: login
+ externalSecretPasswordKey: password
+ selector:
+ app: '{{ include "common.name" . }}'
+ chart: '{{ .Chart.Name }}-{{ .Chart.Version | replace "+" "_" }}'
+ release: '{{ include "common.release" . }}'
+ heritage: '{{ .Release.Service }}'