[CDS-STRIMZI] Migrate cds to use strimzi kafka
Move cds to use strimzi kafka
Signed-off-by: efiacor <fiachra.corcoran@est.tech>
Change-Id: I89a64399d677584829e6408c8f72e9b5ad41cabd
Issue-ID: DMAAP-1706
diff --git a/kubernetes/cds/components/cds-blueprints-processor/resources/config/application.properties b/kubernetes/cds/components/cds-blueprints-processor/resources/config/application.properties
index ea799e2..0beaf4a 100755
--- a/kubernetes/cds/components/cds-blueprints-processor/resources/config/application.properties
+++ b/kubernetes/cds/components/cds-blueprints-processor/resources/config/application.properties
@@ -1,6 +1,6 @@
{{/*
#
-# Copyright (c) 2017-2019 AT&T, IBM, Bell Canada, Nordix Foundation.
+# Copyright (c) 2017-2022 AT&T, IBM, Bell Canada, Nordix Foundation.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
@@ -109,19 +109,70 @@
blueprintsprocessor.restclient.aai-data.additionalHeaders.Accept=application/json
# Self Service Request Kafka Message Consumer
-blueprintsprocessor.messageconsumer.self-service-api.kafkaEnable=false
-blueprintsprocessor.messageconsumer.self-service-api.type=kafka-basic-auth
-blueprintsprocessor.messageconsumer.self-service-api.bootstrapServers=message-router-kafka:9092
-blueprintsprocessor.messageconsumer.self-service-api.groupId=cds-consumer-group
-blueprintsprocessor.messageconsumer.self-service-api.topic=cds-consumer
-blueprintsprocessor.messageconsumer.self-service-api.clientId=cds-client
-blueprintsprocessor.messageconsumer.self-service-api.pollMillSec=1000
+blueprintsprocessor.messageconsumer.self-service-api.kafkaEnable={{ .Values.kafkaRequestConsumer.enabled }}
+blueprintsprocessor.messageconsumer.self-service-api.type={{ .Values.kafkaRequestConsumer.type }}
+{{- if eq .Values.useStrimziKafka true }}
+blueprintsprocessor.messageconsumer.self-service-api.bootstrapServers={{ include "common.release" . }}-strimzi-kafka-bootstrap:9092
+{{- else -}}
+blueprintsprocessor.messageconsumer.self-service-api.bootstrapServers={{ .Values.kafkaRequestConsumer.bootstrapServers }}
+{{- end }}
+blueprintsprocessor.messageconsumer.self-service-api.groupId={{ .Values.kafkaRequestConsumer.groupId }}
+blueprintsprocessor.messageconsumer.self-service-api.topic={{ .Values.kafkaRequestConsumer.topic }}
+blueprintsprocessor.messageconsumer.self-service-api.clientId={{ .Values.kafkaRequestConsumer.clientId }}
+blueprintsprocessor.messageconsumer.self-service-api.pollMillSec={{ .Values.kafkaRequestConsumer.pollMillSec }}
+{{- if and (eq .Values.kafkaRequestConsumer.type "kafka-scram-plain-text-auth") (eq .Values.useStrimziKafka true) }}
+# SCRAM
+blueprintsprocessor.messageconsumer.self-service-api.scramUsername={{ include "common.release" . }}-{{ .Values.cdsKafkaUser }}
+blueprintsprocessor.messageconsumer.self-service-api.scramPassword=${JAAS_PASS}
+{{ end }}
# Self Service Response Kafka Message Producer
-blueprintsprocessor.messageproducer.self-service-api.bootstrapServers=message-router-kafka:9092
+blueprintsprocessor.messageproducer.self-service-api.type={{ .Values.kafkaRequestProducer.type }}
+{{- if eq .Values.useStrimziKafka true }}
+blueprintsprocessor.messageproducer.self-service-api.bootstrapServers={{ include "common.release" . }}-strimzi-kafka-bootstrap:9092
+{{- else -}}
+blueprintsprocessor.messageproducer.self-service-api.bootstrapServers={{ .Values.kafkaRequestProducer.bootstrapServers }}
+{{- end }}
+blueprintsprocessor.messageproducer.self-service-api.clientId={{ .Values.kafkaRequestProducer.clientId }}
+blueprintsprocessor.messageproducer.self-service-api.topic={{ .Values.kafkaRequestProducer.topic }}
+{{- if and (eq .Values.kafkaRequestConsumer.type "kafka-scram-plain-text-auth") (eq .Values.useStrimziKafka true) }}
+# SCRAM
+blueprintsprocessor.messageproducer.self-service-api.scramUsername={{ include "common.release" . }}-{{ .Values.cdsKafkaUser }}
+blueprintsprocessor.messageproducer.self-service-api.scramPassword=${JAAS_PASS}
+{{ end }}
-# Kafka Audit Service Configurations
-blueprintsprocessor.messageproducer.self-service-api.audit.kafkaEnable=false
+# AUDIT KAFKA FEATURE CONFIGURATION
+# Audit feature dumps CDS request to a topic as well as a truncated response message to another topic.
+## Audit request
+blueprintsprocessor.messageproducer.self-service-api.audit.kafkaEnable={{ .Values.kafkaAuditRequest.enabled }}
+blueprintsprocessor.messageproducer.self-service-api.audit.request.type={{ .Values.kafkaAuditRequest.type }}
+{{- if eq .Values.useStrimziKafka true }}
+blueprintsprocessor.messageproducer.self-service-api.audit.request.bootstrapServers={{ include "common.release" . }}-strimzi-kafka-bootstrap:9092
+{{- else -}}
+blueprintsprocessor.messageproducer.self-service-api.audit.request.bootstrapServers={{ .Values.kafkaAuditRequest.bootstrapServers }}
+{{- end }}
+blueprintsprocessor.messageproducer.self-service-api.audit.request.clientId={{ .Values.kafkaAuditRequest.clientId }}
+blueprintsprocessor.messageproducer.self-service-api.audit.request.topic={{ .Values.kafkaAuditRequest.topic }}
+{{- if and (eq .Values.kafkaRequestConsumer.type "kafka-scram-plain-text-auth") (eq .Values.useStrimziKafka true) }}
+# SCRAM
+blueprintsprocessor.messageproducer.self-service-api.audit.request.scramUsername={{ include "common.release" . }}-{{ .Values.cdsKafkaUser }}
+blueprintsprocessor.messageproducer.self-service-api.audit.request.scramPassword=${JAAS_PASS}
+{{ end }}
+
+## Audit response
+blueprintsprocessor.messageproducer.self-service-api.audit.response.type={{ .Values.kafkaAuditResponse.type }}
+{{- if eq .Values.useStrimziKafka true }}
+blueprintsprocessor.messageproducer.self-service-api.audit.response.bootstrapServers={{ include "common.release" . }}-strimzi-kafka-bootstrap:9092
+{{- else -}}
+blueprintsprocessor.messageproducer.self-service-api.audit.response.bootstrapServers={{ .Values.kafkaAuditRequest.bootstrapServers }}
+{{- end }}
+blueprintsprocessor.messageproducer.self-service-api.audit.response.clientId={{ .Values.kafkaAuditResponse.clientId }}
+blueprintsprocessor.messageproducer.self-service-api.audit.response.topic={{ .Values.kafkaAuditResponse.topic }}
+{{- if and (eq .Values.kafkaRequestConsumer.type "kafka-scram-plain-text-auth") (eq .Values.useStrimziKafka true) }}
+# SCRAM
+blueprintsprocessor.messageproducer.self-service-api.audit.response.scramUsername={{ include "common.release" . }}-{{ .Values.cdsKafkaUser }}
+blueprintsprocessor.messageproducer.self-service-api.audit.response.scramPassword=${JAAS_PASS}
+{{ end }}
# Executor Options
blueprintsprocessor.resourceResolution.enabled=true
@@ -132,10 +183,10 @@
## Enable py-executor
blueprintsprocessor.streamingRemoteExecution.enabled=true
-# Used in Health Check
-blueprintsprocessor.messageproducer.self-service-api.type=kafka-basic-auth
-blueprintsprocessor.messageproducer.self-service-api.clientId=cds-client
-blueprintsprocessor.messageproducer.self-service-api.topic=cds-producer
+## Used in Health Check
+#blueprintsprocessor.messageproducer.self-service-api.type=kafka-basic-auth
+#blueprintsprocessor.messageproducer.self-service-api.clientId=cds-client
+#blueprintsprocessor.messageproducer.self-service-api.topic=cds-producer
#Encrypted username and password for health check service
diff --git a/kubernetes/cds/components/cds-blueprints-processor/templates/cds-kafka-topics.yaml b/kubernetes/cds/components/cds-blueprints-processor/templates/cds-kafka-topics.yaml
new file mode 100644
index 0000000..555f4d4
--- /dev/null
+++ b/kubernetes/cds/components/cds-blueprints-processor/templates/cds-kafka-topics.yaml
@@ -0,0 +1,68 @@
+{{/*
+# Copyright © 2022 Nordix Foundation
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+*/}}
+{{ if eq .Values.useStrimziKafka true }}
+apiVersion: kafka.strimzi.io/v1beta2
+kind: KafkaTopic
+metadata:
+ name: {{ .Values.kafkaRequestConsumer.topic }}
+ labels:
+ strimzi.io/cluster: {{ include "common.release" . }}-strimzi
+spec:
+ partitions: 10
+ replicas: 2
+ config:
+ retention.ms: 7200000
+ segment.bytes: 1073741824
+---
+apiVersion: kafka.strimzi.io/v1beta2
+kind: KafkaTopic
+metadata:
+ name: {{ .Values.kafkaRequestProducer.topic }}
+ labels:
+ strimzi.io/cluster: {{ include "common.release" . }}-strimzi
+spec:
+ partitions: 10
+ replicas: 2
+ config:
+ retention.ms: 7200000
+ segment.bytes: 1073741824
+---
+apiVersion: kafka.strimzi.io/v1beta2
+kind: KafkaTopic
+metadata:
+ name: {{ .Values.kafkaAuditRequest.topic }}
+ labels:
+ strimzi.io/cluster: {{ include "common.release" . }}-strimzi
+spec:
+ partitions: 10
+ replicas: 2
+ config:
+ retention.ms: 7200000
+ segment.bytes: 1073741824
+---
+apiVersion: kafka.strimzi.io/v1beta2
+kind: KafkaTopic
+metadata:
+ name: {{ .Values.kafkaAuditResponse.topic }}
+ labels:
+ strimzi.io/cluster: {{ include "common.release" . }}-strimzi
+spec:
+ partitions: 10
+ replicas: 2
+ config:
+ retention.ms: 7200000
+ segment.bytes: 1073741824
+{{ end }}
\ No newline at end of file
diff --git a/kubernetes/cds/components/cds-blueprints-processor/templates/cds-kafka-user.yaml b/kubernetes/cds/components/cds-blueprints-processor/templates/cds-kafka-user.yaml
new file mode 100644
index 0000000..65ee1d2
--- /dev/null
+++ b/kubernetes/cds/components/cds-blueprints-processor/templates/cds-kafka-user.yaml
@@ -0,0 +1,49 @@
+{{/*
+# Copyright © 2022 Nordix Foundation
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+*/}}
+{{ if eq .Values.useStrimziKafka true }}
+apiVersion: kafka.strimzi.io/v1beta2
+kind: KafkaUser
+metadata:
+ name: {{ include "common.release" . }}-{{ .Values.cdsKafkaUser }}
+ labels:
+ strimzi.io/cluster: {{ include "common.release" . }}-strimzi
+spec:
+ authentication:
+ type: scram-sha-512
+ authorization:
+ type: simple
+ acls:
+ - resource:
+ type: group
+ name: {{ .Values.kafkaRequestConsumer.groupId }}
+ operation: All
+ - resource:
+ type: topic
+ name: {{ .Values.kafkaRequestConsumer.topic }}
+ operation: All
+ - resource:
+ type: topic
+ name: {{ .Values.kafkaRequestProducer.topic }}
+ operation: All
+ - resource:
+ type: topic
+ name: {{ .Values.kafkaAuditRequest.topic }}
+ operation: All
+ - resource:
+ type: topic
+ name: {{ .Values.kafkaAuditResponse.topic }}
+ operation: All
+{{ end }}
\ No newline at end of file
diff --git a/kubernetes/cds/components/cds-blueprints-processor/templates/deployment.yaml b/kubernetes/cds/components/cds-blueprints-processor/templates/deployment.yaml
index d92f09a..d68e900 100755
--- a/kubernetes/cds/components/cds-blueprints-processor/templates/deployment.yaml
+++ b/kubernetes/cds/components/cds-blueprints-processor/templates/deployment.yaml
@@ -1,6 +1,7 @@
{{/*
# Copyright (c) 2019 IBM, Bell Canada
# Copyright (c) 2020 Samsung Electronics
+# Modification Copyright © 2022 Nordix Foundation
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
@@ -78,10 +79,6 @@
args:
- --container-name
- cds-db
- {{- if .Values.dmaapEnabled }}
- - --container-name
- - message-router
- {{ end }}
env:
- name: NAMESPACE
valueFrom:
@@ -121,6 +118,10 @@
fieldPath: metadata.name
- name: CLUSTER_CONFIG_FILE
value: {{ .Values.config.appConfigDir }}/hazelcast.yaml
+ {{ if .Values.useStrimziKafka }}
+ - name: JAAS_PASS
+ value: {{- include "common.secret.envFromSecretFast" (dict "global" . "uid" "cds-kafka-secret" "key" "password") | indent 12 }}
+ {{ end }}
ports:
- containerPort: {{ .Values.service.http.internalPort }}
- containerPort: {{ .Values.service.grpc.internalPort }}
diff --git a/kubernetes/cds/components/cds-blueprints-processor/values.yaml b/kubernetes/cds/components/cds-blueprints-processor/values.yaml
index a5180c5..af9482b 100755
--- a/kubernetes/cds/components/cds-blueprints-processor/values.yaml
+++ b/kubernetes/cds/components/cds-blueprints-processor/values.yaml
@@ -1,5 +1,6 @@
# Copyright (c) 2019 IBM, Bell Canada
# Copyright (c) 2020 Samsung Electronics
+# Modification Copyright © 2022 Nordix Foundation
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
@@ -57,6 +58,13 @@
externalSecret: '{{ tpl (default "" .Values.config.sdncDB.dbRootPassExternalSecret) . }}'
password: '{{ .Values.config.sdncDB.dbRootPass }}'
passwordPolicy: required
+ - uid: cds-kafka-secret
+ externalSecret: '{{ tpl (default "" .Values.config.jaasConfExternalSecret) . }}'
+ type: genericKV
+ envs:
+ - name: password
+ value: '{{ .Values.config.someConfig }}'
+ policy: generate
#################################################################
# AAF part
@@ -111,6 +119,7 @@
# dbCredsExternalSecret: <some secret name>
# dbRootPassword: password
# dbRootPassExternalSecret
+ someConfig: blah
# default number of instances
replicaCount: 1
@@ -119,10 +128,40 @@
affinity: {}
-# flag for kafka-listener dependency. Set to true if you are using message-router otherwise set to false if you are using
-# custom kafka cluster.
-dmaapEnabled: true
+# If useStrimziKafka is true, the following also applies:
+# strimzi will create an associated kafka user and the topics defined for Request and Audit elements below.
+# The connection type must be kafka-scram-plain-text-auth
+# The bootstrapServers will target the strimzi kafka cluster by default
+useStrimziKafka: false
+cdsKafkaUser: cds-kafka-user
+kafkaRequestConsumer:
+ enabled: false
+ type: kafka-scram-plain-text-auth
+ bootstrapServers: host:port
+ groupId: cds-consumer
+ topic: cds.blueprint-processor.self-service-api.request
+ clientId: request-receiver-client-id
+ pollMillSec: 1000
+kafkaRequestProducer:
+ type: kafka-scram-plain-text-auth
+ bootstrapServers: host:port
+ clientId: request-producer-client-id
+ topic: cds.blueprint-processor.self-service-api.response
+ enableIdempotence: false
+kafkaAuditRequest:
+ enabled: false
+ type: kafka-scram-plain-text-auth
+ bootstrapServers: host:port
+ clientId: audit-request-producer-client-id
+ topic: cds.blueprint-processor.self-service-api.audit.request
+ enableIdempotence: false
+kafkaAuditResponse:
+ type: kafka-scram-plain-text-auth
+ bootstrapServers: host:port
+ clientId: audit-response-producer-client-id
+ topic: cds.blueprint-processor.self-service-api.audit.response
+ enableIdempotence: false
# probe configuration parameters
startup:
diff --git a/kubernetes/cds/values.yaml b/kubernetes/cds/values.yaml
index edac066..58e6b65 100644
--- a/kubernetes/cds/values.yaml
+++ b/kubernetes/cds/values.yaml
@@ -1,6 +1,7 @@
# Copyright © 2020 Samsung Electronics
# Copyright © 2019 Orange, Bell Canada
# Copyright © 2017 Amdocs, Bell Canada
+# Modification Copyright © 2022 Nordix Foundation
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
@@ -22,6 +23,7 @@
nodePortPrefixExt: 304
persistence:
mountPath: /dockerdata-nfs
+ cdsKafkaUser: cds-kafka-user
#################################################################
# Secrets metaconfig
@@ -212,6 +214,7 @@
dbPort: 3306
dbName: *mysqlDbName
dbCredsExternalSecret: *dbUserSecretName
+ jaasConfExternalSecret: '{{ include "common.release" . }}-{{ .Values.global.kafkaUser }}'
cds-command-executor:
enabled: true