Added elk files plus other updates

Change-Id: Idb2350c22a1c72324f7599171b94ed12d3d84ce0
diff --git a/rapps/chartmuseum.yaml b/rapps/chartmuseum.yaml
index 5904576..6871d46 100644
--- a/rapps/chartmuseum.yaml
+++ b/rapps/chartmuseum.yaml
@@ -1,35 +1,3 @@
-apiVersion: v1
-kind: PersistentVolume
-metadata:
-  name: chartmuseum-storage-pv-volume
-  namespace: default
-  labels:
-    type: local
-    app: chartmuseum 
-spec:
-  storageClassName: manual
-  capacity:
-    storage: 2Gi
-  accessModes:
-    - ReadWriteOnce
-  hostPath:
-    path: "/var/chartmuseum/charts"
----
-apiVersion: v1
-kind: PersistentVolumeClaim
-metadata:
-  name: chartmuseum-storage-pv-claim
-  namespace: default
-  labels:
-    app: chartmuseum 
-spec:
-  storageClassName: manual
-  accessModes:
-    - ReadWriteOnce
-  resources:
-    requests:
-      storage: 2Gi
----
 apiVersion: apps/v1
 kind: Deployment
 metadata:
@@ -67,12 +35,15 @@
             memory: 128Mi
             cpu: "80m"
         volumeMounts:
-        - name: chartmuseum-persistent-storage
+        - name: chartdir
           mountPath: /charts
+          readOnly: true
       volumes:
-      - name: chartmuseum-persistent-storage
-        persistentVolumeClaim:
-          claimName: chartmuseum-storage-pv-claim
+      - name: chartdir 
+        hostPath:
+          # Ensure the file directory is created.
+           path: /var/chartmuseum/charts 
+           type: DirectoryOrCreate
   replicas: 1 
 ---
 apiVersion: v1
diff --git a/rapps/charts/rapp-jwt-invoker/templates/deployment.yaml b/rapps/charts/rapp-jwt-invoker/templates/deployment.yaml
index 3bb60bd..f5be0d6 100644
--- a/rapps/charts/rapp-jwt-invoker/templates/deployment.yaml
+++ b/rapps/charts/rapp-jwt-invoker/templates/deployment.yaml
@@ -53,17 +53,30 @@
             initialDelaySeconds: 5 
             periodSeconds: 60
           readinessProbe:
-              exec:
-                command: ["stat", "init.txt"]
+            initialDelaySeconds: 5 
+            periodSeconds: 10
+            exec:
+              command: ["stat", "init.txt"]
           resources:
             {{- toYaml .Values.resources | nindent 12 }}
           volumeMounts:
-          - name: jwt-invoker-certs-persistent-storage
+          - name: certsdir
             mountPath: /certs
+            readOnly: true
+          - name: cadir
+            mountPath: /ca
+            readOnly: true
       volumes:
-      - name: jwt-invoker-certs-persistent-storage
-        persistentVolumeClaim:
-          claimName: jwt-invoker-certs-storage-pv-claim
+      - name: certsdir
+        hostPath:
+          # Ensure the file directory is created.
+           path: /var/rapps/certs 
+           type: DirectoryOrCreate
+      - name: cadir
+        hostPath:
+          # Ensure the file directory is created.
+           path: /var/elasticsearch/config/certs/ca 
+           type: DirectoryOrCreate
       {{- with .Values.nodeSelector }}
       nodeSelector:
         {{- toYaml . | nindent 8 }}
diff --git a/rapps/charts/rapp-jwt-invoker/templates/persistentvolume.yaml b/rapps/charts/rapp-jwt-invoker/templates/persistentvolume.yaml
deleted file mode 100644
index 5370e21..0000000
--- a/rapps/charts/rapp-jwt-invoker/templates/persistentvolume.yaml
+++ /dev/null
@@ -1,16 +0,0 @@
-apiVersion: v1
-kind: PersistentVolume
-metadata:
-  name: jwt-invoker-certs-storage-pv-volume
-  namespace: istio-nonrtric 
-  labels:
-    type: local
-    app: rapp-jwt-invoker 
-spec:
-  storageClassName: manual
-  capacity:
-    storage: 10Mi
-  accessModes:
-    - ReadOnlyMany 
-  hostPath:
-    path: "/var/rapps/certs"
diff --git a/rapps/charts/rapp-jwt-invoker/templates/persistentvolumeclaim.yaml b/rapps/charts/rapp-jwt-invoker/templates/persistentvolumeclaim.yaml
deleted file mode 100644
index 022d3f9..0000000
--- a/rapps/charts/rapp-jwt-invoker/templates/persistentvolumeclaim.yaml
+++ /dev/null
@@ -1,14 +0,0 @@
-apiVersion: v1
-kind: PersistentVolumeClaim
-metadata:
-  name: jwt-invoker-certs-storage-pv-claim
-  namespace: istio-nonrtric 
-  labels:
-    app: rapp-jwt-invoker 
-spec:
-  storageClassName: manual
-  accessModes:
-    - ReadOnlyMany 
-  resources:
-    requests:
-      storage: 10Mi
diff --git a/rapps/charts/rapp-jwt-provider/templates/persistentvolume.yaml b/rapps/charts/rapp-jwt-provider/templates/persistentvolume.yaml
deleted file mode 100644
index c3d23a9..0000000
--- a/rapps/charts/rapp-jwt-provider/templates/persistentvolume.yaml
+++ /dev/null
@@ -1,16 +0,0 @@
-apiVersion: v1
-kind: PersistentVolume
-metadata:
-  name: jwt-provider-certs-storage-pv-volume
-  namespace: istio-nonrtric 
-  labels:
-    type: local
-    app: rapp-jwt-provider
-spec:
-  storageClassName: manual
-  capacity:
-    storage: 10Mi
-  accessModes:
-    - ReadOnlyMany 
-  hostPath:
-    path: "/var/rapps/certs"
diff --git a/rapps/charts/rapp-x509-invoker/templates/deployment.yaml b/rapps/charts/rapp-x509-invoker/templates/deployment.yaml
index d1fca53..f9e1abb 100644
--- a/rapps/charts/rapp-x509-invoker/templates/deployment.yaml
+++ b/rapps/charts/rapp-x509-invoker/templates/deployment.yaml
@@ -53,17 +53,22 @@
             initialDelaySeconds: 5 
             periodSeconds: 60
           readinessProbe:
-              exec:
-                command: ["stat", "init.txt"]
+            initialDelaySeconds: 5
+            periodSeconds: 10
+            exec:
+              command: ["stat", "init.txt"]
           resources:
             {{- toYaml .Values.resources | nindent 12 }}
           volumeMounts:
-          - name: x509-rapps-certs-persistent-storage
+          - name: certsdir
             mountPath: /certs
+            readOnly: true
       volumes:
-      - name: x509-rapps-certs-persistent-storage
-        persistentVolumeClaim:
-          claimName: x509-rapps-certs-storage-pv-claim
+      - name: certsdir
+        hostPath:
+          # Ensure the file directory is created.
+           path: /var/rapps/certs
+           type: DirectoryOrCreate
       {{- with .Values.nodeSelector }}
       nodeSelector:
         {{- toYaml . | nindent 8 }}
diff --git a/rapps/charts/rapp-x509-invoker/templates/persistentvolume.yaml b/rapps/charts/rapp-x509-invoker/templates/persistentvolume.yaml
deleted file mode 100644
index b64a1a8..0000000
--- a/rapps/charts/rapp-x509-invoker/templates/persistentvolume.yaml
+++ /dev/null
@@ -1,16 +0,0 @@
-apiVersion: v1
-kind: PersistentVolume
-metadata:
-  name: x509-rapps-certs-storage-pv-volume
-  namespace: istio-nonrtric 
-  labels:
-    type: local
-    app: rapp-x509-invoker 
-spec:
-  storageClassName: manual
-  capacity:
-    storage: 10Mi
-  accessModes:
-    - ReadOnlyMany 
-  hostPath:
-    path: "/var/rapps/certs"
diff --git a/rapps/charts/rapp-x509-invoker/templates/persistentvolumeclaim.yaml b/rapps/charts/rapp-x509-invoker/templates/persistentvolumeclaim.yaml
deleted file mode 100644
index 1622ed5..0000000
--- a/rapps/charts/rapp-x509-invoker/templates/persistentvolumeclaim.yaml
+++ /dev/null
@@ -1,14 +0,0 @@
-apiVersion: v1
-kind: PersistentVolumeClaim
-metadata:
-  name: x509-rapps-certs-storage-pv-claim
-  namespace: istio-nonrtric 
-  labels:
-    app: rapp-x509-invoker 
-spec:
-  storageClassName: manual
-  accessModes:
-    - ReadOnlyMany 
-  resources:
-    requests:
-      storage: 10Mi
diff --git a/rapps/elastic-8.1.2.yaml b/rapps/elastic-8.1.2.yaml
new file mode 100644
index 0000000..1ed2a4e
--- /dev/null
+++ b/rapps/elastic-8.1.2.yaml
@@ -0,0 +1,197 @@
+apiVersion: v1
+kind: ConfigMap
+metadata:
+  name: elasticsearch-init-script
+  namespace: logging 
+data:
+  setup_certs.sh: |
+   #!/bin/bash
+   ELASTIC_HOME=/usr/share/elasticsearch
+   # If the ca directory already exists, delete it
+   if [ -d /certs-dir/ca ]; then
+        rm -rf /certs-dir/ca 
+   fi
+   # If the elasticsearch directory already exists, delete it
+   if [ -d /certs-dir/elasticsearch ]; then
+        rm -rf /certs-dir/elasticsearch
+   fi
+   echo "Creating CA";
+   $ELASTIC_HOME/bin/elasticsearch-certutil ca --silent --pem -out /certs-dir/ca.zip;
+   unzip -o /certs-dir/ca.zip -d /certs-dir;
+   echo "Creating certs";
+   echo -ne \
+   "instances:\n"\
+   "  - name: elasticsearch\n"\
+   "    dns:\n"\
+   "      - elasticsearch\n"\
+   "      - elasticsearch.logging\n"\
+   "      - elasticsearch.est.tech\n"\
+   "      - localhost\n"\
+   "    ip:\n"\
+   "      - 127.0.0.1\n"\
+   "      - 192.168.49.2\n"\
+   > /certs-dir/instances.yml;
+   $ELASTIC_HOME/bin/elasticsearch-certutil cert --silent --pem -out /certs-dir/certs.zip --in /certs-dir/instances.yml \
+   --ca-cert /certs-dir/ca/ca.crt --ca-key /certs-dir/ca/ca.key;
+   unzip -o /certs-dir/certs.zip -d /certs-dir;
+  
+   echo "Removing zip files"  
+   rm -f /certs-dir/ca.zip
+   rm -f /certs-dir/certs.zip
+   echo "Setting file permissions"
+   chmod 750 /certs-dir/ca 
+   chmod 750 /certs-dir/elasticsearch
+   chmod 640 /certs-dir/ca/* 
+   chmod 640 /certs-dir/elasticsearch/*
+   echo "All done!"; 
+---
+apiVersion: v1
+kind: ConfigMap
+metadata:
+  name: elasticsearch-config
+  namespace: logging 
+data:
+  elasticsearch.yml: |
+   discovery.type: single-node
+   cluster.name: "docker-cluster"
+   network.host: 0.0.0.0
+   node.name: elasticsearch 
+   ingest.geoip.downloader.enabled: false
+   xpack.license.self_generated.type: basic
+   xpack.security.enabled: true
+   xpack.security.http.ssl.enabled: true
+   xpack.security.http.ssl.key: certs/elasticsearch/elasticsearch.key
+   xpack.security.http.ssl.certificate: certs/elasticsearch/elasticsearch.crt
+   xpack.security.http.ssl.certificate_authorities: certs/ca/ca.crt
+   xpack.security.http.ssl.verification_mode: certificate
+   xpack.security.transport.ssl.enabled: true
+   xpack.security.transport.ssl.key: certs/elasticsearch/elasticsearch.key
+   xpack.security.transport.ssl.certificate: certs/elasticsearch/elasticsearch.crt
+   xpack.security.transport.ssl.certificate_authorities: certs/ca/ca.crt
+   xpack.security.transport.ssl.verification_mode: certificate
+---
+apiVersion: apps/v1 
+kind: Deployment
+metadata:
+  name: elasticsearch
+  namespace: logging
+spec:
+  selector:
+    matchLabels:
+      component: elasticsearch
+  template:
+    metadata:
+      labels:
+        component: elasticsearch
+    spec:
+      containers:
+      - name: elasticsearch
+        imagePullPolicy: IfNotPresent
+        image: docker.elastic.co/elasticsearch/elasticsearch:8.1.2
+        env:
+        - name: ELASTIC_PASSWORD 
+          value: "secret"
+        ports:
+        - containerPort: 9200
+          name: http
+          protocol: TCP 
+        resources:
+          limits:
+            cpu: 500m
+            memory: 4Gi
+          requests:
+            cpu: 500m
+            memory: 4Gi
+        volumeMounts:
+        - name: elasticsearch-storage
+          mountPath: /usr/share/elasticsearch/data 
+          readOnly: false
+        - name: elasticsearch-certs
+          mountPath: /usr/share/elasticsearch/config/certs
+          readOnly: true 
+        - name : config
+          mountPath: /usr/share/elasticsearch/config/elasticsearch.yml 
+          subPath: elasticsearch.yml 
+          readOnly: false
+      initContainers:
+      - name: init-elasticsearch
+        image: docker.elastic.co/elasticsearch/elasticsearch:8.1.2
+        imagePullPolicy: IfNotPresent
+        command: ['/bin/bash', '-c', '/usr/share/elasticsearch/bin/setup_certs.sh']
+        volumeMounts:
+        - name: elasticsearch-certs
+          mountPath: "/certs-dir"
+        - name: elasticsearch-cert-init
+          mountPath: /usr/share/elasticsearch/bin/setup_certs.sh
+          subPath: setup_certs.sh
+      volumes:
+      - name: elasticsearch-storage
+        hostPath:
+          # Ensure the file directory is created.
+           path: /var/elasticsearch/data 
+           type: DirectoryOrCreate
+      - name: elasticsearch-certs
+        hostPath:
+          # Ensure the file directory is created.
+           path: /var/elasticsearch/config/certs 
+           type: DirectoryOrCreate
+      - name: config
+        configMap:
+          name: elasticsearch-config 
+      - name: elasticsearch-cert-init
+        configMap:
+          name: elasticsearch-init-script
+          defaultMode: 0755
+---
+apiVersion: v1
+kind: Service
+metadata:
+  name: elasticsearch
+  namespace: logging
+  labels:
+    service: elasticsearch
+spec:
+  type: NodePort
+  selector:
+    component: elasticsearch
+  ports:
+  - port: 9200
+    targetPort: 9200
+---
+apiVersion: networking.istio.io/v1alpha3
+kind: Gateway
+metadata:
+  name: esgateway
+spec:
+  selector:
+    istio: ingressgateway # use istio default ingress gateway
+  servers:
+  - port:
+      number: 443
+      name: https
+      protocol: HTTPS
+    tls:
+      mode: PASSTHROUGH
+    hosts:
+    - elasticsearch.est.tech
+---
+apiVersion: networking.istio.io/v1alpha3
+kind: VirtualService
+metadata:
+  name: esvirtualservice
+spec:
+  hosts:
+  - "elasticsearch.est.tech"
+  gateways:
+  - esgateway
+  tls:
+  - match:
+    - port: 443
+      sniHosts:
+      - elasticsearch.est.tech
+    route:
+    - destination:
+        host: elasticsearch.logging.svc.cluster.local
+        port:
+          number: 9200
+---
diff --git a/rapps/fluentd.yaml b/rapps/fluentd.yaml
new file mode 100644
index 0000000..c402966
--- /dev/null
+++ b/rapps/fluentd.yaml
@@ -0,0 +1,126 @@
+apiVersion: v1
+kind: ServiceAccount
+metadata:
+  name: fluentd
+  namespace: kube-system
+---
+apiVersion: rbac.authorization.k8s.io/v1
+kind: ClusterRole
+metadata:
+  name: fluentd
+  namespace: kube-system
+rules:
+- apiGroups:
+  - ""
+  resources:
+  - pods
+  - namespaces
+  verbs:
+  - get
+  - list
+  - watch
+---
+kind: ClusterRoleBinding
+apiVersion: rbac.authorization.k8s.io/v1
+metadata:
+  name: fluentd
+roleRef:
+  kind: ClusterRole
+  name: fluentd
+  apiGroup: rbac.authorization.k8s.io
+subjects:
+- kind: ServiceAccount
+  name: fluentd
+  namespace: kube-system
+---
+apiVersion: apps/v1
+kind: DaemonSet
+metadata:
+  name: fluentd
+  namespace: kube-system
+  labels:
+    k8s-app: fluentd-logging
+    version: v1
+spec:
+  selector:
+    matchLabels:
+      k8s-app: fluentd-logging
+      version: v1
+  template:
+    metadata:
+      labels:
+        k8s-app: fluentd-logging
+        version: v1
+    spec:
+      tolerations:
+      - key: node-role.kubernetes.io/master
+        effect: NoSchedule
+      containers:
+      - name: fluentd
+        image: fluent/fluentd-kubernetes-daemonset:v1-debian-elasticsearch
+        env:
+          - name:  FLUENT_ELASTICSEARCH_HOST
+            value: "elasticsearch.logging"
+          - name:  FLUENT_ELASTICSEARCH_PORT
+            value: "9200"
+          - name: FLUENT_ELASTICSEARCH_SCHEME
+            value: "https"
+          - name: FLUENTD_SYSTEMD_CONF
+            value: 'disable'
+          - name: K8S_NODE_NAME 
+            value: 'docker-cluster'
+            #- name: FLUENT_CONTAINER_TAIL_PARSER_TYPE  
+            #value: /^(?<time>.+) (?<stream>stdout|stderr) [^ ]* (?<log>.*)$/ 
+          # Option to configure elasticsearch plugin with self signed certs
+          # ================================================================
+          - name: FLUENT_ELASTICSEARCH_SSL_VERIFY
+            value: "true"
+          # Option to configure elasticsearch plugin with tls
+          # ================================================================
+          - name: FLUENT_ELASTICSEARCH_SSL_VERSION
+            value: "TLSv1_2"
+          # X-Pack Authentication
+          # =====================
+          - name: FLUENT_ELASTICSEARCH_USER
+            value: "elastic"
+          - name: FLUENT_ELASTICSEARCH_PASSWORD
+            value: "secret"
+          # Logz.io Authentication
+          # ======================
+          - name: LOGZIO_TOKEN
+            value: "ThisIsASuperLongToken"
+          - name: LOGZIO_LOGTYPE
+            value: "kubernetes"
+          - name: SSL_CERT_DIR
+            value: /home/fluent/ca
+          - name: SSL_CERT_FILE 
+            value: /home/fluent/ca/ca.crt 
+        resources:
+          limits:
+            memory: 200Mi
+          requests:
+            cpu: 100m
+            memory: 200Mi
+        volumeMounts:
+        - name: varlog
+          mountPath: /var/log
+        - name: dockercontainerlogdirectory
+          mountPath: /var/lib/docker/containers
+          readOnly: true
+        - name: fluentd-ca-cert
+          mountPath: /home/fluent/ca/ca.crt 
+          subPath: ca.crt 
+          readOnly: true
+      terminationGracePeriodSeconds: 30
+      volumes:
+      - name: varlog
+        hostPath:
+          path: /var/log
+      - name: dockercontainerlogdirectory
+        hostPath:
+          path: /var/lib/docker/containers
+      - name: fluentd-ca-cert
+        hostPath:
+          # Ensure the file directory is created.
+           path: /var/elasticsearch/config/certs/ca
+           type: DirectoryOrCreate
diff --git a/rapps/go.mod b/rapps/go.mod
index 5d99959..1700595 100644
--- a/rapps/go.mod
+++ b/rapps/go.mod
@@ -40,6 +40,9 @@
 	github.com/docker/go-connections v0.4.0 // indirect
 	github.com/docker/go-metrics v0.0.1 // indirect
 	github.com/docker/go-units v0.4.0 // indirect
+	github.com/elastic/elastic-transport-go/v8 v8.1.0 // indirect
+	github.com/elastic/go-elasticsearch v0.0.0 // indirect
+	github.com/elastic/go-elasticsearch/v8 v8.1.0 // indirect
 	github.com/evanphx/json-patch v4.12.0+incompatible // indirect
 	github.com/exponent-io/jsonpath v0.0.0-20151013193312-d6023ce2651d // indirect
 	github.com/fatih/color v1.13.0 // indirect
@@ -93,10 +96,10 @@
 	github.com/opentracing/opentracing-go v1.2.0 // indirect
 	github.com/peterbourgon/diskv v2.0.1+incompatible // indirect
 	github.com/pmezard/go-difflib v1.0.0 // indirect
-	github.com/prometheus/client_golang v1.11.0 // indirect
+	github.com/prometheus/client_golang v1.12.1 // indirect
 	github.com/prometheus/client_model v0.2.0 // indirect
-	github.com/prometheus/common v0.28.0 // indirect
-	github.com/prometheus/procfs v0.6.0 // indirect
+	github.com/prometheus/common v0.32.1 // indirect
+	github.com/prometheus/procfs v0.7.3 // indirect
 	github.com/rubenv/sql-migrate v0.0.0-20210614095031-55d5740dbbcc // indirect
 	github.com/russross/blackfriday v1.5.2 // indirect
 	github.com/segmentio/ksuid v1.0.4 // indirect
@@ -114,7 +117,7 @@
 	golang.org/x/net v0.0.0-20220107192237-5cfca573fb4d // indirect
 	golang.org/x/oauth2 v0.0.0-20211104180415-d3ed0bb246c8 // indirect
 	golang.org/x/sync v0.0.0-20210220032951-036812b2e83c // indirect
-	golang.org/x/sys v0.0.0-20211216021012-1d35b9e2eb4e // indirect
+	golang.org/x/sys v0.0.0-20220114195835-da31bd327af9 // indirect
 	golang.org/x/term v0.0.0-20210615171337-6886f2dfbf5b // indirect
 	golang.org/x/text v0.3.7 // indirect
 	golang.org/x/time v0.0.0-20210723032227-1f47c861a9ac // indirect
diff --git a/rapps/go.sum b/rapps/go.sum
index 2caa5ae..818b6fd 100644
--- a/rapps/go.sum
+++ b/rapps/go.sum
@@ -375,6 +375,12 @@
 github.com/docopt/docopt-go v0.0.0-20180111231733-ee0de3bc6815/go.mod h1:WwZ+bS3ebgob9U8Nd0kOddGdZWjyMGR8Wziv+TBNwSE=
 github.com/dustin/go-humanize v0.0.0-20171111073723-bb3d318650d4/go.mod h1:HtrtbFcZ19U5GC7JDqmcUSB87Iq5E25KnS6fMYU6eOk=
 github.com/dustin/go-humanize v1.0.0/go.mod h1:HtrtbFcZ19U5GC7JDqmcUSB87Iq5E25KnS6fMYU6eOk=
+github.com/elastic/elastic-transport-go/v8 v8.1.0 h1:NeqEz1ty4RQz+TVbUrpSU7pZ48XkzGWQj02k5koahIE=
+github.com/elastic/elastic-transport-go/v8 v8.1.0/go.mod h1:87Tcz8IVNe6rVSLdBux1o/PEItLtyabHU3naC7IoqKI=
+github.com/elastic/go-elasticsearch v0.0.0 h1:Pd5fqOuBxKxv83b0+xOAJDAkziWYwFinWnBO0y+TZaA=
+github.com/elastic/go-elasticsearch v0.0.0/go.mod h1:TkBSJBuTyFdBnrNqoPc54FN0vKf5c04IdM4zuStJ7xg=
+github.com/elastic/go-elasticsearch/v8 v8.1.0 h1:6TLhYoes04FRK83GakeuMsOQsx1qRwXdP/LF1nxfx1U=
+github.com/elastic/go-elasticsearch/v8 v8.1.0/go.mod h1:yY52i2Vj0unLz+N3Nwx1gM5LXwoj3h2dgptNGBYkMLA=
 github.com/elazarl/goproxy v0.0.0-20180725130230-947c36da3153 h1:yUdfgN0XgIJw7foRItutHYUIhlcKzcSf5vDpdhQAKTc=
 github.com/elazarl/goproxy v0.0.0-20180725130230-947c36da3153/go.mod h1:/Zj4wYkgs4iZTTu3o/KG3Itv/qCCa8VVMlb3i9OVuzc=
 github.com/emicklei/go-restful v0.0.0-20170410110728-ff4f55a20633/go.mod h1:otzb+WCGbkyDHkqmQmT5YD2WR4BBwUdeQoFo8l/7tVs=
@@ -906,6 +912,8 @@
 github.com/prometheus/client_golang v1.7.1/go.mod h1:PY5Wy2awLA44sXw4AOSfFBetzPP4j5+D6mVACh+pe2M=
 github.com/prometheus/client_golang v1.11.0 h1:HNkLOAEQMIDv/K+04rukrLx6ch7msSRwf3/SASFAGtQ=
 github.com/prometheus/client_golang v1.11.0/go.mod h1:Z6t4BnS23TR94PD6BsDNk8yVqroYurpAkEiz0P2BEV0=
+github.com/prometheus/client_golang v1.12.1 h1:ZiaPsmm9uiBeaSMRznKsCDNtPCS0T3JVDGF+06gjBzk=
+github.com/prometheus/client_golang v1.12.1/go.mod h1:3Z9XVyYiZYEO+YQWt3RD2R3jrbd179Rt297l4aS6nDY=
 github.com/prometheus/client_model v0.0.0-20171117100541-99fa1f4be8e5/go.mod h1:MbSGuTsp3dbXC40dX6PRTWyKYBIrTGTE9sqQNg2J8bo=
 github.com/prometheus/client_model v0.0.0-20180712105110-5c3871d89910/go.mod h1:MbSGuTsp3dbXC40dX6PRTWyKYBIrTGTE9sqQNg2J8bo=
 github.com/prometheus/client_model v0.0.0-20190129233127-fd36f4220a90/go.mod h1:xMI15A0UPsDsEKsMN9yxemIoYk6Tm2C1GtYGdfGttqA=
@@ -922,6 +930,8 @@
 github.com/prometheus/common v0.26.0/go.mod h1:M7rCNAaPfAosfx8veZJCuw84e35h3Cfd9VFqTh1DIvc=
 github.com/prometheus/common v0.28.0 h1:vGVfV9KrDTvWt5boZO0I19g2E3CsWfpPPKZM9dt3mEw=
 github.com/prometheus/common v0.28.0/go.mod h1:vu+V0TpY+O6vW9J44gczi3Ap/oXXR10b+M/gUGO4Hls=
+github.com/prometheus/common v0.32.1 h1:hWIdL3N2HoUx3B8j3YN9mWor0qhY/NlEKZEaXxuIRh4=
+github.com/prometheus/common v0.32.1/go.mod h1:vu+V0TpY+O6vW9J44gczi3Ap/oXXR10b+M/gUGO4Hls=
 github.com/prometheus/procfs v0.0.0-20180125133057-cb4147076ac7/go.mod h1:c3At6R/oaqEKCNdg8wHV1ftS6bRYblBhIjjI8uT2IGk=
 github.com/prometheus/procfs v0.0.0-20181005140218-185b4288413d/go.mod h1:c3At6R/oaqEKCNdg8wHV1ftS6bRYblBhIjjI8uT2IGk=
 github.com/prometheus/procfs v0.0.0-20190507164030-5867b95ac084/go.mod h1:TjEm7ze935MbeOT/UhFTIMYKhuLP4wbCsTZCD3I8kEA=
@@ -934,6 +944,8 @@
 github.com/prometheus/procfs v0.2.0/go.mod h1:lV6e/gmhEcM9IjHGsFOCxxuZ+z1YqCvr4OA4YeYWdaU=
 github.com/prometheus/procfs v0.6.0 h1:mxy4L2jP6qMonqmq+aTtOx1ifVWUgG/TAmntgbh3xv4=
 github.com/prometheus/procfs v0.6.0/go.mod h1:cz+aTbrPOrUb4q7XlbU9ygM+/jj0fzG6c1xBZuNvfVA=
+github.com/prometheus/procfs v0.7.3 h1:4jVXhlkAyzOScmCkXBTOLRLTz8EeU+eyjrwB/EPq0VU=
+github.com/prometheus/procfs v0.7.3/go.mod h1:cz+aTbrPOrUb4q7XlbU9ygM+/jj0fzG6c1xBZuNvfVA=
 github.com/prometheus/tsdb v0.7.1/go.mod h1:qhTCs0VvXwvX/y3TZrWD7rabWM+ijKTux40TwIPHuXU=
 github.com/rogpeppe/fastuuid v0.0.0-20150106093220-6724a57986af/go.mod h1:XWv6SoW27p1b0cqNHllgS5HIMJraePCO15w5zCzIWYg=
 github.com/rogpeppe/fastuuid v1.2.0/go.mod h1:jVj6XXZzXRy/MSR5jhDC/2q6DgLz+nrA6LYCDYWNEvQ=
@@ -1374,6 +1386,8 @@
 golang.org/x/sys v0.0.0-20211205182925-97ca703d548d/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
 golang.org/x/sys v0.0.0-20211216021012-1d35b9e2eb4e h1:fLOSk5Q00efkSvAm+4xcoXD+RRmLmmulPn5I3Y9F2EM=
 golang.org/x/sys v0.0.0-20211216021012-1d35b9e2eb4e/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
+golang.org/x/sys v0.0.0-20220114195835-da31bd327af9 h1:XfKQ4OlFl8okEOr5UvAqFRVj8pY/4yfcXrddB8qAbU0=
+golang.org/x/sys v0.0.0-20220114195835-da31bd327af9/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
 golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo=
 golang.org/x/term v0.0.0-20210615171337-6886f2dfbf5b h1:9zKuko04nR4gjZ4+DNjHqRlAJqbJETHwiNKDqTfOjfE=
 golang.org/x/term v0.0.0-20210615171337-6886f2dfbf5b/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8=
diff --git a/rapps/hw_test.sh b/rapps/hw_test.sh
index 0bec203..aed7e5d 100644
--- a/rapps/hw_test.sh
+++ b/rapps/hw_test.sh
@@ -6,6 +6,9 @@
 FAILED=0
 TEST_TS=$(date +%F-%T)
 TOKEN=""
+ACCESS_TOKEN=""
+REFRESH_TOKEN=""
+CERT="MIICnTCCAYUCBgF/iApKsDANBgkqhkiG9w0BAQsFADASMRAwDgYDVQQDDAdzdGVwLWNhMB4XDTIyMDMxNDEwNDU1MFoXDTMyMDMxNDEwNDczMFowEjEQMA4GA1UEAwwHc3RlcC1jYTCCASIwDQYJKoZIhvcNAQEBBQADggEPADCCAQoCggEBAIreIwbXPJPUDNaljbmI7WQRQqjSVjtSqcLGiUuZKl5HTXIFgWwF5IAstw4hUDs+zGu2xrGvJ6+pzUi0q8gj/ddrYZmR5degJKK/g8FLrFIiq4nIyCYM9WuQZDIe0Hkk4aTa+TJnJ609LuRvJMR8nK/sz+/FOLmcZJgcD5KTcECRIvzBcHzIUxl/Q6wjRl9+tFBlZMS1pDcKlb06YnkJIpawG/3vnl62aU5CKDJrKq0wcQiaWrpmo0eWy+EK3qMPar1AdaJWM5tgOz8uA+AlQtKMv6vRUBseERuU0n6kFLGIpzeO/dvTDuZ43LYLr9x5uJHpj+IYyHbA9lZt62GL/icCAwEAATANBgkqhkiG9w0BAQsFAAOCAQEAYUUzFbXMaIu8ARVOTRdNgrtSAz498CPe2k/ZiGcGAjvfrgmF1nzbE/8yzuKu+x/fdooCE+0rhwH1h9yb43NaKQ59uN/g7tmSm3G4lUwXS04c6g1F5UR0+Bk1QwnhZsUNQZ14gtYSK+P9DOggskF+wHyknTD5KD0tfJJ+qv6wKFZuNXKcXq4vC+s691R3SZjKaNtaoNgdWqb+T/kIEECB3DLA7Hn2ZqrdqDWdvHDuaMDVHnrBuddQaEfrrXU2R4NuA0KenYhAfs9H1HM8M6V3+KNP0C04ngsIjRsGcwJmqK7dHs4k+fXr95KD0IpRkqEuYIIFrsNWPa/A1gwMiTLdpw=="
 
 function get_token
 {
@@ -13,10 +16,20 @@
     url="http://192.168.49.2:31560/auth/realms"
     #     echo $url
          TOKEN=$(curl -s -X POST $url/hwrealm/protocol/openid-connect/token -H \
-		 "Content-Type: application/x-www-form-urlencoded" -d client_secret=FyHk97pt5vgZPekdILgefsKj1SIrej2R \
-		 -d 'grant_type=client_credentials' -d client_id=hwclient | jq -r '.access_token') 
-	 #TOKEN=""
-    echo $TOKEN
+		 "Content-Type: application/x-www-form-urlencoded" -d client_secret=g4kqnQU5urU04OCxoWaGEGdwynFHLhqJ \
+		 -d 'grant_type=client_credentials' -d client_id=step-ca) 
+	echo "TOKEN: $TOKEN"
+        ACCESS_TOKEN=$(echo $TOKEN | jq -r '.access_token')
+	#echo "ACCESS_TOKEN: $ACCESS_TOKEN"
+        REFRESH_TOKEN=$(echo $TOKEN | jq -r '.refresh_token')
+	#echo "REFRESH_TOKEN: $REFRESH_TOKEN"
+         TOKEN2=$(curl -s -X POST $url/hwrealm/protocol/openid-connect/token -H \
+		 "Content-Type: application/x-www-form-urlencoded" \
+		 -d code=$CERT \
+		 -d 'grant_type=authorization_code' -d client_id=step-ca) 
+       echo "TOKEN2 = $TOKEN2"
+	 #ACCESS_TOKEN=""
+    #echo $ACCESS_TOKEN
 }
 
 function run_test
@@ -26,8 +39,8 @@
     echo "Test ${TESTS}: Testing $type /${prefix}"
     get_token $prefix
     url=$INGRESS_HOST:$INGRESS_PORT"/"$prefix
-    echo $url
-    result=$(curl -s -X ${type} -H "Content-type: application/json" -H "Authorization: Bearer $TOKEN" $url)
+    #echo $url
+    result=$(curl -s -X ${type} -H "Content-type: application/json" -H "Authorization: Bearer $ACCESS_TOKEN" $url)
     echo $result
     if [ "$result" != "$msg" ]; then
             echo "FAIL"
@@ -40,7 +53,7 @@
 }
 
 
-run_test "hello-world" "GET" "Hello World!" ""
+run_test "rapp-provider" "GET" "Hello World!" ""
 
 echo
 echo "-----------------------------------------------------------------------"
diff --git a/rapps/kibana-8.1.2.yaml b/rapps/kibana-8.1.2.yaml
new file mode 100644
index 0000000..b5bb88d
--- /dev/null
+++ b/rapps/kibana-8.1.2.yaml
@@ -0,0 +1,108 @@
+apiVersion: v1
+kind: ConfigMap
+metadata:
+  name: kibana-config
+  namespace: logging
+data:
+  kibana.yml: |
+   server.name: kibana
+   server.host: 0.0.0.0
+   elasticsearch.hosts: [ "https://elasticsearch:9200" ]
+   xpack.monitoring.ui.container.elasticsearch.enabled: true
+   xpack.security.encryptionKey: 38747239hdjksdkjasdu291834zejhb38747239hdj
+   xpack.encryptedSavedObjects.encryptionKey: 38747239hdjksdkjasdu291834zejhb38747239hdj
+   elasticsearch.username: "kibana_system"
+   elasticsearch.password: "secret" 
+   elasticsearch.ssl:
+     certificateAuthorities: /usr/share/kibana/config/certs/ca/ca.crt 
+     verificationMode: certificate 
+---
+apiVersion: v1
+kind: ConfigMap
+metadata:
+  name: kibana-pw-script
+  namespace: logging
+data:
+  setup_pw.sh: |
+   #!/bin/bash
+   KIBANA_HOME=/usr/share/kibana
+   ELASTIC_PASSWORD=secret
+   KIBANA_PASSWORD=secret
+   cd $KIBANA_HOME
+   echo "Waiting for Elasticsearch availability";
+   until curl -s --cacert /ca-dir/ca.crt https://elasticsearch:9200 | grep -q "missing authentication credentials"; \
+   do sleep 30; done;
+   echo "Setting kibana_system password";
+   until curl -s -X POST --cacert /ca-dir/ca.crt -u elastic:${ELASTIC_PASSWORD} -H "Content-Type: application/json" \
+   https://elasticsearch:9200/_security/user/kibana_system/_password -d "{\"password\":\"${KIBANA_PASSWORD}\"}" | grep -q "^{}"; \
+   do sleep 10; done;
+   echo "All done!";
+---
+apiVersion: apps/v1
+kind: Deployment
+metadata:
+  name: kibana
+  namespace: logging
+spec:
+  selector:
+    matchLabels:
+      run: kibana
+  template:
+    metadata:
+      labels:
+        run: kibana
+    spec:
+      containers:
+      - name: kibana
+        image: docker.elastic.co/kibana/kibana:8.1.2
+        ports:
+        - containerPort: 5601
+          name: http
+          protocol: TCP
+        volumeMounts:
+        - name : config
+          mountPath: /usr/share/kibana/config/kibana.yml 
+          subPath: kibana.yml
+          readOnly: true 
+        - name : kibana-ca-cert 
+          mountPath: /usr/share/kibana/config/certs/ca
+          readOnly: true
+      initContainers:
+      - name: init-kibana
+        image: docker.elastic.co/kibana/kibana:8.1.2 
+        imagePullPolicy: IfNotPresent
+        command: ['/bin/bash', '-c', '/usr/share/kibana/bin/setup_pw.sh']
+        volumeMounts:
+        - name: kibana-ca-cert 
+          mountPath: "/ca-dir"
+        - name: kibana-pw-init
+          mountPath: /usr/share/kibana/bin/setup_pw.sh
+          subPath: setup_pw.sh
+      volumes:
+      - name: config
+        configMap:
+          name: kibana-config
+      - name: kibana-ca-cert 
+        hostPath:
+          # Ensure the file directory is created.
+           path: /var/elasticsearch/config/certs/ca
+           type: DirectoryOrCreate
+      - name: kibana-pw-init
+        configMap:
+          name: kibana-pw-script
+          defaultMode: 0755
+---
+apiVersion: v1
+kind: Service
+metadata:
+  name: kibana
+  namespace: logging
+  labels:
+    service: kibana
+spec:
+  type: NodePort
+  selector:
+    run: kibana
+  ports:
+  - port: 5601
+    targetPort: 5601
diff --git a/rapps/rapps-keycloak-mgr.go b/rapps/rapps-keycloak-mgr.go
index 82097ff..abfc891 100644
--- a/rapps/rapps-keycloak-mgr.go
+++ b/rapps/rapps-keycloak-mgr.go
@@ -144,6 +144,8 @@
                 Attributes: &map[string]string{"token.endpoint.auth.signing.alg": "RS256",
 		       "use.jwks.string": "true",
                        "jwks.string": jwksString, 
+		       "use.refresh.tokens": "true",
+                       "client_credentials.use_refresh_token": "true",
 		},
         }
 
diff --git a/rapps/rapps-keycloak-mgr.yaml b/rapps/rapps-keycloak-mgr.yaml
index fd00a52..571badf 100644
--- a/rapps/rapps-keycloak-mgr.yaml
+++ b/rapps/rapps-keycloak-mgr.yaml
@@ -1,34 +1,3 @@
-apiVersion: v1
-kind: PersistentVolume
-metadata:
-  name: keycloak-mgr-pv-volume
-  namespace: default 
-  labels:
-    app: rapps-keycloak-mgr 
-spec:
-  storageClassName: manual
-  capacity:
-    storage: 10Mi
-  accessModes:
-    - ReadOnlyMany 
-  hostPath:
-    path: "/var/rapps/certs"
----
-apiVersion: v1
-kind: PersistentVolumeClaim
-metadata:
-  name: keycloak-mgr-pv-claim
-  namespace: default 
-  labels:
-    app: rapps-keycloak-mgr 
-spec:
-  storageClassName: manual
-  accessModes:
-    - ReadOnlyMany 
-  resources:
-    requests:
-      storage: 10Mi
----
 apiVersion: apps/v1
 kind: Deployment
 metadata:
@@ -60,12 +29,15 @@
             memory: 128Mi
             cpu: "80m"
         volumeMounts:
-        - name: keycloak-mgr-cert-storage
+        - name: certsdir
           mountPath: /certs
+          readOnly: true
       volumes:
-      - name: keycloak-mgr-cert-storage 
-        persistentVolumeClaim:
-          claimName: keycloak-mgr-pv-claim
+      - name: certsdir
+        hostPath:
+          # Ensure the file directory is created.
+           path: /var/rapps/certs
+           type: DirectoryOrCreate
       serviceAccountName: helm-app
   replicas: 1 
 ---
diff --git a/rapps/rapps-rapp-jwt-invoker.go b/rapps/rapps-rapp-jwt-invoker.go
index de806ca..7d1a6b5 100644
--- a/rapps/rapps-rapp-jwt-invoker.go
+++ b/rapps/rapps-rapp-jwt-invoker.go
@@ -2,15 +2,24 @@
 
 import (
 	"bytes"
+	"context"
+	"crypto/tls"
 	"encoding/json"
 	"flag"
 	"fmt"
+	"github.com/elastic/go-elasticsearch/v8"
+	"github.com/elastic/go-elasticsearch/esapi"
+	"github.com/google/uuid"
+	"github.com/prometheus/client_golang/prometheus"
+	"github.com/prometheus/client_golang/prometheus/promhttp"
 	"io/ioutil"
+	"net"
 	"net/http"
 	"net/url"
+	"rapps/utils/generatejwt"
 	"strings"
 	"time"
-	"rapps/utils/generatejwt"
+	"log"
 )
 
 type Jwttoken struct {
@@ -29,6 +38,9 @@
 var keycloakHost string
 var keycloakPort string
 var keycloakAlias string
+var elasticHost string
+var elasticPort string
+var elasticAlias string
 var securityEnabled string
 var useGateway string
 var role string
@@ -41,35 +53,140 @@
 var jwt Jwttoken
 
 const (
-	namespace = "istio-nonrtric"
-	scope     = "email"
+	namespace             = "istio-nonrtric"
+	scope                 = "email"
 	client_assertion_type = "urn:ietf:params:oauth:client-assertion-type:jwt-bearer"
 )
 
+var (
+	reqDuration = prometheus.NewHistogramVec(prometheus.HistogramOpts{
+		Name:    "rapp_http_request_duration_seconds",
+		Help:    "Duration of the last request call.",
+		Buckets: []float64{0.05, 0.1, 0.25, 0.5, 1, 2.5, 5, 10},
+	}, []string{"app", "func", "handler", "method", "code"})
+	reqBytes = prometheus.NewSummaryVec(prometheus.SummaryOpts{
+		Name: "rapp_bytes_summary",
+		Help: "Summary of bytes transferred over http",
+	}, []string{"app", "func", "handler", "method", "code"})
+)
+
+type MyDocument struct {
+        Timestamp string `json:"@timestamp"`
+        App string `json:"app"`
+        Func string `json:"func"`
+        Handler string `json:"handler"`
+        Method string `json:"method"`
+        Code string `json:"code"`
+        Bytes int64 `json:"bytes"`
+}
+ 
+var client *elasticsearch.Client 
+
+func connectToElasticsearch() *elasticsearch.Client {
+	clusterURLs := []string{"https://" + elasticAlias + ":" + elasticPort }
+        username := "elastic"
+        password := "secret"
+        cert, _ := ioutil.ReadFile("/ca/ca.crt")
+
+	dialer := &net.Dialer{
+                Timeout:   30 * time.Second,
+                KeepAlive: 30 * time.Second,
+                DualStack: true,
+        }
+
+        // client configuration
+        cfg := elasticsearch.Config{
+                Addresses: clusterURLs,
+                Username:  username,
+                Password:  password,
+                CACert:    cert,
+		Transport: &http.Transport{
+                        DialContext: func(ctx context.Context, network, addr string) (net.Conn, error) {
+                                //fmt.Println("address original =", addr)
+                                if addr == elasticAlias+":"+elasticPort {
+                                        addr = elasticHost + ":" + elasticPort
+                                        //fmt.Println("address modified =", addr)
+                                }
+                                return dialer.DialContext(ctx, network, addr)
+                        },
+		        TLSClientConfig: &tls.Config{
+				MinVersion: tls.VersionTLS12,
+			},
+		},
+        }
+
+        es, err := elasticsearch.NewClient(cfg)
+        if err != nil {
+                log.Fatalf("Error creating the client: %s", err)
+        }
+	log.Println(elasticsearch.Version)
+
+        resp, err := es.Info()
+        if err != nil {
+                log.Fatalf("Error getting response: %s", err)
+        }
+        defer resp.Body.Close()
+        log.Println(resp)
+	return es
+}
+
+func addEsIndex(app, fnc, hnd, meth, code string, bytes int64){
+	ts := time.Now().Format(time.RFC3339)
+	doc := MyDocument{Timestamp: ts, App: app, Func: fnc, Handler: hnd, Method: meth, Code: code, Bytes: bytes }
+        jsonString, _ := json.Marshal(doc)
+	uid := fmt.Sprintf("%v", uuid.New()) 
+	indexName := "gostash-"+time.Now().Format("2006.01.02")
+        request := esapi.IndexRequest{Index: indexName, DocumentID: uid, Body: strings.NewReader(string(jsonString))}
+        _, err := request.Do(context.Background(), client)
+        if err != nil {
+                fmt.Println(err)
+        }
+}
+
 func getToken() string {
 	if ttime.Before(time.Now()) {
+		resp := &http.Response{}
 		client_assertion := getClientAssertion()
 		keycloakUrl := "http://" + keycloakHost + ":" + keycloakPort + "/auth/realms/" + realmName + "/protocol/openid-connect/token"
-		resp, err := http.PostForm(keycloakUrl, url.Values{"client_assertion_type": {client_assertion_type}, 
-		             "client_assertion": {client_assertion}, "grant_type": {"client_credentials"}, "client_id": {clientId}, 
-			     "scope": {scope}})
+		fmt.Printf("Making token request to %s\n", keycloakUrl)
+		timer := prometheus.NewTimer(prometheus.ObserverFunc(func(v float64) {
+			reqDuration.WithLabelValues("rapp-jwt-invoker", "getToken", resp.Request.URL.Path, resp.Request.Method,
+				resp.Status).Observe(v)
+		}))
+		defer timer.ObserveDuration()
+	        var err error
+		if jwt.Refresh_token != "" {
+			resp, err = http.PostForm(keycloakUrl, url.Values{"client_assertion_type": {client_assertion_type},
+				"client_assertion": {client_assertion}, "grant_type": {"refresh_token"},
+				"refresh_token": {jwt.Refresh_token}, "client_id": {clientId}, "scope": {scope}})
+		} else {
+			resp, err = http.PostForm(keycloakUrl, url.Values{"client_assertion_type": {client_assertion_type},
+				"client_assertion": {client_assertion}, "grant_type": {"client_credentials"},
+				"client_id": {clientId}, "scope": {scope}})
+		}
 		if err != nil {
 			fmt.Println(err)
 			panic("Something wrong with the credentials or url ")
 		}
+
 		defer resp.Body.Close()
 		body, err := ioutil.ReadAll(resp.Body)
 		json.Unmarshal([]byte(body), &jwt)
-		ttime = time.Now()
-		ttime = ttime.Add(time.Second * time.Duration(jwt.Expires_in))
+
+		reqBytes.WithLabelValues("rapp-jwt-invoker", "getToken", resp.Request.URL.Path, resp.Request.Method,
+			resp.Status).Observe(float64(resp.ContentLength))
+                addEsIndex("rapp-jwt-invoker", "getToken", resp.Request.URL.Path, resp.Request.Method,
+                            resp.Status, resp.ContentLength)
+	        ttime = time.Now()
+	        ttime = ttime.Add(time.Second * time.Duration(jwt.Expires_in))
 	}
 	return jwt.Access_token
 }
 
 func getClientAssertion() string {
 	realm := "http://" + keycloakHost + ":" + keycloakPort + "/auth/realms/" + realmName
-        clientAssertion := generatejwt.CreateJWT("/certs/client.key", "/certs/client_pub.key", "", clientId, realm)	
-        return clientAssertion 
+	clientAssertion := generatejwt.CreateJWT("/certs/client.key", "/certs/client_pub.key", "", clientId, realm)
+	return clientAssertion
 }
 
 func MakeRequest(client *http.Client, prefix string, method string, ch chan string) {
@@ -91,7 +208,13 @@
 	}
 
 	restUrl = gatewayUrl + prefix
+	resp := &http.Response{}
 
+	timer := prometheus.NewTimer(prometheus.ObserverFunc(func(v float64) {
+		reqDuration.WithLabelValues("rapp-jwt-invoker", "MakeRequest", resp.Request.URL.Path, resp.Request.Method,
+			resp.Status).Observe(v)
+	}))
+	defer timer.ObserveDuration()
 	req, err := http.NewRequest(method, restUrl, bytes.NewBuffer(jsonValue))
 	if err != nil {
 		fmt.Printf("Got error %s", err.Error())
@@ -99,12 +222,16 @@
 	req.Header.Set("Content-type", "application/json")
 	req.Header.Set("Authorization", "Bearer "+token)
 
-	resp, err := client.Do(req)
+	resp, err = client.Do(req)
 	if err != nil {
 		fmt.Printf("Got error %s", err.Error())
 	}
+
 	defer resp.Body.Close()
 	body, _ := ioutil.ReadAll(resp.Body)
+	reqBytes.WithLabelValues("rapp-jwt-invoker", "MakeRequest", req.URL.Path, req.Method,
+		resp.Status).Observe(float64(resp.ContentLength))
+        addEsIndex("rapp-jwt-invoker", "MakeRequest", req.URL.Path, req.Method, resp.Status, resp.ContentLength)
 
 	respString := string(body[:])
 	if respString == "RBAC: access denied" {
@@ -126,11 +253,17 @@
 
 func main() {
 	ttime = time.Now()
-	time.Sleep(1 * time.Second)
+	time.Sleep(3 * time.Second)
+	prometheus.Register(reqDuration)
+	prometheus.Register(reqBytes)
+
 	flag.StringVar(&gatewayHost, "gatewayHost", "istio-ingressgateway.istio-system", "Gateway Host")
 	flag.StringVar(&gatewayPort, "gatewayPort", "80", "Gateway Port")
 	flag.StringVar(&keycloakHost, "keycloakHost", "istio-ingressgateway.istio-system", "Keycloak Host")
 	flag.StringVar(&keycloakPort, "keycloakPort", "80", "Keycloak Port")
+        flag.StringVar(&elasticHost, "elasticHost", "istio-ingressgateway.istio-system", "Elasticsearch Host")
+        flag.StringVar(&elasticPort, "elasticPort", "443", "Elasticsearch Port")
+        flag.StringVar(&elasticAlias, "elasticAlias", "elasticsearch.est.tech", "Elasticsearch URL Alias")
 	flag.StringVar(&useGateway, "useGateway", "Y", "Connect to services through API gateway")
 	flag.StringVar(&securityEnabled, "securityEnabled", "true", "Security is required to use this application")
 	flag.StringVar(&realmName, "realm", "jwt", "Keycloak realm")
@@ -142,6 +275,8 @@
 
 	healthHandler := http.HandlerFunc(health)
 	http.Handle("/health", healthHandler)
+	http.Handle("/metrics", promhttp.Handler())
+	client = connectToElasticsearch() 
 	go func() {
 		http.ListenAndServe(":9000", nil)
 	}()
@@ -170,5 +305,4 @@
 			MakeRequest(client, prefix, method, ch)
 		}(r)
 	}
-
 }
diff --git a/rapps/rp_test.sh b/rapps/rp_test.sh
index 20eca21..e3a7110 100644
--- a/rapps/rp_test.sh
+++ b/rapps/rp_test.sh
@@ -15,20 +15,20 @@
     url="http://192.168.49.2:31560/auth/realms"
     #     echo $url
          TOKEN=$(curl -s -X POST $url/provider/protocol/openid-connect/token -H \
-		 "Content-Type: application/x-www-form-urlencoded" -d client_secret=to6qrqxLj0mwHKS8e3E7s1zzDHsNg1Ua \
+		 "Content-Type: application/x-www-form-urlencoded" -d client_secret=OwTCeahULA21G5TfEVMLG1iMloGiyH3i \
 		 -d 'grant_type=client_credentials' -d client_id=provider-cli) 
 	echo "TOKEN: $TOKEN"
         ACCESS_TOKEN=$(echo $TOKEN | jq -r '.access_token')
 	#echo "ACCESS_TOKEN: $ACCESS_TOKEN"
         REFRESH_TOKEN=$(echo $TOKEN | jq -r '.refresh_token')
 	#echo "REFRESH_TOKEN: $REFRESH_TOKEN"
-         TOKEN2=$(curl -s -X POST $url/provider/protocol/openid-connect/token -H \
-		 "Content-Type: application/x-www-form-urlencoded" -d client_secret=to6qrqxLj0mwHKS8e3E7s1zzDHsNg1Ua \
-		 -d refresh_token=$REFRESH_TOKEN \
-		 -d 'grant_type=refresh_token' -d client_id=provider-cli) 
+        # TOKEN2=$(curl -s -X POST $url/provider/protocol/openid-connect/token -H \
+	#	 "Content-Type: application/x-www-form-urlencoded" -d client_secret= \
+	#	 -d refresh_token=$REFRESH_TOKEN \
+	#	 -d 'grant_type=refresh_token' -d client_id=provider-cli) 
        #echo "TOKEN2 = $TOKEN2"
 	 #ACCESS_TOKEN=""
-    #echo $ACCESS_TOKEN
+    echo $ACCESS_TOKEN
 }
 
 function run_test