From: halil.cakal <halil.cakal@est.tech>
Date: Fri, 31 May 2024 10:18:54 +0000 (+0100)
Subject: Create plots of k6 test result
X-Git-Url: https://gerrit.nordix.org/gitweb?a=commitdiff_plain;h=25ff8e884da6ac6b6b21a5400bbfb4617780e836;p=infra%2Fcicd.git

Create plots of k6 test result

- add shell script to plot k6 test result
- define a new jenkins job to run the shell
- add a timer for the jenkins job

Issue-ID: CPS-2244

Change-Id: I35c6f170b4537dc84fedf0a23ef03a4db60e5de9
Signed-off-by: halil.cakal <halil.cakal@est.tech>
---

diff --git a/jjb/onap/cps/cps-performance-test-jobs.yaml b/jjb/onap/cps/cps-performance-test-jobs.yaml
index 667674279..8d4cb9b38 100644
--- a/jjb/onap/cps/cps-performance-test-jobs.yaml
+++ b/jjb/onap/cps/cps-performance-test-jobs.yaml
@@ -52,3 +52,12 @@
 
       jobs:
           - 'onap-{project-name}-performance-test-k6'
+
+- project:
+    name: cps-k6-tests-plots
+    project: cps
+    project-name: cps
+    timer: "45 7 * * *"
+
+    jobs:
+        - 'onap-{project-name}-k6-performance-tests-plots'
\ No newline at end of file
diff --git a/jjb/onap/cps/prepare-k6-performance-tests-plots.sh b/jjb/onap/cps/prepare-k6-performance-tests-plots.sh
new file mode 100755
index 000000000..0d150e503
--- /dev/null
+++ b/jjb/onap/cps/prepare-k6-performance-tests-plots.sh
@@ -0,0 +1,273 @@
+#!/bin/bash
+#
+# Copyright 2024 Nordix Foundation.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
+set -o errexit  # Exit on most errors
+set -o nounset  # Disallow expansion of unset variables
+set -o pipefail # Use last non-zero exit code in a pipeline
+#set -o xtrace   # Trace logging - disabled to avoid producing gigabytes of logs
+
+#############################################################################################################################
+################################################ F U N C T I O N S ##########################################################
+#############################################################################################################################
+
+# Define the list of "Test Case + Condition" k6 test names to search for in the logs.
+# The Test Case and Condition concatenated with \s* to represent zero or more whitespace characters.
+# Later in this script can use Perl-compatible regular expressions (PCRE) can match to pattern with grep -P flag.
+# The tests which measure http_req_failed excluded since nothing to plot if the rate is always zero.
+k6_test_names=(
+    "1-create-cmhandles.js\s*http_req_duration"
+    "2-wait-for-cmhandles-to-be-ready.js\s*iteration_duration"
+    "3-passthrough-read.js\s*ncmp_overhead"
+    "4-id-search-no-filter.js\s*http_req_duration"
+    "5-search-no-filter.js\s*http_req_duration"
+    "6-id-search-public-property.js\s*http_req_duration"
+    "7-search-public-property.js\s*http_req_duration"
+    "8-id-search-module.js\s*http_req_duration"
+    "9-search-module.js\s*http_req_duration"
+    "10-mixed-load-test.js\s*http_req_duration{scenario:id_search_module}"
+    "10-mixed-load-test.js\s*http_req_duration{scenario:passthrough_read}"
+    "10-mixed-load-test.js\s*http_req_duration{scenario:cm_search_module}"
+    "11-delete-cmhandles.js\s*http_req_duration"
+)
+
+JENKINS_JOB_URL="https://jenkins.nordix.org/job/onap-cps-performance-test-k6"
+
+latestBuildToRecord=""
+consoleText=""
+latestRecordedBuild=""
+timestampOfLatestRecordedBuild=""
+
+# Get last completed build number
+# The number has not been plotted on the graphs yet
+getLastCompletedBuildNumber() {
+  curl -s "${JENKINS_JOB_URL}/lastCompletedBuild/buildNumber"
+}
+
+# Get the last recorded build number from local workspace of the jenkins
+# The number has already been plotted on the graphs
+getLastRecordedBuildNumber() {
+  cd "$WORKSPACE"
+  local file_name="1-create-cmhandles_http_req_duration.txt"
+
+  # Check if the file exists
+  if [ -f "$file_name" ]; then
+    # Get the last line from the file
+    local last_line=$(tail -n 1 "$file_name")
+    local left_side=$(echo "$last_line" | cut -d ',' -f 2)
+    echo "$left_side"
+  else
+    echo "0"
+  fi
+}
+
+# Get all builds numbers
+getAllBuildNumbers() {
+  curl -s "${JENKINS_JOB_URL}/api/json?tree=allBuilds\[id\]" | jq -r '.allBuilds[].id' | sort -n
+}
+
+# Get the console text of k6 performance job
+getConsoleText() {
+  buildToRead=$1
+  consoleURL="${JENKINS_JOB_URL}/${buildToRead}/consoleText"
+  consoleText=$(curl -s "$consoleURL")
+}
+
+# Replace special characters with underscore
+# The return value will be used to create a file for each test combination
+replace_special_characters() {
+  local pattern=".js\s*"
+  local text="$1"
+
+  # Escape special characters in the pattern
+  local escaped_pattern=$(echo "$pattern" | sed 's/[\.*^$]/\\&/g')
+
+  # Replace the pattern with underscores using sed
+  local result=$(echo "$text" | sed "s/$escaped_pattern/_/g")
+
+  echo "$result"
+}
+
+# Get and record the latest k6-job-results with the build number for all tests
+getAndRecordPerformanceJobResultForBuild() {
+  buildNumber="$1"
+  getConsoleText "$buildNumber"
+  # Loop through each test names
+  for k6_test_name in "${k6_test_names[@]}"; do
+    getAndRecordDataResults "$consoleText" "$k6_test_name" "$(replace_special_characters "$k6_test_name").txt" "$buildNumber"
+  done
+}
+
+# Get and record the latest k6-job-results for a single test
+getAndRecordDataResults() {
+  consoleText=$1
+  patternToMatch=$2
+  dataFile=$3
+  buildNumber=$4
+
+  matched_line=""
+  limit_value=""
+  actual=""
+
+  # Find the text combination (Test Case + Condition) in the console log
+  if matched_line=$(echo "$consoleText" | grep -P "${patternToMatch}"); then
+    # Find the first-occurred number in the line matched (limit)
+    limit_value=$(echo "$matched_line" | awk '{for(i=1;i<=NF;i++) if($i ~ /^[0-9]+$/) {print $i; exit}}')
+    # Find the second-occurred number in the line matched (actual)
+    actual=$(echo "$matched_line" | awk '{count=0; for(i=1; i<=NF; i++) if($i ~ /^[0-9]+$/) {count++; if(count==2) {print $i; exit}}}')
+  fi
+
+  # Add a new entry which is a combination of limit, build-number, and actual into related file
+  touch "$dataFile"
+  lastLine=$(tail -n 1 "$dataFile")
+  newLine="$limit_value,$buildNumber,$actual"
+  if [ -z "$actual" ]; then
+    # No entry found for this build, the default added as zero
+    echo "$limit_value,$buildNumber,0" >>"$dataFile"
+    recordLatestRecordedBuild "$buildNumber"
+  elif [ "$newLine" == "$lastLine" ]; then
+    # Entry already exists
+    recordLatestRecordedBuild "$buildNumber"
+  else
+    # New entry added to the file
+    echo "$limit_value,$buildNumber,$actual" >>"$dataFile"
+    recordLatestRecordedBuild "$buildNumber"
+  fi
+}
+
+# Format the date and time of the latest build
+recordLatestRecordedBuild() {
+  latestBuildToRecord="$1"
+  timestampOfLatestRecordedBuild=$(curl -s "${JENKINS_JOB_URL}/${latestBuildToRecord}/api/json?tree=timestamp" | jq -r '.timestamp')
+  formattedTimestampOfLatestRecordedBuild=$(date -d "@$((timestampOfLatestRecordedBuild / 1000))" "+%B %e, %Y at %H:%M")
+  latestRecordedBuild=$latestBuildToRecord
+}
+
+# Plot the image (graph) in png format
+buildPlotImage() {
+  dataFile="$1"
+  chartFileName="$2"
+
+  # Read the first line of the file
+  first_line=$(head -n 1 "$dataFile")
+
+  # Set upper limit of the graphs to %20 to have more space above the plot
+  ten_percent=1.2
+  limit=$(echo "$first_line" | cut -d ',' -f 1)
+  limit=$(expr "$limit * $ten_percent" | bc)
+
+  # Create a temporary Gnuplot script
+  cat <<EOT >gnuplot_script.gp
+set datafile separator ","
+set terminal pngcairo size 1500,600
+set output "${chartFileName}"
+set xlabel "Build"
+set ylabel "Limit (ms)"
+set yrange [0 : ${limit} < *]
+plot '$dataFile' using 2:3:xtic(sprintf("%d", column(2))) with linespoints title "measured", \
+     '$dataFile' using 2:1 with lines linestyle 2 title "limit"
+EOT
+
+  # Run the temporary Gnuplot script
+  gnuplot gnuplot_script.gp
+
+  # Remove the temporary Gnuplot script
+  rm gnuplot_script.gp
+}
+
+# Builds html file
+buildHtmlReport() {
+  # use indirect expansion to get all elements of the array
+  categoryName=("${!1}")
+  reportTitle="$2"
+  outputFile="$3"
+  cat <<EOT >"$outputFile"
+    <!DOCTYPE html>
+    <html>
+    <head>
+    <title>$reportTitle</title>
+    </head>
+    <body>
+      <h1 style="text-align: center;">$reportTitle</h1>
+      <h4>Last updated for performance job build no. $latestRecordedBuild on $formattedTimestampOfLatestRecordedBuild</h4>
+      <table align="center">
+EOT
+# Loop through the tests to generate the HTML rows which consists of the plot-image
+for test_name_in_category in "${categoryName[@]}"; do
+    test_name_in_category="$(replace_special_characters "$test_name_in_category")"
+    cat <<EOF >>"$outputFile"
+        <tr> <!-- Row for $test_name_in_category -->
+            <td align="center" style="padding: 10px;">
+                <figcaption>"$test_name_in_category"</figcaption>
+                <img src="$test_name_in_category.png" width="750" height="300">
+            </td>
+        </tr>
+EOF
+done
+# Close the HTML file
+cat <<EOT >>"$outputFile"
+      </table>
+        <p>The k6 job at 7:15 a.m. once a day, providing performance metrics.</p>
+        <p>The following graphs being updated at 7:45 a.m. once a day.</p>
+        <p>Successful performance tests job build adds new data, but even if a build fails, existing data is retained.</p>
+    </body>
+  </html>
+EOT
+}
+
+#############################################################################################################################
+################################################ M A I N ####################################################################
+#############################################################################################################################
+
+# Install dependencies
+sudo apt-get install -y bc gnuplot jq
+
+# Download data from CPS performance Jenkins job
+cd "$WORKSPACE"
+if [ -z "$(ls -A)" ]; then
+  # If workspace is empty, pull data from all previous performance job runs
+  for buildNumber in $(getAllBuildNumbers); do
+    getAndRecordPerformanceJobResultForBuild "$buildNumber"
+  done
+else
+  # Append new data from latest jobs run
+  lastCompletedBuildNumber=$(getLastCompletedBuildNumber)
+  lastRecordedBuildNumber=$(getLastRecordedBuildNumber)
+  # Check if last completed build number is greater than last recorded build number
+  if [ "$lastCompletedBuildNumber" -gt "$lastRecordedBuildNumber" ]; then
+    for ((i = lastRecordedBuildNumber + 1; i <= lastCompletedBuildNumber; i++)); do
+      # Process the new builds which hasn't been recorded yet
+      getAndRecordPerformanceJobResultForBuild "$i"
+    done
+  else
+    echo "No new builds to process."
+  fi
+fi
+
+# Limit the plots to last 100 builds for each test
+for k6_test_name in "${k6_test_names[@]}"; do
+  k6_file_name="$(replace_special_characters "$k6_test_name")"
+  tail -n 100 "$k6_file_name.txt" > file.tmp && mv file.tmp "$k6_file_name.txt"
+done
+
+# Plot image files in png format
+for k6_test_name in "${k6_test_names[@]}"; do
+  k6_file_name="$(replace_special_characters "$k6_test_name")"
+  buildPlotImage "$k6_file_name.txt" "$k6_file_name.png"
+done
+
+# Build html page
+buildHtmlReport k6_test_names[@] "k6 tests performance review" "k6TestsPerformanceReview.html"
\ No newline at end of file
diff --git a/jjb/onap/global-templates-onap-java.yaml b/jjb/onap/global-templates-onap-java.yaml
index 78e897456..a54bdce9d 100644
--- a/jjb/onap/global-templates-onap-java.yaml
+++ b/jjb/onap/global-templates-onap-java.yaml
@@ -179,6 +179,26 @@
       - shell:
           !include-raw-escape: ./cps/k6-performance-tests-trigger.sh
 
+# A job to plot the result of k6 cps performance test
+- job-template:
+    name: 'onap-{project-name}-k6-performance-tests-plots'
+    disabled_job_var: false
+    node: xerces-cps-hw
+    wipe_workspace: false
+
+    triggers:
+      - timed: '{timer}'
+
+    builders:
+      - shell:
+          !include-raw-escape: ./cps/prepare-k6-performance-tests-plots.sh
+
+    publishers:
+      - html-publisher:
+          name: "K6 Performance Review"
+          dir: "."
+          files: "k6TestsPerformanceReview.html"
+
 # A job to perform performance test which are based on Spock-Groovy Framework on cps project
 - job-template:
     name: 'onap-{project-name}-{stream}-performance-test-java'