From 913c6ea7799455432fc158bee5a61c559ea4d191 Mon Sep 17 00:00:00 2001 From: danielhanrahan Date: Fri, 28 Jun 2024 18:14:14 +0100 Subject: [PATCH] [CPS] k6 plot job using CSV files - k6 plot job scrapes CSV output from performance job Issue-ID: CPS-1975 Signed-off-by: danielhanrahan Change-Id: I584de735ae3de74c40314443449edf4a91b385cf --- .../cps/prepare-k6-performance-tests-plots.sh | 221 ++++++------------ 1 file changed, 74 insertions(+), 147 deletions(-) diff --git a/jjb/onap/cps/prepare-k6-performance-tests-plots.sh b/jjb/onap/cps/prepare-k6-performance-tests-plots.sh index d82bd7ab..df6c5741 100755 --- a/jjb/onap/cps/prepare-k6-performance-tests-plots.sh +++ b/jjb/onap/cps/prepare-k6-performance-tests-plots.sh @@ -18,31 +18,16 @@ set -o errexit # Exit on most errors set -o nounset # Disallow expansion of unset variables set -o pipefail # Use last non-zero exit code in a pipeline -#set -o xtrace # Trace logging - disabled to avoid producing gigabytes of logs +set -o xtrace # Trace logging ############################################################################################################################# ################################################ F U N C T I O N S ########################################################## ############################################################################################################################# -# Define the list of "Test Case + Condition" k6 test names to search for in the logs. -# The Test Case and Condition concatenated with \s* to represent zero or more whitespace characters. -# Later in this script can use Perl-compatible regular expressions (PCRE) can match to pattern with grep -P flag. -# The tests which measure http_req_failed excluded since nothing to plot if the rate is always zero. -k6_test_names=( - "1-create-cmhandles.js\s*http_req_duration" - "2-wait-for-cmhandles-to-be-ready.js\s*iteration_duration" - "10-mixed-load-test.js\s*http_req_duration{scenario:id_search_module}" - "10-mixed-load-test.js\s*http_req_duration{scenario:passthrough_read}" - "10-mixed-load-test.js\s*http_req_duration{scenario:cm_search_module}" - "11-delete-cmhandles.js\s*http_req_duration" -) - JENKINS_JOB_URL="https://jenkins.nordix.org/job/onap-cps-performance-test-k6" -latestBuildToRecord="" -consoleText="" latestRecordedBuild="" -timestampOfLatestRecordedBuild="" +formattedTimestampOfLatestRecordedBuild="" # Get last completed build number # The number has not been plotted on the graphs yet @@ -53,17 +38,11 @@ getLastCompletedBuildNumber() { # Get the last recorded build number from local workspace of the jenkins # The number has already been plotted on the graphs getLastRecordedBuildNumber() { - cd "$WORKSPACE" - local file_name="1-create-cmhandles_http_req_duration.txt" - - # Check if the file exists + local file_name="data/1.csv" if [ -f "$file_name" ]; then - # Get the last line from the file - local last_line=$(tail -n 1 "$file_name") - local left_side=$(echo "$last_line" | cut -d ',' -f 2) - echo "$left_side" + tail -n 1 "$file_name" | cut -d, -f1 else - echo "0" + echo 0 fi } @@ -72,78 +51,38 @@ getAllBuildNumbers() { curl -s "${JENKINS_JOB_URL}/api/json?tree=allBuilds\[id\]" | jq -r '.allBuilds[].id' | sort -n } -# Get the console text of k6 performance job -getConsoleText() { +# Extract CSV results from the console text of k6 performance job +getSummaryCsv() { buildToRead=$1 consoleURL="${JENKINS_JOB_URL}/${buildToRead}/consoleText" - consoleText=$(curl -s "$consoleURL") -} - -# Replace special characters with underscore -# The return value will be used to create a file for each test combination -replace_special_characters() { - local pattern=".js\s*" - local text="$1" - - # Escape special characters in the pattern - local escaped_pattern=$(echo "$pattern" | sed 's/[\.*^$]/\\&/g') - - # Replace the pattern with underscores using sed - local result=$(echo "$text" | sed "s/$escaped_pattern/_/g") - - echo "$result" + # Extract summary CSV from Jenkins console output + curl -s "$consoleURL" | sed -n '/-- BEGIN CSV REPORT/, /-- END CSV REPORT/p' | grep "^[1-9].*,.*,.*,.*,.*$" || true } # Get and record the latest k6-job-results with the build number for all tests getAndRecordPerformanceJobResultForBuild() { buildNumber="$1" - getConsoleText "$buildNumber" - # Loop through each test names - for k6_test_name in "${k6_test_names[@]}"; do - getAndRecordDataResults "$consoleText" "$k6_test_name" "$(replace_special_characters "$k6_test_name").txt" "$buildNumber" - done -} - -# Get and record the latest k6-job-results for a single test -getAndRecordDataResults() { - consoleText=$1 - patternToMatch=$2 - dataFile=$3 - buildNumber=$4 - - matched_line="" - limit_value="" - actual="" - - # Find the text combination (Test Case + Condition) in the console log - if matched_line=$(echo "$consoleText" | grep -P "${patternToMatch}"); then - # Find the first-occurred number in the line matched (limit) - limit_value=$(echo "$matched_line" | awk '{for(i=1;i<=NF;i++) if($i ~ /^[0-9]+$/) {print $i; exit}}') - # Find the second-occurred number in the line matched (actual) - actual=$(echo "$matched_line" | awk '{count=0; for(i=1; i<=NF; i++) if($i ~ /^[0-9]+$/) {count++; if(count==2) {print $i; exit}}}') - fi - - # Add a new entry which is a combination of limit, build-number, and actual into related file - touch "$dataFile" - lastLine=$(tail -n 1 "$dataFile") - newLine="$limit_value,$buildNumber,$actual" - if [ -z "$actual" ]; then - # No entry found for this build, the default added as zero - echo "0,$buildNumber,0" >>"$dataFile" - recordLatestRecordedBuild "$buildNumber" - elif [ "$newLine" == "$lastLine" ]; then - # Entry already exists - recordLatestRecordedBuild "$buildNumber" + getSummaryCsv "$buildNumber" > summary.csv + if [ -s summary.csv ]; then + # Store the test cases, names, and units + cut -d, -f1,2,3 summary.csv > test-cases.csv + # Process each row in the CSV, storing the limit and actual value for this build + while IFS=, read -r test_case test_name units limit actual; do + echo "$buildNumber,$limit,$actual" >> "data/$test_case.csv" + done < summary.csv else - # New entry added to the file - echo "$limit_value,$buildNumber,$actual" >>"$dataFile" - recordLatestRecordedBuild "$buildNumber" + # No CSV data for this build, so record zeroes for this build in each existing data file + for dataFile in data/*.csv; do + echo "$buildNumber,0,0" >> "$dataFile" + done fi + recordLatestRecordedBuild "$buildNumber" } # Format the date and time of the latest build recordLatestRecordedBuild() { - latestBuildToRecord="$1" + local latestBuildToRecord="$1" + local timestampOfLatestRecordedBuild timestampOfLatestRecordedBuild=$(curl -s "${JENKINS_JOB_URL}/${latestBuildToRecord}/api/json?tree=timestamp" | jq -r '.timestamp') formattedTimestampOfLatestRecordedBuild=$(date -d "@$((timestampOfLatestRecordedBuild / 1000))" "+%B %e, %Y at %H:%M") latestRecordedBuild=$latestBuildToRecord @@ -153,13 +92,11 @@ recordLatestRecordedBuild() { buildPlotImage() { dataFile="$1" chartFileName="$2" + units="$3" - # Read the first line of the file - first_line=$(head -n 1 "$dataFile") - # Set upper limit of the graphs to %20 to have more space above the plot - ten_percent=1.2 - limit=$(echo "$first_line" | cut -d ',' -f 1) - limit=$(expr "$limit * $ten_percent" | bc) + # Set upper limit of the graphs to 20% bigger than largest limit to have more space above the plot + limit=$(cut -d, -f2 "$dataFile" | sort -n | tail -n 1) + limit=$(echo "$limit * 1.2" | bc) # Create a temporary Gnuplot script cat <gnuplot_script.gp @@ -167,11 +104,11 @@ set datafile separator "," set terminal pngcairo size 1500,600 set output "${chartFileName}" set xlabel "Build" -set ylabel "Limit (ms)" +set ylabel "${units}" set yrange [0 : ${limit} < *] set xtics rotate -plot '$dataFile' using 2:3:xtic(sprintf("%d", column(2))) with linespoints title "measured", \ - '$dataFile' using 2:1 with lines linestyle 2 title "limit" +plot '$dataFile' using 1:3:xtic(sprintf("%d", column(1))) with linespoints title "measured", \ + '$dataFile' using 1:2 with lines linestyle 2 title "limit" EOT # Run the temporary Gnuplot script @@ -181,46 +118,6 @@ EOT rm gnuplot_script.gp } -# Builds html file -buildHtmlReport() { - # use indirect expansion to get all elements of the array - categoryName=("${!1}") - reportTitle="$2" - outputFile="$3" - cat <"$outputFile" - - - - $reportTitle - - -

$reportTitle

-

Last updated for performance job build no. $latestRecordedBuild on $formattedTimestampOfLatestRecordedBuild

- -EOT -# Loop through the tests to generate the HTML rows which consists of the plot-image -for test_name_in_category in "${categoryName[@]}"; do - test_name_in_category="$(replace_special_characters "$test_name_in_category")" - cat <>"$outputFile" - - - -EOF -done -# Close the HTML file -cat <>"$outputFile" -
-
"$test_name_in_category"
- -
-

The k6 performance tests runs every hour, providing performance metrics to the plots.

-

The k6 plots is being updated 31 minutes past every hour.

-

Successful performance tests job build adds new data, but even if a build fails, existing data is retained.

- - -EOT -} - ############################################################################################################################# ################################################ M A I N #################################################################### ############################################################################################################################# @@ -230,8 +127,9 @@ sudo apt-get install -y bc gnuplot jq # Download data from CPS performance Jenkins job cd "$WORKSPACE" -if [ -z "$(ls -A)" ]; then +if [ ! -d data ]; then # If workspace is empty, pull data from all previous performance job runs + mkdir -p data for buildNumber in $(getAllBuildNumbers); do getAndRecordPerformanceJobResultForBuild "$buildNumber" done @@ -247,20 +145,49 @@ else done else echo "No new builds to process." + exit fi fi -# Limit the plots to last 100 builds for each test -for k6_test_name in "${k6_test_names[@]}"; do - k6_file_name="$(replace_special_characters "$k6_test_name")" - tail -n 100 "$k6_file_name.txt" > file.tmp && mv file.tmp "$k6_file_name.txt" -done +reportTitle="k6 tests performance review" +outputFile="k6TestsPerformanceReview.html" +cat <"$outputFile" + + + + $reportTitle + + +

$reportTitle

+

Last updated for performance job build no. $latestRecordedBuild on $formattedTimestampOfLatestRecordedBuild

+ +EOT -# Plot image files in png format -for k6_test_name in "${k6_test_names[@]}"; do - k6_file_name="$(replace_special_characters "$k6_test_name")" - buildPlotImage "$k6_file_name.txt" "$k6_file_name.png" -done +# Loop through the tests to generate the HTML rows which consists of the plot-image +while IFS=, read -r test_case test_name units; do + dataFile="data/$test_case.csv" + # Limit the plots to last 100 builds for each test + tail -n 100 "$dataFile" > file.tmp && mv file.tmp "$dataFile" + # Plot image files in png format + buildPlotImage "$dataFile" "data/$test_case.png" "$units" + # Output to HTML + cat <>"$outputFile" + + + +EOF +done < test-cases.csv -# Build html page -buildHtmlReport k6_test_names[@] "k6 tests performance review" "k6TestsPerformanceReview.html" \ No newline at end of file +# Close the HTML file +cat <>"$outputFile" +
+
$test_case. $test_name ($units)
+ +
+

The plots are generated from the output of the Nordix Jenkins job: + onap-cps-performance-test-k6.

+

The k6 performance tests run are described in the + NCMP REST Interface Characteristics documentation.

+ + +EOT -- 2.25.1