From 0c5c58d1e5e1fe39fe2797b0be314b5625fe77e0 Mon Sep 17 00:00:00 2001 From: emaclee Date: Mon, 11 Sep 2023 11:53:43 +0200 Subject: [PATCH] Add HTML publisher Issue-Id: CPS-1777 Signed-off-by: emaclee Change-Id: I4d980bb931a0a8c1615f3eb016449872a3da5791 --- .../cps/prepare-performance-tests-data.sh | 538 ++++++++++++++++-- jjb/onap/global-templates-onap-java.yaml | 17 +- 2 files changed, 506 insertions(+), 49 deletions(-) diff --git a/jjb/onap/cps/prepare-performance-tests-data.sh b/jjb/onap/cps/prepare-performance-tests-data.sh index c305cdd9..1a840a88 100644 --- a/jjb/onap/cps/prepare-performance-tests-data.sh +++ b/jjb/onap/cps/prepare-performance-tests-data.sh @@ -1,36 +1,201 @@ #!/bin/bash -cd $WORKSPACE -data_file="results.txt" -data_file_headers="Creating openroadm anchors with large data,Read datatrees using openroadm root,Update 100 data nodes,Batch delete 100 whole lists" +############################################################################################################################# +################################################ F U N C T I O N S ########################################################## +############################################################################################################################# -if [[ ! -f "$data_file" ]]; then - echo "Creating $data_file" - touch "$data_file" - echo "$data_file_headers" > $data_file -else - echo "$data_file already exists" -fi +create_data="create_performance_data.txt" +read_data="read_performance_data.txt" +update_data="update_performance_data.txt" +delete_data="delete_performance_data.txt" + +# Data files headers +create_data_title_1="Creating openroadm anchors with large data" +read_data_title_1="Read datatrees using openroadm root" +update_data_title_1="Update 100 data nodes" +delete_data_title_1="Batch delete 100 whole lists" + +# Text patterns to match in console log +creating_bookstore_pattern="^.*Creating openroadm anchors with large.*" +read_datatreees_pattern="^.*Read datatrees using openroadm root.*" +update_datanodes_pattern="^.*Update 100 data nodes.*" +batch_delete_pattern="^.*Batch delete 100 whole lists.*" -echo "-----------------------------------------------------------------------------------" -echo "-----------------------------------------------------------------------------------" -# Extracting data results from performance job +all_patterns=("$creating_bookstore_pattern" "$read_datatreees_pattern" "$update_datanodes_pattern" "$batch_delete_pattern") +new_data="" + +latestBuild=$(getLatestBuild) +availableBuilds=() +consoleText="" JENKINS_JOB_URL="https://jenkins.nordix.org/job/onap-cps-master-performance-test-java" -latest_build=$(curl -s "${JENKINS_JOB_URL}/lastBuild/api/json" | jq -r '.number') -echo "latest build is $latest_build" -# Get the latest build status -latest_build_status=$(curl -s "${JENKINS_JOB_URL}/api/json" | jq -r '.color') -echo "latest build status is $latest_build_status" +# Create and add header to the file +createAndAddHeader() { + file="$1" + header="$2" + if [[ ! -f "$file" ]]; then + echo "Creating $file" + touch "$file" + echo "Build,$header" > $file + else + echo "$file already exists" + fi +} + +# Get latest build +getLatestBuild() { + latestBuild=$(curl -s "${JENKINS_JOB_URL}/lastBuild/api/json" | jq -r '.number') + echo "latest build is $latestBuild" +} + +getConsoleText() { + buildToRead=$1 + consoleURL="${JENKINS_JOB_URL}/${buildToRead}/consoleText" + consoleText=$(curl -s "$consoleURL") +} + +getAndRecordDataResults() { + consoleText=$1 + patternToMatch=$2 + dataFile=$3 + buildNumber=$4 + new_data="" + matched_line="" + limit_value="" + took_value="" + + # Get and calculate data for plot + if matched_line=$(echo "$consoleText" | grep -o -P "$patternToMatch"); then + echo "Matched line: $matched_line" + limit_value=$(echo "$matched_line" | grep -o -P 'limit\s*\K\d+(,\d+)?' | tr -cd '[:digit:]') + echo $limit_value + took_value=$(echo "$matched_line" | grep -o -P 'took\s*\K\d+(,\d+)?' | tr -cd '[:digit:]') + echo $took_value + percentage=$(($took_value * 100 / $limit_value)) + + echo "Limit value: $limit_value" + echo "Took value: $took_value" + echo "Percentage: $percentage" + + if [ -z "$new_data" ]; then + new_data="$percentage" + else + new_data+=",$percentage" + fi + echo "New data: $new_data" + + else + echo "No match found." + fi + + # Record result + lastLine=$(tail -n 1 "$dataFile") + newLine="$buildNumber,$new_data" + if [ -z "$new_data" ]; then + echo "No new data" + elif [ $newLine == $lastLine ]; then + echo "Data already exists" + else + echo "$buildNumber,$new_data" >> $dataFile + echo "New data added" + fi +} + +buildStaticReport() { + dataFile="$1" # Get the input file name from the function parameter + chartTitle="$2" + + sudo apt-get update + sudo apt-get install -y gnuplot + + # Create a temporary Gnuplot script + cat < gnuplot_script.gp + set datafile separator "," + set terminal pngcairo size 1500,600 + set output "${chartTitle}.png" + + # Set X-axis label + set xlabel "Build" + + # Read titles from the first line of the CSV + plot '$dataFile' using (column(0)+1):2:xtic(1) with linespoints title "Y values" +EOT + + # Run the Gnuplot script + gnuplot gnuplot_script.gp + + # Remove the temporary script + rm gnuplot_script.gp +} + + +buildHTMLReport() { + + chartTitle1="$1" + chartFileName1="$2" # Get the chart file name as the first parameter + + chartTitle2="$3" + chartFileName2="$4" + + chartTitle3="$5" + chartFileName3="$6" + + chartTitle4="$7" + chartFileName4="$8" + + reportTitle="$9" # Get the report title as the second parameter + +cat < "index.html" + + + + $reportTitle + + +

$reportTitle

+ + + + + + + + + +
+
"$chartTitle1"
+ Image 1 + 		+
"$chartTitle2"
+ Image 2 +
+
"$chartTitle3"
+ Image 3 + 		+
"$chartTitle4"
+ Image 4 +
+ + +EOT +} + +#!/bin/bash -# Construct the URL for getting the console text -CONSOLE_URL="${JENKINS_JOB_URL}/${latest_build}/consoleText" +############################################################################################################################# +################################################ F U N C T I O N S ########################################################## +############################################################################################################################# -# Get the console text -console_text=$(curl -s "$CONSOLE_URL") +create_data="create_performance_data.txt" +read_data="read_performance_data.txt" +update_data="update_performance_data.txt" +delete_data="delete_performance_data.txt" -echo "-----------------------------------------------------------------------------------" -echo "-----------------------------------------------------------------------------------" +# Data files headers +create_data_title_1="Creating openroadm anchors with large data" +read_data_title_1="Read datatrees using openroadm root" +update_data_title_1="Update 100 data nodes" +delete_data_title_1="Batch delete 100 whole lists" # Text patterns to match in console log creating_bookstore_pattern="^.*Creating openroadm anchors with large.*" @@ -41,13 +206,54 @@ batch_delete_pattern="^.*Batch delete 100 whole lists.*" all_patterns=("$creating_bookstore_pattern" "$read_datatreees_pattern" "$update_datanodes_pattern" "$batch_delete_pattern") new_data="" -# Extract numerical values as data results -for pattern in "${all_patterns[@]}"; do - if matched_line=$(echo "$console_text" | grep -o -P "$pattern"); then +latestBuild=$(getLatestBuild) +availableBuilds=() +consoleText="" +JENKINS_JOB_URL="https://jenkins.nordix.org/job/onap-cps-master-performance-test-java" + +# Create and add header to the file +createAndAddHeader() { + file="$1" + header="$2" + if [[ ! -f "$file" ]]; then + echo "Creating $file" + touch "$file" + echo "Build,$header" > $file + else + echo "$file already exists" + fi +} + +# Get latest build +getLatestBuild() { + latestBuild=$(curl -s "${JENKINS_JOB_URL}/lastBuild/api/json" | jq -r '.number') + echo "latest build is $latestBuild" +} + +getConsoleText() { + buildToRead=$1 + consoleURL="${JENKINS_JOB_URL}/${buildToRead}/consoleText" + consoleText=$(curl -s "$consoleURL") +} + +getAndRecordDataResults() { + consoleText=$1 + patternToMatch=$2 + dataFile=$3 + buildNumber=$4 + new_data="" + matched_line="" + limit_value="" + took_value="" + + # Get and calculate data for plot + if matched_line=$(echo "$consoleText" | grep -o -P "$patternToMatch"); then echo "Matched line: $matched_line" limit_value=$(echo "$matched_line" | grep -o -P 'limit\s*\K\d+(,\d+)?' | tr -cd '[:digit:]') + echo $limit_value took_value=$(echo "$matched_line" | grep -o -P 'took\s*\K\d+(,\d+)?' | tr -cd '[:digit:]') - percentage=$((took_value * 100 / limit_value)) + echo $took_value + percentage=$(($took_value * 100 / $limit_value)) echo "Limit value: $limit_value" echo "Took value: $took_value" @@ -63,14 +269,274 @@ for pattern in "${all_patterns[@]}"; do else echo "No match found." fi -done -echo "-----------------------------------------------------------------------------------" -echo "-----------------------------------------------------------------------------------" + # Record result + lastLine=$(tail -n 1 "$dataFile") + newLine="$buildNumber,$new_data" + if [ -z "$new_data" ]; then + echo "No new data" + elif [ $newLine == $lastLine ]; then + echo "Data already exists" + else + echo "$buildNumber,$new_data" >> $dataFile + echo "New data added" + fi +} + +buildStaticReport() { + dataFile="$1" # Get the input file name from the function parameter + chartTitle="$2" + + sudo apt-get update + sudo apt-get install -y gnuplot + + # Create a temporary Gnuplot script + cat < gnuplot_script.gp + set datafile separator "," + set terminal pngcairo size 1500,600 + set output "${chartTitle}.png" + + # Set X-axis label + set xlabel "Build" + + # Read titles from the first line of the CSV + plot '$dataFile' using (column(0)+1):2:xtic(1) with linespoints title "Y values" +EOT + + # Run the Gnuplot script + gnuplot gnuplot_script.gp + + # Remove the temporary script + rm gnuplot_script.gp +} + + +buildHTMLReport() { + + chartTitle1="$1" + chartFileName1="$2" # Get the chart file name as the first parameter + + chartTitle2="$3" + chartFileName2="$4" + + chartTitle3="$5" + chartFileName3="$6" + + chartTitle4="$7" + chartFileName4="$8" + + reportTitle="$9" # Get the report title as the second parameter + +cat < "index.html" + + + + $reportTitle + + +

$reportTitle

+ + + + + + + + + +
+
"$chartTitle1"
+ Image 1 + 		+
"$chartTitle2"
+ Image 2 +
+
"$chartTitle3"
+ Image 3 + 		+
"$chartTitle4"
+ Image 4 +
+ + +EOT +} + +buildPageReport() { + chartFileName="$1" + reportTitle="$2" + outputFile="$3" + cat < "$outputFile" + + + + $reportTitle + + +

$reportTitle

+ Graph Image + + +EOT +} + +buildPlotForCreateOperation() { + buildNumber="$1" + createAndAddHeader "$create_data" "$create_data_title_1" + getAndRecordDataResults "$consoleText" "$creating_bookstore_pattern" "$create_data" "$buildNumber" +} + +buildPlotForReadOperation() { + buildNumber="$1" + createAndAddHeader "$read_data" "$read_data_title_1" + getAndRecordDataResults "$consoleText" "$read_datatreees_pattern" "$read_data" "$buildNumber" +} + +buildPlotForUpdateOperation() { + buildNumber="$1" + createAndAddHeader "$update_data" "$update_data_title_1" + getAndRecordDataResults "$consoleText" "$update_datanodes_pattern" "$update_data" "$buildNumber" +} + +buildPlotForBatchOperation() { + buildNumber="$1" + createAndAddHeader "$delete_data" "$delete_data_title_1" + getAndRecordDataResults "$consoleText" "$batch_delete_pattern" "$delete_data" "$buildNumber" +} -# Publish results in file for building plot -if [ -z "$new_data" ]; then - echo "No new data" +############################################################################################################################# +############################################################################################################################# +############################################################################################################################# +############################################################################################################################# + +cd $WORKSPACE + +getLatestBuild +if [ -z "$(ls -A)" ]; then + # Calculate the starting value for the loop + startValue=$((latestBuild - 50)) + + # Start the loop from startValue up to latestBuild + for ((i=startValue; i<=latestBuild; i++)); do + getConsoleText "$i" + buildPlotForCreateOperation "$i" + buildPlotForReadOperation "$i" + buildPlotForUpdateOperation "$i" + buildPlotForBatchOperation "$i" + done + buildStaticReport "$create_data" "createLargeData" + buildStaticReport "$read_data" "readDataTrees" + buildStaticReport "$update_data" "updateDatanodes" + buildStaticReport "$delete_data" "batchDelete" else - echo "$new_data" >> $data_file -fi \ No newline at end of file + getConsoleText $latestBuild + buildPlotForCreateOperation $latestBuild + buildPlotForReadOperation $latestBuild + buildPlotForUpdateOperation $latestBuild + buildPlotForBatchOperation $latestBuild + + buildStaticReport "$create_data" "createLargeData" + buildStaticReport "$read_data" "readDataTrees" + buildStaticReport "$update_data" "updateDatanodes" + buildStaticReport "$delete_data" "batchDelete" +fi + + +touch index.html createOperation.html readOperation.html updateOperation.html deleteOperation.html + +buildHTMLReport \ + "Creating openroadm anchors with large data" \ + "createLargeData.png" \ + "Read datatrees using openroadm root" \ + "readDataTrees.png" \ + "Update 100 data nodes" \ + "updateDatanodes.png" \ + "Batch delete 100 whole lists" \ + "batchDelete.png" \ + "Performance Review" + +buildPageReport "createLargeData.png" "Creating openroadm anchors with large data" "createOperation.html" +buildPageReport "readDataTrees.png" "Read datatrees using openroadm root" "readOperation.html" +buildPageReport "updateDatanodes.png" "Update 100 data nodes" "updateOperation.html" +buildPageReport "batchDelete.png" "Batch delete 100 whole lists" "deleteOperation.html" + + +buildPlotForCreateOperation() { + buildNumber="$1" + createAndAddHeader "$create_data" "$create_data_title_1" + getAndRecordDataResults "$consoleText" "$creating_bookstore_pattern" "$create_data" "$buildNumber" +} + +buildPlotForReadOperation() { + buildNumber="$1" + createAndAddHeader "$read_data" "$read_data_title_1" + getAndRecordDataResults "$consoleText" "$read_datatreees_pattern" "$read_data" "$buildNumber" +} + +buildPlotForUpdateOperation() { + buildNumber="$1" + createAndAddHeader "$update_data" "$update_data_title_1" + getAndRecordDataResults "$consoleText" "$update_datanodes_pattern" "$update_data" "$buildNumber" +} + +buildPlotForBatchOperation() { + buildNumber="$1" + createAndAddHeader "$delete_data" "$delete_data_title_1" + getAndRecordDataResults "$consoleText" "$batch_delete_pattern" "$delete_data" "$buildNumber" +} + +############################################################################################################################# +############################################################################################################################# +############################################################################################################################# +############################################################################################################################# + +cd $WORKSPACE + +getLatestBuild +if [ -z "$(ls -A)" ]; then + # Calculate the starting value for the loop + startValue=$((latestBuild - 50)) + + # Start the loop from startValue up to latestBuild + for ((i=startValue; i<=latestBuild; i++)); do + getConsoleText "$i" + buildPlotForCreateOperation "$i" + buildPlotForReadOperation "$i" + buildPlotForUpdateOperation "$i" + buildPlotForBatchOperation "$i" + done + buildStaticReport "$create_data" "createLargeData" + buildStaticReport "$read_data" "readDataTrees" + buildStaticReport "$update_data" "updateDatanodes" + buildStaticReport "$delete_data" "batchDelete" +else + getConsoleText $latestBuild + buildPlotForCreateOperation $latestBuild + buildPlotForReadOperation $latestBuild + buildPlotForUpdateOperation $latestBuild + buildPlotForBatchOperation $latestBuild + + buildStaticReport "$create_data" "createLargeData" + buildStaticReport "$read_data" "readDataTrees" + buildStaticReport "$update_data" "updateDatanodes" + buildStaticReport "$delete_data" "batchDelete" +fi + + +touch index.html createOperation.html readOperation.html updateOperation.html deleteOperation.html + +buildHTMLReport \ + "Creating openroadm anchors with large data" \ + "createLargeData.png" \ + "Read datatrees using openroadm root" \ + "readDataTrees.png" \ + "Update 100 data nodes" \ + "updateDatanodes.png" \ + "Batch delete 100 whole lists" \ + "batchDelete.png" \ + "Performance Review" + +buildPageReport "createLargeData.png" "Creating openroadm anchors with large data" "createOperation.html" +buildPageReport "readDataTrees.png" "Read datatrees using openroadm root" "readOperation.html" +buildPageReport "updateDatanodes.png" "Update 100 data nodes" "updateOperation.html" +buildPageReport "batchDelete.png" "Batch delete 100 whole lists" "deleteOperation.html" diff --git a/jjb/onap/global-templates-onap-java.yaml b/jjb/onap/global-templates-onap-java.yaml index 91c4a28a..bede6670 100644 --- a/jjb/onap/global-templates-onap-java.yaml +++ b/jjb/onap/global-templates-onap-java.yaml @@ -154,19 +154,10 @@ !include-raw-escape: ./cps/prepare-performance-tests-data.sh publishers: - - plot: - - title: "performance" - group: "performance" - csvFileName: "results.txt" - style: line - exclude-zero-yaxis: true - num-builds: '30' - series: - - file: - url: 'results.txt' - inclusion-flag: 'off' - display-table: false - format: csv + - html-publisher: + name: "Performance Review" + dir: "." + files: "index.html,createOperation.html,readOperation.html,updateOperation.html,deleteOperation.html" - job-template: name: 'onap-{project-name}-{stream}-performance-test-java' -- 2.25.1