#!/bin/bash
-cd $WORKSPACE
-
-data_file="results.txt"
-data_file_headers="Creating openroadm anchors with large data,Read datatrees using openroadm root,Update 100 data nodes,Batch delete 100 whole lists"
-
-if [[ ! -f "$data_file" ]]; then
- echo "Creating $data_file"
- touch "$data_file"
- echo "$data_file_headers" > $data_file
-else
- echo "$data_file already exists"
-fi
-
-echo "-----------------------------------------------------------------------------------"
-echo "-----------------------------------------------------------------------------------"
-# Extracting data results from performance job
-JENKINS_JOB_URL="https://jenkins.nordix.org/job/onap-cps-master-performance-test-java"
-latest_build=$(curl -s "${JENKINS_JOB_URL}/lastBuild/api/json" | jq -r '.number')
-echo "latest build is $latest_build"
-
-# Get the latest build status
-latest_build_status=$(curl -s "${JENKINS_JOB_URL}/api/json" | jq -r '.color')
-echo "latest build status is $latest_build_status"
-
-# Construct the URL for getting the console text
-CONSOLE_URL="${JENKINS_JOB_URL}/${latest_build}/consoleText"
+sudo apt-get install bc
+#############################################################################################################################
+################################################ F U N C T I O N S ##########################################################
+#############################################################################################################################
-# Get the console text
-console_text=$(curl -s "$CONSOLE_URL")
+create_data="create_performance_data.txt"
+read_data="read_performance_data.txt"
+update_data="update_performance_data.txt"
+delete_data="delete_performance_data.txt"
-echo "-----------------------------------------------------------------------------------"
-echo "-----------------------------------------------------------------------------------"
+# Data files headers
+create_data_title_1="Creating openroadm anchors with large data"
+read_data_title_1="Read datatrees using openroadm root"
+update_data_title_1="Update 100 data nodes"
+delete_data_title_1="Batch delete 100 whole lists"
# Text patterns to match in console log
creating_bookstore_pattern="^.*Creating openroadm anchors with large.*"
update_datanodes_pattern="^.*Update 100 data nodes.*"
batch_delete_pattern="^.*Batch delete 100 whole lists.*"
-all_patterns=("$creating_bookstore_pattern" "$read_datatreees_pattern" "$update_datanodes_pattern" "$batch_delete_pattern")
new_data=""
+JENKINS_JOB_URL="https://jenkins.nordix.org/job/onap-cps-master-performance-test-java"
+
+# Get latest build
+getLatestBuild() {
+ latestBuild=$(curl -s "${JENKINS_JOB_URL}/lastBuild/api/json" | jq -r '.number')
+ echo "latest build is $latestBuild"
+}
+
+latestBuild=$(getLatestBuild)
+latestBuildToRecord=""
+consoleText=""
+lastRecordedBuild=""
+latestRecordedBuild=""
+timestampOfLatestRecordedBuild=""
+
+# Create and add header to the file
+createAndAddHeader() {
+ file="$1"
+ header="$2"
+ if [[ ! -f "$file" ]]; then
+ echo "Creating $file"
+ touch "$file"
+ echo "Build,$header" > $file
+ else
+ echo "$file already exists"
+ fi
+}
-# Extract numerical values as data results
-for pattern in "${all_patterns[@]}"; do
- if matched_line=$(echo "$console_text" | grep -o -P "$pattern"); then
+# Get the console text from specific build of the performance job
+getConsoleText() {
+ buildToRead=$1
+ consoleURL="${JENKINS_JOB_URL}/${buildToRead}/consoleText"
+ consoleText=$(curl -s "$consoleURL")
+}
+
+getAndRecordDataResults() {
+ consoleText=$1
+ patternToMatch=$2
+ dataFile=$3
+ buildNumber=$4
+ new_data=""
+ matched_line=""
+ limit_value=""
+ took_value=""
+
+ # Get and calculate data for plot
+ if matched_line=$(echo "$consoleText" | grep -o -P "$patternToMatch"); then
echo "Matched line: $matched_line"
- limit_value=$(echo "$matched_line" | grep -o -P 'limit\s*\K\d+(,\d+)?' | tr -cd '[:digit:]')
- took_value=$(echo "$matched_line" | grep -o -P 'took\s*\K\d+(,\d+)?' | tr -cd '[:digit:]')
- percentage=$((took_value * 100 / limit_value))
+ limit_value=$(echo "$matched_line" | grep -o -P 'limit\s*\K\d+(\.\d+)?' | tr -cd '[:digit:].')
+ echo $limit_value
+ took_value=$(echo "$matched_line" | grep -o -P 'took\s*\K\d+(\.\d+)?' | tr -cd '[:digit:].')
+ echo $took_value
+ percentage=$(echo "scale=2; $took_value * 100.00 / $limit_value" | bc)
+
echo "Limit value: $limit_value"
echo "Took value: $took_value"
else
echo "No match found."
fi
-done
-echo "-----------------------------------------------------------------------------------"
-echo "-----------------------------------------------------------------------------------"
+ # Record result
+ lastLine=$(tail -n 1 "$dataFile")
+ newLine="$buildNumber,$new_data"
+ if [ -z "$new_data" ]; then
+ echo "No new data"
+ lastRecordedBuild=$(echo "$lastLine" | awk -F, '{print $1}')
+ recordLatestRecordedBuild "$lastRecordedBuild"
+ elif [ $newLine == $lastLine ]; then
+ echo "Data already exists"
+ recordLatestRecordedBuild "$buildNumber"
+ else
+ echo "$buildNumber,$new_data" >> $dataFile
+ echo "New data added"
+ recordLatestRecordedBuild "$buildNumber"
+ fi
+}
+
+recordLatestRecordedBuild() {
+ latestBuildToRecord="$1"
+ timestampOfLatestRecordedBuild=$(curl -s "${JENKINS_JOB_URL}/${latestBuildToRecord}/api/json" | jq -r '.timestamp')
+ formattedTimestampOfLatestRecordedBuild=$(date -d "@$((timestampOfLatestRecordedBuild / 1000))" "+%B %e, %Y at %H:%M")
+ echo "Latest recorded build: $latestBuildToRecord"
+ echo "Date of latest recorded build: $formattedTimestampOfLatestRecordedBuild"
+ latestRecordedBuild=$latestBuildToRecord
+}
+
+buildStaticReport() {
+ dataFile="$1" # Get the input file name from the function parameter
+ chartTitle="$2"
+
+ sudo apt-get update
+ sudo apt-get install -y gnuplot
+
+ # Create a temporary Gnuplot script
+ cat <<EOT > gnuplot_script.gp
+ set datafile separator ","
+ set terminal pngcairo size 1500,600
+ set output "${chartTitle}.png"
+
+ # Set X-axis label
+ set xlabel "Build"
+
+ # Set Y-axis label
+ set ylabel "Percentage of limit %"
+
+ set xtics rotate
+ plot '$dataFile' using (column(0)+1):2:xtic(sprintf("%d", column(1))) with linespoints title "limit"
+EOT
+
+ # Run the Gnuplot script
+ gnuplot gnuplot_script.gp
+
+ # Remove the temporary script
+ rm gnuplot_script.gp
+}
+
+buildHTMLReport() {
+
+ chartTitle1="$1" # i.e Get the chart file name as the first parameter
+ chartFileName1="$2"
+
+ chartTitle2="$3"
+ chartFileName2="$4"
+
+ chartTitle3="$5"
+ chartFileName3="$6"
+
+ chartTitle4="$7"
+ chartFileName4="$8"
+
+ reportTitle="$9" # i.e Get the report title as the ninth parameter
+
+cat <<EOT > "index.html"
+<!DOCTYPE html>
+<html>
+<head>
+ <title>$reportTitle</title>
+</head>
+<body>
+ <h1 style="text-align: center;">$reportTitle</h1>
+ <h4>Last updated for performance job build no. $latestRecordedBuild on $formattedTimestampOfLatestRecordedBuild</h4>
+ <p>The performance tests job runs daily at 02:15 UTC, providing performance metrics. The following graphs update at 04:15 UTC.</p>
+ <p>Successful performance tests job build adds new data, but even if a build fails, existing data is retained.</p>
+ <p>Updates occur whenever new successful data is available.</p>
+
+ <table align="center">
+ <tr> <!-- First Row -->
+ <td align="center">
+ <figcaption>"$chartTitle1"</figcaption>
+ <img src="$chartFileName1" alt="Image 1" width="750" height="300">
+ </td>
+ <td align="center" style="padding: 10px;">
+ <figcaption>"$chartTitle2"</figcaption>
+ <img src="$chartFileName2" alt="Image 2" width="750" height="300">
+ </td>
+ </tr>
+ <tr> <!-- Second Row -->
+ <td align="center" style="padding: 10px;">
+ <figcaption>"$chartTitle3"</figcaption>
+ <img src="$chartFileName3" alt="Image 3" width="750" height="300">
+ </td>
+ <td align="center" style="padding: 10px;">
+ <figcaption>"$chartTitle4"</figcaption>
+ <img src="$chartFileName4" alt="Image 4" width="750" height="300">
+ </td>
+ </tr>
+ </table>
+</body>
+</html>
+EOT
+}
-# Publish results in file for building plot
-if [ -z "$new_data" ]; then
- echo "No new data"
+buildPageReport() {
+ chartFileName="$1"
+ reportTitle="$2"
+ outputFile="$3"
+ cat <<EOT > "$outputFile"
+ <!DOCTYPE html>
+ <html>
+ <head>
+ <title>$reportTitle</title>
+ </head>
+ <body>
+ <h1>$reportTitle</h1>
+ <h4>Last updated for performance job build no. $latestRecordedBuild on $formattedTimestampOfLatestRecordedBuild</h4>
+ <img src="$chartFileName" alt="Graph Image">
+ </body>
+ </html>
+EOT
+}
+
+buildPlotForCreateOperation() {
+ buildNumber="$1"
+ createAndAddHeader "$create_data" "$create_data_title_1"
+ getAndRecordDataResults "$consoleText" "$creating_bookstore_pattern" "$create_data" "$buildNumber"
+}
+
+buildPlotForReadOperation() {
+ buildNumber="$1"
+ createAndAddHeader "$read_data" "$read_data_title_1"
+ getAndRecordDataResults "$consoleText" "$read_datatreees_pattern" "$read_data" "$buildNumber"
+}
+
+buildPlotForUpdateOperation() {
+ buildNumber="$1"
+ createAndAddHeader "$update_data" "$update_data_title_1"
+ getAndRecordDataResults "$consoleText" "$update_datanodes_pattern" "$update_data" "$buildNumber"
+}
+
+buildPlotForBatchOperation() {
+ buildNumber="$1"
+ createAndAddHeader "$delete_data" "$delete_data_title_1"
+ getAndRecordDataResults "$consoleText" "$batch_delete_pattern" "$delete_data" "$buildNumber"
+}
+
+#############################################################################################################################
+#############################################################################################################################
+#############################################################################################################################
+#############################################################################################################################
+
+cd $WORKSPACE
+
+getLatestBuild
+if [ -z "$(ls -A)" ]; then
+ # Calculate the starting value for the loop
+ startValue=$((latestBuild - 100))
+
+ # Start the loop from startValue up to latestBuild
+ for ((i=startValue; i<=latestBuild; i++)); do
+ getConsoleText "$i"
+ buildPlotForCreateOperation "$i"
+ buildPlotForReadOperation "$i"
+ buildPlotForUpdateOperation "$i"
+ buildPlotForBatchOperation "$i"
+ done
+ buildStaticReport "$create_data" "createLargeData"
+ buildStaticReport "$read_data" "readDataTrees"
+ buildStaticReport "$update_data" "updateDatanodes"
+ buildStaticReport "$delete_data" "batchDelete"
+else
+ getConsoleText $latestBuild
+ buildPlotForCreateOperation $latestBuild
+ buildPlotForReadOperation $latestBuild
+ buildPlotForUpdateOperation $latestBuild
+ buildPlotForBatchOperation $latestBuild
+
+ buildStaticReport "$create_data" "createLargeData"
+ buildStaticReport "$read_data" "readDataTrees"
+ buildStaticReport "$update_data" "updateDatanodes"
+ buildStaticReport "$delete_data" "batchDelete"
+fi
+
+
+touch index.html createOperation.html readOperation.html updateOperation.html deleteOperation.html
+
+buildHTMLReport \
+ "Creating openroadm anchors with large data" \
+ "createLargeData.png" \
+ "Read datatrees using openroadm root" \
+ "readDataTrees.png" \
+ "Update 100 data nodes" \
+ "updateDatanodes.png" \
+ "Batch delete 100 whole lists" \
+ "batchDelete.png" \
+ "Performance Review"
+
+buildPageReport "createLargeData.png" "Creating openroadm anchors with large data" "createOperation.html"
+buildPageReport "readDataTrees.png" "Read datatrees using openroadm root" "readOperation.html"
+buildPageReport "updateDatanodes.png" "Update 100 data nodes" "updateOperation.html"
+buildPageReport "batchDelete.png" "Batch delete 100 whole lists" "deleteOperation.html"
+
+
+buildPlotForCreateOperation() {
+ buildNumber="$1"
+ createAndAddHeader "$create_data" "$create_data_title_1"
+ getAndRecordDataResults "$consoleText" "$creating_bookstore_pattern" "$create_data" "$buildNumber"
+}
+
+buildPlotForReadOperation() {
+ buildNumber="$1"
+ createAndAddHeader "$read_data" "$read_data_title_1"
+ getAndRecordDataResults "$consoleText" "$read_datatreees_pattern" "$read_data" "$buildNumber"
+}
+
+buildPlotForUpdateOperation() {
+ buildNumber="$1"
+ createAndAddHeader "$update_data" "$update_data_title_1"
+ getAndRecordDataResults "$consoleText" "$update_datanodes_pattern" "$update_data" "$buildNumber"
+}
+
+buildPlotForBatchOperation() {
+ buildNumber="$1"
+ createAndAddHeader "$delete_data" "$delete_data_title_1"
+ getAndRecordDataResults "$consoleText" "$batch_delete_pattern" "$delete_data" "$buildNumber"
+}
+
+#############################################################################################################################
+#############################################################################################################################
+#############################################################################################################################
+#############################################################################################################################
+
+cd $WORKSPACE
+
+getLatestBuild
+if [ -z "$(ls -A)" ]; then
+ # Calculate the starting value for the loop
+ startValue=$((latestBuild - 50))
+
+ # Start the loop from startValue up to latestBuild
+ for ((i=startValue; i<=latestBuild; i++)); do
+ getConsoleText "$i"
+ buildPlotForCreateOperation "$i"
+ buildPlotForReadOperation "$i"
+ buildPlotForUpdateOperation "$i"
+ buildPlotForBatchOperation "$i"
+ done
+ buildStaticReport "$create_data" "createLargeData"
+ buildStaticReport "$read_data" "readDataTrees"
+ buildStaticReport "$update_data" "updateDatanodes"
+ buildStaticReport "$delete_data" "batchDelete"
else
- echo "$new_data" >> $data_file
-fi
\ No newline at end of file
+ getConsoleText $latestBuild
+ buildPlotForCreateOperation $latestBuild
+ buildPlotForReadOperation $latestBuild
+ buildPlotForUpdateOperation $latestBuild
+ buildPlotForBatchOperation $latestBuild
+
+ buildStaticReport "$create_data" "createLargeData"
+ buildStaticReport "$read_data" "readDataTrees"
+ buildStaticReport "$update_data" "updateDatanodes"
+ buildStaticReport "$delete_data" "batchDelete"
+fi
+
+
+touch index.html createOperation.html readOperation.html updateOperation.html deleteOperation.html
+
+buildHTMLReport \
+ "Creating openroadm anchors with large data" \
+ "createLargeData.png" \
+ "Read datatrees using openroadm root" \
+ "readDataTrees.png" \
+ "Update 100 data nodes" \
+ "updateDatanodes.png" \
+ "Batch delete 100 whole lists" \
+ "batchDelete.png" \
+ "Performance Review"
+
+buildPageReport "createLargeData.png" "Creating openroadm anchors with large data" "createOperation.html"
+buildPageReport "readDataTrees.png" "Read datatrees using openroadm root" "readOperation.html"
+buildPageReport "updateDatanodes.png" "Update 100 data nodes" "updateOperation.html"
+buildPageReport "batchDelete.png" "Batch delete 100 whole lists" "deleteOperation.html"
+
+#############################################################################################################################
+##################################################### D A T A ###############################################################
+#############################################################################################################################
+
+cat create_performance_data.txt
+cat read_performance_data.txt
+cat update_performance_data.txt
+cat delete_performance_data.txt
\ No newline at end of file
Code Review / infra / cicd.git / blobdiff