jjb/onap/cps/prepare-performance-tests-data.sh

   1 #!/bin/bash
   2 #############################################################################################################################
   3 ################################################ F U N C T I O N S ##########################################################
   4 #############################################################################################################################
   5
   6 create_data="create_performance_data.txt"
   7 read_data="read_performance_data.txt"
   8 update_data="update_performance_data.txt"
   9 delete_data="delete_performance_data.txt"
  10
  11 # Data files headers
  12 create_data_title_1="Creating openroadm anchors with large data"
  13 read_data_title_1="Read datatrees using openroadm root"
  14 update_data_title_1="Update 100 data nodes"
  15 delete_data_title_1="Batch delete 100 whole lists"
  16
  17 # Text patterns to match in console log
  18 creating_bookstore_pattern="^.*Creating openroadm anchors with large.*"
  19 read_datatreees_pattern="^.*Read datatrees using openroadm root.*"
  20 update_datanodes_pattern="^.*Update 100 data nodes.*"
  21 batch_delete_pattern="^.*Batch delete 100 whole lists.*"
  22
  23 all_patterns=("$creating_bookstore_pattern" "$read_datatreees_pattern" "$update_datanodes_pattern" "$batch_delete_pattern")
  24 new_data=""
  25
  26 # Get latest build
  27 getLatestBuild() {
  28     latestBuild=$(curl -s "${JENKINS_JOB_URL}/lastBuild/api/json" | jq -r '.number')
  29     echo "latest build is $latestBuild"
  30 }
  31
  32 latestBuild=$(getLatestBuild)
  33 consoleText=""
  34 lastRecordedBuild=""
  35 dateOfLastRecordedBuild=""
  36 JENKINS_JOB_URL="https://jenkins.nordix.org/job/onap-cps-master-performance-test-java"
  37
  38 # Create and add header to the file
  39 createAndAddHeader() {
  40     file="$1"
  41     header="$2"
  42     if [[ ! -f "$file" ]]; then
  43         echo "Creating $file"
  44         touch "$file"
  45         echo "Build,$header" > $file
  46     else
  47         echo "$file already exists"
  48     fi
  49 }
  50
  51 getConsoleText() {
  52     buildToRead=$1
  53     consoleURL="${JENKINS_JOB_URL}/${buildToRead}/consoleText"
  54     consoleText=$(curl -s "$consoleURL")
  55 }
  56
  57 getAndRecordDataResults() {
  58     consoleText=$1
  59     patternToMatch=$2
  60     dataFile=$3
  61     buildNumber=$4
  62     new_data=""
  63     matched_line=""
  64     limit_value=""
  65     took_value=""
  66
  67     # Get and calculate data for plot
  68     if matched_line=$(echo "$consoleText" | grep -o -P "$patternToMatch"); then
  69         echo "Matched line: $matched_line"
  70         limit_value=$(echo "$matched_line" | grep -o -P 'limit\s*\K\d+(,\d+)?' | tr -cd '[:digit:]')
  71         echo $limit_value
  72         took_value=$(echo "$matched_line" | grep -o -P 'took\s*\K\d+(,\d+)?' | tr -cd '[:digit:]')
  73         echo $took_value
  74         percentage=$(($took_value * 100 / $limit_value))
  75
  76         echo "Limit value: $limit_value"
  77         echo "Took value: $took_value"
  78         echo "Percentage: $percentage"
  79
  80     if [ -z "$new_data" ]; then
  81         new_data="$percentage"
  82         else
  83         new_data+=",$percentage"
  84         fi
  85         echo "New data: $new_data"
  86
  87         else
  88         echo "No match found."
  89         fi
  90
  91     # Record result
  92     lastLine=$(tail -n 1 "$dataFile")
  93     newLine="$buildNumber,$new_data"
  94     if [ -z "$new_data" ]; then
  95         echo "No new data"
  96     elif [ $newLine == $lastLine ]; then
  97         echo "Data already exists"
  98     else
  99         echo "$buildNumber,$new_data" >> $dataFile
 100         echo "New data added"
 101     fi
 102     lastRecordedBuild=$buildNumber
 103     dateOfLastRecordedBuild=$(curl -s "${JENKINS_JOB_URL}/${lastRecordedBuild}/api/json" | jq -r '.timestamp' | xargs -I {} date -d "@{}")
 104     echo "last Recorded build: $lastRecordedBuild"
 105     echo "date of last recorded build: $dateOfLastRecordedBuild"
 106 }
 107
 108 buildStaticReport() {
 109     dataFile="$1"  # Get the input file name from the function parameter
 110     chartTitle="$2"
 111
 112     sudo apt-get update
 113     sudo apt-get install -y gnuplot
 114
 115     # Create a temporary Gnuplot script
 116     cat <<EOT > gnuplot_script.gp
 117     set datafile separator ","
 118     set terminal pngcairo size 1500,600
 119     set output "${chartTitle}.png"
 120
 121     # Set X-axis label
 122     set xlabel "Build"
 123
 124     # Set Y-axis label
 125     set ylabel "Percentage of limit %"
 126
 127           # Get stats for min and max
 128     stats '$dataFile' using 1
 129           xmin = STATS_min
 130           freq = 5 #frequency
 131           xLabel(x) = (int((x - xmin) / freq) % 2 == 0) ? sprintf("%d", x) : ""
 132
 133         plot '$dataFile' using (column(0)+1):2:xtic(xLabel(column(1))) with linespoints title "limit"
 134 EOT
 135
 136     # Run the Gnuplot script
 137     gnuplot gnuplot_script.gp
 138
 139     # Remove the temporary script
 140     rm gnuplot_script.gp
 141 }
 142
 143 buildHTMLReport() {
 144
 145     chartTitle1="$1"
 146     chartFileName1="$2"   # Get the chart file name as the first parameter
 147
 148     chartTitle2="$3"
 149     chartFileName2="$4"
 150
 151     chartTitle3="$5"
 152     chartFileName3="$6"
 153
 154     chartTitle4="$7"
 155     chartFileName4="$8"
 156
 157     reportTitle="$9"     # Get the report title as the second parameter
 158
 159 cat <<EOT > "index.html"
 160 <!DOCTYPE html>
 161 <html>
 162 <head>
 163   <title>$reportTitle</title>
 164 </head>
 165 <body>
 166     <h1 style="text-align: center;">$reportTitle</h1>
 167     <h4>Last recorded build number:                     $lastRecordedBuild</h4>
 168         <h4>Date of the last recoded build:             $dateOfLastRecordedBuild</h4>
 169     <table align="center">
 170         <tr> <!-- First Row -->
 171             <td align="center">
 172                 <figcaption>"$chartTitle1"</figcaption>
 173                 <img src="$chartFileName1" alt="Image 1" width="750" height="300">
 174             </td>
 175             <td align="center" style="padding: 10px;">
 176                 <figcaption>"$chartTitle2"</figcaption>
 177                 <img src="$chartFileName2" alt="Image 2" width="750" height="300">
 178             </td>
 179         </tr>
 180         <tr> <!-- Second Row -->
 181             <td align="center" style="padding: 10px;">
 182                 <figcaption>"$chartTitle3"</figcaption>
 183                 <img src="$chartFileName3" alt="Image 3" width="750" height="300">
 184             </td>
 185             <td align="center" style="padding: 10px;">
 186                 <figcaption>"$chartTitle4"</figcaption>
 187                 <img src="$chartFileName4" alt="Image 4" width="750" height="300">
 188             </td>
 189         </tr>
 190     </table>
 191 </body>
 192 </html>
 193 EOT
 194 }
 195
 196 buildPageReport() {
 197     chartFileName="$1"
 198     reportTitle="$2"
 199     outputFile="$3"
 200     cat <<EOT > "$outputFile"
 201     <!DOCTYPE html>
 202     <html>
 203     <head>
 204     <title>$reportTitle</title>
 205     </head>
 206     <body>
 207         <h1>$reportTitle</h1>
 208         <h4>Last recorded build number:                 $lastRecordedBuild</h4>
 209             <h4>Date of the last recoded build:         $dateOfLastRecordedBuild</h4>
 210         <img src="$chartFileName" alt="Graph Image">
 211     </body>
 212     </html>
 213 EOT
 214 }
 215
 216 buildPlotForCreateOperation() {
 217         buildNumber="$1"
 218     createAndAddHeader "$create_data" "$create_data_title_1"
 219     getAndRecordDataResults "$consoleText" "$creating_bookstore_pattern" "$create_data" "$buildNumber"
 220 }
 221
 222 buildPlotForReadOperation() {
 223         buildNumber="$1"
 224     createAndAddHeader "$read_data" "$read_data_title_1"
 225     getAndRecordDataResults "$consoleText" "$read_datatreees_pattern" "$read_data"  "$buildNumber"
 226 }
 227
 228 buildPlotForUpdateOperation() {
 229         buildNumber="$1"
 230     createAndAddHeader "$update_data" "$update_data_title_1"
 231     getAndRecordDataResults "$consoleText" "$update_datanodes_pattern" "$update_data"  "$buildNumber"
 232 }
 233
 234 buildPlotForBatchOperation() {
 235         buildNumber="$1"
 236     createAndAddHeader "$delete_data" "$delete_data_title_1"
 237     getAndRecordDataResults "$consoleText" "$batch_delete_pattern" "$delete_data"  "$buildNumber"
 238 }
 239
 240 #############################################################################################################################
 241 #############################################################################################################################
 242 #############################################################################################################################
 243 #############################################################################################################################
 244
 245 cd $WORKSPACE
 246
 247 getLatestBuild
 248 if [ -z "$(ls -A)" ]; then
 249         # Calculate the starting value for the loop
 250         startValue=$((latestBuild - 70))
 251
 252         # Start the loop from startValue up to latestBuild
 253         for ((i=startValue; i<=latestBuild; i++)); do
 254             getConsoleText "$i"
 255             buildPlotForCreateOperation "$i"
 256             buildPlotForReadOperation "$i"
 257             buildPlotForUpdateOperation "$i"
 258             buildPlotForBatchOperation "$i"
 259         done
 260             buildStaticReport "$create_data" "createLargeData"
 261             buildStaticReport "$read_data" "readDataTrees"
 262             buildStaticReport "$update_data" "updateDatanodes"
 263             buildStaticReport "$delete_data" "batchDelete"
 264 else
 265     getConsoleText $latestBuild
 266     buildPlotForCreateOperation $latestBuild
 267     buildPlotForReadOperation $latestBuild
 268     buildPlotForUpdateOperation $latestBuild
 269     buildPlotForBatchOperation $latestBuild
 270
 271     buildStaticReport "$create_data" "createLargeData"
 272     buildStaticReport "$read_data" "readDataTrees"
 273     buildStaticReport "$update_data" "updateDatanodes"
 274     buildStaticReport "$delete_data" "batchDelete"
 275 fi
 276
 277
 278 touch index.html createOperation.html readOperation.html updateOperation.html deleteOperation.html
 279
 280 buildHTMLReport \
 281     "Creating openroadm anchors with large data" \
 282     "createLargeData.png" \
 283     "Read datatrees using openroadm root" \
 284     "readDataTrees.png" \
 285     "Update 100 data nodes" \
 286     "updateDatanodes.png" \
 287     "Batch delete 100 whole lists" \
 288     "batchDelete.png" \
 289     "Performance Review"
 290
 291 buildPageReport "createLargeData.png" "Creating openroadm anchors with large data"  "createOperation.html"
 292 buildPageReport "readDataTrees.png" "Read datatrees using openroadm root"  "readOperation.html"
 293 buildPageReport "updateDatanodes.png" "Update 100 data nodes" "updateOperation.html"
 294 buildPageReport "batchDelete.png" "Batch delete 100 whole lists" "deleteOperation.html"
 295
 296
 297 buildPlotForCreateOperation() {
 298         buildNumber="$1"
 299     createAndAddHeader "$create_data" "$create_data_title_1"
 300     getAndRecordDataResults "$consoleText" "$creating_bookstore_pattern" "$create_data" "$buildNumber"
 301 }
 302
 303 buildPlotForReadOperation() {
 304         buildNumber="$1"
 305     createAndAddHeader "$read_data" "$read_data_title_1"
 306     getAndRecordDataResults "$consoleText" "$read_datatreees_pattern" "$read_data"  "$buildNumber"
 307 }
 308
 309 buildPlotForUpdateOperation() {
 310         buildNumber="$1"
 311     createAndAddHeader "$update_data" "$update_data_title_1"
 312     getAndRecordDataResults "$consoleText" "$update_datanodes_pattern" "$update_data"  "$buildNumber"
 313 }
 314
 315 buildPlotForBatchOperation() {
 316         buildNumber="$1"
 317     createAndAddHeader "$delete_data" "$delete_data_title_1"
 318     getAndRecordDataResults "$consoleText" "$batch_delete_pattern" "$delete_data"  "$buildNumber"
 319 }
 320
 321 #############################################################################################################################
 322 #############################################################################################################################
 323 #############################################################################################################################
 324 #############################################################################################################################
 325
 326 cd $WORKSPACE
 327
 328 getLatestBuild
 329 if [ -z "$(ls -A)" ]; then
 330         # Calculate the starting value for the loop
 331         startValue=$((latestBuild - 50))
 332
 333         # Start the loop from startValue up to latestBuild
 334         for ((i=startValue; i<=latestBuild; i++)); do
 335             getConsoleText "$i"
 336             buildPlotForCreateOperation "$i"
 337             buildPlotForReadOperation "$i"
 338             buildPlotForUpdateOperation "$i"
 339             buildPlotForBatchOperation "$i"
 340         done
 341             buildStaticReport "$create_data" "createLargeData"
 342             buildStaticReport "$read_data" "readDataTrees"
 343             buildStaticReport "$update_data" "updateDatanodes"
 344             buildStaticReport "$delete_data" "batchDelete"
 345 else
 346     getConsoleText $latestBuild
 347     buildPlotForCreateOperation $latestBuild
 348     buildPlotForReadOperation $latestBuild
 349     buildPlotForUpdateOperation $latestBuild
 350     buildPlotForBatchOperation $latestBuild
 351
 352     buildStaticReport "$create_data" "createLargeData"
 353     buildStaticReport "$read_data" "readDataTrees"
 354     buildStaticReport "$update_data" "updateDatanodes"
 355     buildStaticReport "$delete_data" "batchDelete"
 356 fi
 357
 358
 359 touch index.html createOperation.html readOperation.html updateOperation.html deleteOperation.html
 360
 361 buildHTMLReport \
 362     "Creating openroadm anchors with large data" \
 363     "createLargeData.png" \
 364     "Read datatrees using openroadm root" \
 365     "readDataTrees.png" \
 366     "Update 100 data nodes" \
 367     "updateDatanodes.png" \
 368     "Batch delete 100 whole lists" \
 369     "batchDelete.png" \
 370     "Performance Review"
 371
 372 buildPageReport "createLargeData.png" "Creating openroadm anchors with large data"  "createOperation.html"
 373 buildPageReport "readDataTrees.png" "Read datatrees using openroadm root"  "readOperation.html"
 374 buildPageReport "updateDatanodes.png" "Update 100 data nodes" "updateOperation.html"
 375 buildPageReport "batchDelete.png" "Batch delete 100 whole lists" "deleteOperation.html"