jjb/onap/cps/prepare-performance-tests-data.sh

   1 #!/bin/bash
   2
   3 #############################################################################################################################
   4 ################################################ F U N C T I O N S ##########################################################
   5 #############################################################################################################################
   6
   7 create_data="create_performance_data.txt"
   8 read_data="read_performance_data.txt"
   9 update_data="update_performance_data.txt"
  10 delete_data="delete_performance_data.txt"
  11
  12 # Data files headers
  13 create_data_title_1="Creating openroadm anchors with large data"
  14 read_data_title_1="Read datatrees using openroadm root"
  15 update_data_title_1="Update 100 data nodes"
  16 delete_data_title_1="Batch delete 100 whole lists"
  17
  18 # Text patterns to match in console log
  19 creating_bookstore_pattern="^.*Creating openroadm anchors with large.*"
  20 read_datatreees_pattern="^.*Read datatrees using openroadm root.*"
  21 update_datanodes_pattern="^.*Update 100 data nodes.*"
  22 batch_delete_pattern="^.*Batch delete 100 whole lists.*"
  23
  24 all_patterns=("$creating_bookstore_pattern" "$read_datatreees_pattern" "$update_datanodes_pattern" "$batch_delete_pattern")
  25 new_data=""
  26
  27 latestBuild=$(getLatestBuild)
  28 availableBuilds=()
  29 consoleText=""
  30 JENKINS_JOB_URL="https://jenkins.nordix.org/job/onap-cps-master-performance-test-java"
  31
  32 # Create and add header to the file
  33 createAndAddHeader() {
  34     file="$1"
  35     header="$2"
  36     if [[ ! -f "$file" ]]; then
  37         echo "Creating $file"
  38         touch "$file"
  39         echo "Build,$header" > $file
  40     else
  41         echo "$file already exists"
  42     fi
  43 }
  44
  45 # Get latest build
  46 getLatestBuild() {
  47     latestBuild=$(curl -s "${JENKINS_JOB_URL}/lastBuild/api/json" | jq -r '.number')
  48     echo "latest build is $latestBuild"
  49 }
  50
  51 getConsoleText() {
  52     buildToRead=$1
  53     consoleURL="${JENKINS_JOB_URL}/${buildToRead}/consoleText"
  54     consoleText=$(curl -s "$consoleURL")
  55 }
  56
  57 getAndRecordDataResults() {
  58     consoleText=$1
  59     patternToMatch=$2
  60     dataFile=$3
  61     buildNumber=$4
  62     new_data=""
  63     matched_line=""
  64     limit_value=""
  65     took_value=""
  66
  67     # Get and calculate data for plot
  68     if matched_line=$(echo "$consoleText" | grep -o -P "$patternToMatch"); then
  69         echo "Matched line: $matched_line"
  70         limit_value=$(echo "$matched_line" | grep -o -P 'limit\s*\K\d+(,\d+)?' | tr -cd '[:digit:]')
  71         echo $limit_value
  72         took_value=$(echo "$matched_line" | grep -o -P 'took\s*\K\d+(,\d+)?' | tr -cd '[:digit:]')
  73         echo $took_value
  74         percentage=$(($took_value * 100 / $limit_value))
  75
  76         echo "Limit value: $limit_value"
  77         echo "Took value: $took_value"
  78         echo "Percentage: $percentage"
  79
  80     if [ -z "$new_data" ]; then
  81         new_data="$percentage"
  82         else
  83         new_data+=",$percentage"
  84         fi
  85         echo "New data: $new_data"
  86
  87         else
  88         echo "No match found."
  89         fi
  90
  91     # Record result
  92     lastLine=$(tail -n 1 "$dataFile")
  93     newLine="$buildNumber,$new_data"
  94     if [ -z "$new_data" ]; then
  95         echo "No new data"
  96     elif [ $newLine == $lastLine ]; then
  97         echo "Data already exists"
  98     else
  99         echo "$buildNumber,$new_data" >> $dataFile
 100         echo "New data added"
 101     fi
 102 }
 103
 104 buildStaticReport() {
 105     dataFile="$1"  # Get the input file name from the function parameter
 106     chartTitle="$2"
 107
 108     sudo apt-get update
 109     sudo apt-get install -y gnuplot
 110
 111     # Create a temporary Gnuplot script
 112     cat <<EOT > gnuplot_script.gp
 113     set datafile separator ","
 114     set terminal pngcairo size 1500,600
 115     set output "${chartTitle}.png"
 116
 117     # Set X-axis label
 118     set xlabel "Build"
 119
 120     # Read titles from the first line of the CSV
 121     plot '$dataFile' using (column(0)+1):2:xtic(1) with linespoints title "Y values"
 122 EOT
 123
 124     # Run the Gnuplot script
 125     gnuplot gnuplot_script.gp
 126
 127     # Remove the temporary script
 128     rm gnuplot_script.gp
 129 }
 130
 131
 132 buildHTMLReport() {
 133
 134     chartTitle1="$1"
 135     chartFileName1="$2"   # Get the chart file name as the first parameter
 136
 137     chartTitle2="$3"
 138     chartFileName2="$4"
 139
 140     chartTitle3="$5"
 141     chartFileName3="$6"
 142
 143     chartTitle4="$7"
 144     chartFileName4="$8"
 145
 146     reportTitle="$9"     # Get the report title as the second parameter
 147
 148 cat <<EOT > "index.html"
 149 <!DOCTYPE html>
 150 <html>
 151 <head>
 152   <title>$reportTitle</title>
 153 </head>
 154 <body>
 155     <h1 style="text-align: center;">$reportTitle</h1>
 156     <table align="center">
 157         <tr> <!-- First Row -->
 158             <td align="center">
 159                 <figcaption>"$chartTitle1"</figcaption>
 160                 <img src="$chartFileName1" alt="Image 1" width="750" height="300">
 161             </td>
 162             <td align="center" style="padding: 10px;">
 163                 <figcaption>"$chartTitle2"</figcaption>
 164                 <img src="$chartFileName2" alt="Image 2" width="750" height="300">
 165             </td>
 166         </tr>
 167         <tr> <!-- Second Row -->
 168             <td align="center" style="padding: 10px;">
 169                 <figcaption>"$chartTitle3"</figcaption>
 170                 <img src="$chartFileName3" alt="Image 3" width="750" height="300">
 171             </td>
 172             <td align="center" style="padding: 10px;">
 173                 <figcaption>"$chartTitle4"</figcaption>
 174                 <img src="$chartFileName4" alt="Image 4" width="750" height="300">
 175             </td>
 176         </tr>
 177     </table>
 178 </body>
 179 </html>
 180 EOT
 181 }
 182
 183 #!/bin/bash
 184
 185 #############################################################################################################################
 186 ################################################ F U N C T I O N S ##########################################################
 187 #############################################################################################################################
 188
 189 create_data="create_performance_data.txt"
 190 read_data="read_performance_data.txt"
 191 update_data="update_performance_data.txt"
 192 delete_data="delete_performance_data.txt"
 193
 194 # Data files headers
 195 create_data_title_1="Creating openroadm anchors with large data"
 196 read_data_title_1="Read datatrees using openroadm root"
 197 update_data_title_1="Update 100 data nodes"
 198 delete_data_title_1="Batch delete 100 whole lists"
 199
 200 # Text patterns to match in console log
 201 creating_bookstore_pattern="^.*Creating openroadm anchors with large.*"
 202 read_datatreees_pattern="^.*Read datatrees using openroadm root.*"
 203 update_datanodes_pattern="^.*Update 100 data nodes.*"
 204 batch_delete_pattern="^.*Batch delete 100 whole lists.*"
 205
 206 all_patterns=("$creating_bookstore_pattern" "$read_datatreees_pattern" "$update_datanodes_pattern" "$batch_delete_pattern")
 207 new_data=""
 208
 209 latestBuild=$(getLatestBuild)
 210 availableBuilds=()
 211 consoleText=""
 212 JENKINS_JOB_URL="https://jenkins.nordix.org/job/onap-cps-master-performance-test-java"
 213
 214 # Create and add header to the file
 215 createAndAddHeader() {
 216     file="$1"
 217     header="$2"
 218     if [[ ! -f "$file" ]]; then
 219         echo "Creating $file"
 220         touch "$file"
 221         echo "Build,$header" > $file
 222     else
 223         echo "$file already exists"
 224     fi
 225 }
 226
 227 # Get latest build
 228 getLatestBuild() {
 229     latestBuild=$(curl -s "${JENKINS_JOB_URL}/lastBuild/api/json" | jq -r '.number')
 230     echo "latest build is $latestBuild"
 231 }
 232
 233 getConsoleText() {
 234     buildToRead=$1
 235     consoleURL="${JENKINS_JOB_URL}/${buildToRead}/consoleText"
 236     consoleText=$(curl -s "$consoleURL")
 237 }
 238
 239 getAndRecordDataResults() {
 240     consoleText=$1
 241     patternToMatch=$2
 242     dataFile=$3
 243     buildNumber=$4
 244     new_data=""
 245     matched_line=""
 246     limit_value=""
 247     took_value=""
 248
 249     # Get and calculate data for plot
 250     if matched_line=$(echo "$consoleText" | grep -o -P "$patternToMatch"); then
 251         echo "Matched line: $matched_line"
 252         limit_value=$(echo "$matched_line" | grep -o -P 'limit\s*\K\d+(,\d+)?' | tr -cd '[:digit:]')
 253         echo $limit_value
 254         took_value=$(echo "$matched_line" | grep -o -P 'took\s*\K\d+(,\d+)?' | tr -cd '[:digit:]')
 255         echo $took_value
 256         percentage=$(($took_value * 100 / $limit_value))
 257
 258         echo "Limit value: $limit_value"
 259         echo "Took value: $took_value"
 260         echo "Percentage: $percentage"
 261
 262     if [ -z "$new_data" ]; then
 263         new_data="$percentage"
 264         else
 265         new_data+=",$percentage"
 266         fi
 267         echo "New data: $new_data"
 268
 269         else
 270         echo "No match found."
 271         fi
 272
 273     # Record result
 274     lastLine=$(tail -n 1 "$dataFile")
 275     newLine="$buildNumber,$new_data"
 276     if [ -z "$new_data" ]; then
 277         echo "No new data"
 278     elif [ $newLine == $lastLine ]; then
 279         echo "Data already exists"
 280     else
 281         echo "$buildNumber,$new_data" >> $dataFile
 282         echo "New data added"
 283     fi
 284 }
 285
 286 buildStaticReport() {
 287     dataFile="$1"  # Get the input file name from the function parameter
 288     chartTitle="$2"
 289
 290     sudo apt-get update
 291     sudo apt-get install -y gnuplot
 292
 293     # Create a temporary Gnuplot script
 294     cat <<EOT > gnuplot_script.gp
 295     set datafile separator ","
 296     set terminal pngcairo size 1500,600
 297     set output "${chartTitle}.png"
 298
 299     # Set X-axis label
 300     set xlabel "Build"
 301
 302     # Read titles from the first line of the CSV
 303     plot '$dataFile' using (column(0)+1):2:xtic(1) with linespoints title "Y values"
 304 EOT
 305
 306     # Run the Gnuplot script
 307     gnuplot gnuplot_script.gp
 308
 309     # Remove the temporary script
 310     rm gnuplot_script.gp
 311 }
 312
 313
 314 buildHTMLReport() {
 315
 316     chartTitle1="$1"
 317     chartFileName1="$2"   # Get the chart file name as the first parameter
 318
 319     chartTitle2="$3"
 320     chartFileName2="$4"
 321
 322     chartTitle3="$5"
 323     chartFileName3="$6"
 324
 325     chartTitle4="$7"
 326     chartFileName4="$8"
 327
 328     reportTitle="$9"     # Get the report title as the second parameter
 329
 330 cat <<EOT > "index.html"
 331 <!DOCTYPE html>
 332 <html>
 333 <head>
 334   <title>$reportTitle</title>
 335 </head>
 336 <body>
 337     <h1 style="text-align: center;">$reportTitle</h1>
 338     <table align="center">
 339         <tr> <!-- First Row -->
 340             <td align="center">
 341                 <figcaption>"$chartTitle1"</figcaption>
 342                 <img src="$chartFileName1" alt="Image 1" width="750" height="300">
 343             </td>
 344             <td align="center" style="padding: 10px;">
 345                 <figcaption>"$chartTitle2"</figcaption>
 346                 <img src="$chartFileName2" alt="Image 2" width="750" height="300">
 347             </td>
 348         </tr>
 349         <tr> <!-- Second Row -->
 350             <td align="center" style="padding: 10px;">
 351                 <figcaption>"$chartTitle3"</figcaption>
 352                 <img src="$chartFileName3" alt="Image 3" width="750" height="300">
 353             </td>
 354             <td align="center" style="padding: 10px;">
 355                 <figcaption>"$chartTitle4"</figcaption>
 356                 <img src="$chartFileName4" alt="Image 4" width="750" height="300">
 357             </td>
 358         </tr>
 359     </table>
 360 </body>
 361 </html>
 362 EOT
 363 }
 364
 365 buildPageReport() {
 366     chartFileName="$1"
 367     reportTitle="$2"
 368     outputFile="$3"
 369     cat <<EOT > "$outputFile"
 370     <!DOCTYPE html>
 371     <html>
 372     <head>
 373     <title>$reportTitle</title>
 374     </head>
 375     <body>
 376         <h1>$reportTitle</h1>
 377         <img src="$chartFileName" alt="Graph Image">
 378     </body>
 379     </html>
 380 EOT
 381 }
 382
 383 buildPlotForCreateOperation() {
 384         buildNumber="$1"
 385     createAndAddHeader "$create_data" "$create_data_title_1"
 386     getAndRecordDataResults "$consoleText" "$creating_bookstore_pattern" "$create_data" "$buildNumber"
 387 }
 388
 389 buildPlotForReadOperation() {
 390         buildNumber="$1"
 391     createAndAddHeader "$read_data" "$read_data_title_1"
 392     getAndRecordDataResults "$consoleText" "$read_datatreees_pattern" "$read_data"  "$buildNumber"
 393 }
 394
 395 buildPlotForUpdateOperation() {
 396         buildNumber="$1"
 397     createAndAddHeader "$update_data" "$update_data_title_1"
 398     getAndRecordDataResults "$consoleText" "$update_datanodes_pattern" "$update_data"  "$buildNumber"
 399 }
 400
 401 buildPlotForBatchOperation() {
 402         buildNumber="$1"
 403     createAndAddHeader "$delete_data" "$delete_data_title_1"
 404     getAndRecordDataResults "$consoleText" "$batch_delete_pattern" "$delete_data"  "$buildNumber"
 405 }
 406
 407 #############################################################################################################################
 408 #############################################################################################################################
 409 #############################################################################################################################
 410 #############################################################################################################################
 411
 412 cd $WORKSPACE
 413
 414 getLatestBuild
 415 if [ -z "$(ls -A)" ]; then
 416         # Calculate the starting value for the loop
 417         startValue=$((latestBuild - 50))
 418
 419         # Start the loop from startValue up to latestBuild
 420         for ((i=startValue; i<=latestBuild; i++)); do
 421             getConsoleText "$i"
 422             buildPlotForCreateOperation "$i"
 423             buildPlotForReadOperation "$i"
 424             buildPlotForUpdateOperation "$i"
 425             buildPlotForBatchOperation "$i"
 426         done
 427             buildStaticReport "$create_data" "createLargeData"
 428             buildStaticReport "$read_data" "readDataTrees"
 429             buildStaticReport "$update_data" "updateDatanodes"
 430             buildStaticReport "$delete_data" "batchDelete"
 431 else
 432     getConsoleText $latestBuild
 433     buildPlotForCreateOperation $latestBuild
 434     buildPlotForReadOperation $latestBuild
 435     buildPlotForUpdateOperation $latestBuild
 436     buildPlotForBatchOperation $latestBuild
 437
 438     buildStaticReport "$create_data" "createLargeData"
 439     buildStaticReport "$read_data" "readDataTrees"
 440     buildStaticReport "$update_data" "updateDatanodes"
 441     buildStaticReport "$delete_data" "batchDelete"
 442 fi
 443
 444
 445 touch index.html createOperation.html readOperation.html updateOperation.html deleteOperation.html
 446
 447 buildHTMLReport \
 448     "Creating openroadm anchors with large data" \
 449     "createLargeData.png" \
 450     "Read datatrees using openroadm root" \
 451     "readDataTrees.png" \
 452     "Update 100 data nodes" \
 453     "updateDatanodes.png" \
 454     "Batch delete 100 whole lists" \
 455     "batchDelete.png" \
 456     "Performance Review"
 457
 458 buildPageReport "createLargeData.png" "Creating openroadm anchors with large data"  "createOperation.html"
 459 buildPageReport "readDataTrees.png" "Read datatrees using openroadm root"  "readOperation.html"
 460 buildPageReport "updateDatanodes.png" "Update 100 data nodes" "updateOperation.html"
 461 buildPageReport "batchDelete.png" "Batch delete 100 whole lists" "deleteOperation.html"
 462
 463
 464 buildPlotForCreateOperation() {
 465         buildNumber="$1"
 466     createAndAddHeader "$create_data" "$create_data_title_1"
 467     getAndRecordDataResults "$consoleText" "$creating_bookstore_pattern" "$create_data" "$buildNumber"
 468 }
 469
 470 buildPlotForReadOperation() {
 471         buildNumber="$1"
 472     createAndAddHeader "$read_data" "$read_data_title_1"
 473     getAndRecordDataResults "$consoleText" "$read_datatreees_pattern" "$read_data"  "$buildNumber"
 474 }
 475
 476 buildPlotForUpdateOperation() {
 477         buildNumber="$1"
 478     createAndAddHeader "$update_data" "$update_data_title_1"
 479     getAndRecordDataResults "$consoleText" "$update_datanodes_pattern" "$update_data"  "$buildNumber"
 480 }
 481
 482 buildPlotForBatchOperation() {
 483         buildNumber="$1"
 484     createAndAddHeader "$delete_data" "$delete_data_title_1"
 485     getAndRecordDataResults "$consoleText" "$batch_delete_pattern" "$delete_data"  "$buildNumber"
 486 }
 487
 488 #############################################################################################################################
 489 #############################################################################################################################
 490 #############################################################################################################################
 491 #############################################################################################################################
 492
 493 cd $WORKSPACE
 494
 495 getLatestBuild
 496 if [ -z "$(ls -A)" ]; then
 497         # Calculate the starting value for the loop
 498         startValue=$((latestBuild - 50))
 499
 500         # Start the loop from startValue up to latestBuild
 501         for ((i=startValue; i<=latestBuild; i++)); do
 502             getConsoleText "$i"
 503             buildPlotForCreateOperation "$i"
 504             buildPlotForReadOperation "$i"
 505             buildPlotForUpdateOperation "$i"
 506             buildPlotForBatchOperation "$i"
 507         done
 508             buildStaticReport "$create_data" "createLargeData"
 509             buildStaticReport "$read_data" "readDataTrees"
 510             buildStaticReport "$update_data" "updateDatanodes"
 511             buildStaticReport "$delete_data" "batchDelete"
 512 else
 513     getConsoleText $latestBuild
 514     buildPlotForCreateOperation $latestBuild
 515     buildPlotForReadOperation $latestBuild
 516     buildPlotForUpdateOperation $latestBuild
 517     buildPlotForBatchOperation $latestBuild
 518
 519     buildStaticReport "$create_data" "createLargeData"
 520     buildStaticReport "$read_data" "readDataTrees"
 521     buildStaticReport "$update_data" "updateDatanodes"
 522     buildStaticReport "$delete_data" "batchDelete"
 523 fi
 524
 525
 526 touch index.html createOperation.html readOperation.html updateOperation.html deleteOperation.html
 527
 528 buildHTMLReport \
 529     "Creating openroadm anchors with large data" \
 530     "createLargeData.png" \
 531     "Read datatrees using openroadm root" \
 532     "readDataTrees.png" \
 533     "Update 100 data nodes" \
 534     "updateDatanodes.png" \
 535     "Batch delete 100 whole lists" \
 536     "batchDelete.png" \
 537     "Performance Review"
 538
 539 buildPageReport "createLargeData.png" "Creating openroadm anchors with large data"  "createOperation.html"
 540 buildPageReport "readDataTrees.png" "Read datatrees using openroadm root"  "readOperation.html"
 541 buildPageReport "updateDatanodes.png" "Update 100 data nodes" "updateOperation.html"
 542 buildPageReport "batchDelete.png" "Batch delete 100 whole lists" "deleteOperation.html"