jjb/onap/cps/prepare-performance-tests-data.sh

   1 #!/bin/bash
   2 sudo apt-get install bc
   3 #############################################################################################################################
   4 ################################################ F U N C T I O N S ##########################################################
   5 #############################################################################################################################
   6
   7 create_data="create_performance_data.txt"
   8 read_data="read_performance_data.txt"
   9 update_data="update_performance_data.txt"
  10 delete_data="delete_performance_data.txt"
  11
  12 # Data files headers
  13 create_data_title_1="Creating openroadm anchors with large data"
  14 read_data_title_1="Read datatrees using openroadm root"
  15 update_data_title_1="Update 100 data nodes"
  16 delete_data_title_1="Batch delete 100 whole lists"
  17
  18 # Text patterns to match in console log
  19 creating_bookstore_pattern="^.*Creating openroadm anchors with large.*"
  20 read_datatreees_pattern="^.*Read datatrees using openroadm root.*"
  21 update_datanodes_pattern="^.*Update 100 data nodes.*"
  22 batch_delete_pattern="^.*Batch delete 100 whole lists.*"
  23
  24 new_data=""
  25 JENKINS_JOB_URL="https://jenkins.nordix.org/job/onap-cps-master-performance-test-java"
  26
  27 # Get latest build
  28 getLatestBuild() {
  29     latestBuild=$(curl -s "${JENKINS_JOB_URL}/lastBuild/api/json" | jq -r '.number')
  30     echo "latest build is $latestBuild"
  31 }
  32
  33 latestBuild=$(getLatestBuild)
  34 latestBuildToRecord=""
  35 consoleText=""
  36 lastRecordedBuild=""
  37 latestRecordedBuild=""
  38 timestampOfLatestRecordedBuild=""
  39
  40 # Create and add header to the file
  41 createAndAddHeader() {
  42     file="$1"
  43     header="$2"
  44     if [[ ! -f "$file" ]]; then
  45         echo "Creating $file"
  46         touch "$file"
  47         echo "Build,$header" > $file
  48     else
  49         echo "$file already exists"
  50     fi
  51 }
  52
  53 # Get the console text from specific build of the performance job
  54 getConsoleText() {
  55     buildToRead=$1
  56     consoleURL="${JENKINS_JOB_URL}/${buildToRead}/consoleText"
  57     consoleText=$(curl -s "$consoleURL")
  58 }
  59
  60 getAndRecordDataResults() {
  61     consoleText=$1
  62     patternToMatch=$2
  63     dataFile=$3
  64     buildNumber=$4
  65     new_data=""
  66     matched_line=""
  67     limit_value=""
  68     took_value=""
  69
  70     # Get and calculate data for plot
  71     if matched_line=$(echo "$consoleText" | grep -o -P "$patternToMatch"); then
  72         echo "Matched line: $matched_line"
  73         limit_value=$(echo "$matched_line" | grep -o -P 'limit\s*\K\d+(\.\d+)?' | tr -cd '[:digit:].')
  74         echo $limit_value
  75         took_value=$(echo "$matched_line" | grep -o -P 'took\s*\K\d+(\.\d+)?' | tr -cd '[:digit:].')
  76         echo $took_value
  77         percentage=$(echo "scale=2; $took_value * 100.00 / $limit_value" | bc)
  78
  79
  80         echo "Limit value: $limit_value"
  81         echo "Took value: $took_value"
  82         echo "Percentage: $percentage"
  83
  84     if [ -z "$new_data" ]; then
  85         new_data="$percentage"
  86         else
  87         new_data+=",$percentage"
  88         fi
  89         echo "New data: $new_data"
  90
  91         else
  92         echo "No match found."
  93         fi
  94
  95     # Record result
  96     lastLine=$(tail -n 1 "$dataFile")
  97     newLine="$buildNumber,$new_data"
  98     if [ -z "$new_data" ]; then
  99         echo "No new data"
 100         lastRecordedBuild=$(echo "$lastLine" | awk -F, '{print $1}')
 101         recordLatestRecordedBuild "$lastRecordedBuild"
 102     elif [ $newLine == $lastLine ]; then
 103           echo "Data already exists"
 104         recordLatestRecordedBuild "$buildNumber"
 105     else
 106         echo "$buildNumber,$new_data" >> $dataFile
 107         echo "New data added"
 108         recordLatestRecordedBuild "$buildNumber"
 109     fi
 110 }
 111
 112 recordLatestRecordedBuild() {
 113         latestBuildToRecord="$1"
 114   timestampOfLatestRecordedBuild=$(curl -s "${JENKINS_JOB_URL}/${latestBuildToRecord}/api/json" | jq -r '.timestamp')
 115         formattedTimestampOfLatestRecordedBuild=$(date -d "@$((timestampOfLatestRecordedBuild / 1000))" "+%B %e, %Y at %H:%M")
 116     echo "Latest recorded build: $latestBuildToRecord"
 117     echo "Date of latest recorded build: $formattedTimestampOfLatestRecordedBuild"
 118     latestRecordedBuild=$latestBuildToRecord
 119 }
 120
 121 buildStaticReport() {
 122     dataFile="$1"  # Get the input file name from the function parameter
 123     chartTitle="$2"
 124
 125     sudo apt-get update
 126     sudo apt-get install -y gnuplot
 127
 128     # Create a temporary Gnuplot script
 129     cat <<EOT > gnuplot_script.gp
 130     set datafile separator ","
 131     set terminal pngcairo size 1500,600
 132     set output "${chartTitle}.png"
 133
 134     # Set X-axis label
 135     set xlabel "Build"
 136
 137     # Set Y-axis label
 138     set ylabel "Percentage of limit %"
 139
 140         set xtics rotate
 141         plot '$dataFile' using (column(0)+1):2:xtic(sprintf("%d", column(1))) with linespoints title "limit"
 142 EOT
 143
 144     # Run the Gnuplot script
 145     gnuplot gnuplot_script.gp
 146
 147     # Remove the temporary script
 148     rm gnuplot_script.gp
 149 }
 150
 151 buildHTMLReport() {
 152
 153     chartTitle1="$1" # i.e Get the chart file name as the first parameter
 154     chartFileName1="$2"
 155
 156     chartTitle2="$3"
 157     chartFileName2="$4"
 158
 159     chartTitle3="$5"
 160     chartFileName3="$6"
 161
 162     chartTitle4="$7"
 163     chartFileName4="$8"
 164
 165     reportTitle="$9"     # i.e Get the report title as the ninth parameter
 166
 167 cat <<EOT > "index.html"
 168 <!DOCTYPE html>
 169 <html>
 170 <head>
 171   <title>$reportTitle</title>
 172 </head>
 173 <body>
 174     <h1 style="text-align: center;">$reportTitle</h1>
 175     <h4>Last updated for performance job build no. $latestRecordedBuild on $formattedTimestampOfLatestRecordedBuild</h4>
 176     <p>The performance tests job runs daily at 02:15 UTC, providing performance metrics. The following graphs update at 04:15 UTC.</p>
 177     <p>Successful performance tests job build adds new data, but even if a build fails, existing data is retained.</p>
 178     <p>Updates occur whenever new successful data is available.</p>
 179
 180     <table align="center">
 181         <tr> <!-- First Row -->
 182             <td align="center">
 183                 <figcaption>"$chartTitle1"</figcaption>
 184                 <img src="$chartFileName1" alt="Image 1" width="750" height="300">
 185             </td>
 186             <td align="center" style="padding: 10px;">
 187                 <figcaption>"$chartTitle2"</figcaption>
 188                 <img src="$chartFileName2" alt="Image 2" width="750" height="300">
 189             </td>
 190         </tr>
 191         <tr> <!-- Second Row -->
 192             <td align="center" style="padding: 10px;">
 193                 <figcaption>"$chartTitle3"</figcaption>
 194                 <img src="$chartFileName3" alt="Image 3" width="750" height="300">
 195             </td>
 196             <td align="center" style="padding: 10px;">
 197                 <figcaption>"$chartTitle4"</figcaption>
 198                 <img src="$chartFileName4" alt="Image 4" width="750" height="300">
 199             </td>
 200         </tr>
 201     </table>
 202 </body>
 203 </html>
 204 EOT
 205 }
 206
 207 buildPageReport() {
 208     chartFileName="$1"
 209     reportTitle="$2"
 210     outputFile="$3"
 211     cat <<EOT > "$outputFile"
 212     <!DOCTYPE html>
 213     <html>
 214     <head>
 215     <title>$reportTitle</title>
 216     </head>
 217     <body>
 218         <h1>$reportTitle</h1>
 219         <h4>Last updated for performance job build no. $latestRecordedBuild on $formattedTimestampOfLatestRecordedBuild</h4>
 220         <img src="$chartFileName" alt="Graph Image">
 221     </body>
 222     </html>
 223 EOT
 224 }
 225
 226 buildPlotForCreateOperation() {
 227         buildNumber="$1"
 228     createAndAddHeader "$create_data" "$create_data_title_1"
 229     getAndRecordDataResults "$consoleText" "$creating_bookstore_pattern" "$create_data" "$buildNumber"
 230 }
 231
 232 buildPlotForReadOperation() {
 233         buildNumber="$1"
 234     createAndAddHeader "$read_data" "$read_data_title_1"
 235     getAndRecordDataResults "$consoleText" "$read_datatreees_pattern" "$read_data"  "$buildNumber"
 236 }
 237
 238 buildPlotForUpdateOperation() {
 239         buildNumber="$1"
 240     createAndAddHeader "$update_data" "$update_data_title_1"
 241     getAndRecordDataResults "$consoleText" "$update_datanodes_pattern" "$update_data"  "$buildNumber"
 242 }
 243
 244 buildPlotForBatchOperation() {
 245         buildNumber="$1"
 246     createAndAddHeader "$delete_data" "$delete_data_title_1"
 247     getAndRecordDataResults "$consoleText" "$batch_delete_pattern" "$delete_data"  "$buildNumber"
 248 }
 249
 250 #############################################################################################################################
 251 #############################################################################################################################
 252 #############################################################################################################################
 253 #############################################################################################################################
 254
 255 cd $WORKSPACE
 256
 257 getLatestBuild
 258 if [ -z "$(ls -A)" ]; then
 259         # Calculate the starting value for the loop
 260         startValue=$((latestBuild - 100))
 261
 262         # Start the loop from startValue up to latestBuild
 263         for ((i=startValue; i<=latestBuild; i++)); do
 264             getConsoleText "$i"
 265             buildPlotForCreateOperation "$i"
 266             buildPlotForReadOperation "$i"
 267             buildPlotForUpdateOperation "$i"
 268             buildPlotForBatchOperation "$i"
 269         done
 270             buildStaticReport "$create_data" "createLargeData"
 271             buildStaticReport "$read_data" "readDataTrees"
 272             buildStaticReport "$update_data" "updateDatanodes"
 273             buildStaticReport "$delete_data" "batchDelete"
 274 else
 275     getConsoleText $latestBuild
 276     buildPlotForCreateOperation $latestBuild
 277     buildPlotForReadOperation $latestBuild
 278     buildPlotForUpdateOperation $latestBuild
 279     buildPlotForBatchOperation $latestBuild
 280
 281     buildStaticReport "$create_data" "createLargeData"
 282     buildStaticReport "$read_data" "readDataTrees"
 283     buildStaticReport "$update_data" "updateDatanodes"
 284     buildStaticReport "$delete_data" "batchDelete"
 285 fi
 286
 287
 288 touch index.html createOperation.html readOperation.html updateOperation.html deleteOperation.html
 289
 290 buildHTMLReport \
 291     "Creating openroadm anchors with large data" \
 292     "createLargeData.png" \
 293     "Read datatrees using openroadm root" \
 294     "readDataTrees.png" \
 295     "Update 100 data nodes" \
 296     "updateDatanodes.png" \
 297     "Batch delete 100 whole lists" \
 298     "batchDelete.png" \
 299     "Performance Review"
 300
 301 buildPageReport "createLargeData.png" "Creating openroadm anchors with large data"  "createOperation.html"
 302 buildPageReport "readDataTrees.png" "Read datatrees using openroadm root"  "readOperation.html"
 303 buildPageReport "updateDatanodes.png" "Update 100 data nodes" "updateOperation.html"
 304 buildPageReport "batchDelete.png" "Batch delete 100 whole lists" "deleteOperation.html"
 305
 306
 307 buildPlotForCreateOperation() {
 308         buildNumber="$1"
 309     createAndAddHeader "$create_data" "$create_data_title_1"
 310     getAndRecordDataResults "$consoleText" "$creating_bookstore_pattern" "$create_data" "$buildNumber"
 311 }
 312
 313 buildPlotForReadOperation() {
 314         buildNumber="$1"
 315     createAndAddHeader "$read_data" "$read_data_title_1"
 316     getAndRecordDataResults "$consoleText" "$read_datatreees_pattern" "$read_data"  "$buildNumber"
 317 }
 318
 319 buildPlotForUpdateOperation() {
 320         buildNumber="$1"
 321     createAndAddHeader "$update_data" "$update_data_title_1"
 322     getAndRecordDataResults "$consoleText" "$update_datanodes_pattern" "$update_data"  "$buildNumber"
 323 }
 324
 325 buildPlotForBatchOperation() {
 326         buildNumber="$1"
 327     createAndAddHeader "$delete_data" "$delete_data_title_1"
 328     getAndRecordDataResults "$consoleText" "$batch_delete_pattern" "$delete_data"  "$buildNumber"
 329 }
 330
 331 #############################################################################################################################
 332 #############################################################################################################################
 333 #############################################################################################################################
 334 #############################################################################################################################
 335
 336 cd $WORKSPACE
 337
 338 getLatestBuild
 339 if [ -z "$(ls -A)" ]; then
 340         # Calculate the starting value for the loop
 341         startValue=$((latestBuild - 50))
 342
 343         # Start the loop from startValue up to latestBuild
 344         for ((i=startValue; i<=latestBuild; i++)); do
 345             getConsoleText "$i"
 346             buildPlotForCreateOperation "$i"
 347             buildPlotForReadOperation "$i"
 348             buildPlotForUpdateOperation "$i"
 349             buildPlotForBatchOperation "$i"
 350         done
 351             buildStaticReport "$create_data" "createLargeData"
 352             buildStaticReport "$read_data" "readDataTrees"
 353             buildStaticReport "$update_data" "updateDatanodes"
 354             buildStaticReport "$delete_data" "batchDelete"
 355 else
 356     getConsoleText $latestBuild
 357     buildPlotForCreateOperation $latestBuild
 358     buildPlotForReadOperation $latestBuild
 359     buildPlotForUpdateOperation $latestBuild
 360     buildPlotForBatchOperation $latestBuild
 361
 362     buildStaticReport "$create_data" "createLargeData"
 363     buildStaticReport "$read_data" "readDataTrees"
 364     buildStaticReport "$update_data" "updateDatanodes"
 365     buildStaticReport "$delete_data" "batchDelete"
 366 fi
 367
 368
 369 touch index.html createOperation.html readOperation.html updateOperation.html deleteOperation.html
 370
 371 buildHTMLReport \
 372     "Creating openroadm anchors with large data" \
 373     "createLargeData.png" \
 374     "Read datatrees using openroadm root" \
 375     "readDataTrees.png" \
 376     "Update 100 data nodes" \
 377     "updateDatanodes.png" \
 378     "Batch delete 100 whole lists" \
 379     "batchDelete.png" \
 380     "Performance Review"
 381
 382 buildPageReport "createLargeData.png" "Creating openroadm anchors with large data"  "createOperation.html"
 383 buildPageReport "readDataTrees.png" "Read datatrees using openroadm root"  "readOperation.html"
 384 buildPageReport "updateDatanodes.png" "Update 100 data nodes" "updateOperation.html"
 385 buildPageReport "batchDelete.png" "Batch delete 100 whole lists" "deleteOperation.html"
 386
 387 #############################################################################################################################
 388 ##################################################### D A T A ###############################################################
 389 #############################################################################################################################
 390
 391 cat create_performance_data.txt
 392 cat read_performance_data.txt
 393 cat update_performance_data.txt
 394 cat delete_performance_data.txt